From 470bd2e4ca9ecde204ed05f231af195a6ef734b4 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 29 Jul 2021 18:35:19 +0800 Subject: [PATCH 01/37] HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic Signed-off-by: Wellington Chevreuil --- .../hbase/mob/DefaultMobStoreCompactor.java | 22 +- .../hbase/mob/DefaultMobStoreFlusher.java | 4 +- .../CreateStoreFileWriterParams.java | 134 ++++ .../regionserver/DateTieredStoreEngine.java | 5 +- .../regionserver/DefaultStoreEngine.java | 5 +- .../regionserver/DefaultStoreFlusher.java | 11 +- .../hadoop/hbase/regionserver/HMobStore.java | 3 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +- .../hadoop/hbase/regionserver/HStore.java | 699 ++++-------------- .../hbase/regionserver/StoreContext.java | 9 + .../hbase/regionserver/StoreEngine.java | 438 ++++++++++- .../hbase/regionserver/StoreFileManager.java | 9 + .../hbase/regionserver/StoreFlusher.java | 9 +- .../hadoop/hbase/regionserver/StoreUtils.java | 36 +- .../hbase/regionserver/StripeStoreEngine.java | 9 +- .../regionserver/StripeStoreFlusher.java | 9 +- .../AbstractMultiOutputCompactor.java | 7 +- .../regionserver/compactions/Compactor.java | 36 +- .../compactions/DefaultCompactor.java | 16 +- .../DefaultStoreFileTracker.java | 61 ++ .../storefiletracker/StoreFileTracker.java | 75 ++ .../StoreFileTrackerBase.java | 178 +++++ .../StoreFileTrackerFactory.java | 35 + .../compaction/MajorCompactionRequest.java | 1 - .../apache/hadoop/hbase/TestIOFencing.java | 12 +- .../TestCacheOnWriteInSchema.java | 6 +- .../regionserver/TestDefaultStoreEngine.java | 5 +- .../hbase/regionserver/TestHRegion.java | 4 +- .../hadoop/hbase/regionserver/TestHStore.java | 28 +- .../TestRegionMergeTransactionOnCluster.java | 6 +- .../TestStoreFileRefresherChore.java | 3 +- .../regionserver/TestStoreScannerClosure.java | 6 +- .../regionserver/TestStripeStoreEngine.java | 2 + .../compactions/TestDateTieredCompactor.java | 12 +- .../TestStripeCompactionPolicy.java | 12 +- .../compactions/TestStripeCompactor.java | 12 +- 36 files changed, 1224 insertions(+), 705 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java index c45fdff7ca16..01fe0005f048 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map.Entry; import java.util.Optional; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -144,17 +143,16 @@ public InternalScanner createScanner(ScanInfo scanInfo, List s }; private final CellSinkFactory writerFactory = - new CellSinkFactory() { - @Override - public StoreFileWriter createWriter(InternalScanner scanner, - org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, - boolean shouldDropBehind, boolean major) throws IOException { - // make this writer with tags always because of possible new cells with tags. - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, true, true, shouldDropBehind); - } - }; + new CellSinkFactory() { + @Override + public StoreFileWriter createWriter(InternalScanner scanner, + org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, + boolean shouldDropBehind, boolean major) throws IOException { + // make this writer with tags always because of possible new cells with tags. + return store.getStoreEngine().createWriter( + createParams(fd, shouldDropBehind, major).includeMVCCReadpoint(true).includesTag(true)); + } + }; public DefaultMobStoreCompactor(Configuration conf, HStore store) { super(conf, store); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java index 480b85c58dfe..4a1dc7b33a51 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java @@ -25,7 +25,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -127,8 +126,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId, synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk - writer = store.createWriterInTmp(cellsCount, store.getColumnFamilyDescriptor().getCompressionType(), - false, true, true, false); + writer = createWriter(snapshot, true); IOException e = null; try { // It's a mob store, flush the cells in a mob way. This is the difference of flushing diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java new file mode 100644 index 000000000000..10cd9f009e4a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public final class CreateStoreFileWriterParams { + + private long maxKeyCount; + + private Compression.Algorithm compression; + + private boolean isCompaction; + + private boolean includeMVCCReadpoint; + + private boolean includesTag; + + private boolean shouldDropBehind; + + private long totalCompactedFilesSize = -1; + + private String fileStoragePolicy = HConstants.EMPTY_STRING; + + private CreateStoreFileWriterParams() { + } + + public long maxKeyCount() { + return maxKeyCount; + } + + public CreateStoreFileWriterParams maxKeyCount(long maxKeyCount) { + this.maxKeyCount = maxKeyCount; + return this; + } + + public Compression.Algorithm compression() { + return compression; + } + + /** + * Set the compression algorithm to use + */ + public CreateStoreFileWriterParams compression(Compression.Algorithm compression) { + this.compression = compression; + return this; + } + + public boolean isCompaction() { + return isCompaction; + } + + /** + * Whether we are creating a new file in a compaction + */ + public CreateStoreFileWriterParams isCompaction(boolean isCompaction) { + this.isCompaction = isCompaction; + return this; + } + + public boolean includeMVCCReadpoint() { + return includeMVCCReadpoint; + } + + /** + * Whether to include MVCC or not + */ + public CreateStoreFileWriterParams includeMVCCReadpoint(boolean includeMVCCReadpoint) { + this.includeMVCCReadpoint = includeMVCCReadpoint; + return this; + } + + public boolean includesTag() { + return includesTag; + } + + /** + * Whether to includesTag or not + */ + public CreateStoreFileWriterParams includesTag(boolean includesTag) { + this.includesTag = includesTag; + return this; + } + + public boolean shouldDropBehind() { + return shouldDropBehind; + } + + public CreateStoreFileWriterParams shouldDropBehind(boolean shouldDropBehind) { + this.shouldDropBehind = shouldDropBehind; + return this; + } + + public long totalCompactedFilesSize() { + return totalCompactedFilesSize; + } + + public CreateStoreFileWriterParams totalCompactedFilesSize(long totalCompactedFilesSize) { + this.totalCompactedFilesSize = totalCompactedFilesSize; + return this; + } + + public String fileStoragePolicy() { + return fileStoragePolicy; + } + + public CreateStoreFileWriterParams fileStoragePolicy(String fileStoragePolicy) { + this.fileStoragePolicy = fileStoragePolicy; + return this; + } + + public static CreateStoreFileWriterParams create() { + return new CreateStoreFileWriterParams(); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index 1df953d93c96..7422d9112eab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -19,18 +19,17 @@ import java.io.IOException; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.yetus.audience.InterfaceAudience; /** * HBASE-15400 This store engine allows us to store data in date tiered layout with exponential diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java index 58f8bbbb6ac5..693b9c93b9fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; @@ -39,8 +38,8 @@ * their derivatives. */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class DefaultStoreEngine extends StoreEngine< - DefaultStoreFlusher, RatioBasedCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> { +public class DefaultStoreEngine extends StoreEngine { public static final String DEFAULT_STORE_FLUSHER_CLASS_KEY = "hbase.hstore.defaultengine.storeflusher.class"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java index a7d7fb1f3d56..306760d7ce6a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java @@ -21,15 +21,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.util.StringUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Default implementation of StoreFlusher. @@ -60,9 +59,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId, synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk - writer = store.createWriterInTmp(cellsCount, - store.getColumnFamilyDescriptor().getCompressionType(), false, true, - snapshot.isTagsPresent(), false); + writer = createWriter(snapshot, false); IOException e = null; try { performFlush(scanner, writer, throughputController); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index 7ce7f0310c7d..b00a50c522fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -28,7 +28,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -158,7 +157,7 @@ protected KeyValueScanner createScanner(Scan scan, ScanInfo scanInfo, protected StoreEngine createStoreEngine(HStore store, Configuration conf, CellComparator cellComparator) throws IOException { MobStoreEngine engine = new MobStoreEngine(); - engine.createComponents(conf, store, cellComparator); + engine.createComponentsOnce(conf, store, cellComparator); return engine; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 5b26f9aa8ab7..6ab9b0fc3fe8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -142,7 +142,7 @@ public Path getRegionDir() { // Temp Helpers // =========================================================================== /** @return {@link Path} to the region's temp directory, used for file creations */ - Path getTempDir() { + public Path getTempDir() { return new Path(getRegionDir(), REGION_TEMP_DIR); } @@ -237,11 +237,7 @@ public String getStoragePolicyName(String familyName) { * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ - public Collection getStoreFiles(final byte[] familyName) throws IOException { - return getStoreFiles(Bytes.toString(familyName)); - } - - public Collection getStoreFiles(final String familyName) throws IOException { + public List getStoreFiles(final String familyName) throws IOException { return getStoreFiles(familyName, true); } @@ -251,7 +247,7 @@ public Collection getStoreFiles(final String familyName) throws I * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ - public Collection getStoreFiles(final String familyName, final boolean validate) + public List getStoreFiles(final String familyName, final boolean validate) throws IOException { Path familyDir = getStoreDir(familyName); FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index c4e34beca406..34365b944612 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; @@ -47,8 +48,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -70,17 +69,12 @@ import org.apache.hadoop.hbase.conf.ConfigurationManager; import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContext; -import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.InvalidHFileException; -import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.quotas.RegionSizeStore; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; @@ -110,7 +104,6 @@ import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; import org.apache.hbase.thirdparty.org.apache.commons.collections4.IterableUtils; @@ -165,16 +158,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, private boolean cacheOnWriteLogged; - /** - * RWLock for store operations. - * Locked in shared mode when the list of component stores is looked at: - * - all reads/writes to table data - * - checking for split - * Locked in exclusive mode when the list of component stores is modified: - * - closing - * - completing a compaction - */ - final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); /** * Lock specific to archiving compacted store files. This avoids races around * the combination of retrieving the list of compacted files and moving them to @@ -290,14 +273,8 @@ protected HStore(final HRegion region, final ColumnFamilyDescriptor family, } this.storeEngine = createStoreEngine(this, this.conf, region.getCellComparator()); - List hStoreFiles = loadStoreFiles(warmup); - // Move the storeSize calculation out of loadStoreFiles() method, because the secondary read - // replica's refreshStoreFiles() will also use loadStoreFiles() to refresh its store files and - // update the storeSize in the refreshStoreSizeAndTotalBytes() finally (just like compaction) , so - // no need calculate the storeSize twice. - this.storeSize.addAndGet(getStorefilesSize(hStoreFiles, sf -> true)); - this.totalUncompressedBytes.addAndGet(getTotalUncompressedBytes(hStoreFiles)); - this.storeEngine.getStoreFileManager().loadFiles(hStoreFiles); + storeEngine.initialize(warmup); + refreshStoreSizeAndTotalBytes(); flushRetriesNumber = conf.getInt( "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER); @@ -513,102 +490,18 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) { this.dataBlockEncoder = blockEncoder; } - /** - * Creates an unsorted list of StoreFile loaded in parallel - * from the given directory. - */ - private List loadStoreFiles(boolean warmup) throws IOException { - Collection files = getRegionFileSystem().getStoreFiles(getColumnFamilyName()); - return openStoreFiles(files, warmup); - } - - private List openStoreFiles(Collection files, boolean warmup) - throws IOException { - if (CollectionUtils.isEmpty(files)) { - return Collections.emptyList(); - } - // initialize the thread pool for opening store files in parallel.. - ThreadPoolExecutor storeFileOpenerThreadPool = - this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-" - + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - CompletionService completionService = - new ExecutorCompletionService<>(storeFileOpenerThreadPool); - - int totalValidStoreFile = 0; - for (StoreFileInfo storeFileInfo : files) { - // open each store file in parallel - completionService.submit(() -> this.createStoreFileAndReader(storeFileInfo)); - totalValidStoreFile++; - } - - Set compactedStoreFiles = new HashSet<>(); - ArrayList results = new ArrayList<>(files.size()); - IOException ioe = null; - try { - for (int i = 0; i < totalValidStoreFile; i++) { - try { - HStoreFile storeFile = completionService.take().get(); - if (storeFile != null) { - LOG.debug("loaded {}", storeFile); - results.add(storeFile); - compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); - } - } catch (InterruptedException e) { - if (ioe == null) { - ioe = new InterruptedIOException(e.getMessage()); - } - } catch (ExecutionException e) { - if (ioe == null) { - ioe = new IOException(e.getCause()); - } - } - } - } finally { - storeFileOpenerThreadPool.shutdownNow(); - } - if (ioe != null) { - // close StoreFile readers - boolean evictOnClose = - getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; - for (HStoreFile file : results) { - try { - if (file != null) { - file.closeStoreFile(evictOnClose); - } - } catch (IOException e) { - LOG.warn("Could not close store file {}", file, e); - } - } - throw ioe; - } - - // Should not archive the compacted store files when region warmup. See HBASE-22163. - if (!warmup) { - // Remove the compacted files from result - List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); - for (HStoreFile storeFile : results) { - if (compactedStoreFiles.contains(storeFile.getPath().getName())) { - LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); - storeFile.getReader().close(storeFile.getCacheConf() != null ? - storeFile.getCacheConf().shouldEvictOnClose() : true); - filesToRemove.add(storeFile); - } - } - results.removeAll(filesToRemove); - if (!filesToRemove.isEmpty() && this.isPrimaryReplicaStore()) { - LOG.debug("Moving the files {} to archive", filesToRemove); - getRegionFileSystem().removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), - filesToRemove); - } - } - - return results; + private void postRefreshStoreFiles() throws IOException { + // Advance the memstore read point to be at least the new store files seqIds so that + // readers might pick it up. This assumes that the store is not getting any writes (otherwise + // in-flight transactions might be made visible) + getMaxSequenceId().ifPresent(region.getMVCC()::advanceTo); + refreshStoreSizeAndTotalBytes(); } @Override public void refreshStoreFiles() throws IOException { - Collection newFiles = getRegionFileSystem().getStoreFiles(getColumnFamilyName()); - refreshStoreFilesInternal(newFiles); + storeEngine.refreshStoreFiles(); + postRefreshStoreFiles(); } /** @@ -616,89 +509,8 @@ public void refreshStoreFiles() throws IOException { * region replicas to keep up to date with the primary region files. */ public void refreshStoreFiles(Collection newFiles) throws IOException { - List storeFiles = new ArrayList<>(newFiles.size()); - for (String file : newFiles) { - storeFiles.add(getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file)); - } - refreshStoreFilesInternal(storeFiles); - } - - /** - * Checks the underlying store files, and opens the files that have not - * been opened, and removes the store file readers for store files no longer - * available. Mainly used by secondary region replicas to keep up to date with - * the primary region files. - */ - private void refreshStoreFilesInternal(Collection newFiles) throws IOException { - StoreFileManager sfm = storeEngine.getStoreFileManager(); - Collection currentFiles = sfm.getStorefiles(); - Collection compactedFiles = sfm.getCompactedfiles(); - if (currentFiles == null) { - currentFiles = Collections.emptySet(); - } - if (newFiles == null) { - newFiles = Collections.emptySet(); - } - if (compactedFiles == null) { - compactedFiles = Collections.emptySet(); - } - - HashMap currentFilesSet = new HashMap<>(currentFiles.size()); - for (HStoreFile sf : currentFiles) { - currentFilesSet.put(sf.getFileInfo(), sf); - } - HashMap compactedFilesSet = new HashMap<>(compactedFiles.size()); - for (HStoreFile sf : compactedFiles) { - compactedFilesSet.put(sf.getFileInfo(), sf); - } - - Set newFilesSet = new HashSet(newFiles); - // Exclude the files that have already been compacted - newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); - Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); - Set toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet); - - if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) { - return; - } - - LOG.info("Refreshing store files for " + this + " files to add: " - + toBeAddedFiles + " files to remove: " + toBeRemovedFiles); - - Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); - for (StoreFileInfo sfi : toBeRemovedFiles) { - toBeRemovedStoreFiles.add(currentFilesSet.get(sfi)); - } - - // try to open the files - List openedFiles = openStoreFiles(toBeAddedFiles, false); - - // propogate the file changes to the underlying store file manager - replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception - - // Advance the memstore read point to be at least the new store files seqIds so that - // readers might pick it up. This assumes that the store is not getting any writes (otherwise - // in-flight transactions might be made visible) - if (!toBeAddedFiles.isEmpty()) { - // we must have the max sequence id here as we do have several store files - region.getMVCC().advanceTo(this.getMaxSequenceId().getAsLong()); - } - - refreshStoreSizeAndTotalBytes(); - } - - protected HStoreFile createStoreFileAndReader(final Path p) throws IOException { - StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), - p, isPrimaryReplicaStore()); - return createStoreFileAndReader(info); - } - - private HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { - info.setRegionCoprocessorHost(this.region.getCoprocessorHost()); - HStoreFile storeFile = new HStoreFile(info, getColumnFamilyDescriptor().getBloomFilterType(), - getCacheConfig()); - storeFile.initReader(); - return storeFile; + storeEngine.refreshStoreFiles(newFiles); + postRefreshStoreFiles(); } /** @@ -721,7 +533,7 @@ public void stopReplayingFromWAL(){ * Adds a value to the memstore */ public void add(final Cell cell, MemStoreSizing memstoreSizing) { - lock.readLock().lock(); + storeEngine.readLock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace("tableName={}, encodedName={}, columnFamilyName={} is too busy!", @@ -729,7 +541,7 @@ public void add(final Cell cell, MemStoreSizing memstoreSizing) { } this.memstore.add(cell, memstoreSizing); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); currentParallelPutCount.decrementAndGet(); } } @@ -738,7 +550,7 @@ public void add(final Cell cell, MemStoreSizing memstoreSizing) { * Adds the specified value to the memstore */ public void add(final Iterable cells, MemStoreSizing memstoreSizing) { - lock.readLock().lock(); + storeEngine.readLock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace("tableName={}, encodedName={}, columnFamilyName={} is too busy!", @@ -746,7 +558,7 @@ public void add(final Iterable cells, MemStoreSizing memstoreSizing) { } memstore.add(cells, memstoreSizing); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); currentParallelPutCount.decrementAndGet(); } } @@ -869,17 +681,16 @@ public Path bulkLoadHFile(byte[] family, String srcPathStr, Path dstPath) throws LOG.info("Loaded HFile " + srcPath + " into " + this + " as " + dstPath + " - updating store file list."); - HStoreFile sf = createStoreFileAndReader(dstPath); + HStoreFile sf = storeEngine.createStoreFileAndReader(dstPath); bulkLoadHFile(sf); - LOG.info("Successfully loaded {} into {} (new location: {})", - srcPath, this, dstPath); + LOG.info("Successfully loaded {} into {} (new location: {})", srcPath, this, dstPath); return dstPath; } public void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException { - HStoreFile sf = createStoreFileAndReader(fileInfo); + HStoreFile sf = storeEngine.createStoreFileAndReader(fileInfo); bulkLoadHFile(sf); } @@ -887,28 +698,74 @@ private void bulkLoadHFile(HStoreFile sf) throws IOException { StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); - - // Append the new storefile into the list - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); - } finally { - // We need the lock, as long as we are updating the storeFiles - // or changing the memstore. Let us release it before calling - // notifyChangeReadersObservers. See HBASE-4485 for a possible - // deadlock scenario that could have happened if continue to hold - // the lock. - this.lock.writeLock().unlock(); - } + storeEngine.addStoreFiles(Lists.newArrayList(sf)); LOG.info("Loaded HFile " + sf.getFileInfo() + " into " + this); if (LOG.isTraceEnabled()) { - String traceMessage = "BULK LOAD time,size,store size,store files [" - + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize - + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; + String traceMessage = "BULK LOAD time,size,store size,store files [" + + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize + "," + + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage); } } + private ImmutableCollection closeWithoutLock() throws IOException { + // Clear so metrics doesn't find them. + ImmutableCollection result = storeEngine.getStoreFileManager().clearFiles(); + Collection compactedfiles = storeEngine.getStoreFileManager().clearCompactedFiles(); + // clear the compacted files + if (CollectionUtils.isNotEmpty(compactedfiles)) { + removeCompactedfiles(compactedfiles, + getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true); + } + if (!result.isEmpty()) { + // initialize the thread pool for closing store files in parallel. + ThreadPoolExecutor storeFileCloserThreadPool = + this.region.getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" + + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); + + // close each store file in parallel + CompletionService completionService = + new ExecutorCompletionService<>(storeFileCloserThreadPool); + for (HStoreFile f : result) { + completionService.submit(new Callable() { + @Override + public Void call() throws IOException { + boolean evictOnClose = + getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; + f.closeStoreFile(evictOnClose); + return null; + } + }); + } + + IOException ioe = null; + try { + for (int i = 0; i < result.size(); i++) { + try { + Future future = completionService.take(); + future.get(); + } catch (InterruptedException e) { + if (ioe == null) { + ioe = new InterruptedIOException(); + ioe.initCause(e); + } + } catch (ExecutionException e) { + if (ioe == null) { + ioe = new IOException(e.getCause()); + } + } + } + } finally { + storeFileCloserThreadPool.shutdownNow(); + } + if (ioe != null) { + throw ioe; + } + } + LOG.trace("Closed {}", this); + return result; + } + /** * Close all the readers We don't need to worry about subsequent requests because the Region holds * a write lock that will prevent any more reads or writes. @@ -916,67 +773,18 @@ private void bulkLoadHFile(HStoreFile sf) throws IOException { * @throws IOException on failure */ public ImmutableCollection close() throws IOException { + // findbugs can not recognize storeEngine.writeLock is just a lock operation so it will report + // UL_UNRELEASED_LOCK_EXCEPTION_PATH, so here we have to use two try finally... + // Change later if findbugs becomes smarter in the future. this.archiveLock.lock(); - this.lock.writeLock().lock(); try { - // Clear so metrics doesn't find them. - ImmutableCollection result = storeEngine.getStoreFileManager().clearFiles(); - Collection compactedfiles = - storeEngine.getStoreFileManager().clearCompactedFiles(); - // clear the compacted files - if (CollectionUtils.isNotEmpty(compactedfiles)) { - removeCompactedfiles(compactedfiles, getCacheConfig() != null ? - getCacheConfig().shouldEvictOnClose() : true); - } - if (!result.isEmpty()) { - // initialize the thread pool for closing store files in parallel. - ThreadPoolExecutor storeFileCloserThreadPool = this.region - .getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" - + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - - // close each store file in parallel - CompletionService completionService = - new ExecutorCompletionService<>(storeFileCloserThreadPool); - for (HStoreFile f : result) { - completionService.submit(new Callable() { - @Override - public Void call() throws IOException { - boolean evictOnClose = - getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; - f.closeStoreFile(evictOnClose); - return null; - } - }); - } - - IOException ioe = null; - try { - for (int i = 0; i < result.size(); i++) { - try { - Future future = completionService.take(); - future.get(); - } catch (InterruptedException e) { - if (ioe == null) { - ioe = new InterruptedIOException(); - ioe.initCause(e); - } - } catch (ExecutionException e) { - if (ioe == null) { - ioe = new IOException(e.getCause()); - } - } - } - } finally { - storeFileCloserThreadPool.shutdownNow(); - } - if (ioe != null) { - throw ioe; - } + this.storeEngine.writeLock(); + try { + return closeWithoutLock(); + } finally { + this.storeEngine.writeUnlock(); } - LOG.trace("Closed {}", this); - return result; } finally { - this.lock.writeLock().unlock(); this.archiveLock.unlock(); } } @@ -1006,7 +814,7 @@ protected List flushCache(final long logCacheFlushId, MemStoreSnapshot sna try { for (Path pathName : pathNames) { lastPathName = pathName; - validateStoreFile(pathName); + storeEngine.validateStoreFile(pathName); } return pathNames; } catch (Exception e) { @@ -1052,167 +860,18 @@ public HStoreFile tryCommitRecoveredHFile(Path path) throws IOException { } Path dstPath = getRegionFileSystem().commitStoreFile(getColumnFamilyName(), path); - HStoreFile sf = createStoreFileAndReader(dstPath); + HStoreFile sf = storeEngine.createStoreFileAndReader(dstPath); StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); - } finally { - this.lock.writeLock().unlock(); - } + storeEngine.addStoreFiles(Lists.newArrayList(sf)); LOG.info("Loaded recovered hfile to {}, entries={}, sequenceid={}, filesize={}", sf, r.getEntries(), r.getSequenceID(), TraditionalBinaryPrefix.long2String(r.length(), "B", 1)); return sf; } - /** - * Commit the given {@code files}. - *

- * We will move the file into data directory, and open it. - * @param files the files want to commit - * @param validate whether to validate the store files - * @return the committed store files - */ - private List commitStoreFiles(List files, boolean validate) throws IOException { - List committedFiles = new ArrayList<>(files.size()); - HRegionFileSystem hfs = getRegionFileSystem(); - String familyName = getColumnFamilyName(); - for (Path file : files) { - try { - if (validate) { - validateStoreFile(file); - } - Path committedPath = hfs.commitStoreFile(familyName, file); - HStoreFile sf = createStoreFileAndReader(committedPath); - committedFiles.add(sf); - } catch (IOException e) { - LOG.error("Failed to commit store file {}", file, e); - // Try to delete the files we have committed before. - // It is OK to fail when deleting as leaving the file there does not cause any data - // corruption problem. It just introduces some duplicated data which may impact read - // performance a little when reading before compaction. - for (HStoreFile sf : committedFiles) { - Path pathToDelete = sf.getPath(); - try { - sf.deleteStoreFile(); - } catch (IOException deleteEx) { - LOG.warn(HBaseMarkers.FATAL, "Failed to delete committed store file {}", pathToDelete, - deleteEx); - } - } - throw new IOException("Failed to commit the flush", e); - } - } - return committedFiles; - } - - public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, - boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, - boolean shouldDropBehind) throws IOException { - return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint, - includesTag, shouldDropBehind, -1, HConstants.EMPTY_STRING); - } - - /** - * @param compression Compression algorithm to use - * @param isCompaction whether we are creating a new file in a compaction - * @param includeMVCCReadpoint - whether to include MVCC or not - * @param includesTag - includesTag or not - * @return Writer for a new StoreFile in the tmp dir. - */ - // TODO : allow the Writer factory to create Writers of ShipperListener type only in case of - // compaction - public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, - boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, - boolean shouldDropBehind, long totalCompactedFilesSize, String fileStoragePolicy) - throws IOException { - // creating new cache config for each new writer - final CacheConfig cacheConf = getCacheConfig(); - final CacheConfig writerCacheConf = new CacheConfig(cacheConf); - if (isCompaction) { - // Don't cache data on write on compactions, unless specifically configured to do so - // Cache only when total file size remains lower than configured threshold - final boolean cacheCompactedBlocksOnWrite = - getCacheConfig().shouldCacheCompactedBlocksOnWrite(); - // if data blocks are to be cached on write - // during compaction, we should forcefully - // cache index and bloom blocks as well - if (cacheCompactedBlocksOnWrite && totalCompactedFilesSize <= cacheConf - .getCacheCompactedBlocksOnWriteThreshold()) { - writerCacheConf.enableCacheOnWrite(); - if (!cacheOnWriteLogged) { - LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " + - "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this); - cacheOnWriteLogged = true; - } - } else { - writerCacheConf.setCacheDataOnWrite(false); - if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { - // checking condition once again for logging - LOG.debug( - "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted " - + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}", - this, totalCompactedFilesSize, - cacheConf.getCacheCompactedBlocksOnWriteThreshold()); - } - } - } else { - final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite(); - if (shouldCacheDataOnWrite) { - writerCacheConf.enableCacheOnWrite(); - if (!cacheOnWriteLogged) { - LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " + - "Index blocks and Bloom filter blocks", this); - cacheOnWriteLogged = true; - } - } - } - Encryption.Context encryptionContext = storeContext.getEncryptionContext(); - HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag, - encryptionContext); - Path familyTempDir = new Path(getRegionFileSystem().getTempDir(), getColumnFamilyName()); - StoreFileWriter.Builder builder = - new StoreFileWriter.Builder(conf, writerCacheConf, getFileSystem()) - .withOutputDir(familyTempDir) - .withBloomType(storeContext.getBloomFilterType()) - .withMaxKeyCount(maxKeyCount) - .withFavoredNodes(storeContext.getFavoredNodes()) - .withFileContext(hFileContext) - .withShouldDropCacheBehind(shouldDropBehind) - .withCompactedFilesSupplier(storeContext.getCompactedFilesSupplier()) - .withFileStoragePolicy(fileStoragePolicy); - return builder.build(); - } - - private HFileContext createFileContext(Compression.Algorithm compression, - boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { - if (compression == null) { - compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; - } - ColumnFamilyDescriptor family = getColumnFamilyDescriptor(); - HFileContext hFileContext = new HFileContextBuilder() - .withIncludesMvcc(includeMVCCReadpoint) - .withIncludesTags(includesTag) - .withCompression(compression) - .withCompressTags(family.isCompressTags()) - .withChecksumType(StoreUtils.getChecksumType(conf)) - .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)) - .withBlockSize(family.getBlocksize()) - .withHBaseCheckSum(true) - .withDataBlockEncoding(family.getDataBlockEncoding()) - .withEncryptionContext(encryptionContext) - .withCreateTime(EnvironmentEdgeManager.currentTime()) - .withColumnFamily(getColumnFamilyDescriptor().getName()) - .withTableName(getTableName().getName()) - .withCellComparator(getComparator()) - .build(); - return hFileContext; - } - private long getTotalSize(Collection sfs) { return sfs.stream().mapToLong(sf -> sf.getReader().length()).sum(); } @@ -1222,18 +881,8 @@ private long getTotalSize(Collection sfs) { * @param sfs Store files * @return Whether compaction is required. */ - private boolean updateStorefiles(List sfs, long snapshotId) throws IOException { - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(sfs); - } finally { - // We need the lock, as long as we are updating the storeFiles - // or changing the memstore. Let us release it before calling - // notifyChangeReadersObservers. See HBASE-4485 for a possible - // deadlock scenario that could have happened if continue to hold - // the lock. - this.lock.writeLock().unlock(); - } + private boolean completeFlush(List sfs, long snapshotId) throws IOException { + storeEngine.addStoreFiles(sfs); // We do not need to call clearSnapshot method inside the write lock. // The clearSnapshot itself is thread safe, which can be called at the same time with other // memstore operations expect snapshot and clearSnapshot. And for these two methods, in HRegion @@ -1259,11 +908,11 @@ private boolean updateStorefiles(List sfs, long snapshotId) throws I private void notifyChangedReadersObservers(List sfs) throws IOException { for (ChangedReadersObserver o : this.changedReaderObservers) { List memStoreScanners; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { memStoreScanners = this.memstore.getScanners(o.getReadPoint()); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } o.updateReaders(sfs, memStoreScanners); } @@ -1305,13 +954,13 @@ public List getScanners(boolean cacheBlocks, boolean usePread, byte[] stopRow, boolean includeStopRow, long readPt) throws IOException { Collection storeFilesToScan; List memStoreScanners; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { storeFilesToScan = this.storeEngine.getStoreFileManager().getFilesForScan(startRow, includeStartRow, stopRow, includeStopRow); memStoreScanners = this.memstore.getScanners(readPt); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } try { @@ -1388,11 +1037,11 @@ public List getScanners(List files, boolean cacheBl boolean includeMemstoreScanner) throws IOException { List memStoreScanners = null; if (includeMemstoreScanner) { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { memStoreScanners = this.memstore.getScanners(readPt); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } try { @@ -1508,14 +1157,13 @@ protected List doCompaction(CompactionRequestImpl cr, List newFiles) throws IOException { // Do the steps necessary to complete the compaction. setStoragePolicyFromFileName(newFiles); - List sfs = commitStoreFiles(newFiles, true); + List sfs = storeEngine.commitStoreFiles(newFiles, true); if (this.getCoprocessorHost() != null) { for (HStoreFile sf : sfs) { getCoprocessorHost().postCompact(this, sf, cr.getTracker(), cr, user); } } - writeCompactionWalRecord(filesToCompact, sfs); - replaceStoreFiles(filesToCompact, sfs); + replaceStoreFiles(filesToCompact, sfs, true); if (cr.isMajor()) { majorCompactedCellsCount.addAndGet(getCompactionProgress().getTotalCompactingKVs()); majorCompactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize); @@ -1579,25 +1227,24 @@ private void writeCompactionWalRecord(Collection filesCompacted, this.region.getRegionInfo(), compactionDescriptor, this.region.getMVCC()); } - void replaceStoreFiles(Collection compactedFiles, Collection result) - throws IOException { - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result); - synchronized (filesCompacting) { - filesCompacting.removeAll(compactedFiles); - } - - // These may be null when the RS is shutting down. The space quota Chores will fix the Region - // sizes later so it's not super-critical if we miss these. - RegionServerServices rsServices = region.getRegionServerServices(); - if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { - updateSpaceQuotaAfterFileReplacement( - rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), - compactedFiles, result); - } - } finally { - this.lock.writeLock().unlock(); + @RestrictedApi(explanation = "Should only be called in TestHStore", link = "", + allowedOnPath = ".*/(HStore|TestHStore).java") + void replaceStoreFiles(Collection compactedFiles, Collection result, + boolean writeCompactionMarker) throws IOException { + storeEngine.replaceStoreFiles(compactedFiles, result); + if (writeCompactionMarker) { + writeCompactionWalRecord(compactedFiles, result); + } + synchronized (filesCompacting) { + filesCompacting.removeAll(compactedFiles); + } + // These may be null when the RS is shutting down. The space quota Chores will fix the Region + // sizes later so it's not super-critical if we miss these. + RegionServerServices rsServices = region.getRegionServerServices(); + if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { + updateSpaceQuotaAfterFileReplacement( + rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), + compactedFiles, result); } } @@ -1720,7 +1367,7 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick for (String compactionOutput : compactionOutputs) { StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput); - HStoreFile storeFile = createStoreFileAndReader(storeFileInfo); + HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); outputStoreFiles.add(storeFile); } } @@ -1728,7 +1375,7 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) { LOG.info("Replaying compaction marker, replacing input files: " + inputStoreFiles + " with output files : " + outputStoreFiles); - this.replaceStoreFiles(inputStoreFiles, outputStoreFiles); + this.replaceStoreFiles(inputStoreFiles, outputStoreFiles, false); this.refreshStoreSizeAndTotalBytes(); } } @@ -1737,14 +1384,14 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick public boolean hasReferences() { // Grab the read lock here, because we need to ensure that: only when the atomic // replaceStoreFiles(..) finished, we can get all the complete store file list. - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { // Merge the current store files with compacted files here due to HBASE-20940. Collection allStoreFiles = new ArrayList<>(getStorefiles()); allStoreFiles.addAll(getCompactedFiles()); return StoreUtils.hasReferences(allStoreFiles); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -1784,7 +1431,7 @@ public Optional requestCompaction(int priority, final CompactionContext compaction = storeEngine.createCompaction(); CompactionRequestImpl request = null; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { synchronized (filesCompacting) { // First, see if coprocessor would want to override selection. @@ -1857,7 +1504,7 @@ public Optional requestCompaction(int priority, request.setTracker(tracker); } } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } if (LOG.isDebugEnabled()) { @@ -1890,7 +1537,7 @@ private void removeUnneededFiles() throws IOException { this, getColumnFamilyDescriptor().getMinVersions()); return; } - this.lock.readLock().lock(); + this.storeEngine.readLock(); Collection delSfs = null; try { synchronized (filesCompacting) { @@ -1902,7 +1549,7 @@ private void removeUnneededFiles() throws IOException { } } } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } if (CollectionUtils.isEmpty(delSfs)) { @@ -1910,8 +1557,7 @@ private void removeUnneededFiles() throws IOException { } Collection newFiles = Collections.emptyList(); // No new files. - writeCompactionWalRecord(delSfs, newFiles); - replaceStoreFiles(delSfs, newFiles); + replaceStoreFiles(delSfs, newFiles, true); refreshStoreSizeAndTotalBytes(); LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in " + this + "; total size is " @@ -1933,25 +1579,6 @@ protected void finishCompactionRequest(CompactionRequestImpl cr) { } } - /** - * Validates a store file by opening and closing it. In HFileV2 this should not be an expensive - * operation. - * @param path the path to the store file - */ - private void validateStoreFile(Path path) throws IOException { - HStoreFile storeFile = null; - try { - storeFile = createStoreFileAndReader(path); - } catch (IOException e) { - LOG.error("Failed to open store file : {}, keeping it in tmp location", path, e); - throw e; - } finally { - if (storeFile != null) { - storeFile.closeStoreFile(false); - } - } - } - /** * Update counts. */ @@ -1997,7 +1624,7 @@ public boolean canSplit() { * Determines if Store should be split. */ public Optional getSplitPoint() { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { // Should already be enforced by the split policy! assert !this.getRegionInfo().isMetaRegion(); @@ -2010,7 +1637,7 @@ public Optional getSplitPoint() { } catch(IOException e) { LOG.warn("Failed getting store size for {}", this, e); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } return Optional.empty(); } @@ -2043,7 +1670,7 @@ public void triggerMajorCompaction() { */ public KeyValueScanner getScanner(Scan scan, final NavigableSet targetCols, long readPt) throws IOException { - lock.readLock().lock(); + storeEngine.readLock(); try { ScanInfo scanInfo; if (this.getCoprocessorHost() != null) { @@ -2053,7 +1680,7 @@ public KeyValueScanner getScanner(Scan scan, final NavigableSet targetCo } return createScanner(scan, scanInfo, targetCols, readPt); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); } } @@ -2083,7 +1710,7 @@ public List recreateScanners(List currentFileS boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, boolean includeStartRow, byte[] stopRow, boolean includeStopRow, long readPt, boolean includeMemstoreScanner) throws IOException { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { Map name2File = new HashMap<>(getStorefilesCount() + getCompactedFilesCount()); @@ -2108,7 +1735,7 @@ public List recreateScanners(List currentFileS return getScanners(filesToReopen, cacheBlocks, false, false, matcher, startRow, includeStartRow, stopRow, includeStopRow, readPt, false); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -2174,41 +1801,20 @@ public long getStoreSizeUncompressed() { @Override public long getStorefilesSize() { // Include all StoreFiles - return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), sf -> true); + return StoreUtils.getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), + sf -> true); } @Override public long getHFilesSize() { // Include only StoreFiles which are HFiles - return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), + return StoreUtils.getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), HStoreFile::isHFile); } - private long getTotalUncompressedBytes(List files) { - return files.stream() - .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) - .sum(); - } - - private long getStorefilesSize(Collection files, Predicate predicate) { - return files.stream().filter(predicate) - .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::length)).sum(); - } - - private long getStorefileFieldSize(HStoreFile file, ToLongFunction f) { - if (file == null) { - return 0L; - } - StoreFileReader reader = file.getReader(); - if (reader == null) { - return 0L; - } - return f.applyAsLong(reader); - } - private long getStorefilesFieldSize(ToLongFunction f) { return this.storeEngine.getStoreFileManager().getStorefiles().stream() - .mapToLong(file -> getStorefileFieldSize(file, f)).sum(); + .mapToLong(file -> StoreUtils.getStorefileFieldSize(file, f)).sum(); } @Override @@ -2279,11 +1885,11 @@ public long getSmallestReadPoint() { */ public void upsert(Iterable cells, long readpoint, MemStoreSizing memstoreSizing) throws IOException { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { this.memstore.upsert(cells, readpoint, memstoreSizing); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -2336,7 +1942,7 @@ public boolean commit(MonitoredTask status) throws IOException { return false; } status.setStatus("Flushing " + this + ": reopening flushed file"); - List storeFiles = commitStoreFiles(tempFiles, false); + List storeFiles = storeEngine.commitStoreFiles(tempFiles, false); for (HStoreFile sf : storeFiles) { StoreFileReader r = sf.getReader(); if (LOG.isInfoEnabled()) { @@ -2359,7 +1965,7 @@ public boolean commit(MonitoredTask status) throws IOException { } } // Add new file to store files. Clear snapshot too while we have the Store write lock. - return updateStorefiles(storeFiles, snapshot.getId()); + return completeFlush(storeFiles, snapshot.getId()); } @Override @@ -2387,7 +1993,7 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) // open the file as a store file (hfile link, etc) StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file); - HStoreFile storeFile = createStoreFileAndReader(storeFileInfo); + HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); storeFiles.add(storeFile); HStore.this.storeSize.addAndGet(storeFile.getReader().length()); HStore.this.totalUncompressedBytes @@ -2404,7 +2010,7 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) snapshotId = snapshot.getId(); snapshot.close(); } - HStore.this.updateStorefiles(storeFiles, snapshotId); + HStore.this.completeFlush(storeFiles, snapshotId); } /** @@ -2417,7 +2023,7 @@ public void abort() throws IOException { //won't be closed. If we are using MSLAB, the chunk referenced by those scanners //can't be released, thus memory leak snapshot.close(); - HStore.this.updateStorefiles(Collections.emptyList(), snapshot.getId()); + HStore.this.completeFlush(Collections.emptyList(), snapshot.getId()); } } } @@ -2583,7 +2189,7 @@ public synchronized void closeAndArchiveCompactedFiles() throws IOException { // ensure other threads do not attempt to archive the same files on close() archiveLock.lock(); try { - lock.readLock().lock(); + storeEngine.readLock(); Collection copyCompactedfiles = null; try { Collection compactedfiles = @@ -2595,7 +2201,7 @@ public synchronized void closeAndArchiveCompactedFiles() throws IOException { LOG.trace("No compacted files to archive"); } } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); } if (CollectionUtils.isNotEmpty(copyCompactedfiles)) { removeCompactedfiles(copyCompactedfiles, true); @@ -2730,12 +2336,7 @@ public boolean isSloppyMemStore() { private void clearCompactedfiles(List filesToRemove) throws IOException { LOG.trace("Clearing the compacted file {} from this store", filesToRemove); - try { - lock.writeLock().lock(); - this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove); - } finally { - lock.writeLock().unlock(); - } + storeEngine.removeCompactedFiles(filesToRemove); } void reportArchivedFilesForQuota(List archivedFiles, List fileSizes) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java index 26233505db73..2a9f96859361 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -108,6 +109,14 @@ public RegionCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + public RegionInfo getRegionInfo() { + return regionFileSystem.getRegionInfo(); + } + + public boolean isPrimaryReplicaStore() { + return getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID; + } + public static Builder getBuilder() { return new Builder(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index 60b3c3d0d20f..f9d6c294f779 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -20,37 +20,129 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; - +import java.util.Set; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.Compactor; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; /** - * StoreEngine is a factory that can create the objects necessary for HStore to operate. - * Since not all compaction policies, compactors and store file managers are compatible, - * they are tied together and replaced together via StoreEngine-s. + * StoreEngine is a factory that can create the objects necessary for HStore to operate. Since not + * all compaction policies, compactors and store file managers are compatible, they are tied + * together and replaced together via StoreEngine-s. + *

+ * We expose read write lock methods to upper layer for store operations:
+ *

    + *
  • Locked in shared mode when the list of component stores is looked at: + *
      + *
    • all reads/writes to table data
      • + *
    • checking for split
      • + *
    + *
    • + *
  • Locked in exclusive mode when the list of component stores is modified: + *
      + *
    • closing
      • + *
    • completing a compaction
      • + *
    + *
    • + *
+ *

+ * It is a bit confusing that we have a StoreFileManager(SFM) and then a StoreFileTracker(SFT). As + * its name says, SFT is used to track the store files list. The reason why we have a SFT beside SFM + * is that, when introducing stripe compaction, we introduced the StoreEngine and also the SFM, but + * actually, the SFM here is not a general 'Manager', it is only designed to manage the in memory + * 'stripes', so we can select different store files when scanning or compacting. The 'tracking' of + * store files is actually done in {@link org.apache.hadoop.hbase.regionserver.HRegionFileSystem} + * and {@link HStore} before we have SFT. And since SFM is designed to only holds in memory states, + * we will hold write lock when updating it, the lock is also used to protect the normal read/write + * requests. This means we'd better not add IO operations to SFM. And also, no matter what the in + * memory state is, stripe or not, it does not effect how we track the store files. So consider all + * these facts, here we introduce a separated SFT to track the store files. + *

+ * Here, since we always need to update SFM and SFT almost at the same time, we introduce methods in + * StoreEngine directly to update them both, so upper layer just need to update StoreEngine once, to + * reduce the possible misuse. */ @InterfaceAudience.Private -public abstract class StoreEngine { +public abstract class StoreEngine { + + private static final Logger LOG = LoggerFactory.getLogger(StoreEngine.class); + protected SF storeFlusher; protected CP compactionPolicy; protected C compactor; protected SFM storeFileManager; + private Configuration conf; + private StoreContext ctx; + private RegionCoprocessorHost coprocessorHost; + private Function openStoreFileThreadPoolCreator; + private StoreFileTracker storeFileTracker; + + private final ReadWriteLock storeLock = new ReentrantReadWriteLock(); /** - * The name of the configuration parameter that specifies the class of - * a store engine that is used to manage and compact HBase store files. + * The name of the configuration parameter that specifies the class of a store engine that is used + * to manage and compact HBase store files. */ public static final String STORE_ENGINE_CLASS_KEY = "hbase.hstore.engine.class"; - private static final Class> - DEFAULT_STORE_ENGINE_CLASS = DefaultStoreEngine.class; + private static final Class> DEFAULT_STORE_ENGINE_CLASS = + DefaultStoreEngine.class; + + /** + * Acquire read lock of this store. + */ + public void readLock() { + storeLock.readLock().lock(); + } + + /** + * Release read lock of this store. + */ + public void readUnlock() { + storeLock.readLock().unlock(); + } + + /** + * Acquire write lock of this store. + */ + public void writeLock() { + storeLock.writeLock().lock(); + } + + /** + * Release write lock of this store. + */ + public void writeUnlock() { + storeLock.writeLock().unlock(); + } /** * @return Compaction policy to use. @@ -80,6 +172,11 @@ public StoreFlusher getStoreFlusher() { return this.storeFlusher; } + private StoreFileTracker createStoreFileTracker(HStore store) { + return StoreFileTrackerFactory.create(store.conf, store.getRegionInfo().getTable(), + store.isPrimaryReplicaStore(), store.getStoreContext()); + } + /** * @param filesCompacting Files currently compacting * @return whether a compaction selection is possible @@ -87,8 +184,8 @@ public StoreFlusher getStoreFlusher() { public abstract boolean needsCompaction(List filesCompacting); /** - * Creates an instance of a compaction context specific to this engine. - * Doesn't actually select or start a compaction. See CompactionContext class comment. + * Creates an instance of a compaction context specific to this engine. Doesn't actually select or + * start a compaction. See CompactionContext class comment. * @return New CompactionContext object. */ public abstract CompactionContext createCompaction() throws IOException; @@ -96,32 +193,321 @@ public StoreFlusher getStoreFlusher() { /** * Create the StoreEngine's components. */ - protected abstract void createComponents( - Configuration conf, HStore store, CellComparator cellComparator) throws IOException; + protected abstract void createComponents(Configuration conf, HStore store, + CellComparator cellComparator) throws IOException; - private void createComponentsOnce( - Configuration conf, HStore store, CellComparator cellComparator) throws IOException { - assert compactor == null && compactionPolicy == null - && storeFileManager == null && storeFlusher == null; + protected final void createComponentsOnce(Configuration conf, HStore store, + CellComparator cellComparator) throws IOException { + assert compactor == null && compactionPolicy == null && storeFileManager == null && + storeFlusher == null && storeFileTracker == null; createComponents(conf, store, cellComparator); - assert compactor != null && compactionPolicy != null - && storeFileManager != null && storeFlusher != null; + this.conf = conf; + this.ctx = store.getStoreContext(); + this.coprocessorHost = store.getHRegion().getCoprocessorHost(); + this.openStoreFileThreadPoolCreator = store.getHRegion()::getStoreFileOpenAndCloseThreadPool; + this.storeFileTracker = createStoreFileTracker(store); + assert compactor != null && compactionPolicy != null && storeFileManager != null && + storeFlusher != null && storeFileTracker != null; + } + + /** + * Create a writer for writing new store files. + * @return Writer for a new StoreFile + */ + public StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException { + return storeFileTracker.createWriter(params); + } + + public HStoreFile createStoreFileAndReader(Path p) throws IOException { + StoreFileInfo info = new StoreFileInfo(conf, ctx.getRegionFileSystem().getFileSystem(), p, + ctx.isPrimaryReplicaStore()); + return createStoreFileAndReader(info); + } + + public HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { + info.setRegionCoprocessorHost(coprocessorHost); + HStoreFile storeFile = + new HStoreFile(info, ctx.getFamily().getBloomFilterType(), ctx.getCacheConf()); + storeFile.initReader(); + return storeFile; + } + + /** + * Validates a store file by opening and closing it. In HFileV2 this should not be an expensive + * operation. + * @param path the path to the store file + */ + public void validateStoreFile(Path path) throws IOException { + HStoreFile storeFile = null; + try { + storeFile = createStoreFileAndReader(path); + } catch (IOException e) { + LOG.error("Failed to open store file : {}, keeping it in tmp location", path, e); + throw e; + } finally { + if (storeFile != null) { + storeFile.closeStoreFile(false); + } + } + } + + private List openStoreFiles(Collection files, boolean warmup) + throws IOException { + if (CollectionUtils.isEmpty(files)) { + return Collections.emptyList(); + } + // initialize the thread pool for opening store files in parallel.. + ExecutorService storeFileOpenerThreadPool = + openStoreFileThreadPoolCreator.apply("StoreFileOpener-" + + ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString()); + CompletionService completionService = + new ExecutorCompletionService<>(storeFileOpenerThreadPool); + + int totalValidStoreFile = 0; + for (StoreFileInfo storeFileInfo : files) { + // open each store file in parallel + completionService.submit(() -> createStoreFileAndReader(storeFileInfo)); + totalValidStoreFile++; + } + + Set compactedStoreFiles = new HashSet<>(); + ArrayList results = new ArrayList<>(files.size()); + IOException ioe = null; + try { + for (int i = 0; i < totalValidStoreFile; i++) { + try { + HStoreFile storeFile = completionService.take().get(); + if (storeFile != null) { + LOG.debug("loaded {}", storeFile); + results.add(storeFile); + compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); + } + } catch (InterruptedException e) { + if (ioe == null) { + ioe = new InterruptedIOException(e.getMessage()); + } + } catch (ExecutionException e) { + if (ioe == null) { + ioe = new IOException(e.getCause()); + } + } + } + } finally { + storeFileOpenerThreadPool.shutdownNow(); + } + if (ioe != null) { + // close StoreFile readers + boolean evictOnClose = + ctx.getCacheConf() != null ? ctx.getCacheConf().shouldEvictOnClose() : true; + for (HStoreFile file : results) { + try { + if (file != null) { + file.closeStoreFile(evictOnClose); + } + } catch (IOException e) { + LOG.warn("Could not close store file {}", file, e); + } + } + throw ioe; + } + + // Should not archive the compacted store files when region warmup. See HBASE-22163. + if (!warmup) { + // Remove the compacted files from result + List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); + for (HStoreFile storeFile : results) { + if (compactedStoreFiles.contains(storeFile.getPath().getName())) { + LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); + storeFile.getReader().close( + storeFile.getCacheConf() != null ? storeFile.getCacheConf().shouldEvictOnClose() : + true); + filesToRemove.add(storeFile); + } + } + results.removeAll(filesToRemove); + if (!filesToRemove.isEmpty() && ctx.isPrimaryReplicaStore()) { + LOG.debug("Moving the files {} to archive", filesToRemove); + ctx.getRegionFileSystem().removeStoreFiles(ctx.getFamily().getNameAsString(), + filesToRemove); + } + } + + return results; + } + + public void initialize(boolean warmup) throws IOException { + List fileInfos = storeFileTracker.load(); + List files = openStoreFiles(fileInfos, warmup); + storeFileManager.loadFiles(files); + } + + public void refreshStoreFiles() throws IOException { + List fileInfos = storeFileTracker.load(); + refreshStoreFilesInternal(fileInfos); + } + + public void refreshStoreFiles(Collection newFiles) throws IOException { + List storeFiles = new ArrayList<>(newFiles.size()); + for (String file : newFiles) { + storeFiles + .add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), file)); + } + refreshStoreFilesInternal(storeFiles); + } + + /** + * Checks the underlying store files, and opens the files that have not been opened, and removes + * the store file readers for store files no longer available. Mainly used by secondary region + * replicas to keep up to date with the primary region files. + */ + private void refreshStoreFilesInternal(Collection newFiles) throws IOException { + Collection currentFiles = storeFileManager.getStorefiles(); + Collection compactedFiles = storeFileManager.getCompactedfiles(); + if (currentFiles == null) { + currentFiles = Collections.emptySet(); + } + if (newFiles == null) { + newFiles = Collections.emptySet(); + } + if (compactedFiles == null) { + compactedFiles = Collections.emptySet(); + } + + HashMap currentFilesSet = new HashMap<>(currentFiles.size()); + for (HStoreFile sf : currentFiles) { + currentFilesSet.put(sf.getFileInfo(), sf); + } + HashMap compactedFilesSet = new HashMap<>(compactedFiles.size()); + for (HStoreFile sf : compactedFiles) { + compactedFilesSet.put(sf.getFileInfo(), sf); + } + + Set newFilesSet = new HashSet(newFiles); + // Exclude the files that have already been compacted + newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); + Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); + Set toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet); + + if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) { + return; + } + + LOG.info("Refreshing store files for " + this + " files to add: " + toBeAddedFiles + + " files to remove: " + toBeRemovedFiles); + + Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); + for (StoreFileInfo sfi : toBeRemovedFiles) { + toBeRemovedStoreFiles.add(currentFilesSet.get(sfi)); + } + + // try to open the files + List openedFiles = openStoreFiles(toBeAddedFiles, false); + + // propogate the file changes to the underlying store file manager + replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); // won't throw an exception + } + + /** + * Commit the given {@code files}. + *

+ * We will move the file into data directory, and open it. + * @param files the files want to commit + * @param validate whether to validate the store files + * @return the committed store files + */ + public List commitStoreFiles(List files, boolean validate) throws IOException { + List committedFiles = new ArrayList<>(files.size()); + HRegionFileSystem hfs = ctx.getRegionFileSystem(); + String familyName = ctx.getFamily().getNameAsString(); + Path storeDir = hfs.getStoreDir(familyName); + for (Path file : files) { + try { + if (validate) { + validateStoreFile(file); + } + Path committedPath; + // As we want to support writing to data directory directly, here we need to check whether + // the store file is already in the right place + if (file.getParent() != null && file.getParent().equals(storeDir)) { + // already in the right place, skip renmaing + committedPath = file; + } else { + // Write-out finished successfully, move into the right spot + committedPath = hfs.commitStoreFile(familyName, file); + } + HStoreFile sf = createStoreFileAndReader(committedPath); + committedFiles.add(sf); + } catch (IOException e) { + LOG.error("Failed to commit store file {}", file, e); + // Try to delete the files we have committed before. + // It is OK to fail when deleting as leaving the file there does not cause any data + // corruption problem. It just introduces some duplicated data which may impact read + // performance a little when reading before compaction. + for (HStoreFile sf : committedFiles) { + Path pathToDelete = sf.getPath(); + try { + sf.deleteStoreFile(); + } catch (IOException deleteEx) { + LOG.warn(HBaseMarkers.FATAL, "Failed to delete committed store file {}", pathToDelete, + deleteEx); + } + } + throw new IOException("Failed to commit the flush", e); + } + } + return committedFiles; + } + + public void addStoreFiles(Collection storeFiles) throws IOException { + storeFileTracker.add(StoreUtils.toStoreFileInfo(storeFiles)); + writeLock(); + try { + storeFileManager.insertNewFiles(storeFiles); + } finally { + // We need the lock, as long as we are updating the storeFiles + // or changing the memstore. Let us release it before calling + // notifyChangeReadersObservers. See HBASE-4485 for a possible + // deadlock scenario that could have happened if continue to hold + // the lock. + writeUnlock(); + } + } + + public void replaceStoreFiles(Collection compactedFiles, + Collection newFiles) throws IOException { + storeFileTracker.replace(StoreUtils.toStoreFileInfo(compactedFiles), + StoreUtils.toStoreFileInfo(newFiles)); + writeLock(); + try { + storeFileManager.addCompactionResults(compactedFiles, newFiles); + } finally { + writeUnlock(); + } + } + + public void removeCompactedFiles(Collection compactedFiles) { + writeLock(); + try { + storeFileManager.removeCompactedFiles(compactedFiles); + } finally { + writeUnlock(); + } } /** * Create the StoreEngine configured for the given Store. - * @param store The store. An unfortunate dependency needed due to it - * being passed to coprocessors via the compactor. + * @param store The store. An unfortunate dependency needed due to it being passed to coprocessors + * via the compactor. * @param conf Store configuration. * @param cellComparator CellComparator for storeFileManager. * @return StoreEngine to use. */ - public static StoreEngine create( - HStore store, Configuration conf, CellComparator cellComparator) throws IOException { + public static StoreEngine create(HStore store, Configuration conf, + CellComparator cellComparator) throws IOException { String className = conf.get(STORE_ENGINE_CLASS_KEY, DEFAULT_STORE_ENGINE_CLASS.getName()); try { - StoreEngine se = ReflectionUtils.instantiateWithCustomCtor( - className, new Class[] { }, new Object[] { }); + StoreEngine se = + ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {}, new Object[] {}); se.createComponentsOnce(conf, store, cellComparator); return se; } catch (Exception e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java index 27127f3a6c64..a40b209c6ebb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.util.Collection; import java.util.Comparator; @@ -49,12 +50,16 @@ public interface StoreFileManager { * Loads the initial store files into empty StoreFileManager. * @param storeFiles The files to load. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void loadFiles(List storeFiles); /** * Adds new files, either for from MemStore flush or bulk insert, into the structure. * @param sfs New store files. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void insertNewFiles(Collection sfs); /** @@ -62,12 +67,16 @@ public interface StoreFileManager { * @param compactedFiles The input files for the compaction. * @param results The resulting files for the compaction. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void addCompactionResults(Collection compactedFiles, Collection results); /** * Remove the compacted files * @param compactedFiles the list of compacted files */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void removeCompactedFiles(Collection compactedFiles); /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java index 1064b6c70547..58031288f751 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java @@ -70,10 +70,17 @@ protected void finalizeWriter(StoreFileWriter writer, long cacheFlushSeqNum, writer.close(); } + protected final StoreFileWriter createWriter(MemStoreSnapshot snapshot, boolean alwaysIncludesTag) + throws IOException { + return store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(snapshot.getCellsCount()) + .compression(store.getColumnFamilyDescriptor().getCompressionType()).isCompaction(false) + .includeMVCCReadpoint(true).includesTag(alwaysIncludesTag || snapshot.isTagsPresent()) + .shouldDropBehind(false)); + } /** * Creates the scanner for flushing snapshot. Also calls coprocessors. - * @param snapshotScanners * @return The scanner; null if coprocessor is canceling the flush. */ protected final InternalScanner createScanner(List snapshotScanners, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java index ac5955feca7e..e464b86254d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java @@ -20,10 +20,13 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; - +import java.util.function.Predicate; +import java.util.function.ToLongFunction; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; @@ -39,10 +42,13 @@ * Utility functions for region server storage layer. */ @InterfaceAudience.Private -public class StoreUtils { +public final class StoreUtils { private static final Logger LOG = LoggerFactory.getLogger(StoreUtils.class); + private StoreUtils() { + } + /** * Creates a deterministic hash code for store file collection. */ @@ -161,4 +167,30 @@ public static int getBytesPerChecksum(Configuration conf) { HFile.DEFAULT_BYTES_PER_CHECKSUM); } + public static List toStoreFileInfo(Collection storefiles) { + return storefiles.stream().map(HStoreFile::getFileInfo).collect(Collectors.toList()); + } + + public static long getTotalUncompressedBytes(List files) { + return files.stream() + .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) + .sum(); + } + + public static long getStorefilesSize(Collection files, + Predicate predicate) { + return files.stream().filter(predicate) + .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::length)).sum(); + } + + public static long getStorefileFieldSize(HStoreFile file, ToLongFunction f) { + if (file == null) { + return 0L; + } + StoreFileReader reader = file.getReader(); + if (reader == null) { + return 0L; + } + return f.applyAsLong(reader); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java index 14863a69a9b1..bfb3f649ff27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java @@ -20,20 +20,19 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java index 1560aef5f6b3..f8183b7645a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java @@ -70,7 +70,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushSeqNum StripeMultiFileWriter mw = null; try { mw = req.createWriter(); // Writer according to the policy. - StripeMultiFileWriter.WriterFactory factory = createWriterFactory(cellsCount); + StripeMultiFileWriter.WriterFactory factory = createWriterFactory(snapshot); StoreScanner storeScanner = (scanner instanceof StoreScanner) ? (StoreScanner)scanner : null; mw.init(storeScanner, factory); @@ -98,13 +98,12 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushSeqNum return result; } - private StripeMultiFileWriter.WriterFactory createWriterFactory(final long kvCount) { + private StripeMultiFileWriter.WriterFactory createWriterFactory(MemStoreSnapshot snapshot) { return new StripeMultiFileWriter.WriterFactory() { @Override public StoreFileWriter createWriter() throws IOException { - StoreFileWriter writer = store.createWriterInTmp(kvCount, - store.getColumnFamilyDescriptor().getCompressionType(), false, true, true, false); - return writer; + // XXX: it used to always pass true for includesTag, re-consider? + return StripeStoreFlusher.this.createWriter(snapshot, true); } }; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java index 42841bfee531..533be176e7a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java @@ -51,13 +51,14 @@ protected void initMultiWriter(AbstractMultiFileWriter writer, InternalScanner s WriterFactory writerFactory = new WriterFactory() { @Override public StoreFileWriter createWriter() throws IOException { - return createTmpWriter(fd, shouldDropBehind, major); + return AbstractMultiOutputCompactor.this.createWriter(fd, shouldDropBehind, major); } @Override public StoreFileWriter createWriterWithStoragePolicy(String fileStoragePolicy) - throws IOException { - return createTmpWriter(fd, shouldDropBehind, fileStoragePolicy, major); + throws IOException { + return AbstractMultiOutputCompactor.this.createWriter(fd, shouldDropBehind, + fileStoragePolicy, major); } }; // Prepare multi-writer, and perform the compaction using scanner and writer. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index e524f7dfd5fd..47ef0f290251 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -28,7 +28,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -39,6 +38,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.regionserver.CellSink; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -61,6 +61,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.io.Closeables; /** @@ -261,29 +262,32 @@ public InternalScanner createScanner(ScanInfo scanInfo, List s } }; + protected final CreateStoreFileWriterParams createParams(FileDetails fd, boolean shouldDropBehind, + boolean major) { + return CreateStoreFileWriterParams.create().maxKeyCount(fd.maxKeyCount) + .compression(major ? majorCompactionCompression : minorCompactionCompression) + .isCompaction(true).includeMVCCReadpoint(fd.maxMVCCReadpoint > 0) + .includesTag(fd.maxTagsLength > 0).shouldDropBehind(shouldDropBehind) + .totalCompactedFilesSize(fd.totalCompactedFilesSize); + } + /** - * Creates a writer for a new file in a temporary directory. + * Creates a writer for a new file. * @param fd The file details. - * @return Writer for a new StoreFile in the tmp dir. + * @return Writer for a new StoreFile * @throws IOException if creation failed */ - protected final StoreFileWriter createTmpWriter(FileDetails fd, boolean shouldDropBehind, boolean major) - throws IOException { + protected final StoreFileWriter createWriter(FileDetails fd, boolean shouldDropBehind, + boolean major) throws IOException { // When all MVCC readpoints are 0, don't write them. // See HBASE-8166, HBASE-12600, and HBASE-13389. - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, fd.maxMVCCReadpoint > 0, - fd.maxTagsLength > 0, shouldDropBehind, fd.totalCompactedFilesSize, - HConstants.EMPTY_STRING); + return store.getStoreEngine().createWriter(createParams(fd, shouldDropBehind, major)); } - protected final StoreFileWriter createTmpWriter(FileDetails fd, boolean shouldDropBehind, - String fileStoragePolicy, boolean major) throws IOException { - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, fd.maxMVCCReadpoint > 0, - fd.maxTagsLength > 0, shouldDropBehind, fd.totalCompactedFilesSize, fileStoragePolicy); + protected final StoreFileWriter createWriter(FileDetails fd, boolean shouldDropBehind, + String fileStoragePolicy, boolean major) throws IOException { + return store.getStoreEngine() + .createWriter(createParams(fd, shouldDropBehind, major).fileStoragePolicy(fileStoragePolicy)); } private ScanInfo preCompactScannerOpen(CompactionRequestImpl request, ScanType scanType, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java index 49d3e8ee01e7..afa2429cb6e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java @@ -45,14 +45,14 @@ public DefaultCompactor(Configuration conf, HStore store) { } private final CellSinkFactory writerFactory = - new CellSinkFactory() { - @Override - public StoreFileWriter createWriter(InternalScanner scanner, - org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, - boolean shouldDropBehind, boolean major) throws IOException { - return createTmpWriter(fd, shouldDropBehind, major); - } - }; + new CellSinkFactory() { + @Override + public StoreFileWriter createWriter(InternalScanner scanner, + org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, + boolean shouldDropBehind, boolean major) throws IOException { + return DefaultCompactor.this.createWriter(fd, shouldDropBehind, major); + } + }; /** * Do a minor/major compaction on an explicit set of storefiles from a Store. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java new file mode 100644 index 000000000000..d4c9a868eb48 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * The default implementation for store file tracker, where we do not persist the store file list, + * and use listing when loading store files. + */ +@InterfaceAudience.Private +class DefaultStoreFileTracker extends StoreFileTrackerBase { + + public DefaultStoreFileTracker(Configuration conf, TableName tableName, boolean isPrimaryReplica, + StoreContext ctx) { + super(conf, tableName, isPrimaryReplica, ctx); + } + + @Override + public List load() throws IOException { + return ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + } + + @Override + public boolean requireWritingToTmpDirFirst() { + return true; + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + // NOOP + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + // NOOP + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java new file mode 100644 index 000000000000..aadedc8ef727 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * An interface to define how we track the store files for a give store. + *

+ * In the old time, we will write store to a tmp directory first, and then rename it to the actual + * data file. And once a store file is under data directory, we will consider it as 'committed'. And + * we need to do listing when loading store files. + *

+ * When cloud age is coming, now we want to store the store files on object storage, where rename + * and list are not as cheap as on HDFS, especially rename. Although introducing a metadata + * management layer for object storage could solve the problem, but we still want HBase to run on + * pure object storage, so here we introduce this interface to abstract how we track the store + * files. For the old implementation, we just persist nothing here, and do listing to load store + * files. When running on object storage, we could persist the store file list in a system region, + * or in a file on the object storage, to make it possible to write directly into the data directory + * to avoid renaming, and also avoid listing when loading store files. + *

+ * The implementation requires to be thread safe as flush and compaction may occur as the same time, + * and we could also do multiple compactions at the same time. As the implementation may choose to + * persist the store file list to external storage, which could be slow, it is the duty for the + * callers to not call it inside a lock which may block normal read/write requests. + */ +@InterfaceAudience.Private +public interface StoreFileTracker { + + /** + * Load the store files list when opening a region. + */ + List load() throws IOException; + + /** + * Add new store files. + *

+ * Used for flush and bulk load. + */ + void add(Collection newFiles) throws IOException; + + /** + * Add new store files and remove compacted store files after compaction. + */ + void replace(Collection compactedFiles, Collection newFiles) + throws IOException; + + /** + * Create a writer for writing new store files. + * @return Writer for a new StoreFile + */ + StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java new file mode 100644 index 000000000000..2451f45bdc7b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for all store file tracker. + *

+ * Mainly used to place the common logic to skip persistent for secondary replicas. + */ +@InterfaceAudience.Private +abstract class StoreFileTrackerBase implements StoreFileTracker { + + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerBase.class); + + protected final Configuration conf; + + protected final TableName tableName; + + protected final boolean isPrimaryReplica; + + protected final StoreContext ctx; + + private volatile boolean cacheOnWriteLogged; + + protected StoreFileTrackerBase(Configuration conf, TableName tableName, boolean isPrimaryReplica, + StoreContext ctx) { + this.conf = conf; + this.tableName = tableName; + this.isPrimaryReplica = isPrimaryReplica; + this.ctx = ctx; + } + + @Override + public final void add(Collection newFiles) throws IOException { + if (isPrimaryReplica) { + doAddNewStoreFiles(newFiles); + } + } + + @Override + public final void replace(Collection compactedFiles, + Collection newFiles) throws IOException { + if (isPrimaryReplica) { + doAddCompactionResults(compactedFiles, newFiles); + } + } + + private HFileContext createFileContext(Compression.Algorithm compression, + boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { + if (compression == null) { + compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; + } + ColumnFamilyDescriptor family = ctx.getFamily(); + HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(includeMVCCReadpoint) + .withIncludesTags(includesTag).withCompression(compression) + .withCompressTags(family.isCompressTags()).withChecksumType(StoreUtils.getChecksumType(conf)) + .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)) + .withBlockSize(family.getBlocksize()).withHBaseCheckSum(true) + .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(encryptionContext) + .withCreateTime(EnvironmentEdgeManager.currentTime()).withColumnFamily(family.getName()) + .withTableName(tableName.getName()).withCellComparator(ctx.getComparator()).build(); + return hFileContext; + } + + @Override + public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) + throws IOException { + if (!isPrimaryReplica) { + throw new IllegalStateException("Should not call create writer on secondary replicas"); + } + // creating new cache config for each new writer + final CacheConfig cacheConf = ctx.getCacheConf(); + final CacheConfig writerCacheConf = new CacheConfig(cacheConf); + long totalCompactedFilesSize = params.totalCompactedFilesSize(); + if (params.isCompaction()) { + // Don't cache data on write on compactions, unless specifically configured to do so + // Cache only when total file size remains lower than configured threshold + final boolean cacheCompactedBlocksOnWrite = cacheConf.shouldCacheCompactedBlocksOnWrite(); + // if data blocks are to be cached on write + // during compaction, we should forcefully + // cache index and bloom blocks as well + if (cacheCompactedBlocksOnWrite && + totalCompactedFilesSize <= cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { + writerCacheConf.enableCacheOnWrite(); + if (!cacheOnWriteLogged) { + LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " + + "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this); + cacheOnWriteLogged = true; + } + } else { + writerCacheConf.setCacheDataOnWrite(false); + if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { + // checking condition once again for logging + LOG.debug( + "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted " + + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}", + this, totalCompactedFilesSize, cacheConf.getCacheCompactedBlocksOnWriteThreshold()); + } + } + } else { + final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite(); + if (shouldCacheDataOnWrite) { + writerCacheConf.enableCacheOnWrite(); + if (!cacheOnWriteLogged) { + LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " + + "Index blocks and Bloom filter blocks", this); + cacheOnWriteLogged = true; + } + } + } + Encryption.Context encryptionContext = ctx.getEncryptionContext(); + HFileContext hFileContext = createFileContext(params.compression(), + params.includeMVCCReadpoint(), params.includesTag(), encryptionContext); + Path outputDir; + if (requireWritingToTmpDirFirst()) { + outputDir = + new Path(ctx.getRegionFileSystem().getTempDir(), ctx.getFamily().getNameAsString()); + } else { + throw new UnsupportedOperationException("not supported yet"); + } + StoreFileWriter.Builder builder = + new StoreFileWriter.Builder(conf, writerCacheConf, ctx.getRegionFileSystem().getFileSystem()) + .withOutputDir(outputDir).withBloomType(ctx.getBloomFilterType()) + .withMaxKeyCount(params.maxKeyCount()).withFavoredNodes(ctx.getFavoredNodes()) + .withFileContext(hFileContext).withShouldDropCacheBehind(params.shouldDropBehind()) + .withCompactedFilesSupplier(ctx.getCompactedFilesSupplier()) + .withFileStoragePolicy(params.fileStoragePolicy()); + return builder.build(); + } + + /** + * Whether the implementation of this tracker requires you to write to temp directory first, i.e, + * does not allow broken store files under the actual data directory. + */ + protected abstract boolean requireWritingToTmpDirFirst(); + + protected abstract void doAddNewStoreFiles(Collection newFiles) throws IOException; + + protected abstract void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java new file mode 100644 index 000000000000..4f7231bc3b9a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Factory method for creating store file tracker. + */ +@InterfaceAudience.Private +public final class StoreFileTrackerFactory { + + public static StoreFileTracker create(Configuration conf, TableName tableName, + boolean isPrimaryReplica, StoreContext ctx) { + return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java index 22ec6cb89ec4..291b909f69bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java @@ -105,7 +105,6 @@ Set getStoresRequiringCompaction(Set requestedStores, long times boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts) throws IOException { - // do we have any store files? Collection storeFiles = fileSystem.getStoreFiles(family); if (storeFiles == null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index 3c2bc3f0cb12..9314d7e9827f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -211,11 +211,13 @@ protected BlockCompactionsInCompletionHStore(HRegion region, ColumnFamilyDescrip @Override protected void refreshStoreSizeAndTotalBytes() throws IOException { - try { - r.compactionsWaiting.countDown(); - r.compactionsBlocked.await(); - } catch (InterruptedException ex) { - throw new IOException(ex); + if (r != null) { + try { + r.compactionsWaiting.countDown(); + r.compactionsBlocked.await(); + } catch (InterruptedException ex) { + throw new IOException(ex); + } } super.refreshStoreSizeAndTotalBytes(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index cd83dc8c2494..0a6b37bbba4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -215,8 +215,10 @@ public void tearDown() throws IOException { @Test public void testCacheOnWriteInSchema() throws IOException { // Write some random data into the store - StoreFileWriter writer = store.createWriterInTmp(Integer.MAX_VALUE, - HFile.DEFAULT_COMPRESSION_ALGORITHM, false, true, false, false); + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(Integer.MAX_VALUE) + .compression(HFile.DEFAULT_COMPRESSION_ALGORITHM).isCompaction(false) + .includeMVCCReadpoint(true).includesTag(false).shouldDropBehind(false)); writeStoreFile(writer); writer.close(); // Verify the block types of interest were cached on write diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java index e832c47aac81..3784876a59f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java @@ -65,9 +65,12 @@ public void testCustomParts() throws Exception { DummyCompactionPolicy.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, DummyStoreFlusher.class.getName()); + HRegion mockRegion = Mockito.mock(HRegion.class); HStore mockStore = Mockito.mock(HStore.class); Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); - StoreEngine se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); + Mockito.when(mockStore.getHRegion()).thenReturn(mockRegion); + StoreEngine se = + StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); Assert.assertTrue(se instanceof DefaultStoreEngine); Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy); Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index a5584ff32599..7fb57686c35d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -5744,7 +5744,7 @@ public void testCompactionFromPrimary() throws IOException { Collection storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); Collection storeFileInfos = primaryRegion.getRegionFileSystem() - .getStoreFiles(families[0]); + .getStoreFiles(Bytes.toString(families[0])); Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); verifyData(secondaryRegion, 0, 1000, cq, families); @@ -7648,7 +7648,7 @@ protected List doCompaction(CompactionRequestImpl cr, getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; for (Path newFile : newFiles) { // Create storefile around what we wrote with a reader on it. - HStoreFile sf = createStoreFileAndReader(newFile); + HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); sf.closeStoreFile(evictOnClose); sfs.add(sf); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index b64f52192740..8e4c4a385de3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -310,7 +310,7 @@ public Object run() throws Exception { /** * Verify that compression and data block encoding are respected by the - * Store.createWriterInTmp() method, used on store flush. + * createWriter method, used on store flush. */ @Test public void testCreateWriter() throws Exception { @@ -322,9 +322,11 @@ public void testCreateWriter() throws Exception { .build(); init(name.getMethodName(), conf, hcd); - // Test createWriterInTmp() - StoreFileWriter writer = - store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false); + // Test createWriter + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(4) + .compression(hcd.getCompressionType()).isCompaction(false).includeMVCCReadpoint(true) + .includesTag(false).shouldDropBehind(false)); Path path = writer.getPath(); writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); @@ -1021,19 +1023,19 @@ public void testRefreshStoreFilesNotChanged() throws IOException { // add one more file addStoreFile(); - HStore spiedStore = spy(store); + StoreEngine spiedStoreEngine = spy(store.getStoreEngine()); // call first time after files changed - spiedStore.refreshStoreFiles(); + spiedStoreEngine.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); - verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); + verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any()); // call second time - spiedStore.refreshStoreFiles(); + spiedStoreEngine.refreshStoreFiles(); // ensure that replaceStoreFiles is not called, i.e, the times does not change, if files are not // refreshed, - verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); + verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any()); } private long countMemStoreScanner(StoreScanner scanner) { @@ -1644,7 +1646,7 @@ public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger() throws // Do compaction MyThread thread = new MyThread(storeScanner); thread.start(); - store.replaceStoreFiles(actualStorefiles, actualStorefiles1); + store.replaceStoreFiles(actualStorefiles, actualStorefiles1, false); thread.join(); KeyValueHeap heap2 = thread.getHeap(); assertFalse(heap.equals(heap2)); @@ -1710,8 +1712,10 @@ public void testSpaceQuotaChangeAfterReplacement() throws IOException { @Test public void testHFileContextSetWithCFAndTable() throws Exception { init(this.name.getMethodName()); - StoreFileWriter writer = store.createWriterInTmp(10000L, - Compression.Algorithm.NONE, false, true, false, true); + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(10000L) + .compression(Compression.Algorithm.NONE).isCompaction(true).includeMVCCReadpoint(true) + .includesTag(false).shouldDropBehind(true)); HFileContext hFileContext = writer.getHFileWriter().getFileContext(); assertArrayEquals(family, hFileContext.getColumnFamily()); assertArrayEquals(table, hFileContext.getTableName()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 60f65be66c47..1dbf37aa8063 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -245,7 +245,7 @@ public void testCleanMergeReference() throws Exception { TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo); int count = 0; for(ColumnFamilyDescriptor colFamily : columnFamilies) { - count += hrfs.getStoreFiles(colFamily.getName()).size(); + count += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } ADMIN.compactRegion(mergedRegionInfo.getRegionName()); // clean up the merged region store files @@ -254,7 +254,7 @@ public void testCleanMergeReference() throws Exception { int newcount = 0; while (EnvironmentEdgeManager.currentTime() < timeout) { for(ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount += hrfs.getStoreFiles(colFamily.getName()).size(); + newcount += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } if(newcount > count) { break; @@ -273,7 +273,7 @@ public void testCleanMergeReference() throws Exception { while (EnvironmentEdgeManager.currentTime() < timeout) { int newcount1 = 0; for(ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount1 += hrfs.getStoreFiles(colFamily.getName()).size(); + newcount1 += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } if(newcount1 <= 1) { break; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java index 9141327d26d4..2fab050446ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -98,7 +97,7 @@ static class FailingHRegionFileSystem extends HRegionFileSystem { } @Override - public Collection getStoreFiles(String familyName) throws IOException { + public List getStoreFiles(String familyName) throws IOException { if (fail) { throw new IOException("simulating FS failure"); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java index 4cb023f9b9d6..121f8301751f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java @@ -29,7 +29,6 @@ import java.util.Random; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -128,13 +127,12 @@ public void testScannerCloseAndUpdateReadersWithMemstoreScanner() throws Excepti p.addColumn(fam, Bytes.toBytes("q1"), Bytes.toBytes("val")); region.put(p); HStore store = region.getStore(fam); - ReentrantReadWriteLock lock = store.lock; // use the lock to manually get a new memstore scanner. this is what // HStore#notifyChangedReadersObservers does under the lock.(lock is not needed here //since it is just a testcase). - lock.readLock().lock(); + store.getStoreEngine().readLock(); final List memScanners = store.memstore.getScanners(Long.MAX_VALUE); - lock.readLock().unlock(); + store.getStoreEngine().readUnlock(); Thread closeThread = new Thread() { public void run() { // close should be completed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index f5330f6faa42..eb0b1c1ca694 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -118,8 +118,10 @@ private static HStoreFile createFile() throws Exception { } private static TestStoreEngine createEngine(Configuration conf) throws Exception { + HRegion region = mock(HRegion.class); HStore store = mock(HStore.class); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); + when(store.getHRegion()).thenReturn(region); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); return (TestStoreEngine) StoreEngine.create(store, conf, kvComparator); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java index d7b7ba760156..0ea82c57be1a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java @@ -22,9 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,11 +42,13 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.Scanner; @@ -110,10 +109,9 @@ private DateTieredCompactor createCompactor(StoreFileWritersCapture writers, when(store.areWritesEnabled()).thenReturn(true); when(store.getFileSystem()).thenReturn(mock(FileSystem.class)); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.newBuilder(TABLE_NAME).build()); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); when(store.getComparator()).thenReturn(CellComparatorImpl.COMPARATOR); OptionalLong maxSequenceId = StoreUtils.getMaxSequenceIdInList(storefiles); when(store.getMaxSequenceId()).thenReturn(maxSequenceId); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java index 2ba15d1be18e..bdab20ead8a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java @@ -30,7 +30,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; @@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StripeMultiFileWriter; @@ -864,12 +865,9 @@ private StripeCompactor createCompactor() throws Exception { when(info.getRegionNameAsString()).thenReturn("testRegion"); when(store.getColumnFamilyDescriptor()).thenReturn(familyDescriptor); when(store.getRegionInfo()).thenReturn(info); - when( - store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when( - store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); Configuration conf = HBaseConfiguration.create(); conf.setBoolean("hbase.regionserver.compaction.private.readers", usePrivateReaders); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java index e49174e6afe3..ae59c74bad95 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java @@ -21,9 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.createDummyRequest; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,10 +40,12 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.Scanner; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.StoreFileWritersCapture; @@ -209,10 +208,9 @@ private StripeCompactor createCompactor(StoreFileWritersCapture writers, KeyValu when(store.areWritesEnabled()).thenReturn(true); when(store.getFileSystem()).thenReturn(mock(FileSystem.class)); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.newBuilder(TABLE_NAME).build()); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); when(store.getComparator()).thenReturn(CellComparatorImpl.COMPARATOR); return new StripeCompactor(conf, store) { From 8b26dcca6a00334240ed62146a030f7fe3c03a8f Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 26 Aug 2021 18:51:12 +0800 Subject: [PATCH 02/37] HBASE-25988 Store the store file list by a file (#3578) Signed-off-by: Wellington Chevreuil --- .../server/region/StoreFileTracker.proto | 36 +++++ .../hbase/regionserver/StoreContext.java | 5 + .../hbase/regionserver/StoreEngine.java | 8 +- .../DefaultStoreFileTracker.java | 5 +- .../FileBasedStoreFileTracker.java | 142 ++++++++++++++++++ .../storefiletracker/StoreFileListFile.java | 142 ++++++++++++++++++ .../StoreFileTrackerBase.java | 11 +- .../StoreFileTrackerFactory.java | 12 +- ...stRegionWithFileBasedStoreFileTracker.java | 109 ++++++++++++++ 9 files changed, 451 insertions(+), 19 deletions(-) create mode 100644 hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto new file mode 100644 index 000000000000..2a269ea4ac4e --- /dev/null +++ b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; +// This file contains protocol buffers that are used for store file tracker. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "StoreFileTrackerProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message StoreFileEntry { + required string name = 1; + required uint64 size = 2; +} + +message StoreFileList { + required uint64 timestamp = 1; + repeated StoreFileEntry store_file = 2; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java index 2a9f96859361..588f8f4027a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java @@ -22,6 +22,7 @@ import java.util.function.Supplier; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.io.HeapSize; @@ -109,6 +110,10 @@ public RegionCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + public TableName getTableName() { + return getRegionInfo().getTable(); + } + public RegionInfo getRegionInfo() { return regionFileSystem.getRegionInfo(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index f9d6c294f779..61b7dfa66848 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -172,9 +172,9 @@ public StoreFlusher getStoreFlusher() { return this.storeFlusher; } - private StoreFileTracker createStoreFileTracker(HStore store) { - return StoreFileTrackerFactory.create(store.conf, store.getRegionInfo().getTable(), - store.isPrimaryReplicaStore(), store.getStoreContext()); + private StoreFileTracker createStoreFileTracker(Configuration conf, HStore store) { + return StoreFileTrackerFactory.create(conf, store.isPrimaryReplicaStore(), + store.getStoreContext()); } /** @@ -205,7 +205,7 @@ protected final void createComponentsOnce(Configuration conf, HStore store, this.ctx = store.getStoreContext(); this.coprocessorHost = store.getHRegion().getCoprocessorHost(); this.openStoreFileThreadPoolCreator = store.getHRegion()::getStoreFileOpenAndCloseThreadPool; - this.storeFileTracker = createStoreFileTracker(store); + this.storeFileTracker = createStoreFileTracker(conf, store); assert compactor != null && compactionPolicy != null && storeFileManager != null && storeFlusher != null && storeFileTracker != null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index d4c9a868eb48..fa044818336f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -21,7 +21,6 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -33,9 +32,9 @@ @InterfaceAudience.Private class DefaultStoreFileTracker extends StoreFileTrackerBase { - public DefaultStoreFileTracker(Configuration conf, TableName tableName, boolean isPrimaryReplica, + public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - super(conf, tableName, isPrimaryReplica, ctx); + super(conf, isPrimaryReplica, ctx); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java new file mode 100644 index 000000000000..de28b0eb9996 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry; +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; + +/** + * A file based store file tracker. + *

+ * For this tracking way, the store file list will be persistent into a file, so we can write the + * new store files directly to the final data directory, as we will not load the broken files. This + * will greatly reduce the time for flush and compaction on some object storages as a rename is + * actual a copy on them. And it also avoid listing when loading store file list, which could also + * speed up the loading of store files as listing is also not a fast operation on most object + * storages. + */ +@InterfaceAudience.Private +public class FileBasedStoreFileTracker extends StoreFileTrackerBase { + + private final StoreFileListFile backedFile; + + private final Map storefiles = new HashMap<>(); + + public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + backedFile = new StoreFileListFile(ctx); + } + + @Override + public List load() throws IOException { + StoreFileList list = backedFile.load(); + if (list == null) { + return Collections.emptyList(); + } + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + List infos = new ArrayList<>(); + for (StoreFileEntry entry : list.getStoreFileList()) { + infos.add(ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, ctx.getRegionInfo(), + ctx.getRegionFileSystem().getRegionInfoForFS(), ctx.getFamily().getNameAsString(), + new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()))); + } + // In general, for primary replica, the load method should only be called once when + // initialization, so we do not need synchronized here. And for secondary replicas, though the + // load method could be called multiple times, we will never call other methods so no + // synchronized is also fine. + // But we have a refreshStoreFiles method in the Region interface, which can be called by CPs, + // and we have a RefreshHFilesEndpoint example to expose the refreshStoreFiles method as RPC, so + // for safety, let's still keep the synchronized here. + synchronized (storefiles) { + for (StoreFileInfo info : infos) { + storefiles.put(info.getPath().getName(), info); + } + } + return infos; + } + + @Override + protected boolean requireWritingToTmpDirFirst() { + return false; + } + + private StoreFileEntry toStoreFileEntry(StoreFileInfo info) { + return StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize()) + .build(); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + synchronized (storefiles) { + StoreFileList.Builder builder = StoreFileList.newBuilder(); + for (StoreFileInfo info : storefiles.values()) { + builder.addStoreFile(toStoreFileEntry(info)); + } + for (StoreFileInfo info : newFiles) { + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + for (StoreFileInfo info : newFiles) { + storefiles.put(info.getPath().getName(), info); + } + } + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + Set compactedFileNames = + compactedFiles.stream().map(info -> info.getPath().getName()).collect(Collectors.toSet()); + synchronized (storefiles) { + StoreFileList.Builder builder = StoreFileList.newBuilder(); + storefiles.forEach((name, info) -> { + if (compactedFileNames.contains(name)) { + return; + } + builder.addStoreFile(toStoreFileEntry(info)); + }); + for (StoreFileInfo info : newFiles) { + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + for (String name : compactedFileNames) { + storefiles.remove(name); + } + for (StoreFileInfo info : newFiles) { + storefiles.put(info.getPath().getName(), info); + } + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java new file mode 100644 index 000000000000..c778bfc51deb --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.FileNotFoundException; +import java.io.IOException; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; +import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; + +/** + * To fully avoid listing, here we use two files for tracking. When loading, we will try to read + * both the two files, if only one exists, we will trust this one, if both exist, we will compare + * the timestamp to see which one is newer and trust that one. And we will record in memory that + * which one is trusted by us, and when we need to update the store file list, we will write to the + * other file. + *

+ * So in this way, we could avoid listing when we want to load the store file list file. + */ +@InterfaceAudience.Private +class StoreFileListFile { + + private static final Logger LOG = LoggerFactory.getLogger(StoreFileListFile.class); + + private static final String TRACK_FILE_DIR = ".filelist"; + + private static final String TRACK_FILE = "f1"; + + private static final String TRACK_FILE_ROTATE = "f2"; + + private final StoreContext ctx; + + private final Path trackFileDir; + + private final Path[] trackFiles = new Path[2]; + + // this is used to make sure that we do not go backwards + private long prevTimestamp = -1; + + private int nextTrackFile = -1; + + StoreFileListFile(StoreContext ctx) { + this.ctx = ctx; + trackFileDir = new Path(ctx.getFamilyStoreDirectoryPath(), TRACK_FILE_DIR); + trackFiles[0] = new Path(trackFileDir, TRACK_FILE); + trackFiles[1] = new Path(trackFileDir, TRACK_FILE_ROTATE); + } + + private StoreFileList load(Path path) throws IOException { + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + byte[] bytes; + try (FSDataInputStream in = fs.open(path)) { + bytes = ByteStreams.toByteArray(in); + } + // Read all the bytes and then parse it, so we will only throw InvalidProtocolBufferException + // here. This is very important for upper layer to determine whether this is the normal case, + // where the file does not exist or is incomplete. If there is another type of exception, the + // upper layer should throw it out instead of just ignoring it, otherwise it will lead to data + // loss. + return StoreFileList.parseFrom(bytes); + } + + private int select(StoreFileList[] lists) { + if (lists[0] == null) { + return 1; + } + if (lists[1] == null) { + return 0; + } + return lists[0].getTimestamp() >= lists[1].getTimestamp() ? 0 : 1; + } + + StoreFileList load() throws IOException { + StoreFileList[] lists = new StoreFileList[2]; + for (int i = 0; i < 2; i++) { + try { + lists[i] = load(trackFiles[i]); + } catch (FileNotFoundException | InvalidProtocolBufferException e) { + // this is normal case, so use info and do not log stacktrace + LOG.info("Failed to load track file {}: {}", trackFiles[i], e); + } + } + int winnerIndex = select(lists); + if (lists[winnerIndex] != null) { + nextTrackFile = 1 - winnerIndex; + prevTimestamp = lists[winnerIndex].getTimestamp(); + } else { + nextTrackFile = 0; + } + return lists[winnerIndex]; + } + + /** + * We will set the timestamp in this method so just pass the builder in + */ + void update(StoreFileList.Builder builder) throws IOException { + Preconditions.checkState(nextTrackFile >= 0, "should call load first before calling update"); + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + long timestamp = Math.max(prevTimestamp + 1, EnvironmentEdgeManager.currentTime()); + try (FSDataOutputStream out = fs.create(trackFiles[nextTrackFile], true)) { + builder.setTimestamp(timestamp).build().writeTo(out); + } + // record timestamp + prevTimestamp = timestamp; + // rotate the file + nextTrackFile = 1 - nextTrackFile; + try { + fs.delete(trackFiles[nextTrackFile], false); + } catch (IOException e) { + // we will create new file with overwrite = true, so not a big deal here, only for speed up + // loading as we do not need to read this file when loading(we will hit FileNotFoundException) + LOG.debug("failed to delete old track file {}, not a big deal, just ignore", e); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 2451f45bdc7b..92c699278c2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -21,7 +21,6 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -51,18 +50,14 @@ abstract class StoreFileTrackerBase implements StoreFileTracker { protected final Configuration conf; - protected final TableName tableName; - protected final boolean isPrimaryReplica; protected final StoreContext ctx; private volatile boolean cacheOnWriteLogged; - protected StoreFileTrackerBase(Configuration conf, TableName tableName, boolean isPrimaryReplica, - StoreContext ctx) { + protected StoreFileTrackerBase(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { this.conf = conf; - this.tableName = tableName; this.isPrimaryReplica = isPrimaryReplica; this.ctx = ctx; } @@ -95,7 +90,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, .withBlockSize(family.getBlocksize()).withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(encryptionContext) .withCreateTime(EnvironmentEdgeManager.currentTime()).withColumnFamily(family.getName()) - .withTableName(tableName.getName()).withCellComparator(ctx.getComparator()).build(); + .withTableName(ctx.getTableName().getName()).withCellComparator(ctx.getComparator()).build(); return hFileContext; } @@ -153,7 +148,7 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) outputDir = new Path(ctx.getRegionFileSystem().getTempDir(), ctx.getFamily().getNameAsString()); } else { - throw new UnsupportedOperationException("not supported yet"); + outputDir = ctx.getFamilyStoreDirectoryPath(); } StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf, ctx.getRegionFileSystem().getFileSystem()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 4f7231bc3b9a..6cdfaf4a0fe1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; /** @@ -28,8 +28,12 @@ @InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static StoreFileTracker create(Configuration conf, TableName tableName, - boolean isPrimaryReplica, StoreContext ctx) { - return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx); + public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; + + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, + StoreContext ctx) { + Class tracker = + conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java new file mode 100644 index 000000000000..3bc60d13fd94 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestRegionWithFileBasedStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionWithFileBasedStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final TableDescriptor TD = + TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build(); + + @Rule + public TestName name = new TestName(); + + private HRegion region; + + @Before + public void setUp() throws IOException { + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, FileBasedStoreFileTracker.class, + StoreFileTracker.class); + region = + HBaseTestingUtil.createRegionAndWAL(RI, UTIL.getDataTestDir(name.getMethodName()), conf, TD); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + HBaseTestingUtil.closeRegionAndWAL(region); + } + UTIL.cleanupTestDir(); + } + + @Test + public void testFlushAndCompaction() throws IOException { + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int v = i * 10 + j; + region.put(new Put(Bytes.toBytes(v)).addColumn(CF, CQ, Bytes.toBytes(v))); + } + region.flush(true); + if (i % 3 == 2) { + region.compact(true); + } + } + // reopen the region, make sure the store file tracker works, i.e, we can get all the records + // back + region.close(); + region = HRegion.openHRegion(region, null); + for (int i = 0; i < 100; i++) { + Result result = region.get(new Get(Bytes.toBytes(i))); + assertEquals(i, Bytes.toInt(result.getValue(CF, CQ))); + } + } +} From 3b9936d37c1bfc1269e9ad066033262214b6e7c9 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 8 Sep 2021 10:31:49 +0100 Subject: [PATCH 03/37] HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) Signed-off-by: Duo Zhang --- .../MergeTableRegionsProcedure.java | 23 +- .../assignment/SplitTableRegionProcedure.java | 41 +-- .../hbase/regionserver/HRegionFileSystem.java | 42 ++- .../DefaultStoreFileTracker.java | 4 +- .../storefiletracker/StoreFileTracker.java | 1 - .../StoreFileTrackerFactory.java | 33 ++- .../regionserver/TestDefaultStoreEngine.java | 1 + .../TestDirectStoreSplitsMerges.java | 32 ++- .../hbase/regionserver/TestHStoreFile.java | 19 +- .../TestMergesSplitsAddToTracker.java | 262 ++++++++++++++++++ .../regionserver/TestStripeStoreEngine.java | 1 + .../TestStoreFileTracker.java | 56 ++++ 12 files changed, 473 insertions(+), 42 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index fb57cb92dc2f..4b25a28a6055 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -24,6 +24,8 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -53,6 +55,8 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WALSplitUtil; @@ -584,40 +588,47 @@ private void createMergedRegion(final MasterProcedureEnv env) throws IOException final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); - + List mergedFiles = new ArrayList<>(); HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem( env.getMasterConfiguration(), fs, tableDir, mergedRegion); for (RegionInfo ri: this.regionsToMerge) { HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tableDir, ri, false); - mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion); + mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); } assert mergeRegionFs != null; - mergeRegionFs.commitMergedRegion(); + mergeRegionFs.commitMergedRegion(mergedFiles, env); // Prepare to create merged regions env.getAssignmentManager().getRegionStates(). getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); } - private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, + private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { final TableDescriptor htd = env.getMasterServices().getTableDescriptors() .get(mergedRegion.getTable()); + List mergedFiles = new ArrayList<>(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); - final Collection storeFiles = regionFs.getStoreFiles(family); + Configuration trackerConfig = + StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, + family, regionFs); + final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { // Create reference file(s) to parent region file here in mergedDir. // As this procedure is running on master, use CacheConfig.DISABLED means // don't cache any block. - mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, + Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + mergedFiles.add(refFile); } } } + return mergedFiles; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 0a15e36a16af..4d53df4bc670 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -64,6 +64,8 @@ import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.RegionSplitRestriction; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -618,21 +620,20 @@ public void createDaughterRegions(final MasterProcedureEnv env) throws IOExcepti final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); - regionFs.createSplitsDir(daughterOneRI, daughterTwoRI); - Pair expectedReferences = splitStoreFiles(env, regionFs); + Pair, List> expectedReferences = splitStoreFiles(env, regionFs); - assertReferenceFileCount(fs, expectedReferences.getFirst(), + assertReferenceFileCount(fs, expectedReferences.getFirst().size(), regionFs.getSplitsDir(daughterOneRI)); - regionFs.commitDaughterRegion(daughterOneRI); - assertReferenceFileCount(fs, expectedReferences.getFirst(), + regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst(), env); + assertReferenceFileCount(fs, expectedReferences.getFirst().size(), new Path(tabledir, daughterOneRI.getEncodedName())); - assertReferenceFileCount(fs, expectedReferences.getSecond(), + assertReferenceFileCount(fs, expectedReferences.getSecond().size(), regionFs.getSplitsDir(daughterTwoRI)); - regionFs.commitDaughterRegion(daughterTwoRI); - assertReferenceFileCount(fs, expectedReferences.getSecond(), + regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond(), env); + assertReferenceFileCount(fs, expectedReferences.getSecond().size(), new Path(tabledir, daughterTwoRI.getEncodedName())); } @@ -649,7 +650,7 @@ private void deleteDaughterRegions(final MasterProcedureEnv env) throws IOExcept * Create Split directory * @param env MasterProcedureEnv */ - private Pair splitStoreFiles(final MasterProcedureEnv env, + private Pair, List> splitStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws IOException { final Configuration conf = env.getMasterConfiguration(); TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); @@ -665,7 +666,11 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, new HashMap>(htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); - Collection sfis = regionFs.getStoreFiles(family); + Configuration trackerConfig = StoreFileTrackerFactory. + mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, + family, regionFs); + Collection sfis = tracker.load(); if (sfis == null) { continue; } @@ -691,7 +696,7 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, } if (nbFiles == 0) { // no file needs to be splitted. - return new Pair(0, 0); + return new Pair<>(Collections.emptyList(), Collections.emptyList()); } // Max #threads is the smaller of the number of storefiles or the default max determined above. int maxThreads = Math.min( @@ -744,14 +749,18 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, throw (InterruptedIOException) new InterruptedIOException().initCause(e); } - int daughterA = 0; - int daughterB = 0; + List daughterA = new ArrayList<>(); + List daughterB = new ArrayList<>(); // Look for any exception for (Future> future : futures) { try { Pair p = future.get(); - daughterA += p.getFirst() != null ? 1 : 0; - daughterB += p.getSecond() != null ? 1 : 0; + if(p.getFirst() != null){ + daughterA.add(p.getFirst()); + } + if(p.getSecond() != null){ + daughterB.add(p.getSecond()); + } } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } catch (ExecutionException e) { @@ -764,7 +773,7 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, getParentRegion().getShortNameToLog() + " Daughter A: " + daughterA + " storefiles, Daughter B: " + daughterB + " storefiles."); } - return new Pair(daughterA, daughterB); + return new Pair<>(daughterA, daughterB); } private void assertReferenceFileCount(final FileSystem fs, final int expectedReferenceFileCount, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 6ab9b0fc3fe8..3b053a349c2c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -23,7 +23,9 @@ import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -45,6 +47,9 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; @@ -592,19 +597,46 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} * @throws IOException */ - public Path commitDaughterRegion(final RegionInfo regionInfo) - throws IOException { + public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, + MasterProcedureEnv env) throws IOException { Path regionDir = this.getSplitsDir(regionInfo); if (fs.exists(regionDir)) { // Write HRI to a file in case we need to recover hbase:meta Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); + HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( + env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false); + insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); } - return regionDir; } + private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProcedureEnv env, + HRegionFileSystem regionFs) throws IOException { + TableDescriptor tblDesc = env.getMasterServices().getTableDescriptors(). + get(regionInfo.getTable()); + //we need to map trackers per store + Map trackerMap = new HashMap<>(); + //we need to map store files per store + Map> fileInfoMap = new HashMap<>(); + for(Path file : allFiles) { + String familyName = file.getParent().getName(); + trackerMap.computeIfAbsent(familyName, t -> { + Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, + tblDesc.getColumnFamily(Bytes.toBytes(familyName))); + return StoreFileTrackerFactory. + create(config, true, familyName, regionFs); + }); + fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); + List infos = fileInfoMap.get(familyName); + infos.add(new StoreFileInfo(conf, fs, file, true)); + } + for(Map.Entry entry : trackerMap.entrySet()) { + entry.getValue().add(fileInfoMap.get(entry.getKey())); + } + } + /** * Creates region split daughter directories under the table dir. If the daughter regions already * exist, for example, in the case of a recovery from a previous failed split procedure, this @@ -755,13 +787,15 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi * Commit a merged region, making it ready for use. * @throws IOException */ - public void commitMergedRegion() throws IOException { + public void commitMergedRegion(List allMergedFiles, MasterProcedureEnv env) + throws IOException { Path regionDir = getMergesDir(regionInfoForFs); if (regionDir != null && fs.exists(regionDir)) { // Write HRI to a file in case we need to recover hbase:meta Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); + insertRegionFilesIntoStoreTracker(allMergedFiles, env, this); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index fa044818336f..22e05132bf91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -32,8 +33,7 @@ @InterfaceAudience.Private class DefaultStoreFileTracker extends StoreFileTrackerBase { - public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index aadedc8ef727..0a85abb00a4e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -48,7 +48,6 @@ */ @InterfaceAudience.Private public interface StoreFileTracker { - /** * Load the store files list when opening a region. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 6cdfaf4a0fe1..c446d5ae9a31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,22 +18,51 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompoundConfiguration; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Factory method for creating store file tracker. */ @InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + StoreContext ctx) { Class tracker = conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, + HRegionFileSystem regionFs) { + ColumnFamilyDescriptorBuilder fDescBuilder = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); + StoreContext ctx = StoreContext.getBuilder(). + withColumnFamilyDescriptor(fDescBuilder.build()). + withRegionFileSystem(regionFs). + build(); + return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + } + + public static Configuration mergeConfigurations(Configuration global, + TableDescriptor table, ColumnFamilyDescriptor family) { + return new CompoundConfiguration() + .add(global) + .addBytesMap(table.getValues()) + .addStringMap(family.getConfiguration()) + .addBytesMap(family.getValues()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java index 3784876a59f3..523f27782362 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java @@ -67,6 +67,7 @@ public void testCustomParts() throws Exception { DummyStoreFlusher.class.getName()); HRegion mockRegion = Mockito.mock(HRegion.class); HStore mockStore = Mockito.mock(HStore.class); + mockStore.conf = conf; Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); Mockito.when(mockStore.getHRegion()).thenReturn(mockRegion); StoreEngine se = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java index bd24f1b22293..0eba8aa541ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.Path; @@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -139,7 +141,9 @@ public void testCommitDaughterRegionNoFiles() throws Exception { setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime()).build(); Path splitDir = regionFS.getSplitsDir(daughterA); - Path result = regionFS.commitDaughterRegion(daughterA); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path result = regionFS.commitDaughterRegion(daughterA, new ArrayList<>(), env); assertEquals(splitDir, result); } @@ -162,14 +166,18 @@ public void testCommitDaughterRegionWithFiles() throws Exception { Path splitDirA = regionFS.getSplitsDir(daughterA); Path splitDirB = regionFS.getSplitsDir(daughterB); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - regionFS + List filesA = new ArrayList<>(); + filesA.add(regionFS .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy()); - regionFS + Bytes.toBytes("002"), false, region.getSplitPolicy())); + List filesB = new ArrayList<>(); + filesB.add(regionFS .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), true, region.getSplitPolicy()); - Path resultA = regionFS.commitDaughterRegion(daughterA); - Path resultB = regionFS.commitDaughterRegion(daughterB); + Bytes.toBytes("002"), true, region.getSplitPolicy())); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path resultA = regionFS.commitDaughterRegion(daughterA, filesA, env); + Path resultB = regionFS.commitDaughterRegion(daughterB, filesB, env); assertEquals(splitDirA, resultA); assertEquals(splitDirB, resultB); } @@ -203,8 +211,11 @@ public void testCommitMergedRegion() throws Exception { mergeFileFromRegion(mergeRegionFs, first, file); //merge file from second region file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - mergeFileFromRegion(mergeRegionFs, second, file); - mergeRegionFs.commitMergedRegion(); + List mergedFiles = new ArrayList<>(); + mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file)); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + mergeRegionFs.commitMergedRegion(mergedFiles, env); } private void waitForSplitProcComplete(int attempts, int waitTime) throws Exception { @@ -223,11 +234,12 @@ private void waitForSplitProcComplete(int attempts, int waitTime) throws Excepti } } - private void mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, + private Path mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, HStoreFile file) throws IOException { Path mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile); + return mergedFile; } private void validateResultingFile(String originalRegion, Path result){ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index e6f0357204fc..c7203a928aa7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -49,12 +50,14 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -69,6 +72,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.BloomFilterFactory; @@ -1055,7 +1060,19 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h if (null == path) { return null; } - Path regionDir = regionFs.commitDaughterRegion(hri); + List splitFiles = new ArrayList<>(); + splitFiles.add(path); + MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class); + MasterServices mockServices = mock(MasterServices.class); + when(mockEnv.getMasterServices()).thenReturn(mockServices); + when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration()); + TableDescriptors mockTblDescs = mock(TableDescriptors.class); + when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs); + TableDescriptor mockTblDesc = mock(TableDescriptor.class); + when(mockTblDescs.get(any())).thenReturn(mockTblDesc); + ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class); + when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc); + Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); return new Path(new Path(regionDir, family), path.getName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java new file mode 100644 index 000000000000..c6205cb18493 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory. + TRACK_IMPL; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + + +@Category({RegionServerTests.class, LargeTests.class}) +public class TestMergesSplitsAddToTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMergesSplitsAddToTracker.class); + + private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + + public static final byte[] FAMILY_NAME = Bytes.toBytes("info"); + + @Rule + public TestName name = new TestName(); + + @BeforeClass + public static void setupClass() throws Exception { + TEST_UTIL.getConfiguration().set(TRACK_IMPL, TestStoreFileTracker.class.getName()); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup(){ + TestStoreFileTracker.trackedFiles = new HashMap<>(); + } + + @Test + public void testCommitDaughterRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //first put some data in order to have a store file created + putThreeRowsAndFlush(table); + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0); + HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem(); + RegionInfo daughterA = + RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey()). + setEndKey(Bytes.toBytes("002")).setSplit(false). + setRegionId(region.getRegionInfo().getRegionId() + + EnvironmentEdgeManager.currentTime()). + build(); + RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002")) + .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false) + .setRegionId(region.getRegionInfo().getRegionId()).build(); + HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + List splitFilesA = new ArrayList<>(); + splitFilesA.add(regionFS + .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, + Bytes.toBytes("002"), false, region.getSplitPolicy())); + List splitFilesB = new ArrayList<>(); + splitFilesB.add(regionFS + .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, + Bytes.toBytes("002"), true, region.getSplitPolicy())); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env); + Path resultB = regionFS.commitDaughterRegion(daughterB, splitFilesB, env); + FileSystem fs = regionFS.getFileSystem(); + verifyFilesAreTracked(resultA, fs); + verifyFilesAreTracked(resultB, fs); + } + + @Test + public void testCommitMergedRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //splitting the table first + TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002")); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + HRegion second = regions.get(1); + HRegionFileSystem regionFS = first.getRegionFileSystem(); + + RegionInfo mergeResult = + RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey()) + .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false) + .setRegionId(first.getRegionInfo().getRegionId() + + EnvironmentEdgeManager.currentTime()).build(); + + HRegionFileSystem mergeFS = HRegionFileSystem.createRegionOnFileSystem( + TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), + regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult); + + List mergedFiles = new ArrayList<>(); + //merge file from first region + mergedFiles.add(mergeFileFromRegion(first, mergeFS)); + //merge file from second region + mergedFiles.add(mergeFileFromRegion(second, mergeFS)); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + mergeFS.commitMergedRegion(mergedFiles, env); + //validate + FileSystem fs = first.getRegionFileSystem().getFileSystem(); + Path finalMergeDir = new Path(first.getRegionFileSystem().getTableDir(), + mergeResult.getEncodedName()); + verifyFilesAreTracked(finalMergeDir, fs); + } + + @Test + public void testSplitLoadsFromTracker() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0); + Pair copyResult = copyFileInTheStoreDir(region); + StoreFileInfo fileInfo = copyResult.getFirst(); + String copyName = copyResult.getSecond(); + //Now splits the region + TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002")); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + validateDaughterRegionsFiles(first, fileInfo.getActiveFileName(), copyName); + HRegion second = regions.get(1); + validateDaughterRegionsFiles(second, fileInfo.getActiveFileName(), copyName); + } + + @Test + public void testMergeLoadsFromTracker() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, new byte[][]{FAMILY_NAME}, + new byte[][]{Bytes.toBytes("002")}); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + Pair copyResult = copyFileInTheStoreDir(first); + StoreFileInfo fileInfo = copyResult.getFirst(); + String copyName = copyResult.getSecond(); + //Now merges the first two regions + TEST_UTIL.getAdmin().mergeRegionsAsync(new byte[][]{ + first.getRegionInfo().getEncodedNameAsBytes(), + regions.get(1).getRegionInfo().getEncodedNameAsBytes() + }, true).get(10, TimeUnit.SECONDS); + regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion merged = regions.get(0); + validateDaughterRegionsFiles(merged, fileInfo.getActiveFileName(), copyName); + } + + private Pair copyFileInTheStoreDir(HRegion region) throws IOException { + Path storeDir = region.getRegionFileSystem().getStoreDir("info"); + //gets the single file + StoreFileInfo fileInfo = region.getRegionFileSystem().getStoreFiles("info").get(0); + //make a copy of the valid file staight into the store dir, so that it's not tracked. + String copyName = UUID.randomUUID().toString().replaceAll("-", ""); + Path copy = new Path(storeDir, copyName); + FileUtil.copy(region.getFilesystem(), fileInfo.getFileStatus(), region.getFilesystem(), + copy , false, false, TEST_UTIL.getConfiguration()); + return new Pair<>(fileInfo, copyName); + } + + private void validateDaughterRegionsFiles(HRegion region, String orignalFileName, + String untrackedFile) throws IOException { + //verify there's no link for the untracked, copied file in first region + List infos = region.getRegionFileSystem().getStoreFiles("info"); + final MutableBoolean foundLink = new MutableBoolean(false); + infos.stream().forEach(i -> { + i.getActiveFileName().contains(orignalFileName); + if(i.getActiveFileName().contains(untrackedFile)){ + fail(); + } + if(i.getActiveFileName().contains(orignalFileName)){ + foundLink.setTrue(); + } + }); + assertTrue(foundLink.booleanValue()); + } + + private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Exception { + String storeId = regionDir.getName() + "-info"; + for(FileStatus f : fs.listStatus(new Path(regionDir, Bytes.toString(FAMILY_NAME)))){ + assertTrue(TestStoreFileTracker.trackedFiles.get(storeId).stream().filter(s -> + s.getPath().equals(f.getPath())).findFirst().isPresent()); + } + } + + private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS) + throws IOException { + HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); + } + + private void putThreeRowsAndFlush(TableName table) throws IOException { + Table tbl = TEST_UTIL.getConnection().getTable(table); + Put put = new Put(Bytes.toBytes("001")); + byte[] qualifier = Bytes.toBytes("1"); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1)); + tbl.put(put); + put = new Put(Bytes.toBytes("002")); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2)); + tbl.put(put); + put = new Put(Bytes.toBytes("003")); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2)); + tbl.put(put); + TEST_UTIL.flush(table); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index eb0b1c1ca694..80012dfcd461 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -120,6 +120,7 @@ private static HStoreFile createFile() throws Exception { private static TestStoreEngine createEngine(Configuration conf) throws Exception { HRegion region = mock(HRegion.class); HStore store = mock(HStore.class); + store.conf = conf; when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); when(store.getHRegion()).thenReturn(region); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java new file mode 100644 index 000000000000..05ca1fcb419b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestStoreFileTracker extends DefaultStoreFileTracker { + + private static final Logger LOG = LoggerFactory.getLogger(TestStoreFileTracker.class); + public static Map> trackedFiles = new HashMap<>(); + private String storeId; + + public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); + LOG.info("created storeId: {}", storeId); + trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + LOG.info("adding to storeId: {}", storeId); + trackedFiles.get(storeId).addAll(newFiles); + } + + @Override + public List load() throws IOException { + return trackedFiles.get(storeId); + } +} From 17565843e8fa0d0d9f47439c926cf1884fe7d4c4 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 9 Sep 2021 12:11:25 +0800 Subject: [PATCH 04/37] HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656) Signed-off-by: Wellington Chevreuil --- .../DefaultStoreFileTracker.java | 10 +- .../FileBasedStoreFileTracker.java | 15 +- .../MigrationStoreFileTracker.java | 88 ++++++++ .../storefiletracker/StoreFileListFile.java | 6 +- .../StoreFileTrackerBase.java | 12 +- .../StoreFileTrackerFactory.java | 39 ++-- .../TestMigrationStoreFileTracker.java | 193 ++++++++++++++++++ 7 files changed, 343 insertions(+), 20 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index 22e05132bf91..a13b75b0dcc4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -39,7 +40,9 @@ public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, Sto @Override public List load() throws IOException { - return ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + List files = + ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + return files != null ? files : Collections.emptyList(); } @Override @@ -57,4 +60,9 @@ protected void doAddCompactionResults(Collection compactedFiles, Collection newFiles) throws IOException { // NOOP } + + @Override + void set(List files) { + // NOOP + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index de28b0eb9996..c370b87c1154 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -48,7 +48,7 @@ * storages. */ @InterfaceAudience.Private -public class FileBasedStoreFileTracker extends StoreFileTrackerBase { +class FileBasedStoreFileTracker extends StoreFileTrackerBase { private final StoreFileListFile backedFile; @@ -139,4 +139,17 @@ protected void doAddCompactionResults(Collection compactedFiles, } } } + + @Override + void set(List files) throws IOException { + synchronized (storefiles) { + storefiles.clear(); + StoreFileList.Builder builder = StoreFileList.newBuilder(); + for (StoreFileInfo info : files) { + storefiles.put(info.getPath().getName(), info); + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java new file mode 100644 index 000000000000..e486e6d563af --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + +/** + * A store file tracker used for migrating between store file tracker implementations. + */ +@InterfaceAudience.Private +class MigrationStoreFileTracker extends StoreFileTrackerBase { + + public static final String SRC_IMPL = "hbase.store.file-tracker.migration.src.impl"; + + public static final String DST_IMPL = "hbase.store.file-tracker.migration.dst.impl"; + + private final StoreFileTrackerBase src; + + private final StoreFileTrackerBase dst; + + public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + this.src = StoreFileTrackerFactory.create(conf, SRC_IMPL, isPrimaryReplica, ctx); + this.dst = StoreFileTrackerFactory.create(conf, DST_IMPL, isPrimaryReplica, ctx); + Preconditions.checkArgument(!src.getClass().equals(dst.getClass()), + "src and dst is the same: %s", src.getClass()); + } + + @Override + public List load() throws IOException { + List files = src.load(); + dst.set(files); + return files; + } + + @Override + protected boolean requireWritingToTmpDirFirst() { + // Returns true if either of the two StoreFileTracker returns true. + // For example, if we want to migrate from a tracker implementation which can ignore the broken + // files under data directory to a tracker implementation which can not, if we still allow + // writing in tmp directory directly, we may have some broken files under the data directory and + // then after we finally change the implementation which can not ignore the broken files, we + // will be in trouble. + return src.requireWritingToTmpDirFirst() || dst.requireWritingToTmpDirFirst(); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + src.doAddNewStoreFiles(newFiles); + dst.doAddNewStoreFiles(newFiles); + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + src.doAddCompactionResults(compactedFiles, newFiles); + dst.doAddCompactionResults(compactedFiles, newFiles); + } + + @Override + void set(List files) { + throw new UnsupportedOperationException( + "Should not call this method on " + getClass().getSimpleName()); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java index c778bfc51deb..ffb3647e6259 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; @@ -121,7 +120,10 @@ StoreFileList load() throws IOException { * We will set the timestamp in this method so just pass the builder in */ void update(StoreFileList.Builder builder) throws IOException { - Preconditions.checkState(nextTrackFile >= 0, "should call load first before calling update"); + if (nextTrackFile < 0) { + // we need to call load first to load the prevTimestamp and also the next file + load(); + } FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); long timestamp = Math.max(prevTimestamp + 1, EnvironmentEdgeManager.currentTime()); try (FSDataOutputStream out = fs.create(trackFiles[nextTrackFile], true)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 92c699278c2e..d860f8e3812d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -95,8 +96,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, } @Override - public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) - throws IOException { + public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException { if (!isPrimaryReplica) { throw new IllegalStateException("Should not call create writer on secondary replicas"); } @@ -170,4 +170,12 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) protected abstract void doAddCompactionResults(Collection compactedFiles, Collection newFiles) throws IOException; + + /** + * used to mirror the store file list after loading when migration. + *

+ * Do not add this method to the {@link StoreFileTracker} interface since we do not need this + * method in upper layer. + */ + abstract void set(List files) throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index c446d5ae9a31..2c2b71db6f8c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -30,6 +30,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + /** * Factory method for creating store file tracker. */ @@ -39,7 +41,7 @@ public final class StoreFileTrackerFactory { private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + StoreContext ctx) { Class tracker = conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); @@ -47,22 +49,31 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli } public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, - HRegionFileSystem regionFs) { + HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); - StoreContext ctx = StoreContext.getBuilder(). - withColumnFamilyDescriptor(fDescBuilder.build()). - withRegionFileSystem(regionFs). - build(); - return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) + .withRegionFileSystem(regionFs).build(); + return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx); + } + + public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, + ColumnFamilyDescriptor family) { + return new CompoundConfiguration().add(global).addBytesMap(table.getValues()) + .addStringMap(family.getConfiguration()).addBytesMap(family.getValues()); } - public static Configuration mergeConfigurations(Configuration global, - TableDescriptor table, ColumnFamilyDescriptor family) { - return new CompoundConfiguration() - .add(global) - .addBytesMap(table.getValues()) - .addStringMap(family.getConfiguration()) - .addBytesMap(family.getValues()); + static StoreFileTrackerBase create(Configuration conf, String configName, + boolean isPrimaryReplica, StoreContext ctx) { + String className = + Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); + Class tracker; + try { + tracker = Class.forName(className).asSubclass(StoreFileTrackerBase.class); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); + return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java new file mode 100644 index 000000000000..e9ebb4c792e4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.ChunkCreator; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WAL; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.io.Closeables; + +@RunWith(Parameterized.class) +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestMigrationStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMigrationStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final TableDescriptor TD = + TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build(); + + @Rule + public TestName name = new TestName(); + + @Parameter(0) + public Class srcImplClass; + + @Parameter(1) + public Class dstImplClass; + + private HRegion region; + + private Path rootDir; + + private WAL wal; + + @Parameters(name = "{index}: src={0}, dst={1}") + public static List params() { + List> impls = + Arrays.asList(DefaultStoreFileTracker.class, FileBasedStoreFileTracker.class); + List params = new ArrayList<>(); + for (Class src : impls) { + for (Class dst : impls) { + if (src.equals(dst)) { + continue; + } + params.add(new Object[] { src, dst }); + } + } + return params; + } + + @BeforeClass + public static void setUpBeforeClass() { + ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, + MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); + } + + @Before + public void setUp() throws IOException { + Configuration conf = UTIL.getConfiguration(); + conf.setClass(MigrationStoreFileTracker.SRC_IMPL, srcImplClass, StoreFileTrackerBase.class); + conf.setClass(MigrationStoreFileTracker.DST_IMPL, dstImplClass, StoreFileTrackerBase.class); + rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_")); + wal = HBaseTestingUtil.createWal(conf, rootDir, RI); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + region.close(); + } + Closeables.close(wal, true); + UTIL.cleanupTestDir(); + } + + private List getStoreFiles() { + return Iterables.getOnlyElement(region.getStores()).getStorefiles().stream() + .map(s -> s.getFileInfo().getPath().getName()).collect(Collectors.toList()); + } + + private HRegion createRegion(Class trackerImplClass) + throws IOException { + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true); + } + + private void reopenRegion(Class trackerImplClass) + throws IOException { + region.flush(true); + List before = getStoreFiles(); + region.close(); + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + region = HRegion.openHRegion(rootDir, RI, TD, wal, conf); + List after = getStoreFiles(); + assertEquals(before.size(), after.size()); + assertThat(after, hasItems(before.toArray(new String[0]))); + } + + private void putData(int start, int end) throws IOException { + for (int i = start; i < end; i++) { + region.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + if (i % 30 == 0) { + region.flush(true); + } + } + } + + private void verifyData(int start, int end) throws IOException { + for (int i = start; i < end; i++) { + Result result = region.get(new Get(Bytes.toBytes(i))); + assertEquals(i, Bytes.toInt(result.getValue(CF, CQ))); + } + } + + @Test + public void testMigration() throws IOException { + region = createRegion(srcImplClass); + putData(0, 100); + verifyData(0, 100); + reopenRegion(MigrationStoreFileTracker.class); + verifyData(0, 100); + region.compact(true); + putData(100, 200); + reopenRegion(dstImplClass); + verifyData(0, 200); + } +} From 0ce891a88f08179e4802bdc11cfb45d803c76079 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Sun, 12 Sep 2021 14:14:03 +0100 Subject: [PATCH 05/37] HBASE-26246 Persist the StoreFileTracker configurations to TableDescriptor when creating table (#3666) Signed-off-by: Duo Zhang --- .../procedure/CreateTableProcedure.java | 7 ++++++ .../hbase/regionserver/HRegionFileSystem.java | 2 +- .../MigrationStoreFileTracker.java | 8 ++++++ .../storefiletracker/StoreFileTracker.java | 8 ++++++ .../StoreFileTrackerBase.java | 13 ++++++++++ .../StoreFileTrackerFactory.java | 25 +++++++++++++------ .../apache/hadoop/hbase/client/TestAdmin.java | 6 +++++ .../hadoop/hbase/client/TestAdmin3.java | 6 +++++ .../hbase/client/TestAsyncTableAdminApi.java | 6 +++++ .../hbase/client/TestAsyncTableAdminApi3.java | 6 +++++ .../MasterProcedureTestingUtility.java | 7 ++++++ .../procedure/TestCreateTableProcedure.java | 17 +++++++++++++ .../TestStoreFileTracker.java | 14 ++++++++--- 13 files changed, 113 insertions(+), 12 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 2313e70f75bb..dccea5512c1f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -33,10 +34,12 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -285,6 +288,10 @@ private void preCreate(final MasterProcedureEnv env) getTableName(), (newRegions != null ? newRegions.size() : 0)); } + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); + StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder); + tableDescriptor = builder.build(); + final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { final RegionInfo[] regions = newRegions == null ? null : diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 3b053a349c2c..40a42ab10383 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -607,7 +607,7 @@ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegi writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false); - insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); + insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); } return regionDir; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index e486e6d563af..483a240baded 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -85,4 +86,11 @@ void set(List files) { throw new UnsupportedOperationException( "Should not call this method on " + getClass().getSimpleName()); } + + @Override + public void persistConfiguration(TableDescriptorBuilder builder) { + super.persistConfiguration(builder); + builder.setValue(SRC_IMPL, src.getClass().getName()); + builder.setValue(DST_IMPL, dst.getClass().getName()); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 0a85abb00a4e..81fa1a9be5b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.Collection; import java.util.List; + +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -71,4 +73,10 @@ void replace(Collection compactedFiles, Collection * @return Writer for a new StoreFile */ StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; + + /** + * Saves StoreFileTracker implementations specific configs into the table descriptors. + * @param builder The table descriptor builder for the given table. + */ + void persistConfiguration(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index d860f8e3812d..83ebbc78ab35 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -17,18 +17,22 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; + import java.io.IOException; import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -78,6 +82,15 @@ public final void replace(Collection compactedFiles, } } + @Override + public void persistConfiguration(TableDescriptorBuilder builder) { + if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) { + String trackerImpl = StoreFileTrackerFactory. + getStoreFileTrackerImpl(conf).getName(); + builder.setValue(TRACK_IMPL, trackerImpl).build(); + } + } + private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { if (compression == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 2c2b71db6f8c..daec74c2af62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -6,9 +6,7 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,6 +20,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.util.Bytes; @@ -35,15 +34,17 @@ /** * Factory method for creating store file tracker. */ -@InterfaceAudience.Private -public final class StoreFileTrackerFactory { +@InterfaceAudience.Private public final class StoreFileTrackerFactory { public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); + public static Class getStoreFileTrackerImpl(Configuration conf) { + return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + } + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = - conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + Class tracker = getStoreFileTrackerImpl(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } @@ -54,7 +55,7 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx); + return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); } public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, @@ -76,4 +77,12 @@ static StoreFileTrackerBase create(Configuration conf, String configName, LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + + public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) { + TableDescriptor tableDescriptor = builder.build(); + ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; + StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); + tracker.persistConfiguration(builder); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 22302629dc70..10ab4961a5b4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -423,6 +425,10 @@ private void testCloneTableSchema(final TableName tableName, final TableName new assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + // HBASE-26246 introduced persist of store file tracker into table descriptor + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); if (preserveSplits) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java index e7be08e41edf..7d40fd12bda5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -236,6 +238,10 @@ public void testGetTableDescriptor() throws IOException { ADMIN.createTable(htd); Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); TableDescriptor confirmedHtd = table.getDescriptor(); + //HBASE-26246 introduced persist of store file tracker into table descriptor + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); table.close(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 572a1d5a6226..a1832f49956a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -372,6 +374,10 @@ private void testCloneTableSchema(final TableName tableName, assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + //HBASE-26246 introduced persist of store file tracker into table descriptor + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); if (preserveSplits) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index 4a71baf44a13..cb89bf564720 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -35,6 +36,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -147,6 +149,10 @@ public void testGetTableDescriptor() throws Exception { TableDescriptor desc = builder.build(); admin.createTable(desc).join(); TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get(); + //HBASE-26246 introduced persist of store file tracker into table descriptor + desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index a3206432b810..2ceec5f241c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -60,6 +61,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -226,6 +228,11 @@ public static void validateTableCreation(final HMaster master, final TableName t assertTrue("family not found " + family[i], htd.getColumnFamily(Bytes.toBytes(family[i])) != null); } assertEquals(family.length, htd.getColumnFamilyCount()); + + // checks store file tracker impl has been properly set in htd + String storeFileTrackerImpl = + StoreFileTrackerFactory.getStoreFileTrackerImpl(master.getConfiguration()).getName(); + assertEquals(storeFileTrackerImpl, htd.getValue(TRACK_IMPL)); } public static void validateTableDeletion( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 3b57ed56744b..0bc77f0ef8d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -88,6 +90,21 @@ private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) thr MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); } + @Test + public void testCreateWithTrackImpl() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + ProcedureExecutor procExec = getMasterProcedureExecutor(); + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); + String trackerName = TestStoreFileTracker.class.getName(); + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, trackerName).build(); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + htd = getMaster().getTableDescriptors().get(tableName); + assertEquals(trackerName, htd.getValue(TRACK_IMPL)); + } + @Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor procExec = getMasterProcedureExecutor(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java index 05ca1fcb419b..1dc9c4e53fc5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,15 +40,21 @@ public class TestStoreFileTracker extends DefaultStoreFileTracker { public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); - LOG.info("created storeId: {}", storeId); - trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + if (ctx.getRegionFileSystem() != null) { + this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); + LOG.info("created storeId: {}", storeId); + trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + } else { + LOG.info("ctx.getRegionFileSystem() returned null. Leaving storeId null."); + } + } @Override protected void doAddNewStoreFiles(Collection newFiles) throws IOException { LOG.info("adding to storeId: {}", storeId); trackedFiles.get(storeId).addAll(newFiles); + trackedFiles.putIfAbsent(storeId, (List)newFiles); } @Override From 989028bf92ba19528077002eed2d123c0e143e83 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 14 Sep 2021 16:28:21 +0800 Subject: [PATCH 06/37] HBASE-26248 Should find a suitable way to let users specify the store file tracker implementation (#3665) Signed-off-by: Wellington Chevreuil --- .../MigrationStoreFileTracker.java | 13 ++- .../storefiletracker/StoreFileTracker.java | 7 +- .../StoreFileTrackerBase.java | 12 ++- .../StoreFileTrackerFactory.java | 97 +++++++++++++++++-- .../apache/hadoop/hbase/client/TestAdmin.java | 6 +- .../hadoop/hbase/client/TestAdmin3.java | 6 +- .../hbase/client/TestAsyncTableAdminApi.java | 6 +- .../hbase/client/TestAsyncTableAdminApi3.java | 6 +- .../MasterProcedureTestingUtility.java | 6 +- .../procedure/TestCreateTableProcedure.java | 6 +- .../TestMergesSplitsAddToTracker.java | 4 +- .../TestMigrationStoreFileTracker.java | 27 +++--- ...stRegionWithFileBasedStoreFileTracker.java | 3 +- .../TestStoreFileTrackerFactory.java | 58 +++++++++++ 14 files changed, 202 insertions(+), 55 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 483a240baded..3eeef9000576 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -44,8 +45,8 @@ class MigrationStoreFileTracker extends StoreFileTrackerBase { public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - this.src = StoreFileTrackerFactory.create(conf, SRC_IMPL, isPrimaryReplica, ctx); - this.dst = StoreFileTrackerFactory.create(conf, DST_IMPL, isPrimaryReplica, ctx); + this.src = StoreFileTrackerFactory.createForMigration(conf, SRC_IMPL, isPrimaryReplica, ctx); + this.dst = StoreFileTrackerFactory.createForMigration(conf, DST_IMPL, isPrimaryReplica, ctx); Preconditions.checkArgument(!src.getClass().equals(dst.getClass()), "src and dst is the same: %s", src.getClass()); } @@ -90,7 +91,11 @@ void set(List files) { @Override public void persistConfiguration(TableDescriptorBuilder builder) { super.persistConfiguration(builder); - builder.setValue(SRC_IMPL, src.getClass().getName()); - builder.setValue(DST_IMPL, dst.getClass().getName()); + if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { + builder.setValue(SRC_IMPL, src.getTrackerName()); + } + if (StringUtils.isEmpty(builder.getValue(DST_IMPL))) { + builder.setValue(DST_IMPL, dst.getTrackerName()); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 81fa1a9be5b2..59fe7ef52f96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -75,7 +75,12 @@ void replace(Collection compactedFiles, Collection StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; /** - * Saves StoreFileTracker implementations specific configs into the table descriptors. + * Saves StoreFileTracker implementations specific configurations into the table descriptors. + *

+ * This is used to avoid accidentally data loss when changing the cluster level store file tracker + * implementation, and also possible misconfiguration between master and region servers. + *

+ * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ void persistConfiguration(TableDescriptorBuilder builder); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 83ebbc78ab35..a786add49b21 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import java.io.IOException; import java.util.Collection; @@ -84,13 +84,15 @@ public final void replace(Collection compactedFiles, @Override public void persistConfiguration(TableDescriptorBuilder builder) { - if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) { - String trackerImpl = StoreFileTrackerFactory. - getStoreFileTrackerImpl(conf).getName(); - builder.setValue(TRACK_IMPL, trackerImpl).build(); + if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { + builder.setValue(TRACKER_IMPL, getTrackerName()); } } + protected final String getTrackerName() { + return StoreFileTrackerFactory.getStoreFileTrackerName(getClass()); + } + private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { if (compression == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index daec74c2af62..95c1b91d78a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -15,6 +15,9 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompoundConfiguration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -33,22 +36,81 @@ /** * Factory method for creating store file tracker. + *

+ * The current implementations are: + *

    + *
  • default: DefaultStoreFileTracker, see {@link DefaultStoreFileTracker}.
    • + *
  • file:FileBasedStoreFileTracker, see {@link FileBasedStoreFileTracker}.
    • + *
  • migration:MigrationStoreFileTracker, see {@link MigrationStoreFileTracker}.
    • + *
+ * @see DefaultStoreFileTracker + * @see FileBasedStoreFileTracker + * @see MigrationStoreFileTracker */ -@InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; +@InterfaceAudience.Private +public final class StoreFileTrackerFactory { + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); - public static Class getStoreFileTrackerImpl(Configuration conf) { - return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + public static final String TRACKER_IMPL = "hbase.store.file-tracker.impl"; + + /** + * Maps between configuration names for trackers and implementation classes. + */ + public enum Trackers { + DEFAULT(DefaultStoreFileTracker.class), FILE(FileBasedStoreFileTracker.class), + MIGRATION(MigrationStoreFileTracker.class); + + final Class clazz; + + Trackers(Class clazz) { + this.clazz = clazz; + } + } + + private static final Map, Trackers> CLASS_TO_ENUM = reverse(); + + private static Map, Trackers> reverse() { + Map, Trackers> map = new HashMap<>(); + for (Trackers tracker : Trackers.values()) { + map.put(tracker.clazz, tracker); + } + return Collections.unmodifiableMap(map); + } + + private StoreFileTrackerFactory() { + } + + public static String getStoreFileTrackerName(Configuration conf) { + return conf.get(TRACKER_IMPL, Trackers.DEFAULT.name()); + } + + static String getStoreFileTrackerName(Class clazz) { + Trackers name = CLASS_TO_ENUM.get(clazz); + return name != null ? name.name() : clazz.getName(); + } + + private static Class getTrackerClass(Configuration conf) { + try { + Trackers tracker = Trackers.valueOf(getStoreFileTrackerName(conf).toUpperCase()); + return tracker.clazz; + } catch (IllegalArgumentException e) { + // Fall back to them specifying a class name + return conf.getClass(TRACKER_IMPL, Trackers.DEFAULT.clazz, StoreFileTracker.class); + } } public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = getStoreFileTrackerImpl(conf); + Class tracker = getTrackerClass(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + /** + * Used at master side when splitting/merging regions, as we do not have a Store, thus no + * StoreContext at master side. + */ public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = @@ -64,15 +126,30 @@ public static Configuration mergeConfigurations(Configuration global, TableDescr .addStringMap(family.getConfiguration()).addBytesMap(family.getValues()); } - static StoreFileTrackerBase create(Configuration conf, String configName, + /** + * Create store file tracker to be used as source or destination for + * {@link MigrationStoreFileTracker}. + */ + static StoreFileTrackerBase createForMigration(Configuration conf, String configName, boolean isPrimaryReplica, StoreContext ctx) { - String className = + String trackerName = Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); Class tracker; try { - tracker = Class.forName(className).asSubclass(StoreFileTrackerBase.class); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); + tracker = + Trackers.valueOf(trackerName.toUpperCase()).clazz.asSubclass(StoreFileTrackerBase.class); + } catch (IllegalArgumentException e) { + // Fall back to them specifying a class name + try { + tracker = Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException(cnfe); + } + } + // prevent nest of MigrationStoreFileTracker, it will cause infinite recursion. + if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { + throw new IllegalArgumentException("Should not specify " + configName + " as " + + Trackers.MIGRATION + " because it can not be nested"); } LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 10ab4961a5b4..ba843fe43460 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -426,8 +426,8 @@ private void testCloneTableSchema(final TableName tableName, final TableName new assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); // HBASE-26246 introduced persist of store file tracker into table descriptor - tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java index 7d40fd12bda5..c2de0fbd3555 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -239,8 +239,8 @@ public void testGetTableDescriptor() throws IOException { Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); TableDescriptor confirmedHtd = table.getDescriptor(); //HBASE-26246 introduced persist of store file tracker into table descriptor - htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index a1832f49956a..147630b53a38 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -375,8 +375,8 @@ private void testCloneTableSchema(final TableName tableName, assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); //HBASE-26246 introduced persist of store file tracker into table descriptor - tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index cb89bf564720..67377b07c180 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -150,8 +150,8 @@ public void testGetTableDescriptor() throws Exception { admin.createTable(desc).join(); TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get(); //HBASE-26246 introduced persist of store file tracker into table descriptor - desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index 2ceec5f241c6..726e117ef1ac 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -231,8 +231,8 @@ public static void validateTableCreation(final HMaster master, final TableName t // checks store file tracker impl has been properly set in htd String storeFileTrackerImpl = - StoreFileTrackerFactory.getStoreFileTrackerImpl(master.getConfiguration()).getName(); - assertEquals(storeFileTrackerImpl, htd.getValue(TRACK_IMPL)); + StoreFileTrackerFactory.getStoreFileTrackerName(master.getConfiguration()); + assertEquals(storeFileTrackerImpl, htd.getValue(TRACKER_IMPL)); } public static void validateTableDeletion( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 0bc77f0ef8d9..f432c8060d3d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -96,13 +96,13 @@ public void testCreateWithTrackImpl() throws Exception { ProcedureExecutor procExec = getMasterProcedureExecutor(); TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); String trackerName = TestStoreFileTracker.class.getName(); - htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, trackerName).build(); + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build(); RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); long procId = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); htd = getMaster().getTableDescriptors().get(tableName); - assertEquals(trackerName, htd.getValue(TRACK_IMPL)); + assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java index c6205cb18493..68fc444493c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory. - TRACK_IMPL; + TRACKER_IMPL; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -74,7 +74,7 @@ public class TestMergesSplitsAddToTracker { @BeforeClass public static void setupClass() throws Exception { - TEST_UTIL.getConfiguration().set(TRACK_IMPL, TestStoreFileTracker.class.getName()); + TEST_UTIL.getConfiguration().set(TRACKER_IMPL, TestStoreFileTracker.class.getName()); TEST_UTIL.startMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java index e9ebb4c792e4..119f8fb553f8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -86,10 +85,10 @@ public class TestMigrationStoreFileTracker { public TestName name = new TestName(); @Parameter(0) - public Class srcImplClass; + public StoreFileTrackerFactory.Trackers srcImpl; @Parameter(1) - public Class dstImplClass; + public StoreFileTrackerFactory.Trackers dstImpl; private HRegion region; @@ -99,11 +98,13 @@ public class TestMigrationStoreFileTracker { @Parameters(name = "{index}: src={0}, dst={1}") public static List params() { - List> impls = - Arrays.asList(DefaultStoreFileTracker.class, FileBasedStoreFileTracker.class); List params = new ArrayList<>(); - for (Class src : impls) { - for (Class dst : impls) { + for (StoreFileTrackerFactory.Trackers src : StoreFileTrackerFactory.Trackers.values()) { + for (StoreFileTrackerFactory.Trackers dst : StoreFileTrackerFactory.Trackers.values()) { + if (src == StoreFileTrackerFactory.Trackers.MIGRATION + || dst == StoreFileTrackerFactory.Trackers.MIGRATION) { + continue; + } if (src.equals(dst)) { continue; } @@ -122,8 +123,8 @@ public static void setUpBeforeClass() { @Before public void setUp() throws IOException { Configuration conf = UTIL.getConfiguration(); - conf.setClass(MigrationStoreFileTracker.SRC_IMPL, srcImplClass, StoreFileTrackerBase.class); - conf.setClass(MigrationStoreFileTracker.DST_IMPL, dstImplClass, StoreFileTrackerBase.class); + conf.set(MigrationStoreFileTracker.SRC_IMPL, srcImpl.name().toLowerCase()); + conf.set(MigrationStoreFileTracker.DST_IMPL, dstImpl.name().toLowerCase()); rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_")); wal = HBaseTestingUtil.createWal(conf, rootDir, RI); } @@ -145,7 +146,7 @@ private List getStoreFiles() { private HRegion createRegion(Class trackerImplClass) throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class); return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true); } @@ -155,7 +156,7 @@ private void reopenRegion(Class trackerImplClass List before = getStoreFiles(); region.close(); Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class); region = HRegion.openHRegion(rootDir, RI, TD, wal, conf); List after = getStoreFiles(); assertEquals(before.size(), after.size()); @@ -180,14 +181,14 @@ private void verifyData(int start, int end) throws IOException { @Test public void testMigration() throws IOException { - region = createRegion(srcImplClass); + region = createRegion(srcImpl.clazz.asSubclass(StoreFileTrackerBase.class)); putData(0, 100); verifyData(0, 100); reopenRegion(MigrationStoreFileTracker.class); verifyData(0, 100); region.compact(true); putData(100, 200); - reopenRegion(dstImplClass); + reopenRegion(dstImpl.clazz.asSubclass(StoreFileTrackerBase.class)); verifyData(0, 200); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java index 3bc60d13fd94..77cd3c448fa0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java @@ -71,8 +71,7 @@ public class TestRegionWithFileBasedStoreFileTracker { @Before public void setUp() throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, FileBasedStoreFileTracker.class, - StoreFileTracker.class); + conf.set(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()); region = HBaseTestingUtil.createRegionAndWAL(RI, UTIL.getDataTestDir(name.getMethodName()), conf, TD); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java new file mode 100644 index 000000000000..41f2afdfa421 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertThrows; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestStoreFileTrackerFactory { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStoreFileTrackerFactory.class); + + @Test + public void testCreateForMigration() { + Configuration conf = HBaseConfiguration.create(); + String configName = "config"; + + // no config + assertThrows(NullPointerException.class, () -> StoreFileTrackerFactory.createForMigration(conf, + configName, false, StoreContext.getBuilder().build())); + + // class not found + conf.set(configName, "config"); + assertThrows(RuntimeException.class, () -> StoreFileTrackerFactory.createForMigration(conf, + configName, false, StoreContext.getBuilder().build())); + + // nested MigrationStoreFileTracker + conf.setClass(configName, MigrationStoreFileTracker.class, StoreFileTrackerBase.class); + assertThrows(IllegalArgumentException.class, () -> StoreFileTrackerFactory + .createForMigration(conf, configName, false, StoreContext.getBuilder().build())); + } +} From 6581fef9a289c4037e0bf25c1ee4b6c38484d82f Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Wed, 15 Sep 2021 23:00:03 +0800 Subject: [PATCH 07/37] HBASE-26264 Add more checks to prevent misconfiguration on store file tracker (#3681) Signed-off-by: Josh Elser --- .../MergeTableRegionsProcedure.java | 3 +- .../assignment/SplitTableRegionProcedure.java | 3 +- .../procedure/CreateTableProcedure.java | 8 +- .../procedure/ModifyTableProcedure.java | 5 + .../hbase/regionserver/HRegionFileSystem.java | 2 +- .../MigrationStoreFileTracker.java | 8 + .../StoreFileTrackerFactory.java | 173 ++++++++++++- .../TestChangeStoreFileTracker.java | 242 ++++++++++++++++++ 8 files changed, 422 insertions(+), 22 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 4b25a28a6055..9a00fe26a8f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -614,8 +614,7 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg String family = hcd.getNameAsString(); Configuration trackerConfig = StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, - family, regionFs); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 4d53df4bc670..44136dc7cbee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -668,8 +668,7 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en String family = cfd.getNameAsString(); Configuration trackerConfig = StoreFileTrackerFactory. mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, - family, regionFs); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); Collection sfis = tracker.load(); if (sfis == null) { continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index dccea5512c1f..ee8e51ffe86b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -277,15 +277,17 @@ private boolean prepareCreate(final MasterProcedureEnv env) throws IOException { MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom); } + // check for store file tracker configurations + StoreFileTrackerFactory.checkForCreateTable(env.getMasterConfiguration(), tableDescriptor); + return true; } private void preCreate(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { - ProcedureSyncWait.getMasterQuotaManager(env) - .checkNamespaceTableAndRegionQuota( - getTableName(), (newRegions != null ? newRegions.size() : 0)); + ProcedureSyncWait.getMasterQuotaManager(env).checkNamespaceTableAndRegionQuota(getTableName(), + (newRegions != null ? newRegions.size() : 0)); } TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index 247dd9c202f4..1640644328f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.Bytes; @@ -325,6 +326,10 @@ private void prepareModify(final MasterProcedureEnv env) throws IOException { modifiedTableDescriptor.getRegionServerGroup(), forWhom); MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom); } + + // check for store file tracker configurations + StoreFileTrackerFactory.checkForModifyTable(env.getMasterConfiguration(), + unmodifiedTableDescriptor, modifiedTableDescriptor); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 40a42ab10383..db18d4e50fd6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -626,7 +626,7 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, tblDesc.getColumnFamily(Bytes.toBytes(familyName))); return StoreFileTrackerFactory. - create(config, true, familyName, regionFs); + create(config, familyName, regionFs); }); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 3eeef9000576..1946d4bf7eef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -98,4 +98,12 @@ public void persistConfiguration(TableDescriptorBuilder builder) { builder.setValue(DST_IMPL, dst.getTrackerName()); } } + + static Class getSrcTrackerClass(Configuration conf) { + return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, SRC_IMPL); + } + + static Class getDstTrackerClass(Configuration conf) { + return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, DST_IMPL); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 95c1b91d78a4..885ef74fd1d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -15,11 +15,13 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompoundConfiguration; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -111,13 +113,13 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. */ - public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, + public static StoreFileTracker create(Configuration conf, String family, HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + return StoreFileTrackerFactory.create(conf, true, ctx); } public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, @@ -126,30 +128,35 @@ public static Configuration mergeConfigurations(Configuration global, TableDescr .addStringMap(family.getConfiguration()).addBytesMap(family.getValues()); } - /** - * Create store file tracker to be used as source or destination for - * {@link MigrationStoreFileTracker}. - */ - static StoreFileTrackerBase createForMigration(Configuration conf, String configName, - boolean isPrimaryReplica, StoreContext ctx) { + static Class + getStoreFileTrackerClassForMigration(Configuration conf, String configName) { String trackerName = Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); - Class tracker; try { - tracker = - Trackers.valueOf(trackerName.toUpperCase()).clazz.asSubclass(StoreFileTrackerBase.class); + return Trackers.valueOf(trackerName.toUpperCase()).clazz + .asSubclass(StoreFileTrackerBase.class); } catch (IllegalArgumentException e) { // Fall back to them specifying a class name try { - tracker = Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); + return Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } } + } + + /** + * Create store file tracker to be used as source or destination for + * {@link MigrationStoreFileTracker}. + */ + static StoreFileTrackerBase createForMigration(Configuration conf, String configName, + boolean isPrimaryReplica, StoreContext ctx) { + Class tracker = + getStoreFileTrackerClassForMigration(conf, configName); // prevent nest of MigrationStoreFileTracker, it will cause infinite recursion. if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { - throw new IllegalArgumentException("Should not specify " + configName + " as " + - Trackers.MIGRATION + " because it can not be nested"); + throw new IllegalArgumentException("Should not specify " + configName + " as " + + Trackers.MIGRATION + " because it can not be nested"); } LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); @@ -162,4 +169,142 @@ public static void persistTrackerConfig(Configuration conf, TableDescriptorBuild StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); tracker.persistConfiguration(builder); } + + // should not use MigrationStoreFileTracker for new family + private static void checkForNewFamily(Configuration conf, TableDescriptor table, + ColumnFamilyDescriptor family) throws IOException { + Configuration mergedConf = mergeConfigurations(conf, table, family); + Class tracker = getTrackerClass(mergedConf); + if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { + throw new DoNotRetryIOException( + "Should not use " + Trackers.MIGRATION + " as store file tracker for new family " + + family.getNameAsString() + " of table " + table.getTableName()); + } + } + + /** + * Pre check when creating a new table. + *

+ * For now, only make sure that we do not use {@link Trackers#MIGRATION} for newly created tables. + * @throws IOException when there are check errors, the upper layer should fail the + * {@code CreateTableProcedure}. + */ + public static void checkForCreateTable(Configuration conf, TableDescriptor table) + throws IOException { + for (ColumnFamilyDescriptor family : table.getColumnFamilies()) { + checkForNewFamily(conf, table, family); + } + } + + + /** + * Pre check when modifying a table. + *

+ * The basic idea is when you want to change the store file tracker implementation, you should use + * {@link Trackers#MIGRATION} first and then change to the destination store file tracker + * implementation. + *

+ * There are several rules: + *

    + *
  • For newly added family, you should not use {@link Trackers#MIGRATION}.
    • + *
  • For modifying a family: + *
      + *
    • If old tracker is {@link Trackers#MIGRATION}, then: + *
        + *
      • The new tracker is also {@link Trackers#MIGRATION}, then they must have the same src and + * dst tracker.
        • + *
      • The new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the dst + * tracker of the old tracker.
        • + *
      + *
      • + *
    • If the old tracker is not {@link Trackers#MIGRATION}, then: + *
        + *
      • If the new tracker is {@link Trackers#MIGRATION}, then the old tracker must be the src + * tracker of the new tracker.
        • + *
      • If the new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the same + * with old tracker.
        • + *
      + *
      • + *
    + *
    • + *
+ * @throws IOException when there are check errors, the upper layer should fail the + * {@code ModifyTableProcedure}. + */ + public static void checkForModifyTable(Configuration conf, TableDescriptor oldTable, + TableDescriptor newTable) throws IOException { + for (ColumnFamilyDescriptor newFamily : newTable.getColumnFamilies()) { + ColumnFamilyDescriptor oldFamily = oldTable.getColumnFamily(newFamily.getName()); + if (oldFamily == null) { + checkForNewFamily(conf, newTable, newFamily); + continue; + } + Configuration oldConf = mergeConfigurations(conf, oldTable, oldFamily); + Configuration newConf = mergeConfigurations(conf, newTable, newFamily); + + Class oldTracker = getTrackerClass(oldConf); + Class newTracker = getTrackerClass(newConf); + + if (MigrationStoreFileTracker.class.isAssignableFrom(oldTracker)) { + Class oldSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(oldConf); + Class oldDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(oldConf); + if (oldTracker.equals(newTracker)) { + // confirm that we have the same src tracker and dst tracker + Class newSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(newConf); + if (!oldSrcTracker.equals(newSrcTracker)) { + throw new DoNotRetryIOException( + "The src tracker has been changed from " + getStoreFileTrackerName(oldSrcTracker) + + " to " + getStoreFileTrackerName(newSrcTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + Class newDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(newConf); + if (!oldDstTracker.equals(newDstTracker)) { + throw new DoNotRetryIOException( + "The dst tracker has been changed from " + getStoreFileTrackerName(oldDstTracker) + + " to " + getStoreFileTrackerName(newDstTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + } else { + // we can only change to the dst tracker + if (!newTracker.equals(oldDstTracker)) { + throw new DoNotRetryIOException( + "Should migrate tracker to " + getStoreFileTrackerName(oldDstTracker) + " but got " + + getStoreFileTrackerName(newTracker) + " for family " + newFamily.getNameAsString() + + " of table " + newTable.getTableName()); + } + } + } else { + if (!oldTracker.equals(newTracker)) { + // can only change to MigrationStoreFileTracker and the src tracker should be the old + // tracker + if (!MigrationStoreFileTracker.class.isAssignableFrom(newTracker)) { + throw new DoNotRetryIOException("Should change to " + Trackers.MIGRATION + + " first when migrating from " + getStoreFileTrackerName(oldTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + Class newSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(newConf); + if (!oldTracker.equals(newSrcTracker)) { + throw new DoNotRetryIOException( + "Should use src tracker " + getStoreFileTrackerName(oldTracker) + " first but got " + + getStoreFileTrackerName(newSrcTracker) + " when migrating from " + + getStoreFileTrackerName(oldTracker) + " for family " + newFamily.getNameAsString() + + " of table " + newTable.getTableName()); + } + Class newDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(newConf); + // the src and dst tracker should not be the same + if (newSrcTracker.equals(newDstTracker)) { + throw new DoNotRetryIOException("The src tracker and dst tracker are both " + + getStoreFileTrackerName(newSrcTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + } + } + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java new file mode 100644 index 000000000000..70f62c02ed28 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNameTestRule; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; + +/** + * Test changing store file tracker implementation by altering table. + */ +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestChangeStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestChangeStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + @Rule + public final TableNameTestRule tableName = new TableNameTestRule(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws IOException { + UTIL.shutdownMiniCluster(); + } + + @Test(expected = DoNotRetryIOException.class) + public void testCreateError() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().createTable(td); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError1() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError2() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError3() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + // return the TableDescriptor for creating table + private TableDescriptor createTableAndChangeToMigrationTracker() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + return td; + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError4() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError5() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError6() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = + TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.DEFAULT.name()).build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError7() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("family1")) + .setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .build()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + // actually a NPE as we do not specify the src and dst impl for migration store file tracker + @Test(expected = IOException.class) + public void testModifyError8() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = + TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()).build(); + UTIL.getAdmin().modifyTable(newTd); + } + + private String getStoreFileName(TableName table, byte[] family) { + return Iterables + .getOnlyElement(Iterables.getOnlyElement(UTIL.getMiniHBaseCluster().getRegions(table)) + .getStore(family).getStorefiles()) + .getPath().getName(); + } + + @Test + public void testModify() throws IOException { + TableName tn = tableName.getTableName(); + byte[] row = Bytes.toBytes("row"); + byte[] family = Bytes.toBytes("family"); + byte[] qualifier = Bytes.toBytes("qualifier"); + byte[] value = Bytes.toBytes("value"); + TableDescriptor td = TableDescriptorBuilder.newBuilder(tn) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); + UTIL.getAdmin().createTable(td); + try (Table table = UTIL.getConnection().getTable(tn)) { + table.put(new Put(row).addColumn(family, qualifier, value)); + } + UTIL.flush(tn); + String fileName = getStoreFileName(tn, family); + + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + assertEquals(fileName, getStoreFileName(tn, family)); + try (Table table = UTIL.getConnection().getTable(tn)) { + assertArrayEquals(value, table.get(new Get(row)).getValue(family, qualifier)); + } + + TableDescriptor newTd2 = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd2); + assertEquals(fileName, getStoreFileName(tn, family)); + try (Table table = UTIL.getConnection().getTable(tn)) { + assertArrayEquals(value, table.get(new Get(row)).getValue(family, qualifier)); + } + } +} From ab03ee1e97233916cf850efae87193884309f731 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 17 Sep 2021 09:40:44 +0800 Subject: [PATCH 08/37] HBASE-26280 Use store file tracker when snapshoting (#3685) Signed-off-by: Wellington Chevreuil Reviewed-by: Josh Elser --- .../MergeTableRegionsProcedure.java | 7 +- .../assignment/SplitTableRegionProcedure.java | 5 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +-- .../StoreFileTrackerFactory.java | 17 ++--- .../hbase/snapshot/SnapshotManifest.java | 42 +++++------ ...apshotFromClientCloneLinksAfterDelete.java | 4 +- .../client/TestMobSnapshotFromClient.java | 7 +- .../hbase/client/TestSnapshotFromClient.java | 30 ++++++-- .../hbase/regionserver/TestHStoreFile.java | 6 +- .../snapshot/MobSnapshotTestingUtils.java | 74 ++++++++++--------- .../hbase/snapshot/SnapshotTestingUtils.java | 16 ++-- 11 files changed, 107 insertions(+), 111 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 9a00fe26a8f0..85a433af81e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -24,8 +24,6 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; - -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -612,9 +610,8 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg List mergedFiles = new ArrayList<>(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); - Configuration trackerConfig = - StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 44136dc7cbee..60ff48985f42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -666,9 +666,8 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en new HashMap>(htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); - Configuration trackerConfig = StoreFileTrackerFactory. - mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, cfd, regionFs); Collection sfis = tracker.load(); if (sfis == null) { continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index db18d4e50fd6..03a7c5c8baaf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -595,7 +595,6 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * to the proper location in the filesystem. * * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} - * @throws IOException */ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, MasterProcedureEnv env) throws IOException { @@ -622,12 +621,8 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced Map> fileInfoMap = new HashMap<>(); for(Path file : allFiles) { String familyName = file.getParent().getName(); - trackerMap.computeIfAbsent(familyName, t -> { - Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, - tblDesc.getColumnFamily(Bytes.toBytes(familyName))); - return StoreFileTrackerFactory. - create(config, familyName, regionFs); - }); + trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc, + tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs)); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); infos.add(new StoreFileInfo(conf, fs, file, true)); @@ -673,7 +668,6 @@ public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws I * this method is invoked on the Master side, then the RegionSplitPolicy will * NOT have a reference to a Region. * @return Path to created reference. - * @throws IOException */ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, boolean top, RegionSplitPolicy splitPolicy) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 885ef74fd1d7..974238cb971e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -23,12 +23,10 @@ import org.apache.hadoop.hbase.CompoundConfiguration; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -113,16 +111,15 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. */ - public static StoreFileTracker create(Configuration conf, String family, - HRegionFileSystem regionFs) { - ColumnFamilyDescriptorBuilder fDescBuilder = - ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); - StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) - .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, true, ctx); + public static StoreFileTracker create(Configuration conf, TableDescriptor td, + ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs) { + StoreContext ctx = + StoreContext.getBuilder().withColumnFamilyDescriptor(cfd).withRegionFileSystem(regionFs) + .withFamilyStoreDirectoryPath(regionFs.getStoreDir(cfd.getNameAsString())).build(); + return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), true, ctx); } - public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, + private static Configuration mergeConfigurations(Configuration global, TableDescriptor table, ColumnFamilyDescriptor family) { return new CompoundConfiguration().add(global).addBytesMap(table.getValues()) .addStringMap(family.getConfiguration()).addBytesMap(family.getValues()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 5e82cad6b494..f154aa92cd6e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -47,7 +47,8 @@ import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.Threads; @@ -291,8 +292,8 @@ public void addRegion(final Path tableDir, final RegionInfo regionInfo) throws I addRegion(tableDir, regionInfo, visitor); } - protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor) - throws IOException { + protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor) + throws IOException { boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo); try { Path baseDir = tableDir; @@ -300,8 +301,8 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio if (isMobRegion) { baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable()); } - HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, - baseDir, regionInfo, true); + HRegionFileSystem regionFs = + HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true); monitor.rethrowException(); // 1. dump region meta info into the snapshot directory @@ -317,26 +318,19 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio // in batches and may miss files being added/deleted. This could be more robust (iteratively // checking to see if we have all the files until we are sure), but the limit is currently // 1000 files/batch, far more than the number of store files under a single column family. - Collection familyNames = regionFs.getFamilies(); - if (familyNames != null) { - for (String familyName: familyNames) { - Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName)); - monitor.rethrowException(); - - Collection storeFiles = regionFs.getStoreFiles(familyName); - if (storeFiles == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("No files under family: " + familyName); - } - continue; - } - - // 2.1. build the snapshot reference for the store - // iterate through all the store's files and create "references". - addReferenceFiles(visitor, regionData, familyData, storeFiles, false); - - visitor.familyClose(regionData, familyData); + for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { + Object familyData = visitor.familyOpen(regionData, cfd.getName()); + monitor.rethrowException(); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs); + List storeFiles = tracker.load(); + if (storeFiles.isEmpty()) { + LOG.debug("No files under family: {}", cfd.getNameAsString()); + continue; } + // 2.1. build the snapshot reference for the store + // iterate through all the store's files and create "references". + addReferenceFiles(visitor, regionData, familyData, storeFiles, false); + visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); } catch (IOException e) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java index c2087a935198..e352303f76ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; @@ -93,7 +94,8 @@ public static void setUpBeforeClass() throws Exception { @Override protected void createTable() throws IOException, InterruptedException { MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, - SnapshotTestingUtils.getSplitKeys(), getNumReplicas(), DelayFlushCoprocessor.class.getName(), + SnapshotTestingUtils.getSplitKeys(), getNumReplicas(), + StoreFileTrackerFactory.Trackers.DEFAULT.name(), DelayFlushCoprocessor.class.getName(), FAMILY); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java index cdc41b01e87d..0695be12d0c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java @@ -26,8 +26,6 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Test create/using/deleting snapshots from the client @@ -41,8 +39,6 @@ public class TestMobSnapshotFromClient extends TestSnapshotFromClient { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestMobSnapshotFromClient.class); - private static final Logger LOG = LoggerFactory.getLogger(TestMobSnapshotFromClient.class); - /** * Setup the config for the cluster * @throws Exception on failure @@ -60,6 +56,7 @@ protected static void setupConf(Configuration conf) { @Override protected void createTable() throws Exception { - MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), TEST_FAM); + MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), trackerImpl.name(), + TEST_FAM); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java index 08e33ac78d05..56a48c122a14 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; @@ -32,9 +33,11 @@ import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNameTestRule; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1; @@ -51,7 +54,10 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +70,8 @@ *

* This is an end-to-end test for the snapshot utility */ -@Category({LargeTests.class, ClientTests.class}) +@RunWith(Parameterized.class) +@Category({ LargeTests.class, ClientTests.class }) public class TestSnapshotFromClient { @ClassRule @@ -82,7 +89,16 @@ public class TestSnapshotFromClient { private static final Pattern MATCH_ALL = Pattern.compile(".*"); @Rule - public TestName name = new TestName(); + public TableNameTestRule name = new TableNameTestRule(); + + @Parameter + public StoreFileTrackerFactory.Trackers trackerImpl; + + @Parameters(name = "{index}: tracker={0}") + public static List params() { + return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT }, + new Object[] { StoreFileTrackerFactory.Trackers.FILE }); + } /** * Setup the config for the cluster @@ -109,7 +125,6 @@ protected static void setupConf(Configuration conf) { conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); - } @Before @@ -119,7 +134,8 @@ public void setup() throws Exception { protected void createTable() throws Exception { TableDescriptor htd = - TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()).build(); + TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build(); UTIL.createTable(htd, new byte[][] { TEST_FAM }, null); } @@ -316,7 +332,7 @@ public void testOfflineTableSnapshotWithEmptyRegions() throws Exception { @Test public void testListTableSnapshots() throws Exception { Admin admin = null; - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = name.getTableName(); try { admin = UTIL.getAdmin(); @@ -401,7 +417,7 @@ public void testListTableSnapshotsWithRegex() throws Exception { @Test public void testDeleteTableSnapshots() throws Exception { Admin admin = null; - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = name.getTableName(); try { admin = UTIL.getAdmin(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index c7203a928aa7..b4f628b358b5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -1068,10 +1069,9 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration()); TableDescriptors mockTblDescs = mock(TableDescriptors.class); when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs); - TableDescriptor mockTblDesc = mock(TableDescriptor.class); + TableDescriptor mockTblDesc = TableDescriptorBuilder.newBuilder(hri.getTable()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); when(mockTblDescs.get(any())).thenReturn(mockTblDesc); - ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class); - when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc); Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); return new Path(new Path(regionDir, family), path.getName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index fea4fb4ba583..7523ae8fa0ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; @@ -45,29 +46,40 @@ public class MobSnapshotTestingUtils { /** * Create the Mob Table. */ - public static void createMobTable(final HBaseTestingUtil util, - final TableName tableName, int regionReplication, - final byte[]... families) throws IOException, InterruptedException { - createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), - regionReplication, families); + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + int regionReplication, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), regionReplication, + StoreFileTrackerFactory.Trackers.DEFAULT.name(), families); + } + + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + int regionReplication, String storeFileTracker, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), regionReplication, + storeFileTracker, families); } - public static void createPreSplitMobTable(final HBaseTestingUtil util, - final TableName tableName, int nRegions, final byte[]... families) - throws IOException, InterruptedException { - createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), - 1, families); + public static void createPreSplitMobTable(final HBaseTestingUtil util, final TableName tableName, + int nRegions, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), 1, families); + } + + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + final byte[][] splitKeys, int regionReplication, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, splitKeys, regionReplication, + StoreFileTrackerFactory.Trackers.DEFAULT.name(), families); } public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, - final byte[][] splitKeys, int regionReplication, final byte[]... families) - throws IOException, InterruptedException { - createMobTable(util, tableName, splitKeys, regionReplication, null, families); + final byte[][] splitKeys, int regionReplication, String storeFileTracker, + final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, splitKeys, regionReplication, storeFileTracker, null, families); } - public static void createMobTable(HBaseTestingUtil util, TableName tableName, - byte[][] splitKeys, int regionReplication, String cpClassName, byte[]... families) - throws IOException, InterruptedException { + public static void createMobTable(HBaseTestingUtil util, TableName tableName, byte[][] splitKeys, + int regionReplication, String storeFileTracker, String cpClassName, byte[]... families) + throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); for (byte[] family : families) { @@ -77,6 +89,7 @@ public static void createMobTable(HBaseTestingUtil util, TableName tableName, if (!StringUtils.isBlank(cpClassName)) { builder.setCoprocessor(cpClassName); } + builder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, storeFileTracker); util.getAdmin().createTable(builder.build(), splitKeys); SnapshotTestingUtils.waitForTableToBeOnline(util, tableName); assertEquals((splitKeys.length + 1) * regionReplication, @@ -85,15 +98,10 @@ public static void createMobTable(HBaseTestingUtil util, TableName tableName, /** * Create a Mob table. - * - * @param util - * @param tableName - * @param families * @return An Table instance for the created table. - * @throws IOException */ - public static Table createMobTable(final HBaseTestingUtil util, - final TableName tableName, final byte[]... families) throws IOException { + public static Table createMobTable(final HBaseTestingUtil util, final TableName tableName, + final byte[]... families) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); for (byte[] family : families) { // Disable blooms (they are on by default as of 0.95) but we disable them @@ -102,10 +110,7 @@ public static Table createMobTable(final HBaseTestingUtil util, // and blooms being // on is interfering. builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family) - .setBloomFilterType(BloomType.NONE) - .setMobEnabled(true) - .setMobThreshold(0L) - .build()); + .setBloomFilterType(BloomType.NONE).setMobEnabled(true).setMobThreshold(0L).build()); } util.getAdmin().createTable(builder.build()); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait @@ -135,8 +140,8 @@ public static int countMobRows(final Table table, final byte[]... families) thro } } - public static void verifyMobRowCount(final HBaseTestingUtil util, - final TableName tableName, long expectedRows) throws IOException { + public static void verifyMobRowCount(final HBaseTestingUtil util, final TableName tableName, + long expectedRows) throws IOException { Table table = ConnectionFactory.createConnection(util.getConfiguration()).getTable(tableName); try { @@ -156,13 +161,10 @@ public SnapshotMock(final Configuration conf, final FileSystem fs, final Path ro @Override public TableDescriptor createHtd(final String tableName) { - return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)) - .setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(Bytes.toBytes(TEST_FAMILY)) - .setMobEnabled(true) - .setMobThreshold(0L) - .build()) - .build(); + return TableDescriptorBuilder + .newBuilder(TableName.valueOf(tableName)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(Bytes.toBytes(TEST_FAMILY)).setMobEnabled(true).setMobThreshold(0L).build()) + .build(); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index d8d2a5eed0b1..00d2e84a464b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -480,9 +480,8 @@ public SnapshotBuilder(final Configuration conf, final FileSystem fs, this.desc = desc; this.tableRegions = tableRegions; this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); - new FSTableDescriptors(conf) - .createTableDescriptorForTableDirectory(this.snapshotDir.getFileSystem(conf), - snapshotDir, htd, false); + FSTableDescriptors.createTableDescriptorForTableDirectory( + this.snapshotDir.getFileSystem(conf), snapshotDir, htd, false); } public TableDescriptor getTableDescriptor() { @@ -502,15 +501,13 @@ public Path[] addRegion() throws IOException { } public Path[] addRegionV1() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) - .build()); + return addRegion( + desc.toBuilder().setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION).build()); } public Path[] addRegionV2() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) - .build()); + return addRegion( + desc.toBuilder().setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION).build()); } private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { @@ -521,6 +518,7 @@ private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws I RegionData regionData = tableRegions[this.snapshotted++]; ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); + manifest.addTableDescriptor(htd); manifest.addRegion(regionData.tableDir, regionData.hri); return regionData.files; } From fe91ad4b81961350496a49ce461b7ca9de273cdb Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 29 Jul 2021 18:35:19 +0800 Subject: [PATCH 09/37] HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic Signed-off-by: Wellington Chevreuil --- .../hbase/mob/DefaultMobStoreCompactor.java | 22 +- .../hbase/mob/DefaultMobStoreFlusher.java | 4 +- .../CreateStoreFileWriterParams.java | 134 ++++ .../regionserver/DateTieredStoreEngine.java | 5 +- .../regionserver/DefaultStoreEngine.java | 5 +- .../regionserver/DefaultStoreFlusher.java | 11 +- .../hadoop/hbase/regionserver/HMobStore.java | 3 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +- .../hadoop/hbase/regionserver/HStore.java | 699 ++++-------------- .../hbase/regionserver/StoreContext.java | 9 + .../hbase/regionserver/StoreEngine.java | 438 ++++++++++- .../hbase/regionserver/StoreFileManager.java | 9 + .../hbase/regionserver/StoreFlusher.java | 9 +- .../hadoop/hbase/regionserver/StoreUtils.java | 37 +- .../hbase/regionserver/StripeStoreEngine.java | 9 +- .../regionserver/StripeStoreFlusher.java | 9 +- .../AbstractMultiOutputCompactor.java | 7 +- .../regionserver/compactions/Compactor.java | 36 +- .../compactions/DefaultCompactor.java | 16 +- .../DefaultStoreFileTracker.java | 61 ++ .../storefiletracker/StoreFileTracker.java | 75 ++ .../StoreFileTrackerBase.java | 178 +++++ .../StoreFileTrackerFactory.java | 35 + .../compaction/MajorCompactionRequest.java | 1 - .../apache/hadoop/hbase/TestIOFencing.java | 12 +- .../TestCacheOnWriteInSchema.java | 6 +- .../regionserver/TestDefaultStoreEngine.java | 5 +- .../hbase/regionserver/TestHRegion.java | 4 +- .../hadoop/hbase/regionserver/TestHStore.java | 28 +- .../TestRegionMergeTransactionOnCluster.java | 6 +- .../TestStoreFileRefresherChore.java | 3 +- .../regionserver/TestStoreScannerClosure.java | 6 +- .../regionserver/TestStripeStoreEngine.java | 2 + .../compactions/TestDateTieredCompactor.java | 12 +- .../TestStripeCompactionPolicy.java | 12 +- .../compactions/TestStripeCompactor.java | 12 +- 36 files changed, 1225 insertions(+), 705 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java index c45fdff7ca16..01fe0005f048 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map.Entry; import java.util.Optional; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -144,17 +143,16 @@ public InternalScanner createScanner(ScanInfo scanInfo, List s }; private final CellSinkFactory writerFactory = - new CellSinkFactory() { - @Override - public StoreFileWriter createWriter(InternalScanner scanner, - org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, - boolean shouldDropBehind, boolean major) throws IOException { - // make this writer with tags always because of possible new cells with tags. - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, true, true, shouldDropBehind); - } - }; + new CellSinkFactory() { + @Override + public StoreFileWriter createWriter(InternalScanner scanner, + org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, + boolean shouldDropBehind, boolean major) throws IOException { + // make this writer with tags always because of possible new cells with tags. + return store.getStoreEngine().createWriter( + createParams(fd, shouldDropBehind, major).includeMVCCReadpoint(true).includesTag(true)); + } + }; public DefaultMobStoreCompactor(Configuration conf, HStore store) { super(conf, store); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java index 480b85c58dfe..4a1dc7b33a51 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java @@ -25,7 +25,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -127,8 +126,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId, synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk - writer = store.createWriterInTmp(cellsCount, store.getColumnFamilyDescriptor().getCompressionType(), - false, true, true, false); + writer = createWriter(snapshot, true); IOException e = null; try { // It's a mob store, flush the cells in a mob way. This is the difference of flushing diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java new file mode 100644 index 000000000000..10cd9f009e4a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public final class CreateStoreFileWriterParams { + + private long maxKeyCount; + + private Compression.Algorithm compression; + + private boolean isCompaction; + + private boolean includeMVCCReadpoint; + + private boolean includesTag; + + private boolean shouldDropBehind; + + private long totalCompactedFilesSize = -1; + + private String fileStoragePolicy = HConstants.EMPTY_STRING; + + private CreateStoreFileWriterParams() { + } + + public long maxKeyCount() { + return maxKeyCount; + } + + public CreateStoreFileWriterParams maxKeyCount(long maxKeyCount) { + this.maxKeyCount = maxKeyCount; + return this; + } + + public Compression.Algorithm compression() { + return compression; + } + + /** + * Set the compression algorithm to use + */ + public CreateStoreFileWriterParams compression(Compression.Algorithm compression) { + this.compression = compression; + return this; + } + + public boolean isCompaction() { + return isCompaction; + } + + /** + * Whether we are creating a new file in a compaction + */ + public CreateStoreFileWriterParams isCompaction(boolean isCompaction) { + this.isCompaction = isCompaction; + return this; + } + + public boolean includeMVCCReadpoint() { + return includeMVCCReadpoint; + } + + /** + * Whether to include MVCC or not + */ + public CreateStoreFileWriterParams includeMVCCReadpoint(boolean includeMVCCReadpoint) { + this.includeMVCCReadpoint = includeMVCCReadpoint; + return this; + } + + public boolean includesTag() { + return includesTag; + } + + /** + * Whether to includesTag or not + */ + public CreateStoreFileWriterParams includesTag(boolean includesTag) { + this.includesTag = includesTag; + return this; + } + + public boolean shouldDropBehind() { + return shouldDropBehind; + } + + public CreateStoreFileWriterParams shouldDropBehind(boolean shouldDropBehind) { + this.shouldDropBehind = shouldDropBehind; + return this; + } + + public long totalCompactedFilesSize() { + return totalCompactedFilesSize; + } + + public CreateStoreFileWriterParams totalCompactedFilesSize(long totalCompactedFilesSize) { + this.totalCompactedFilesSize = totalCompactedFilesSize; + return this; + } + + public String fileStoragePolicy() { + return fileStoragePolicy; + } + + public CreateStoreFileWriterParams fileStoragePolicy(String fileStoragePolicy) { + this.fileStoragePolicy = fileStoragePolicy; + return this; + } + + public static CreateStoreFileWriterParams create() { + return new CreateStoreFileWriterParams(); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index 1df953d93c96..7422d9112eab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -19,18 +19,17 @@ import java.io.IOException; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.yetus.audience.InterfaceAudience; /** * HBASE-15400 This store engine allows us to store data in date tiered layout with exponential diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java index 58f8bbbb6ac5..693b9c93b9fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; @@ -39,8 +38,8 @@ * their derivatives. */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class DefaultStoreEngine extends StoreEngine< - DefaultStoreFlusher, RatioBasedCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> { +public class DefaultStoreEngine extends StoreEngine { public static final String DEFAULT_STORE_FLUSHER_CLASS_KEY = "hbase.hstore.defaultengine.storeflusher.class"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java index a7d7fb1f3d56..306760d7ce6a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java @@ -21,15 +21,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.util.StringUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Default implementation of StoreFlusher. @@ -60,9 +59,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId, synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk - writer = store.createWriterInTmp(cellsCount, - store.getColumnFamilyDescriptor().getCompressionType(), false, true, - snapshot.isTagsPresent(), false); + writer = createWriter(snapshot, false); IOException e = null; try { performFlush(scanner, writer, throughputController); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index 7ce7f0310c7d..b00a50c522fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -28,7 +28,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -158,7 +157,7 @@ protected KeyValueScanner createScanner(Scan scan, ScanInfo scanInfo, protected StoreEngine createStoreEngine(HStore store, Configuration conf, CellComparator cellComparator) throws IOException { MobStoreEngine engine = new MobStoreEngine(); - engine.createComponents(conf, store, cellComparator); + engine.createComponentsOnce(conf, store, cellComparator); return engine; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 5b26f9aa8ab7..6ab9b0fc3fe8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -142,7 +142,7 @@ public Path getRegionDir() { // Temp Helpers // =========================================================================== /** @return {@link Path} to the region's temp directory, used for file creations */ - Path getTempDir() { + public Path getTempDir() { return new Path(getRegionDir(), REGION_TEMP_DIR); } @@ -237,11 +237,7 @@ public String getStoragePolicyName(String familyName) { * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ - public Collection getStoreFiles(final byte[] familyName) throws IOException { - return getStoreFiles(Bytes.toString(familyName)); - } - - public Collection getStoreFiles(final String familyName) throws IOException { + public List getStoreFiles(final String familyName) throws IOException { return getStoreFiles(familyName, true); } @@ -251,7 +247,7 @@ public Collection getStoreFiles(final String familyName) throws I * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ - public Collection getStoreFiles(final String familyName, final boolean validate) + public List getStoreFiles(final String familyName, final boolean validate) throws IOException { Path familyDir = getStoreDir(familyName); FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index e78e74ce0890..253605ab48e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; @@ -47,8 +48,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -72,17 +71,12 @@ import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; import org.apache.hadoop.hbase.coprocessor.ReadOnlyConfiguration; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContext; -import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.InvalidHFileException; -import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.quotas.RegionSizeStore; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; @@ -112,7 +106,6 @@ import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; import org.apache.hbase.thirdparty.org.apache.commons.collections4.IterableUtils; @@ -167,16 +160,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, private boolean cacheOnWriteLogged; - /** - * RWLock for store operations. - * Locked in shared mode when the list of component stores is looked at: - * - all reads/writes to table data - * - checking for split - * Locked in exclusive mode when the list of component stores is modified: - * - closing - * - completing a compaction - */ - final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); /** * Lock specific to archiving compacted store files. This avoids races around * the combination of retrieving the list of compacted files and moving them to @@ -285,14 +268,8 @@ protected HStore(final HRegion region, final ColumnFamilyDescriptor family, } this.storeEngine = createStoreEngine(this, this.conf, region.getCellComparator()); - List hStoreFiles = loadStoreFiles(warmup); - // Move the storeSize calculation out of loadStoreFiles() method, because the secondary read - // replica's refreshStoreFiles() will also use loadStoreFiles() to refresh its store files and - // update the storeSize in the refreshStoreSizeAndTotalBytes() finally (just like compaction) , so - // no need calculate the storeSize twice. - this.storeSize.addAndGet(getStorefilesSize(hStoreFiles, sf -> true)); - this.totalUncompressedBytes.addAndGet(getTotalUncompressedBytes(hStoreFiles)); - this.storeEngine.getStoreFileManager().loadFiles(hStoreFiles); + storeEngine.initialize(warmup); + refreshStoreSizeAndTotalBytes(); flushRetriesNumber = conf.getInt( "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER); @@ -508,102 +485,18 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) { this.dataBlockEncoder = blockEncoder; } - /** - * Creates an unsorted list of StoreFile loaded in parallel - * from the given directory. - */ - private List loadStoreFiles(boolean warmup) throws IOException { - Collection files = getRegionFileSystem().getStoreFiles(getColumnFamilyName()); - return openStoreFiles(files, warmup); - } - - private List openStoreFiles(Collection files, boolean warmup) - throws IOException { - if (CollectionUtils.isEmpty(files)) { - return Collections.emptyList(); - } - // initialize the thread pool for opening store files in parallel.. - ThreadPoolExecutor storeFileOpenerThreadPool = - this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-" - + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - CompletionService completionService = - new ExecutorCompletionService<>(storeFileOpenerThreadPool); - - int totalValidStoreFile = 0; - for (StoreFileInfo storeFileInfo : files) { - // open each store file in parallel - completionService.submit(() -> this.createStoreFileAndReader(storeFileInfo)); - totalValidStoreFile++; - } - - Set compactedStoreFiles = new HashSet<>(); - ArrayList results = new ArrayList<>(files.size()); - IOException ioe = null; - try { - for (int i = 0; i < totalValidStoreFile; i++) { - try { - HStoreFile storeFile = completionService.take().get(); - if (storeFile != null) { - LOG.debug("loaded {}", storeFile); - results.add(storeFile); - compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); - } - } catch (InterruptedException e) { - if (ioe == null) { - ioe = new InterruptedIOException(e.getMessage()); - } - } catch (ExecutionException e) { - if (ioe == null) { - ioe = new IOException(e.getCause()); - } - } - } - } finally { - storeFileOpenerThreadPool.shutdownNow(); - } - if (ioe != null) { - // close StoreFile readers - boolean evictOnClose = - getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; - for (HStoreFile file : results) { - try { - if (file != null) { - file.closeStoreFile(evictOnClose); - } - } catch (IOException e) { - LOG.warn("Could not close store file {}", file, e); - } - } - throw ioe; - } - - // Should not archive the compacted store files when region warmup. See HBASE-22163. - if (!warmup) { - // Remove the compacted files from result - List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); - for (HStoreFile storeFile : results) { - if (compactedStoreFiles.contains(storeFile.getPath().getName())) { - LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); - storeFile.getReader().close(storeFile.getCacheConf() != null ? - storeFile.getCacheConf().shouldEvictOnClose() : true); - filesToRemove.add(storeFile); - } - } - results.removeAll(filesToRemove); - if (!filesToRemove.isEmpty() && this.isPrimaryReplicaStore()) { - LOG.debug("Moving the files {} to archive", filesToRemove); - getRegionFileSystem().removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), - filesToRemove); - } - } - - return results; + private void postRefreshStoreFiles() throws IOException { + // Advance the memstore read point to be at least the new store files seqIds so that + // readers might pick it up. This assumes that the store is not getting any writes (otherwise + // in-flight transactions might be made visible) + getMaxSequenceId().ifPresent(region.getMVCC()::advanceTo); + refreshStoreSizeAndTotalBytes(); } @Override public void refreshStoreFiles() throws IOException { - Collection newFiles = getRegionFileSystem().getStoreFiles(getColumnFamilyName()); - refreshStoreFilesInternal(newFiles); + storeEngine.refreshStoreFiles(); + postRefreshStoreFiles(); } /** @@ -611,89 +504,8 @@ public void refreshStoreFiles() throws IOException { * region replicas to keep up to date with the primary region files. */ public void refreshStoreFiles(Collection newFiles) throws IOException { - List storeFiles = new ArrayList<>(newFiles.size()); - for (String file : newFiles) { - storeFiles.add(getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file)); - } - refreshStoreFilesInternal(storeFiles); - } - - /** - * Checks the underlying store files, and opens the files that have not - * been opened, and removes the store file readers for store files no longer - * available. Mainly used by secondary region replicas to keep up to date with - * the primary region files. - */ - private void refreshStoreFilesInternal(Collection newFiles) throws IOException { - StoreFileManager sfm = storeEngine.getStoreFileManager(); - Collection currentFiles = sfm.getStorefiles(); - Collection compactedFiles = sfm.getCompactedfiles(); - if (currentFiles == null) { - currentFiles = Collections.emptySet(); - } - if (newFiles == null) { - newFiles = Collections.emptySet(); - } - if (compactedFiles == null) { - compactedFiles = Collections.emptySet(); - } - - HashMap currentFilesSet = new HashMap<>(currentFiles.size()); - for (HStoreFile sf : currentFiles) { - currentFilesSet.put(sf.getFileInfo(), sf); - } - HashMap compactedFilesSet = new HashMap<>(compactedFiles.size()); - for (HStoreFile sf : compactedFiles) { - compactedFilesSet.put(sf.getFileInfo(), sf); - } - - Set newFilesSet = new HashSet(newFiles); - // Exclude the files that have already been compacted - newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); - Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); - Set toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet); - - if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) { - return; - } - - LOG.info("Refreshing store files for " + this + " files to add: " - + toBeAddedFiles + " files to remove: " + toBeRemovedFiles); - - Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); - for (StoreFileInfo sfi : toBeRemovedFiles) { - toBeRemovedStoreFiles.add(currentFilesSet.get(sfi)); - } - - // try to open the files - List openedFiles = openStoreFiles(toBeAddedFiles, false); - - // propogate the file changes to the underlying store file manager - replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception - - // Advance the memstore read point to be at least the new store files seqIds so that - // readers might pick it up. This assumes that the store is not getting any writes (otherwise - // in-flight transactions might be made visible) - if (!toBeAddedFiles.isEmpty()) { - // we must have the max sequence id here as we do have several store files - region.getMVCC().advanceTo(this.getMaxSequenceId().getAsLong()); - } - - refreshStoreSizeAndTotalBytes(); - } - - protected HStoreFile createStoreFileAndReader(final Path p) throws IOException { - StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), - p, isPrimaryReplicaStore()); - return createStoreFileAndReader(info); - } - - private HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { - info.setRegionCoprocessorHost(this.region.getCoprocessorHost()); - HStoreFile storeFile = new HStoreFile(info, getColumnFamilyDescriptor().getBloomFilterType(), - getCacheConfig()); - storeFile.initReader(); - return storeFile; + storeEngine.refreshStoreFiles(newFiles); + postRefreshStoreFiles(); } /** @@ -716,7 +528,7 @@ public void stopReplayingFromWAL(){ * Adds a value to the memstore */ public void add(final Cell cell, MemStoreSizing memstoreSizing) { - lock.readLock().lock(); + storeEngine.readLock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace("tableName={}, encodedName={}, columnFamilyName={} is too busy!", @@ -724,7 +536,7 @@ public void add(final Cell cell, MemStoreSizing memstoreSizing) { } this.memstore.add(cell, memstoreSizing); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); currentParallelPutCount.decrementAndGet(); } } @@ -733,7 +545,7 @@ public void add(final Cell cell, MemStoreSizing memstoreSizing) { * Adds the specified value to the memstore */ public void add(final Iterable cells, MemStoreSizing memstoreSizing) { - lock.readLock().lock(); + storeEngine.readLock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace("tableName={}, encodedName={}, columnFamilyName={} is too busy!", @@ -741,7 +553,7 @@ public void add(final Iterable cells, MemStoreSizing memstoreSizing) { } memstore.add(cells, memstoreSizing); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); currentParallelPutCount.decrementAndGet(); } } @@ -864,17 +676,16 @@ public Path bulkLoadHFile(byte[] family, String srcPathStr, Path dstPath) throws LOG.info("Loaded HFile " + srcPath + " into " + this + " as " + dstPath + " - updating store file list."); - HStoreFile sf = createStoreFileAndReader(dstPath); + HStoreFile sf = storeEngine.createStoreFileAndReader(dstPath); bulkLoadHFile(sf); - LOG.info("Successfully loaded {} into {} (new location: {})", - srcPath, this, dstPath); + LOG.info("Successfully loaded {} into {} (new location: {})", srcPath, this, dstPath); return dstPath; } public void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException { - HStoreFile sf = createStoreFileAndReader(fileInfo); + HStoreFile sf = storeEngine.createStoreFileAndReader(fileInfo); bulkLoadHFile(sf); } @@ -882,28 +693,74 @@ private void bulkLoadHFile(HStoreFile sf) throws IOException { StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); - - // Append the new storefile into the list - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); - } finally { - // We need the lock, as long as we are updating the storeFiles - // or changing the memstore. Let us release it before calling - // notifyChangeReadersObservers. See HBASE-4485 for a possible - // deadlock scenario that could have happened if continue to hold - // the lock. - this.lock.writeLock().unlock(); - } + storeEngine.addStoreFiles(Lists.newArrayList(sf)); LOG.info("Loaded HFile " + sf.getFileInfo() + " into " + this); if (LOG.isTraceEnabled()) { - String traceMessage = "BULK LOAD time,size,store size,store files [" - + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize - + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; + String traceMessage = "BULK LOAD time,size,store size,store files [" + + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize + "," + + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage); } } + private ImmutableCollection closeWithoutLock() throws IOException { + // Clear so metrics doesn't find them. + ImmutableCollection result = storeEngine.getStoreFileManager().clearFiles(); + Collection compactedfiles = storeEngine.getStoreFileManager().clearCompactedFiles(); + // clear the compacted files + if (CollectionUtils.isNotEmpty(compactedfiles)) { + removeCompactedfiles(compactedfiles, + getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true); + } + if (!result.isEmpty()) { + // initialize the thread pool for closing store files in parallel. + ThreadPoolExecutor storeFileCloserThreadPool = + this.region.getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" + + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); + + // close each store file in parallel + CompletionService completionService = + new ExecutorCompletionService<>(storeFileCloserThreadPool); + for (HStoreFile f : result) { + completionService.submit(new Callable() { + @Override + public Void call() throws IOException { + boolean evictOnClose = + getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; + f.closeStoreFile(evictOnClose); + return null; + } + }); + } + + IOException ioe = null; + try { + for (int i = 0; i < result.size(); i++) { + try { + Future future = completionService.take(); + future.get(); + } catch (InterruptedException e) { + if (ioe == null) { + ioe = new InterruptedIOException(); + ioe.initCause(e); + } + } catch (ExecutionException e) { + if (ioe == null) { + ioe = new IOException(e.getCause()); + } + } + } + } finally { + storeFileCloserThreadPool.shutdownNow(); + } + if (ioe != null) { + throw ioe; + } + } + LOG.trace("Closed {}", this); + return result; + } + /** * Close all the readers We don't need to worry about subsequent requests because the Region holds * a write lock that will prevent any more reads or writes. @@ -911,67 +768,18 @@ private void bulkLoadHFile(HStoreFile sf) throws IOException { * @throws IOException on failure */ public ImmutableCollection close() throws IOException { + // findbugs can not recognize storeEngine.writeLock is just a lock operation so it will report + // UL_UNRELEASED_LOCK_EXCEPTION_PATH, so here we have to use two try finally... + // Change later if findbugs becomes smarter in the future. this.archiveLock.lock(); - this.lock.writeLock().lock(); try { - // Clear so metrics doesn't find them. - ImmutableCollection result = storeEngine.getStoreFileManager().clearFiles(); - Collection compactedfiles = - storeEngine.getStoreFileManager().clearCompactedFiles(); - // clear the compacted files - if (CollectionUtils.isNotEmpty(compactedfiles)) { - removeCompactedfiles(compactedfiles, getCacheConfig() != null ? - getCacheConfig().shouldEvictOnClose() : true); - } - if (!result.isEmpty()) { - // initialize the thread pool for closing store files in parallel. - ThreadPoolExecutor storeFileCloserThreadPool = this.region - .getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" - + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - - // close each store file in parallel - CompletionService completionService = - new ExecutorCompletionService<>(storeFileCloserThreadPool); - for (HStoreFile f : result) { - completionService.submit(new Callable() { - @Override - public Void call() throws IOException { - boolean evictOnClose = - getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; - f.closeStoreFile(evictOnClose); - return null; - } - }); - } - - IOException ioe = null; - try { - for (int i = 0; i < result.size(); i++) { - try { - Future future = completionService.take(); - future.get(); - } catch (InterruptedException e) { - if (ioe == null) { - ioe = new InterruptedIOException(); - ioe.initCause(e); - } - } catch (ExecutionException e) { - if (ioe == null) { - ioe = new IOException(e.getCause()); - } - } - } - } finally { - storeFileCloserThreadPool.shutdownNow(); - } - if (ioe != null) { - throw ioe; - } + this.storeEngine.writeLock(); + try { + return closeWithoutLock(); + } finally { + this.storeEngine.writeUnlock(); } - LOG.trace("Closed {}", this); - return result; } finally { - this.lock.writeLock().unlock(); this.archiveLock.unlock(); } } @@ -1001,7 +809,7 @@ protected List flushCache(final long logCacheFlushId, MemStoreSnapshot sna try { for (Path pathName : pathNames) { lastPathName = pathName; - validateStoreFile(pathName); + storeEngine.validateStoreFile(pathName); } return pathNames; } catch (Exception e) { @@ -1047,167 +855,18 @@ public HStoreFile tryCommitRecoveredHFile(Path path) throws IOException { } Path dstPath = getRegionFileSystem().commitStoreFile(getColumnFamilyName(), path); - HStoreFile sf = createStoreFileAndReader(dstPath); + HStoreFile sf = storeEngine.createStoreFileAndReader(dstPath); StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); - } finally { - this.lock.writeLock().unlock(); - } + storeEngine.addStoreFiles(Lists.newArrayList(sf)); LOG.info("Loaded recovered hfile to {}, entries={}, sequenceid={}, filesize={}", sf, r.getEntries(), r.getSequenceID(), TraditionalBinaryPrefix.long2String(r.length(), "B", 1)); return sf; } - /** - * Commit the given {@code files}. - *

- * We will move the file into data directory, and open it. - * @param files the files want to commit - * @param validate whether to validate the store files - * @return the committed store files - */ - private List commitStoreFiles(List files, boolean validate) throws IOException { - List committedFiles = new ArrayList<>(files.size()); - HRegionFileSystem hfs = getRegionFileSystem(); - String familyName = getColumnFamilyName(); - for (Path file : files) { - try { - if (validate) { - validateStoreFile(file); - } - Path committedPath = hfs.commitStoreFile(familyName, file); - HStoreFile sf = createStoreFileAndReader(committedPath); - committedFiles.add(sf); - } catch (IOException e) { - LOG.error("Failed to commit store file {}", file, e); - // Try to delete the files we have committed before. - // It is OK to fail when deleting as leaving the file there does not cause any data - // corruption problem. It just introduces some duplicated data which may impact read - // performance a little when reading before compaction. - for (HStoreFile sf : committedFiles) { - Path pathToDelete = sf.getPath(); - try { - sf.deleteStoreFile(); - } catch (IOException deleteEx) { - LOG.warn(HBaseMarkers.FATAL, "Failed to delete committed store file {}", pathToDelete, - deleteEx); - } - } - throw new IOException("Failed to commit the flush", e); - } - } - return committedFiles; - } - - public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, - boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, - boolean shouldDropBehind) throws IOException { - return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint, - includesTag, shouldDropBehind, -1, HConstants.EMPTY_STRING); - } - - /** - * @param compression Compression algorithm to use - * @param isCompaction whether we are creating a new file in a compaction - * @param includeMVCCReadpoint - whether to include MVCC or not - * @param includesTag - includesTag or not - * @return Writer for a new StoreFile in the tmp dir. - */ - // TODO : allow the Writer factory to create Writers of ShipperListener type only in case of - // compaction - public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, - boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, - boolean shouldDropBehind, long totalCompactedFilesSize, String fileStoragePolicy) - throws IOException { - // creating new cache config for each new writer - final CacheConfig cacheConf = getCacheConfig(); - final CacheConfig writerCacheConf = new CacheConfig(cacheConf); - if (isCompaction) { - // Don't cache data on write on compactions, unless specifically configured to do so - // Cache only when total file size remains lower than configured threshold - final boolean cacheCompactedBlocksOnWrite = - getCacheConfig().shouldCacheCompactedBlocksOnWrite(); - // if data blocks are to be cached on write - // during compaction, we should forcefully - // cache index and bloom blocks as well - if (cacheCompactedBlocksOnWrite && totalCompactedFilesSize <= cacheConf - .getCacheCompactedBlocksOnWriteThreshold()) { - writerCacheConf.enableCacheOnWrite(); - if (!cacheOnWriteLogged) { - LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " + - "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this); - cacheOnWriteLogged = true; - } - } else { - writerCacheConf.setCacheDataOnWrite(false); - if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { - // checking condition once again for logging - LOG.debug( - "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted " - + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}", - this, totalCompactedFilesSize, - cacheConf.getCacheCompactedBlocksOnWriteThreshold()); - } - } - } else { - final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite(); - if (shouldCacheDataOnWrite) { - writerCacheConf.enableCacheOnWrite(); - if (!cacheOnWriteLogged) { - LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " + - "Index blocks and Bloom filter blocks", this); - cacheOnWriteLogged = true; - } - } - } - Encryption.Context encryptionContext = storeContext.getEncryptionContext(); - HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag, - encryptionContext); - Path familyTempDir = new Path(getRegionFileSystem().getTempDir(), getColumnFamilyName()); - StoreFileWriter.Builder builder = - new StoreFileWriter.Builder(conf, writerCacheConf, getFileSystem()) - .withOutputDir(familyTempDir) - .withBloomType(storeContext.getBloomFilterType()) - .withMaxKeyCount(maxKeyCount) - .withFavoredNodes(storeContext.getFavoredNodes()) - .withFileContext(hFileContext) - .withShouldDropCacheBehind(shouldDropBehind) - .withCompactedFilesSupplier(storeContext.getCompactedFilesSupplier()) - .withFileStoragePolicy(fileStoragePolicy); - return builder.build(); - } - - private HFileContext createFileContext(Compression.Algorithm compression, - boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { - if (compression == null) { - compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; - } - ColumnFamilyDescriptor family = getColumnFamilyDescriptor(); - HFileContext hFileContext = new HFileContextBuilder() - .withIncludesMvcc(includeMVCCReadpoint) - .withIncludesTags(includesTag) - .withCompression(compression) - .withCompressTags(family.isCompressTags()) - .withChecksumType(StoreUtils.getChecksumType(conf)) - .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)) - .withBlockSize(family.getBlocksize()) - .withHBaseCheckSum(true) - .withDataBlockEncoding(family.getDataBlockEncoding()) - .withEncryptionContext(encryptionContext) - .withCreateTime(EnvironmentEdgeManager.currentTime()) - .withColumnFamily(getColumnFamilyDescriptor().getName()) - .withTableName(getTableName().getName()) - .withCellComparator(getComparator()) - .build(); - return hFileContext; - } - private long getTotalSize(Collection sfs) { return sfs.stream().mapToLong(sf -> sf.getReader().length()).sum(); } @@ -1217,18 +876,8 @@ private long getTotalSize(Collection sfs) { * @param sfs Store files * @return Whether compaction is required. */ - private boolean updateStorefiles(List sfs, long snapshotId) throws IOException { - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(sfs); - } finally { - // We need the lock, as long as we are updating the storeFiles - // or changing the memstore. Let us release it before calling - // notifyChangeReadersObservers. See HBASE-4485 for a possible - // deadlock scenario that could have happened if continue to hold - // the lock. - this.lock.writeLock().unlock(); - } + private boolean completeFlush(List sfs, long snapshotId) throws IOException { + storeEngine.addStoreFiles(sfs); // We do not need to call clearSnapshot method inside the write lock. // The clearSnapshot itself is thread safe, which can be called at the same time with other // memstore operations expect snapshot and clearSnapshot. And for these two methods, in HRegion @@ -1254,11 +903,11 @@ private boolean updateStorefiles(List sfs, long snapshotId) throws I private void notifyChangedReadersObservers(List sfs) throws IOException { for (ChangedReadersObserver o : this.changedReaderObservers) { List memStoreScanners; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { memStoreScanners = this.memstore.getScanners(o.getReadPoint()); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } o.updateReaders(sfs, memStoreScanners); } @@ -1300,13 +949,13 @@ public List getScanners(boolean cacheBlocks, boolean usePread, byte[] stopRow, boolean includeStopRow, long readPt) throws IOException { Collection storeFilesToScan; List memStoreScanners; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { storeFilesToScan = this.storeEngine.getStoreFileManager().getFilesForScan(startRow, includeStartRow, stopRow, includeStopRow); memStoreScanners = this.memstore.getScanners(readPt); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } try { @@ -1383,11 +1032,11 @@ public List getScanners(List files, boolean cacheBl boolean includeMemstoreScanner) throws IOException { List memStoreScanners = null; if (includeMemstoreScanner) { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { memStoreScanners = this.memstore.getScanners(readPt); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } try { @@ -1503,14 +1152,13 @@ protected List doCompaction(CompactionRequestImpl cr, List newFiles) throws IOException { // Do the steps necessary to complete the compaction. setStoragePolicyFromFileName(newFiles); - List sfs = commitStoreFiles(newFiles, true); + List sfs = storeEngine.commitStoreFiles(newFiles, true); if (this.getCoprocessorHost() != null) { for (HStoreFile sf : sfs) { getCoprocessorHost().postCompact(this, sf, cr.getTracker(), cr, user); } } - writeCompactionWalRecord(filesToCompact, sfs); - replaceStoreFiles(filesToCompact, sfs); + replaceStoreFiles(filesToCompact, sfs, true); if (cr.isMajor()) { majorCompactedCellsCount.addAndGet(getCompactionProgress().getTotalCompactingKVs()); majorCompactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize); @@ -1574,25 +1222,24 @@ private void writeCompactionWalRecord(Collection filesCompacted, this.region.getRegionInfo(), compactionDescriptor, this.region.getMVCC()); } - void replaceStoreFiles(Collection compactedFiles, Collection result) - throws IOException { - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result); - synchronized (filesCompacting) { - filesCompacting.removeAll(compactedFiles); - } - - // These may be null when the RS is shutting down. The space quota Chores will fix the Region - // sizes later so it's not super-critical if we miss these. - RegionServerServices rsServices = region.getRegionServerServices(); - if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { - updateSpaceQuotaAfterFileReplacement( - rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), - compactedFiles, result); - } - } finally { - this.lock.writeLock().unlock(); + @RestrictedApi(explanation = "Should only be called in TestHStore", link = "", + allowedOnPath = ".*/(HStore|TestHStore).java") + void replaceStoreFiles(Collection compactedFiles, Collection result, + boolean writeCompactionMarker) throws IOException { + storeEngine.replaceStoreFiles(compactedFiles, result); + if (writeCompactionMarker) { + writeCompactionWalRecord(compactedFiles, result); + } + synchronized (filesCompacting) { + filesCompacting.removeAll(compactedFiles); + } + // These may be null when the RS is shutting down. The space quota Chores will fix the Region + // sizes later so it's not super-critical if we miss these. + RegionServerServices rsServices = region.getRegionServerServices(); + if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { + updateSpaceQuotaAfterFileReplacement( + rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), + compactedFiles, result); } } @@ -1715,7 +1362,7 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick for (String compactionOutput : compactionOutputs) { StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput); - HStoreFile storeFile = createStoreFileAndReader(storeFileInfo); + HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); outputStoreFiles.add(storeFile); } } @@ -1723,7 +1370,7 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) { LOG.info("Replaying compaction marker, replacing input files: " + inputStoreFiles + " with output files : " + outputStoreFiles); - this.replaceStoreFiles(inputStoreFiles, outputStoreFiles); + this.replaceStoreFiles(inputStoreFiles, outputStoreFiles, false); this.refreshStoreSizeAndTotalBytes(); } } @@ -1732,14 +1379,14 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick public boolean hasReferences() { // Grab the read lock here, because we need to ensure that: only when the atomic // replaceStoreFiles(..) finished, we can get all the complete store file list. - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { // Merge the current store files with compacted files here due to HBASE-20940. Collection allStoreFiles = new ArrayList<>(getStorefiles()); allStoreFiles.addAll(getCompactedFiles()); return StoreUtils.hasReferences(allStoreFiles); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -1779,7 +1426,7 @@ public Optional requestCompaction(int priority, final CompactionContext compaction = storeEngine.createCompaction(); CompactionRequestImpl request = null; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { synchronized (filesCompacting) { // First, see if coprocessor would want to override selection. @@ -1852,7 +1499,7 @@ public Optional requestCompaction(int priority, request.setTracker(tracker); } } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } if (LOG.isDebugEnabled()) { @@ -1885,7 +1532,7 @@ private void removeUnneededFiles() throws IOException { this, getColumnFamilyDescriptor().getMinVersions()); return; } - this.lock.readLock().lock(); + this.storeEngine.readLock(); Collection delSfs = null; try { synchronized (filesCompacting) { @@ -1897,7 +1544,7 @@ private void removeUnneededFiles() throws IOException { } } } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } if (CollectionUtils.isEmpty(delSfs)) { @@ -1905,8 +1552,7 @@ private void removeUnneededFiles() throws IOException { } Collection newFiles = Collections.emptyList(); // No new files. - writeCompactionWalRecord(delSfs, newFiles); - replaceStoreFiles(delSfs, newFiles); + replaceStoreFiles(delSfs, newFiles, true); refreshStoreSizeAndTotalBytes(); LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in " + this + "; total size is " @@ -1928,25 +1574,6 @@ protected void finishCompactionRequest(CompactionRequestImpl cr) { } } - /** - * Validates a store file by opening and closing it. In HFileV2 this should not be an expensive - * operation. - * @param path the path to the store file - */ - private void validateStoreFile(Path path) throws IOException { - HStoreFile storeFile = null; - try { - storeFile = createStoreFileAndReader(path); - } catch (IOException e) { - LOG.error("Failed to open store file : {}, keeping it in tmp location", path, e); - throw e; - } finally { - if (storeFile != null) { - storeFile.closeStoreFile(false); - } - } - } - /** * Update counts. */ @@ -1992,7 +1619,7 @@ public boolean canSplit() { * Determines if Store should be split. */ public Optional getSplitPoint() { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { // Should already be enforced by the split policy! assert !this.getRegionInfo().isMetaRegion(); @@ -2005,7 +1632,7 @@ public Optional getSplitPoint() { } catch(IOException e) { LOG.warn("Failed getting store size for {}", this, e); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } return Optional.empty(); } @@ -2038,7 +1665,7 @@ public void triggerMajorCompaction() { */ public KeyValueScanner getScanner(Scan scan, final NavigableSet targetCols, long readPt) throws IOException { - lock.readLock().lock(); + storeEngine.readLock(); try { ScanInfo scanInfo; if (this.getCoprocessorHost() != null) { @@ -2048,7 +1675,7 @@ public KeyValueScanner getScanner(Scan scan, final NavigableSet targetCo } return createScanner(scan, scanInfo, targetCols, readPt); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); } } @@ -2078,7 +1705,7 @@ public List recreateScanners(List currentFileS boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, boolean includeStartRow, byte[] stopRow, boolean includeStopRow, long readPt, boolean includeMemstoreScanner) throws IOException { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { Map name2File = new HashMap<>(getStorefilesCount() + getCompactedFilesCount()); @@ -2103,7 +1730,7 @@ public List recreateScanners(List currentFileS return getScanners(filesToReopen, cacheBlocks, false, false, matcher, startRow, includeStartRow, stopRow, includeStopRow, readPt, false); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -2169,41 +1796,20 @@ public long getStoreSizeUncompressed() { @Override public long getStorefilesSize() { // Include all StoreFiles - return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), sf -> true); + return StoreUtils.getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), + sf -> true); } @Override public long getHFilesSize() { // Include only StoreFiles which are HFiles - return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), + return StoreUtils.getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), HStoreFile::isHFile); } - private long getTotalUncompressedBytes(List files) { - return files.stream() - .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) - .sum(); - } - - private long getStorefilesSize(Collection files, Predicate predicate) { - return files.stream().filter(predicate) - .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::length)).sum(); - } - - private long getStorefileFieldSize(HStoreFile file, ToLongFunction f) { - if (file == null) { - return 0L; - } - StoreFileReader reader = file.getReader(); - if (reader == null) { - return 0L; - } - return f.applyAsLong(reader); - } - private long getStorefilesFieldSize(ToLongFunction f) { return this.storeEngine.getStoreFileManager().getStorefiles().stream() - .mapToLong(file -> getStorefileFieldSize(file, f)).sum(); + .mapToLong(file -> StoreUtils.getStorefileFieldSize(file, f)).sum(); } @Override @@ -2274,11 +1880,11 @@ public long getSmallestReadPoint() { */ public void upsert(Iterable cells, long readpoint, MemStoreSizing memstoreSizing) throws IOException { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { this.memstore.upsert(cells, readpoint, memstoreSizing); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -2331,7 +1937,7 @@ public boolean commit(MonitoredTask status) throws IOException { return false; } status.setStatus("Flushing " + this + ": reopening flushed file"); - List storeFiles = commitStoreFiles(tempFiles, false); + List storeFiles = storeEngine.commitStoreFiles(tempFiles, false); for (HStoreFile sf : storeFiles) { StoreFileReader r = sf.getReader(); if (LOG.isInfoEnabled()) { @@ -2354,7 +1960,7 @@ public boolean commit(MonitoredTask status) throws IOException { } } // Add new file to store files. Clear snapshot too while we have the Store write lock. - return updateStorefiles(storeFiles, snapshot.getId()); + return completeFlush(storeFiles, snapshot.getId()); } @Override @@ -2382,7 +1988,7 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) // open the file as a store file (hfile link, etc) StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file); - HStoreFile storeFile = createStoreFileAndReader(storeFileInfo); + HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); storeFiles.add(storeFile); HStore.this.storeSize.addAndGet(storeFile.getReader().length()); HStore.this.totalUncompressedBytes @@ -2399,7 +2005,7 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) snapshotId = snapshot.getId(); snapshot.close(); } - HStore.this.updateStorefiles(storeFiles, snapshotId); + HStore.this.completeFlush(storeFiles, snapshotId); } /** @@ -2412,7 +2018,7 @@ public void abort() throws IOException { //won't be closed. If we are using MSLAB, the chunk referenced by those scanners //can't be released, thus memory leak snapshot.close(); - HStore.this.updateStorefiles(Collections.emptyList(), snapshot.getId()); + HStore.this.completeFlush(Collections.emptyList(), snapshot.getId()); } } } @@ -2574,7 +2180,7 @@ public synchronized void closeAndArchiveCompactedFiles() throws IOException { // ensure other threads do not attempt to archive the same files on close() archiveLock.lock(); try { - lock.readLock().lock(); + storeEngine.readLock(); Collection copyCompactedfiles = null; try { Collection compactedfiles = @@ -2586,7 +2192,7 @@ public synchronized void closeAndArchiveCompactedFiles() throws IOException { LOG.trace("No compacted files to archive"); } } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); } if (CollectionUtils.isNotEmpty(copyCompactedfiles)) { removeCompactedfiles(copyCompactedfiles, true); @@ -2721,12 +2327,7 @@ public boolean isSloppyMemStore() { private void clearCompactedfiles(List filesToRemove) throws IOException { LOG.trace("Clearing the compacted file {} from this store", filesToRemove); - try { - lock.writeLock().lock(); - this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove); - } finally { - lock.writeLock().unlock(); - } + storeEngine.removeCompactedFiles(filesToRemove); } void reportArchivedFilesForQuota(List archivedFiles, List fileSizes) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java index 26233505db73..2a9f96859361 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -108,6 +109,14 @@ public RegionCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + public RegionInfo getRegionInfo() { + return regionFileSystem.getRegionInfo(); + } + + public boolean isPrimaryReplicaStore() { + return getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID; + } + public static Builder getBuilder() { return new Builder(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index 60b3c3d0d20f..f9d6c294f779 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -20,37 +20,129 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; - +import java.util.Set; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.Compactor; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; /** - * StoreEngine is a factory that can create the objects necessary for HStore to operate. - * Since not all compaction policies, compactors and store file managers are compatible, - * they are tied together and replaced together via StoreEngine-s. + * StoreEngine is a factory that can create the objects necessary for HStore to operate. Since not + * all compaction policies, compactors and store file managers are compatible, they are tied + * together and replaced together via StoreEngine-s. + *

+ * We expose read write lock methods to upper layer for store operations:
+ *

    + *
  • Locked in shared mode when the list of component stores is looked at: + *
      + *
    • all reads/writes to table data
      • + *
    • checking for split
      • + *
    + *
    • + *
  • Locked in exclusive mode when the list of component stores is modified: + *
      + *
    • closing
      • + *
    • completing a compaction
      • + *
    + *
    • + *
+ *

+ * It is a bit confusing that we have a StoreFileManager(SFM) and then a StoreFileTracker(SFT). As + * its name says, SFT is used to track the store files list. The reason why we have a SFT beside SFM + * is that, when introducing stripe compaction, we introduced the StoreEngine and also the SFM, but + * actually, the SFM here is not a general 'Manager', it is only designed to manage the in memory + * 'stripes', so we can select different store files when scanning or compacting. The 'tracking' of + * store files is actually done in {@link org.apache.hadoop.hbase.regionserver.HRegionFileSystem} + * and {@link HStore} before we have SFT. And since SFM is designed to only holds in memory states, + * we will hold write lock when updating it, the lock is also used to protect the normal read/write + * requests. This means we'd better not add IO operations to SFM. And also, no matter what the in + * memory state is, stripe or not, it does not effect how we track the store files. So consider all + * these facts, here we introduce a separated SFT to track the store files. + *

+ * Here, since we always need to update SFM and SFT almost at the same time, we introduce methods in + * StoreEngine directly to update them both, so upper layer just need to update StoreEngine once, to + * reduce the possible misuse. */ @InterfaceAudience.Private -public abstract class StoreEngine { +public abstract class StoreEngine { + + private static final Logger LOG = LoggerFactory.getLogger(StoreEngine.class); + protected SF storeFlusher; protected CP compactionPolicy; protected C compactor; protected SFM storeFileManager; + private Configuration conf; + private StoreContext ctx; + private RegionCoprocessorHost coprocessorHost; + private Function openStoreFileThreadPoolCreator; + private StoreFileTracker storeFileTracker; + + private final ReadWriteLock storeLock = new ReentrantReadWriteLock(); /** - * The name of the configuration parameter that specifies the class of - * a store engine that is used to manage and compact HBase store files. + * The name of the configuration parameter that specifies the class of a store engine that is used + * to manage and compact HBase store files. */ public static final String STORE_ENGINE_CLASS_KEY = "hbase.hstore.engine.class"; - private static final Class> - DEFAULT_STORE_ENGINE_CLASS = DefaultStoreEngine.class; + private static final Class> DEFAULT_STORE_ENGINE_CLASS = + DefaultStoreEngine.class; + + /** + * Acquire read lock of this store. + */ + public void readLock() { + storeLock.readLock().lock(); + } + + /** + * Release read lock of this store. + */ + public void readUnlock() { + storeLock.readLock().unlock(); + } + + /** + * Acquire write lock of this store. + */ + public void writeLock() { + storeLock.writeLock().lock(); + } + + /** + * Release write lock of this store. + */ + public void writeUnlock() { + storeLock.writeLock().unlock(); + } /** * @return Compaction policy to use. @@ -80,6 +172,11 @@ public StoreFlusher getStoreFlusher() { return this.storeFlusher; } + private StoreFileTracker createStoreFileTracker(HStore store) { + return StoreFileTrackerFactory.create(store.conf, store.getRegionInfo().getTable(), + store.isPrimaryReplicaStore(), store.getStoreContext()); + } + /** * @param filesCompacting Files currently compacting * @return whether a compaction selection is possible @@ -87,8 +184,8 @@ public StoreFlusher getStoreFlusher() { public abstract boolean needsCompaction(List filesCompacting); /** - * Creates an instance of a compaction context specific to this engine. - * Doesn't actually select or start a compaction. See CompactionContext class comment. + * Creates an instance of a compaction context specific to this engine. Doesn't actually select or + * start a compaction. See CompactionContext class comment. * @return New CompactionContext object. */ public abstract CompactionContext createCompaction() throws IOException; @@ -96,32 +193,321 @@ public StoreFlusher getStoreFlusher() { /** * Create the StoreEngine's components. */ - protected abstract void createComponents( - Configuration conf, HStore store, CellComparator cellComparator) throws IOException; + protected abstract void createComponents(Configuration conf, HStore store, + CellComparator cellComparator) throws IOException; - private void createComponentsOnce( - Configuration conf, HStore store, CellComparator cellComparator) throws IOException { - assert compactor == null && compactionPolicy == null - && storeFileManager == null && storeFlusher == null; + protected final void createComponentsOnce(Configuration conf, HStore store, + CellComparator cellComparator) throws IOException { + assert compactor == null && compactionPolicy == null && storeFileManager == null && + storeFlusher == null && storeFileTracker == null; createComponents(conf, store, cellComparator); - assert compactor != null && compactionPolicy != null - && storeFileManager != null && storeFlusher != null; + this.conf = conf; + this.ctx = store.getStoreContext(); + this.coprocessorHost = store.getHRegion().getCoprocessorHost(); + this.openStoreFileThreadPoolCreator = store.getHRegion()::getStoreFileOpenAndCloseThreadPool; + this.storeFileTracker = createStoreFileTracker(store); + assert compactor != null && compactionPolicy != null && storeFileManager != null && + storeFlusher != null && storeFileTracker != null; + } + + /** + * Create a writer for writing new store files. + * @return Writer for a new StoreFile + */ + public StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException { + return storeFileTracker.createWriter(params); + } + + public HStoreFile createStoreFileAndReader(Path p) throws IOException { + StoreFileInfo info = new StoreFileInfo(conf, ctx.getRegionFileSystem().getFileSystem(), p, + ctx.isPrimaryReplicaStore()); + return createStoreFileAndReader(info); + } + + public HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { + info.setRegionCoprocessorHost(coprocessorHost); + HStoreFile storeFile = + new HStoreFile(info, ctx.getFamily().getBloomFilterType(), ctx.getCacheConf()); + storeFile.initReader(); + return storeFile; + } + + /** + * Validates a store file by opening and closing it. In HFileV2 this should not be an expensive + * operation. + * @param path the path to the store file + */ + public void validateStoreFile(Path path) throws IOException { + HStoreFile storeFile = null; + try { + storeFile = createStoreFileAndReader(path); + } catch (IOException e) { + LOG.error("Failed to open store file : {}, keeping it in tmp location", path, e); + throw e; + } finally { + if (storeFile != null) { + storeFile.closeStoreFile(false); + } + } + } + + private List openStoreFiles(Collection files, boolean warmup) + throws IOException { + if (CollectionUtils.isEmpty(files)) { + return Collections.emptyList(); + } + // initialize the thread pool for opening store files in parallel.. + ExecutorService storeFileOpenerThreadPool = + openStoreFileThreadPoolCreator.apply("StoreFileOpener-" + + ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString()); + CompletionService completionService = + new ExecutorCompletionService<>(storeFileOpenerThreadPool); + + int totalValidStoreFile = 0; + for (StoreFileInfo storeFileInfo : files) { + // open each store file in parallel + completionService.submit(() -> createStoreFileAndReader(storeFileInfo)); + totalValidStoreFile++; + } + + Set compactedStoreFiles = new HashSet<>(); + ArrayList results = new ArrayList<>(files.size()); + IOException ioe = null; + try { + for (int i = 0; i < totalValidStoreFile; i++) { + try { + HStoreFile storeFile = completionService.take().get(); + if (storeFile != null) { + LOG.debug("loaded {}", storeFile); + results.add(storeFile); + compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); + } + } catch (InterruptedException e) { + if (ioe == null) { + ioe = new InterruptedIOException(e.getMessage()); + } + } catch (ExecutionException e) { + if (ioe == null) { + ioe = new IOException(e.getCause()); + } + } + } + } finally { + storeFileOpenerThreadPool.shutdownNow(); + } + if (ioe != null) { + // close StoreFile readers + boolean evictOnClose = + ctx.getCacheConf() != null ? ctx.getCacheConf().shouldEvictOnClose() : true; + for (HStoreFile file : results) { + try { + if (file != null) { + file.closeStoreFile(evictOnClose); + } + } catch (IOException e) { + LOG.warn("Could not close store file {}", file, e); + } + } + throw ioe; + } + + // Should not archive the compacted store files when region warmup. See HBASE-22163. + if (!warmup) { + // Remove the compacted files from result + List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); + for (HStoreFile storeFile : results) { + if (compactedStoreFiles.contains(storeFile.getPath().getName())) { + LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); + storeFile.getReader().close( + storeFile.getCacheConf() != null ? storeFile.getCacheConf().shouldEvictOnClose() : + true); + filesToRemove.add(storeFile); + } + } + results.removeAll(filesToRemove); + if (!filesToRemove.isEmpty() && ctx.isPrimaryReplicaStore()) { + LOG.debug("Moving the files {} to archive", filesToRemove); + ctx.getRegionFileSystem().removeStoreFiles(ctx.getFamily().getNameAsString(), + filesToRemove); + } + } + + return results; + } + + public void initialize(boolean warmup) throws IOException { + List fileInfos = storeFileTracker.load(); + List files = openStoreFiles(fileInfos, warmup); + storeFileManager.loadFiles(files); + } + + public void refreshStoreFiles() throws IOException { + List fileInfos = storeFileTracker.load(); + refreshStoreFilesInternal(fileInfos); + } + + public void refreshStoreFiles(Collection newFiles) throws IOException { + List storeFiles = new ArrayList<>(newFiles.size()); + for (String file : newFiles) { + storeFiles + .add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), file)); + } + refreshStoreFilesInternal(storeFiles); + } + + /** + * Checks the underlying store files, and opens the files that have not been opened, and removes + * the store file readers for store files no longer available. Mainly used by secondary region + * replicas to keep up to date with the primary region files. + */ + private void refreshStoreFilesInternal(Collection newFiles) throws IOException { + Collection currentFiles = storeFileManager.getStorefiles(); + Collection compactedFiles = storeFileManager.getCompactedfiles(); + if (currentFiles == null) { + currentFiles = Collections.emptySet(); + } + if (newFiles == null) { + newFiles = Collections.emptySet(); + } + if (compactedFiles == null) { + compactedFiles = Collections.emptySet(); + } + + HashMap currentFilesSet = new HashMap<>(currentFiles.size()); + for (HStoreFile sf : currentFiles) { + currentFilesSet.put(sf.getFileInfo(), sf); + } + HashMap compactedFilesSet = new HashMap<>(compactedFiles.size()); + for (HStoreFile sf : compactedFiles) { + compactedFilesSet.put(sf.getFileInfo(), sf); + } + + Set newFilesSet = new HashSet(newFiles); + // Exclude the files that have already been compacted + newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); + Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); + Set toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet); + + if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) { + return; + } + + LOG.info("Refreshing store files for " + this + " files to add: " + toBeAddedFiles + + " files to remove: " + toBeRemovedFiles); + + Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); + for (StoreFileInfo sfi : toBeRemovedFiles) { + toBeRemovedStoreFiles.add(currentFilesSet.get(sfi)); + } + + // try to open the files + List openedFiles = openStoreFiles(toBeAddedFiles, false); + + // propogate the file changes to the underlying store file manager + replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); // won't throw an exception + } + + /** + * Commit the given {@code files}. + *

+ * We will move the file into data directory, and open it. + * @param files the files want to commit + * @param validate whether to validate the store files + * @return the committed store files + */ + public List commitStoreFiles(List files, boolean validate) throws IOException { + List committedFiles = new ArrayList<>(files.size()); + HRegionFileSystem hfs = ctx.getRegionFileSystem(); + String familyName = ctx.getFamily().getNameAsString(); + Path storeDir = hfs.getStoreDir(familyName); + for (Path file : files) { + try { + if (validate) { + validateStoreFile(file); + } + Path committedPath; + // As we want to support writing to data directory directly, here we need to check whether + // the store file is already in the right place + if (file.getParent() != null && file.getParent().equals(storeDir)) { + // already in the right place, skip renmaing + committedPath = file; + } else { + // Write-out finished successfully, move into the right spot + committedPath = hfs.commitStoreFile(familyName, file); + } + HStoreFile sf = createStoreFileAndReader(committedPath); + committedFiles.add(sf); + } catch (IOException e) { + LOG.error("Failed to commit store file {}", file, e); + // Try to delete the files we have committed before. + // It is OK to fail when deleting as leaving the file there does not cause any data + // corruption problem. It just introduces some duplicated data which may impact read + // performance a little when reading before compaction. + for (HStoreFile sf : committedFiles) { + Path pathToDelete = sf.getPath(); + try { + sf.deleteStoreFile(); + } catch (IOException deleteEx) { + LOG.warn(HBaseMarkers.FATAL, "Failed to delete committed store file {}", pathToDelete, + deleteEx); + } + } + throw new IOException("Failed to commit the flush", e); + } + } + return committedFiles; + } + + public void addStoreFiles(Collection storeFiles) throws IOException { + storeFileTracker.add(StoreUtils.toStoreFileInfo(storeFiles)); + writeLock(); + try { + storeFileManager.insertNewFiles(storeFiles); + } finally { + // We need the lock, as long as we are updating the storeFiles + // or changing the memstore. Let us release it before calling + // notifyChangeReadersObservers. See HBASE-4485 for a possible + // deadlock scenario that could have happened if continue to hold + // the lock. + writeUnlock(); + } + } + + public void replaceStoreFiles(Collection compactedFiles, + Collection newFiles) throws IOException { + storeFileTracker.replace(StoreUtils.toStoreFileInfo(compactedFiles), + StoreUtils.toStoreFileInfo(newFiles)); + writeLock(); + try { + storeFileManager.addCompactionResults(compactedFiles, newFiles); + } finally { + writeUnlock(); + } + } + + public void removeCompactedFiles(Collection compactedFiles) { + writeLock(); + try { + storeFileManager.removeCompactedFiles(compactedFiles); + } finally { + writeUnlock(); + } } /** * Create the StoreEngine configured for the given Store. - * @param store The store. An unfortunate dependency needed due to it - * being passed to coprocessors via the compactor. + * @param store The store. An unfortunate dependency needed due to it being passed to coprocessors + * via the compactor. * @param conf Store configuration. * @param cellComparator CellComparator for storeFileManager. * @return StoreEngine to use. */ - public static StoreEngine create( - HStore store, Configuration conf, CellComparator cellComparator) throws IOException { + public static StoreEngine create(HStore store, Configuration conf, + CellComparator cellComparator) throws IOException { String className = conf.get(STORE_ENGINE_CLASS_KEY, DEFAULT_STORE_ENGINE_CLASS.getName()); try { - StoreEngine se = ReflectionUtils.instantiateWithCustomCtor( - className, new Class[] { }, new Object[] { }); + StoreEngine se = + ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {}, new Object[] {}); se.createComponentsOnce(conf, store, cellComparator); return se; } catch (Exception e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java index 27127f3a6c64..a40b209c6ebb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.util.Collection; import java.util.Comparator; @@ -49,12 +50,16 @@ public interface StoreFileManager { * Loads the initial store files into empty StoreFileManager. * @param storeFiles The files to load. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void loadFiles(List storeFiles); /** * Adds new files, either for from MemStore flush or bulk insert, into the structure. * @param sfs New store files. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void insertNewFiles(Collection sfs); /** @@ -62,12 +67,16 @@ public interface StoreFileManager { * @param compactedFiles The input files for the compaction. * @param results The resulting files for the compaction. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void addCompactionResults(Collection compactedFiles, Collection results); /** * Remove the compacted files * @param compactedFiles the list of compacted files */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void removeCompactedFiles(Collection compactedFiles); /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java index 1064b6c70547..58031288f751 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java @@ -70,10 +70,17 @@ protected void finalizeWriter(StoreFileWriter writer, long cacheFlushSeqNum, writer.close(); } + protected final StoreFileWriter createWriter(MemStoreSnapshot snapshot, boolean alwaysIncludesTag) + throws IOException { + return store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(snapshot.getCellsCount()) + .compression(store.getColumnFamilyDescriptor().getCompressionType()).isCompaction(false) + .includeMVCCReadpoint(true).includesTag(alwaysIncludesTag || snapshot.isTagsPresent()) + .shouldDropBehind(false)); + } /** * Creates the scanner for flushing snapshot. Also calls coprocessors. - * @param snapshotScanners * @return The scanner; null if coprocessor is canceling the flush. */ protected final InternalScanner createScanner(List snapshotScanners, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java index 454b244fb512..10a9330f8326 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java @@ -20,10 +20,13 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; - +import java.util.function.Predicate; +import java.util.function.ToLongFunction; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; @@ -42,10 +45,13 @@ * Utility functions for region server storage layer. */ @InterfaceAudience.Private -public class StoreUtils { +public final class StoreUtils { private static final Logger LOG = LoggerFactory.getLogger(StoreUtils.class); + private StoreUtils() { + } + /** * Creates a deterministic hash code for store file collection. */ @@ -171,4 +177,31 @@ public static Configuration createStoreConfiguration(Configuration conf, TableDe return new CompoundConfiguration().add(conf).addBytesMap(td.getValues()) .addStringMap(cfd.getConfiguration()).addBytesMap(cfd.getValues()); } + + public static List toStoreFileInfo(Collection storefiles) { + return storefiles.stream().map(HStoreFile::getFileInfo).collect(Collectors.toList()); + } + + public static long getTotalUncompressedBytes(List files) { + return files.stream() + .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) + .sum(); + } + + public static long getStorefilesSize(Collection files, + Predicate predicate) { + return files.stream().filter(predicate) + .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::length)).sum(); + } + + public static long getStorefileFieldSize(HStoreFile file, ToLongFunction f) { + if (file == null) { + return 0L; + } + StoreFileReader reader = file.getReader(); + if (reader == null) { + return 0L; + } + return f.applyAsLong(reader); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java index 14863a69a9b1..bfb3f649ff27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java @@ -20,20 +20,19 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java index 1560aef5f6b3..f8183b7645a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java @@ -70,7 +70,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushSeqNum StripeMultiFileWriter mw = null; try { mw = req.createWriter(); // Writer according to the policy. - StripeMultiFileWriter.WriterFactory factory = createWriterFactory(cellsCount); + StripeMultiFileWriter.WriterFactory factory = createWriterFactory(snapshot); StoreScanner storeScanner = (scanner instanceof StoreScanner) ? (StoreScanner)scanner : null; mw.init(storeScanner, factory); @@ -98,13 +98,12 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushSeqNum return result; } - private StripeMultiFileWriter.WriterFactory createWriterFactory(final long kvCount) { + private StripeMultiFileWriter.WriterFactory createWriterFactory(MemStoreSnapshot snapshot) { return new StripeMultiFileWriter.WriterFactory() { @Override public StoreFileWriter createWriter() throws IOException { - StoreFileWriter writer = store.createWriterInTmp(kvCount, - store.getColumnFamilyDescriptor().getCompressionType(), false, true, true, false); - return writer; + // XXX: it used to always pass true for includesTag, re-consider? + return StripeStoreFlusher.this.createWriter(snapshot, true); } }; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java index 42841bfee531..533be176e7a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java @@ -51,13 +51,14 @@ protected void initMultiWriter(AbstractMultiFileWriter writer, InternalScanner s WriterFactory writerFactory = new WriterFactory() { @Override public StoreFileWriter createWriter() throws IOException { - return createTmpWriter(fd, shouldDropBehind, major); + return AbstractMultiOutputCompactor.this.createWriter(fd, shouldDropBehind, major); } @Override public StoreFileWriter createWriterWithStoragePolicy(String fileStoragePolicy) - throws IOException { - return createTmpWriter(fd, shouldDropBehind, fileStoragePolicy, major); + throws IOException { + return AbstractMultiOutputCompactor.this.createWriter(fd, shouldDropBehind, + fileStoragePolicy, major); } }; // Prepare multi-writer, and perform the compaction using scanner and writer. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index e524f7dfd5fd..47ef0f290251 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -28,7 +28,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -39,6 +38,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.regionserver.CellSink; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -61,6 +61,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.io.Closeables; /** @@ -261,29 +262,32 @@ public InternalScanner createScanner(ScanInfo scanInfo, List s } }; + protected final CreateStoreFileWriterParams createParams(FileDetails fd, boolean shouldDropBehind, + boolean major) { + return CreateStoreFileWriterParams.create().maxKeyCount(fd.maxKeyCount) + .compression(major ? majorCompactionCompression : minorCompactionCompression) + .isCompaction(true).includeMVCCReadpoint(fd.maxMVCCReadpoint > 0) + .includesTag(fd.maxTagsLength > 0).shouldDropBehind(shouldDropBehind) + .totalCompactedFilesSize(fd.totalCompactedFilesSize); + } + /** - * Creates a writer for a new file in a temporary directory. + * Creates a writer for a new file. * @param fd The file details. - * @return Writer for a new StoreFile in the tmp dir. + * @return Writer for a new StoreFile * @throws IOException if creation failed */ - protected final StoreFileWriter createTmpWriter(FileDetails fd, boolean shouldDropBehind, boolean major) - throws IOException { + protected final StoreFileWriter createWriter(FileDetails fd, boolean shouldDropBehind, + boolean major) throws IOException { // When all MVCC readpoints are 0, don't write them. // See HBASE-8166, HBASE-12600, and HBASE-13389. - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, fd.maxMVCCReadpoint > 0, - fd.maxTagsLength > 0, shouldDropBehind, fd.totalCompactedFilesSize, - HConstants.EMPTY_STRING); + return store.getStoreEngine().createWriter(createParams(fd, shouldDropBehind, major)); } - protected final StoreFileWriter createTmpWriter(FileDetails fd, boolean shouldDropBehind, - String fileStoragePolicy, boolean major) throws IOException { - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, fd.maxMVCCReadpoint > 0, - fd.maxTagsLength > 0, shouldDropBehind, fd.totalCompactedFilesSize, fileStoragePolicy); + protected final StoreFileWriter createWriter(FileDetails fd, boolean shouldDropBehind, + String fileStoragePolicy, boolean major) throws IOException { + return store.getStoreEngine() + .createWriter(createParams(fd, shouldDropBehind, major).fileStoragePolicy(fileStoragePolicy)); } private ScanInfo preCompactScannerOpen(CompactionRequestImpl request, ScanType scanType, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java index 49d3e8ee01e7..afa2429cb6e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java @@ -45,14 +45,14 @@ public DefaultCompactor(Configuration conf, HStore store) { } private final CellSinkFactory writerFactory = - new CellSinkFactory() { - @Override - public StoreFileWriter createWriter(InternalScanner scanner, - org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, - boolean shouldDropBehind, boolean major) throws IOException { - return createTmpWriter(fd, shouldDropBehind, major); - } - }; + new CellSinkFactory() { + @Override + public StoreFileWriter createWriter(InternalScanner scanner, + org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, + boolean shouldDropBehind, boolean major) throws IOException { + return DefaultCompactor.this.createWriter(fd, shouldDropBehind, major); + } + }; /** * Do a minor/major compaction on an explicit set of storefiles from a Store. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java new file mode 100644 index 000000000000..d4c9a868eb48 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * The default implementation for store file tracker, where we do not persist the store file list, + * and use listing when loading store files. + */ +@InterfaceAudience.Private +class DefaultStoreFileTracker extends StoreFileTrackerBase { + + public DefaultStoreFileTracker(Configuration conf, TableName tableName, boolean isPrimaryReplica, + StoreContext ctx) { + super(conf, tableName, isPrimaryReplica, ctx); + } + + @Override + public List load() throws IOException { + return ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + } + + @Override + public boolean requireWritingToTmpDirFirst() { + return true; + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + // NOOP + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + // NOOP + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java new file mode 100644 index 000000000000..aadedc8ef727 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * An interface to define how we track the store files for a give store. + *

+ * In the old time, we will write store to a tmp directory first, and then rename it to the actual + * data file. And once a store file is under data directory, we will consider it as 'committed'. And + * we need to do listing when loading store files. + *

+ * When cloud age is coming, now we want to store the store files on object storage, where rename + * and list are not as cheap as on HDFS, especially rename. Although introducing a metadata + * management layer for object storage could solve the problem, but we still want HBase to run on + * pure object storage, so here we introduce this interface to abstract how we track the store + * files. For the old implementation, we just persist nothing here, and do listing to load store + * files. When running on object storage, we could persist the store file list in a system region, + * or in a file on the object storage, to make it possible to write directly into the data directory + * to avoid renaming, and also avoid listing when loading store files. + *

+ * The implementation requires to be thread safe as flush and compaction may occur as the same time, + * and we could also do multiple compactions at the same time. As the implementation may choose to + * persist the store file list to external storage, which could be slow, it is the duty for the + * callers to not call it inside a lock which may block normal read/write requests. + */ +@InterfaceAudience.Private +public interface StoreFileTracker { + + /** + * Load the store files list when opening a region. + */ + List load() throws IOException; + + /** + * Add new store files. + *

+ * Used for flush and bulk load. + */ + void add(Collection newFiles) throws IOException; + + /** + * Add new store files and remove compacted store files after compaction. + */ + void replace(Collection compactedFiles, Collection newFiles) + throws IOException; + + /** + * Create a writer for writing new store files. + * @return Writer for a new StoreFile + */ + StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java new file mode 100644 index 000000000000..2451f45bdc7b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for all store file tracker. + *

+ * Mainly used to place the common logic to skip persistent for secondary replicas. + */ +@InterfaceAudience.Private +abstract class StoreFileTrackerBase implements StoreFileTracker { + + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerBase.class); + + protected final Configuration conf; + + protected final TableName tableName; + + protected final boolean isPrimaryReplica; + + protected final StoreContext ctx; + + private volatile boolean cacheOnWriteLogged; + + protected StoreFileTrackerBase(Configuration conf, TableName tableName, boolean isPrimaryReplica, + StoreContext ctx) { + this.conf = conf; + this.tableName = tableName; + this.isPrimaryReplica = isPrimaryReplica; + this.ctx = ctx; + } + + @Override + public final void add(Collection newFiles) throws IOException { + if (isPrimaryReplica) { + doAddNewStoreFiles(newFiles); + } + } + + @Override + public final void replace(Collection compactedFiles, + Collection newFiles) throws IOException { + if (isPrimaryReplica) { + doAddCompactionResults(compactedFiles, newFiles); + } + } + + private HFileContext createFileContext(Compression.Algorithm compression, + boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { + if (compression == null) { + compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; + } + ColumnFamilyDescriptor family = ctx.getFamily(); + HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(includeMVCCReadpoint) + .withIncludesTags(includesTag).withCompression(compression) + .withCompressTags(family.isCompressTags()).withChecksumType(StoreUtils.getChecksumType(conf)) + .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)) + .withBlockSize(family.getBlocksize()).withHBaseCheckSum(true) + .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(encryptionContext) + .withCreateTime(EnvironmentEdgeManager.currentTime()).withColumnFamily(family.getName()) + .withTableName(tableName.getName()).withCellComparator(ctx.getComparator()).build(); + return hFileContext; + } + + @Override + public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) + throws IOException { + if (!isPrimaryReplica) { + throw new IllegalStateException("Should not call create writer on secondary replicas"); + } + // creating new cache config for each new writer + final CacheConfig cacheConf = ctx.getCacheConf(); + final CacheConfig writerCacheConf = new CacheConfig(cacheConf); + long totalCompactedFilesSize = params.totalCompactedFilesSize(); + if (params.isCompaction()) { + // Don't cache data on write on compactions, unless specifically configured to do so + // Cache only when total file size remains lower than configured threshold + final boolean cacheCompactedBlocksOnWrite = cacheConf.shouldCacheCompactedBlocksOnWrite(); + // if data blocks are to be cached on write + // during compaction, we should forcefully + // cache index and bloom blocks as well + if (cacheCompactedBlocksOnWrite && + totalCompactedFilesSize <= cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { + writerCacheConf.enableCacheOnWrite(); + if (!cacheOnWriteLogged) { + LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " + + "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this); + cacheOnWriteLogged = true; + } + } else { + writerCacheConf.setCacheDataOnWrite(false); + if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { + // checking condition once again for logging + LOG.debug( + "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted " + + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}", + this, totalCompactedFilesSize, cacheConf.getCacheCompactedBlocksOnWriteThreshold()); + } + } + } else { + final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite(); + if (shouldCacheDataOnWrite) { + writerCacheConf.enableCacheOnWrite(); + if (!cacheOnWriteLogged) { + LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " + + "Index blocks and Bloom filter blocks", this); + cacheOnWriteLogged = true; + } + } + } + Encryption.Context encryptionContext = ctx.getEncryptionContext(); + HFileContext hFileContext = createFileContext(params.compression(), + params.includeMVCCReadpoint(), params.includesTag(), encryptionContext); + Path outputDir; + if (requireWritingToTmpDirFirst()) { + outputDir = + new Path(ctx.getRegionFileSystem().getTempDir(), ctx.getFamily().getNameAsString()); + } else { + throw new UnsupportedOperationException("not supported yet"); + } + StoreFileWriter.Builder builder = + new StoreFileWriter.Builder(conf, writerCacheConf, ctx.getRegionFileSystem().getFileSystem()) + .withOutputDir(outputDir).withBloomType(ctx.getBloomFilterType()) + .withMaxKeyCount(params.maxKeyCount()).withFavoredNodes(ctx.getFavoredNodes()) + .withFileContext(hFileContext).withShouldDropCacheBehind(params.shouldDropBehind()) + .withCompactedFilesSupplier(ctx.getCompactedFilesSupplier()) + .withFileStoragePolicy(params.fileStoragePolicy()); + return builder.build(); + } + + /** + * Whether the implementation of this tracker requires you to write to temp directory first, i.e, + * does not allow broken store files under the actual data directory. + */ + protected abstract boolean requireWritingToTmpDirFirst(); + + protected abstract void doAddNewStoreFiles(Collection newFiles) throws IOException; + + protected abstract void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java new file mode 100644 index 000000000000..4f7231bc3b9a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Factory method for creating store file tracker. + */ +@InterfaceAudience.Private +public final class StoreFileTrackerFactory { + + public static StoreFileTracker create(Configuration conf, TableName tableName, + boolean isPrimaryReplica, StoreContext ctx) { + return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java index 22ec6cb89ec4..291b909f69bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java @@ -105,7 +105,6 @@ Set getStoresRequiringCompaction(Set requestedStores, long times boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts) throws IOException { - // do we have any store files? Collection storeFiles = fileSystem.getStoreFiles(family); if (storeFiles == null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index 3c2bc3f0cb12..9314d7e9827f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -211,11 +211,13 @@ protected BlockCompactionsInCompletionHStore(HRegion region, ColumnFamilyDescrip @Override protected void refreshStoreSizeAndTotalBytes() throws IOException { - try { - r.compactionsWaiting.countDown(); - r.compactionsBlocked.await(); - } catch (InterruptedException ex) { - throw new IOException(ex); + if (r != null) { + try { + r.compactionsWaiting.countDown(); + r.compactionsBlocked.await(); + } catch (InterruptedException ex) { + throw new IOException(ex); + } } super.refreshStoreSizeAndTotalBytes(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index cd83dc8c2494..0a6b37bbba4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -215,8 +215,10 @@ public void tearDown() throws IOException { @Test public void testCacheOnWriteInSchema() throws IOException { // Write some random data into the store - StoreFileWriter writer = store.createWriterInTmp(Integer.MAX_VALUE, - HFile.DEFAULT_COMPRESSION_ALGORITHM, false, true, false, false); + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(Integer.MAX_VALUE) + .compression(HFile.DEFAULT_COMPRESSION_ALGORITHM).isCompaction(false) + .includeMVCCReadpoint(true).includesTag(false).shouldDropBehind(false)); writeStoreFile(writer); writer.close(); // Verify the block types of interest were cached on write diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java index e832c47aac81..3784876a59f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java @@ -65,9 +65,12 @@ public void testCustomParts() throws Exception { DummyCompactionPolicy.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, DummyStoreFlusher.class.getName()); + HRegion mockRegion = Mockito.mock(HRegion.class); HStore mockStore = Mockito.mock(HStore.class); Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); - StoreEngine se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); + Mockito.when(mockStore.getHRegion()).thenReturn(mockRegion); + StoreEngine se = + StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); Assert.assertTrue(se instanceof DefaultStoreEngine); Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy); Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index a5584ff32599..7fb57686c35d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -5744,7 +5744,7 @@ public void testCompactionFromPrimary() throws IOException { Collection storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); Collection storeFileInfos = primaryRegion.getRegionFileSystem() - .getStoreFiles(families[0]); + .getStoreFiles(Bytes.toString(families[0])); Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); verifyData(secondaryRegion, 0, 1000, cq, families); @@ -7648,7 +7648,7 @@ protected List doCompaction(CompactionRequestImpl cr, getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; for (Path newFile : newFiles) { // Create storefile around what we wrote with a reader on it. - HStoreFile sf = createStoreFileAndReader(newFile); + HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); sf.closeStoreFile(evictOnClose); sfs.add(sf); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index b64f52192740..8e4c4a385de3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -310,7 +310,7 @@ public Object run() throws Exception { /** * Verify that compression and data block encoding are respected by the - * Store.createWriterInTmp() method, used on store flush. + * createWriter method, used on store flush. */ @Test public void testCreateWriter() throws Exception { @@ -322,9 +322,11 @@ public void testCreateWriter() throws Exception { .build(); init(name.getMethodName(), conf, hcd); - // Test createWriterInTmp() - StoreFileWriter writer = - store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false); + // Test createWriter + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(4) + .compression(hcd.getCompressionType()).isCompaction(false).includeMVCCReadpoint(true) + .includesTag(false).shouldDropBehind(false)); Path path = writer.getPath(); writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); @@ -1021,19 +1023,19 @@ public void testRefreshStoreFilesNotChanged() throws IOException { // add one more file addStoreFile(); - HStore spiedStore = spy(store); + StoreEngine spiedStoreEngine = spy(store.getStoreEngine()); // call first time after files changed - spiedStore.refreshStoreFiles(); + spiedStoreEngine.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); - verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); + verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any()); // call second time - spiedStore.refreshStoreFiles(); + spiedStoreEngine.refreshStoreFiles(); // ensure that replaceStoreFiles is not called, i.e, the times does not change, if files are not // refreshed, - verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); + verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any()); } private long countMemStoreScanner(StoreScanner scanner) { @@ -1644,7 +1646,7 @@ public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger() throws // Do compaction MyThread thread = new MyThread(storeScanner); thread.start(); - store.replaceStoreFiles(actualStorefiles, actualStorefiles1); + store.replaceStoreFiles(actualStorefiles, actualStorefiles1, false); thread.join(); KeyValueHeap heap2 = thread.getHeap(); assertFalse(heap.equals(heap2)); @@ -1710,8 +1712,10 @@ public void testSpaceQuotaChangeAfterReplacement() throws IOException { @Test public void testHFileContextSetWithCFAndTable() throws Exception { init(this.name.getMethodName()); - StoreFileWriter writer = store.createWriterInTmp(10000L, - Compression.Algorithm.NONE, false, true, false, true); + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(10000L) + .compression(Compression.Algorithm.NONE).isCompaction(true).includeMVCCReadpoint(true) + .includesTag(false).shouldDropBehind(true)); HFileContext hFileContext = writer.getHFileWriter().getFileContext(); assertArrayEquals(family, hFileContext.getColumnFamily()); assertArrayEquals(table, hFileContext.getTableName()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 60f65be66c47..1dbf37aa8063 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -245,7 +245,7 @@ public void testCleanMergeReference() throws Exception { TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo); int count = 0; for(ColumnFamilyDescriptor colFamily : columnFamilies) { - count += hrfs.getStoreFiles(colFamily.getName()).size(); + count += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } ADMIN.compactRegion(mergedRegionInfo.getRegionName()); // clean up the merged region store files @@ -254,7 +254,7 @@ public void testCleanMergeReference() throws Exception { int newcount = 0; while (EnvironmentEdgeManager.currentTime() < timeout) { for(ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount += hrfs.getStoreFiles(colFamily.getName()).size(); + newcount += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } if(newcount > count) { break; @@ -273,7 +273,7 @@ public void testCleanMergeReference() throws Exception { while (EnvironmentEdgeManager.currentTime() < timeout) { int newcount1 = 0; for(ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount1 += hrfs.getStoreFiles(colFamily.getName()).size(); + newcount1 += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } if(newcount1 <= 1) { break; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java index 9141327d26d4..2fab050446ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -98,7 +97,7 @@ static class FailingHRegionFileSystem extends HRegionFileSystem { } @Override - public Collection getStoreFiles(String familyName) throws IOException { + public List getStoreFiles(String familyName) throws IOException { if (fail) { throw new IOException("simulating FS failure"); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java index 4cb023f9b9d6..121f8301751f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java @@ -29,7 +29,6 @@ import java.util.Random; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -128,13 +127,12 @@ public void testScannerCloseAndUpdateReadersWithMemstoreScanner() throws Excepti p.addColumn(fam, Bytes.toBytes("q1"), Bytes.toBytes("val")); region.put(p); HStore store = region.getStore(fam); - ReentrantReadWriteLock lock = store.lock; // use the lock to manually get a new memstore scanner. this is what // HStore#notifyChangedReadersObservers does under the lock.(lock is not needed here //since it is just a testcase). - lock.readLock().lock(); + store.getStoreEngine().readLock(); final List memScanners = store.memstore.getScanners(Long.MAX_VALUE); - lock.readLock().unlock(); + store.getStoreEngine().readUnlock(); Thread closeThread = new Thread() { public void run() { // close should be completed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index f5330f6faa42..eb0b1c1ca694 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -118,8 +118,10 @@ private static HStoreFile createFile() throws Exception { } private static TestStoreEngine createEngine(Configuration conf) throws Exception { + HRegion region = mock(HRegion.class); HStore store = mock(HStore.class); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); + when(store.getHRegion()).thenReturn(region); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); return (TestStoreEngine) StoreEngine.create(store, conf, kvComparator); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java index d7b7ba760156..0ea82c57be1a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java @@ -22,9 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,11 +42,13 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.Scanner; @@ -110,10 +109,9 @@ private DateTieredCompactor createCompactor(StoreFileWritersCapture writers, when(store.areWritesEnabled()).thenReturn(true); when(store.getFileSystem()).thenReturn(mock(FileSystem.class)); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.newBuilder(TABLE_NAME).build()); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); when(store.getComparator()).thenReturn(CellComparatorImpl.COMPARATOR); OptionalLong maxSequenceId = StoreUtils.getMaxSequenceIdInList(storefiles); when(store.getMaxSequenceId()).thenReturn(maxSequenceId); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java index 2ba15d1be18e..bdab20ead8a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java @@ -30,7 +30,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; @@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StripeMultiFileWriter; @@ -864,12 +865,9 @@ private StripeCompactor createCompactor() throws Exception { when(info.getRegionNameAsString()).thenReturn("testRegion"); when(store.getColumnFamilyDescriptor()).thenReturn(familyDescriptor); when(store.getRegionInfo()).thenReturn(info); - when( - store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when( - store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); Configuration conf = HBaseConfiguration.create(); conf.setBoolean("hbase.regionserver.compaction.private.readers", usePrivateReaders); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java index e49174e6afe3..ae59c74bad95 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java @@ -21,9 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.createDummyRequest; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,10 +40,12 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.Scanner; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.StoreFileWritersCapture; @@ -209,10 +208,9 @@ private StripeCompactor createCompactor(StoreFileWritersCapture writers, KeyValu when(store.areWritesEnabled()).thenReturn(true); when(store.getFileSystem()).thenReturn(mock(FileSystem.class)); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.newBuilder(TABLE_NAME).build()); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); when(store.getComparator()).thenReturn(CellComparatorImpl.COMPARATOR); return new StripeCompactor(conf, store) { From dd7949f5249bbf91e820df12ced0c36fcd09bdd7 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 26 Aug 2021 18:51:12 +0800 Subject: [PATCH 10/37] HBASE-25988 Store the store file list by a file (#3578) Signed-off-by: Wellington Chevreuil --- .../server/region/StoreFileTracker.proto | 36 +++++ .../hbase/regionserver/StoreContext.java | 5 + .../hbase/regionserver/StoreEngine.java | 8 +- .../DefaultStoreFileTracker.java | 5 +- .../FileBasedStoreFileTracker.java | 142 ++++++++++++++++++ .../storefiletracker/StoreFileListFile.java | 142 ++++++++++++++++++ .../StoreFileTrackerBase.java | 11 +- .../StoreFileTrackerFactory.java | 12 +- ...stRegionWithFileBasedStoreFileTracker.java | 109 ++++++++++++++ 9 files changed, 451 insertions(+), 19 deletions(-) create mode 100644 hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto new file mode 100644 index 000000000000..2a269ea4ac4e --- /dev/null +++ b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; +// This file contains protocol buffers that are used for store file tracker. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "StoreFileTrackerProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message StoreFileEntry { + required string name = 1; + required uint64 size = 2; +} + +message StoreFileList { + required uint64 timestamp = 1; + repeated StoreFileEntry store_file = 2; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java index 2a9f96859361..588f8f4027a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java @@ -22,6 +22,7 @@ import java.util.function.Supplier; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.io.HeapSize; @@ -109,6 +110,10 @@ public RegionCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + public TableName getTableName() { + return getRegionInfo().getTable(); + } + public RegionInfo getRegionInfo() { return regionFileSystem.getRegionInfo(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index f9d6c294f779..61b7dfa66848 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -172,9 +172,9 @@ public StoreFlusher getStoreFlusher() { return this.storeFlusher; } - private StoreFileTracker createStoreFileTracker(HStore store) { - return StoreFileTrackerFactory.create(store.conf, store.getRegionInfo().getTable(), - store.isPrimaryReplicaStore(), store.getStoreContext()); + private StoreFileTracker createStoreFileTracker(Configuration conf, HStore store) { + return StoreFileTrackerFactory.create(conf, store.isPrimaryReplicaStore(), + store.getStoreContext()); } /** @@ -205,7 +205,7 @@ protected final void createComponentsOnce(Configuration conf, HStore store, this.ctx = store.getStoreContext(); this.coprocessorHost = store.getHRegion().getCoprocessorHost(); this.openStoreFileThreadPoolCreator = store.getHRegion()::getStoreFileOpenAndCloseThreadPool; - this.storeFileTracker = createStoreFileTracker(store); + this.storeFileTracker = createStoreFileTracker(conf, store); assert compactor != null && compactionPolicy != null && storeFileManager != null && storeFlusher != null && storeFileTracker != null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index d4c9a868eb48..fa044818336f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -21,7 +21,6 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -33,9 +32,9 @@ @InterfaceAudience.Private class DefaultStoreFileTracker extends StoreFileTrackerBase { - public DefaultStoreFileTracker(Configuration conf, TableName tableName, boolean isPrimaryReplica, + public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - super(conf, tableName, isPrimaryReplica, ctx); + super(conf, isPrimaryReplica, ctx); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java new file mode 100644 index 000000000000..de28b0eb9996 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry; +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; + +/** + * A file based store file tracker. + *

+ * For this tracking way, the store file list will be persistent into a file, so we can write the + * new store files directly to the final data directory, as we will not load the broken files. This + * will greatly reduce the time for flush and compaction on some object storages as a rename is + * actual a copy on them. And it also avoid listing when loading store file list, which could also + * speed up the loading of store files as listing is also not a fast operation on most object + * storages. + */ +@InterfaceAudience.Private +public class FileBasedStoreFileTracker extends StoreFileTrackerBase { + + private final StoreFileListFile backedFile; + + private final Map storefiles = new HashMap<>(); + + public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + backedFile = new StoreFileListFile(ctx); + } + + @Override + public List load() throws IOException { + StoreFileList list = backedFile.load(); + if (list == null) { + return Collections.emptyList(); + } + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + List infos = new ArrayList<>(); + for (StoreFileEntry entry : list.getStoreFileList()) { + infos.add(ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, ctx.getRegionInfo(), + ctx.getRegionFileSystem().getRegionInfoForFS(), ctx.getFamily().getNameAsString(), + new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()))); + } + // In general, for primary replica, the load method should only be called once when + // initialization, so we do not need synchronized here. And for secondary replicas, though the + // load method could be called multiple times, we will never call other methods so no + // synchronized is also fine. + // But we have a refreshStoreFiles method in the Region interface, which can be called by CPs, + // and we have a RefreshHFilesEndpoint example to expose the refreshStoreFiles method as RPC, so + // for safety, let's still keep the synchronized here. + synchronized (storefiles) { + for (StoreFileInfo info : infos) { + storefiles.put(info.getPath().getName(), info); + } + } + return infos; + } + + @Override + protected boolean requireWritingToTmpDirFirst() { + return false; + } + + private StoreFileEntry toStoreFileEntry(StoreFileInfo info) { + return StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize()) + .build(); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + synchronized (storefiles) { + StoreFileList.Builder builder = StoreFileList.newBuilder(); + for (StoreFileInfo info : storefiles.values()) { + builder.addStoreFile(toStoreFileEntry(info)); + } + for (StoreFileInfo info : newFiles) { + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + for (StoreFileInfo info : newFiles) { + storefiles.put(info.getPath().getName(), info); + } + } + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + Set compactedFileNames = + compactedFiles.stream().map(info -> info.getPath().getName()).collect(Collectors.toSet()); + synchronized (storefiles) { + StoreFileList.Builder builder = StoreFileList.newBuilder(); + storefiles.forEach((name, info) -> { + if (compactedFileNames.contains(name)) { + return; + } + builder.addStoreFile(toStoreFileEntry(info)); + }); + for (StoreFileInfo info : newFiles) { + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + for (String name : compactedFileNames) { + storefiles.remove(name); + } + for (StoreFileInfo info : newFiles) { + storefiles.put(info.getPath().getName(), info); + } + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java new file mode 100644 index 000000000000..c778bfc51deb --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.FileNotFoundException; +import java.io.IOException; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; +import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; + +/** + * To fully avoid listing, here we use two files for tracking. When loading, we will try to read + * both the two files, if only one exists, we will trust this one, if both exist, we will compare + * the timestamp to see which one is newer and trust that one. And we will record in memory that + * which one is trusted by us, and when we need to update the store file list, we will write to the + * other file. + *

+ * So in this way, we could avoid listing when we want to load the store file list file. + */ +@InterfaceAudience.Private +class StoreFileListFile { + + private static final Logger LOG = LoggerFactory.getLogger(StoreFileListFile.class); + + private static final String TRACK_FILE_DIR = ".filelist"; + + private static final String TRACK_FILE = "f1"; + + private static final String TRACK_FILE_ROTATE = "f2"; + + private final StoreContext ctx; + + private final Path trackFileDir; + + private final Path[] trackFiles = new Path[2]; + + // this is used to make sure that we do not go backwards + private long prevTimestamp = -1; + + private int nextTrackFile = -1; + + StoreFileListFile(StoreContext ctx) { + this.ctx = ctx; + trackFileDir = new Path(ctx.getFamilyStoreDirectoryPath(), TRACK_FILE_DIR); + trackFiles[0] = new Path(trackFileDir, TRACK_FILE); + trackFiles[1] = new Path(trackFileDir, TRACK_FILE_ROTATE); + } + + private StoreFileList load(Path path) throws IOException { + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + byte[] bytes; + try (FSDataInputStream in = fs.open(path)) { + bytes = ByteStreams.toByteArray(in); + } + // Read all the bytes and then parse it, so we will only throw InvalidProtocolBufferException + // here. This is very important for upper layer to determine whether this is the normal case, + // where the file does not exist or is incomplete. If there is another type of exception, the + // upper layer should throw it out instead of just ignoring it, otherwise it will lead to data + // loss. + return StoreFileList.parseFrom(bytes); + } + + private int select(StoreFileList[] lists) { + if (lists[0] == null) { + return 1; + } + if (lists[1] == null) { + return 0; + } + return lists[0].getTimestamp() >= lists[1].getTimestamp() ? 0 : 1; + } + + StoreFileList load() throws IOException { + StoreFileList[] lists = new StoreFileList[2]; + for (int i = 0; i < 2; i++) { + try { + lists[i] = load(trackFiles[i]); + } catch (FileNotFoundException | InvalidProtocolBufferException e) { + // this is normal case, so use info and do not log stacktrace + LOG.info("Failed to load track file {}: {}", trackFiles[i], e); + } + } + int winnerIndex = select(lists); + if (lists[winnerIndex] != null) { + nextTrackFile = 1 - winnerIndex; + prevTimestamp = lists[winnerIndex].getTimestamp(); + } else { + nextTrackFile = 0; + } + return lists[winnerIndex]; + } + + /** + * We will set the timestamp in this method so just pass the builder in + */ + void update(StoreFileList.Builder builder) throws IOException { + Preconditions.checkState(nextTrackFile >= 0, "should call load first before calling update"); + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + long timestamp = Math.max(prevTimestamp + 1, EnvironmentEdgeManager.currentTime()); + try (FSDataOutputStream out = fs.create(trackFiles[nextTrackFile], true)) { + builder.setTimestamp(timestamp).build().writeTo(out); + } + // record timestamp + prevTimestamp = timestamp; + // rotate the file + nextTrackFile = 1 - nextTrackFile; + try { + fs.delete(trackFiles[nextTrackFile], false); + } catch (IOException e) { + // we will create new file with overwrite = true, so not a big deal here, only for speed up + // loading as we do not need to read this file when loading(we will hit FileNotFoundException) + LOG.debug("failed to delete old track file {}, not a big deal, just ignore", e); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 2451f45bdc7b..92c699278c2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -21,7 +21,6 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -51,18 +50,14 @@ abstract class StoreFileTrackerBase implements StoreFileTracker { protected final Configuration conf; - protected final TableName tableName; - protected final boolean isPrimaryReplica; protected final StoreContext ctx; private volatile boolean cacheOnWriteLogged; - protected StoreFileTrackerBase(Configuration conf, TableName tableName, boolean isPrimaryReplica, - StoreContext ctx) { + protected StoreFileTrackerBase(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { this.conf = conf; - this.tableName = tableName; this.isPrimaryReplica = isPrimaryReplica; this.ctx = ctx; } @@ -95,7 +90,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, .withBlockSize(family.getBlocksize()).withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(encryptionContext) .withCreateTime(EnvironmentEdgeManager.currentTime()).withColumnFamily(family.getName()) - .withTableName(tableName.getName()).withCellComparator(ctx.getComparator()).build(); + .withTableName(ctx.getTableName().getName()).withCellComparator(ctx.getComparator()).build(); return hFileContext; } @@ -153,7 +148,7 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) outputDir = new Path(ctx.getRegionFileSystem().getTempDir(), ctx.getFamily().getNameAsString()); } else { - throw new UnsupportedOperationException("not supported yet"); + outputDir = ctx.getFamilyStoreDirectoryPath(); } StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf, ctx.getRegionFileSystem().getFileSystem()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 4f7231bc3b9a..6cdfaf4a0fe1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; /** @@ -28,8 +28,12 @@ @InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static StoreFileTracker create(Configuration conf, TableName tableName, - boolean isPrimaryReplica, StoreContext ctx) { - return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx); + public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; + + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, + StoreContext ctx) { + Class tracker = + conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java new file mode 100644 index 000000000000..3bc60d13fd94 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestRegionWithFileBasedStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionWithFileBasedStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final TableDescriptor TD = + TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build(); + + @Rule + public TestName name = new TestName(); + + private HRegion region; + + @Before + public void setUp() throws IOException { + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, FileBasedStoreFileTracker.class, + StoreFileTracker.class); + region = + HBaseTestingUtil.createRegionAndWAL(RI, UTIL.getDataTestDir(name.getMethodName()), conf, TD); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + HBaseTestingUtil.closeRegionAndWAL(region); + } + UTIL.cleanupTestDir(); + } + + @Test + public void testFlushAndCompaction() throws IOException { + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int v = i * 10 + j; + region.put(new Put(Bytes.toBytes(v)).addColumn(CF, CQ, Bytes.toBytes(v))); + } + region.flush(true); + if (i % 3 == 2) { + region.compact(true); + } + } + // reopen the region, make sure the store file tracker works, i.e, we can get all the records + // back + region.close(); + region = HRegion.openHRegion(region, null); + for (int i = 0; i < 100; i++) { + Result result = region.get(new Get(Bytes.toBytes(i))); + assertEquals(i, Bytes.toInt(result.getValue(CF, CQ))); + } + } +} From cb35f18de8a59b55fddb973729e557d327592f9f Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 8 Sep 2021 10:31:49 +0100 Subject: [PATCH 11/37] HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) Signed-off-by: Duo Zhang --- .../MergeTableRegionsProcedure.java | 23 +- .../assignment/SplitTableRegionProcedure.java | 41 +-- .../hbase/regionserver/HRegionFileSystem.java | 42 ++- .../DefaultStoreFileTracker.java | 4 +- .../storefiletracker/StoreFileTracker.java | 1 - .../StoreFileTrackerFactory.java | 33 ++- .../regionserver/TestDefaultStoreEngine.java | 1 + .../TestDirectStoreSplitsMerges.java | 32 ++- .../hbase/regionserver/TestHStoreFile.java | 19 +- .../TestMergesSplitsAddToTracker.java | 262 ++++++++++++++++++ .../regionserver/TestStripeStoreEngine.java | 1 + .../TestStoreFileTracker.java | 56 ++++ 12 files changed, 473 insertions(+), 42 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index fb57cb92dc2f..4b25a28a6055 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -24,6 +24,8 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -53,6 +55,8 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WALSplitUtil; @@ -584,40 +588,47 @@ private void createMergedRegion(final MasterProcedureEnv env) throws IOException final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); - + List mergedFiles = new ArrayList<>(); HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem( env.getMasterConfiguration(), fs, tableDir, mergedRegion); for (RegionInfo ri: this.regionsToMerge) { HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tableDir, ri, false); - mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion); + mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); } assert mergeRegionFs != null; - mergeRegionFs.commitMergedRegion(); + mergeRegionFs.commitMergedRegion(mergedFiles, env); // Prepare to create merged regions env.getAssignmentManager().getRegionStates(). getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); } - private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, + private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { final TableDescriptor htd = env.getMasterServices().getTableDescriptors() .get(mergedRegion.getTable()); + List mergedFiles = new ArrayList<>(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); - final Collection storeFiles = regionFs.getStoreFiles(family); + Configuration trackerConfig = + StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, + family, regionFs); + final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { // Create reference file(s) to parent region file here in mergedDir. // As this procedure is running on master, use CacheConfig.DISABLED means // don't cache any block. - mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, + Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + mergedFiles.add(refFile); } } } + return mergedFiles; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 0a15e36a16af..4d53df4bc670 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -64,6 +64,8 @@ import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.RegionSplitRestriction; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -618,21 +620,20 @@ public void createDaughterRegions(final MasterProcedureEnv env) throws IOExcepti final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); - regionFs.createSplitsDir(daughterOneRI, daughterTwoRI); - Pair expectedReferences = splitStoreFiles(env, regionFs); + Pair, List> expectedReferences = splitStoreFiles(env, regionFs); - assertReferenceFileCount(fs, expectedReferences.getFirst(), + assertReferenceFileCount(fs, expectedReferences.getFirst().size(), regionFs.getSplitsDir(daughterOneRI)); - regionFs.commitDaughterRegion(daughterOneRI); - assertReferenceFileCount(fs, expectedReferences.getFirst(), + regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst(), env); + assertReferenceFileCount(fs, expectedReferences.getFirst().size(), new Path(tabledir, daughterOneRI.getEncodedName())); - assertReferenceFileCount(fs, expectedReferences.getSecond(), + assertReferenceFileCount(fs, expectedReferences.getSecond().size(), regionFs.getSplitsDir(daughterTwoRI)); - regionFs.commitDaughterRegion(daughterTwoRI); - assertReferenceFileCount(fs, expectedReferences.getSecond(), + regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond(), env); + assertReferenceFileCount(fs, expectedReferences.getSecond().size(), new Path(tabledir, daughterTwoRI.getEncodedName())); } @@ -649,7 +650,7 @@ private void deleteDaughterRegions(final MasterProcedureEnv env) throws IOExcept * Create Split directory * @param env MasterProcedureEnv */ - private Pair splitStoreFiles(final MasterProcedureEnv env, + private Pair, List> splitStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws IOException { final Configuration conf = env.getMasterConfiguration(); TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); @@ -665,7 +666,11 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, new HashMap>(htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); - Collection sfis = regionFs.getStoreFiles(family); + Configuration trackerConfig = StoreFileTrackerFactory. + mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, + family, regionFs); + Collection sfis = tracker.load(); if (sfis == null) { continue; } @@ -691,7 +696,7 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, } if (nbFiles == 0) { // no file needs to be splitted. - return new Pair(0, 0); + return new Pair<>(Collections.emptyList(), Collections.emptyList()); } // Max #threads is the smaller of the number of storefiles or the default max determined above. int maxThreads = Math.min( @@ -744,14 +749,18 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, throw (InterruptedIOException) new InterruptedIOException().initCause(e); } - int daughterA = 0; - int daughterB = 0; + List daughterA = new ArrayList<>(); + List daughterB = new ArrayList<>(); // Look for any exception for (Future> future : futures) { try { Pair p = future.get(); - daughterA += p.getFirst() != null ? 1 : 0; - daughterB += p.getSecond() != null ? 1 : 0; + if(p.getFirst() != null){ + daughterA.add(p.getFirst()); + } + if(p.getSecond() != null){ + daughterB.add(p.getSecond()); + } } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } catch (ExecutionException e) { @@ -764,7 +773,7 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, getParentRegion().getShortNameToLog() + " Daughter A: " + daughterA + " storefiles, Daughter B: " + daughterB + " storefiles."); } - return new Pair(daughterA, daughterB); + return new Pair<>(daughterA, daughterB); } private void assertReferenceFileCount(final FileSystem fs, final int expectedReferenceFileCount, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 6ab9b0fc3fe8..3b053a349c2c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -23,7 +23,9 @@ import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -45,6 +47,9 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; @@ -592,19 +597,46 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} * @throws IOException */ - public Path commitDaughterRegion(final RegionInfo regionInfo) - throws IOException { + public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, + MasterProcedureEnv env) throws IOException { Path regionDir = this.getSplitsDir(regionInfo); if (fs.exists(regionDir)) { // Write HRI to a file in case we need to recover hbase:meta Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); + HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( + env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false); + insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); } - return regionDir; } + private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProcedureEnv env, + HRegionFileSystem regionFs) throws IOException { + TableDescriptor tblDesc = env.getMasterServices().getTableDescriptors(). + get(regionInfo.getTable()); + //we need to map trackers per store + Map trackerMap = new HashMap<>(); + //we need to map store files per store + Map> fileInfoMap = new HashMap<>(); + for(Path file : allFiles) { + String familyName = file.getParent().getName(); + trackerMap.computeIfAbsent(familyName, t -> { + Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, + tblDesc.getColumnFamily(Bytes.toBytes(familyName))); + return StoreFileTrackerFactory. + create(config, true, familyName, regionFs); + }); + fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); + List infos = fileInfoMap.get(familyName); + infos.add(new StoreFileInfo(conf, fs, file, true)); + } + for(Map.Entry entry : trackerMap.entrySet()) { + entry.getValue().add(fileInfoMap.get(entry.getKey())); + } + } + /** * Creates region split daughter directories under the table dir. If the daughter regions already * exist, for example, in the case of a recovery from a previous failed split procedure, this @@ -755,13 +787,15 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi * Commit a merged region, making it ready for use. * @throws IOException */ - public void commitMergedRegion() throws IOException { + public void commitMergedRegion(List allMergedFiles, MasterProcedureEnv env) + throws IOException { Path regionDir = getMergesDir(regionInfoForFs); if (regionDir != null && fs.exists(regionDir)) { // Write HRI to a file in case we need to recover hbase:meta Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); + insertRegionFilesIntoStoreTracker(allMergedFiles, env, this); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index fa044818336f..22e05132bf91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -32,8 +33,7 @@ @InterfaceAudience.Private class DefaultStoreFileTracker extends StoreFileTrackerBase { - public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index aadedc8ef727..0a85abb00a4e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -48,7 +48,6 @@ */ @InterfaceAudience.Private public interface StoreFileTracker { - /** * Load the store files list when opening a region. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 6cdfaf4a0fe1..c446d5ae9a31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,22 +18,51 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompoundConfiguration; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Factory method for creating store file tracker. */ @InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + StoreContext ctx) { Class tracker = conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, + HRegionFileSystem regionFs) { + ColumnFamilyDescriptorBuilder fDescBuilder = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); + StoreContext ctx = StoreContext.getBuilder(). + withColumnFamilyDescriptor(fDescBuilder.build()). + withRegionFileSystem(regionFs). + build(); + return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + } + + public static Configuration mergeConfigurations(Configuration global, + TableDescriptor table, ColumnFamilyDescriptor family) { + return new CompoundConfiguration() + .add(global) + .addBytesMap(table.getValues()) + .addStringMap(family.getConfiguration()) + .addBytesMap(family.getValues()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java index 3784876a59f3..523f27782362 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java @@ -67,6 +67,7 @@ public void testCustomParts() throws Exception { DummyStoreFlusher.class.getName()); HRegion mockRegion = Mockito.mock(HRegion.class); HStore mockStore = Mockito.mock(HStore.class); + mockStore.conf = conf; Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); Mockito.when(mockStore.getHRegion()).thenReturn(mockRegion); StoreEngine se = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java index bd24f1b22293..0eba8aa541ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.Path; @@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -139,7 +141,9 @@ public void testCommitDaughterRegionNoFiles() throws Exception { setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime()).build(); Path splitDir = regionFS.getSplitsDir(daughterA); - Path result = regionFS.commitDaughterRegion(daughterA); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path result = regionFS.commitDaughterRegion(daughterA, new ArrayList<>(), env); assertEquals(splitDir, result); } @@ -162,14 +166,18 @@ public void testCommitDaughterRegionWithFiles() throws Exception { Path splitDirA = regionFS.getSplitsDir(daughterA); Path splitDirB = regionFS.getSplitsDir(daughterB); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - regionFS + List filesA = new ArrayList<>(); + filesA.add(regionFS .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy()); - regionFS + Bytes.toBytes("002"), false, region.getSplitPolicy())); + List filesB = new ArrayList<>(); + filesB.add(regionFS .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), true, region.getSplitPolicy()); - Path resultA = regionFS.commitDaughterRegion(daughterA); - Path resultB = regionFS.commitDaughterRegion(daughterB); + Bytes.toBytes("002"), true, region.getSplitPolicy())); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path resultA = regionFS.commitDaughterRegion(daughterA, filesA, env); + Path resultB = regionFS.commitDaughterRegion(daughterB, filesB, env); assertEquals(splitDirA, resultA); assertEquals(splitDirB, resultB); } @@ -203,8 +211,11 @@ public void testCommitMergedRegion() throws Exception { mergeFileFromRegion(mergeRegionFs, first, file); //merge file from second region file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - mergeFileFromRegion(mergeRegionFs, second, file); - mergeRegionFs.commitMergedRegion(); + List mergedFiles = new ArrayList<>(); + mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file)); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + mergeRegionFs.commitMergedRegion(mergedFiles, env); } private void waitForSplitProcComplete(int attempts, int waitTime) throws Exception { @@ -223,11 +234,12 @@ private void waitForSplitProcComplete(int attempts, int waitTime) throws Excepti } } - private void mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, + private Path mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, HStoreFile file) throws IOException { Path mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile); + return mergedFile; } private void validateResultingFile(String originalRegion, Path result){ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index e6f0357204fc..c7203a928aa7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -49,12 +50,14 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -69,6 +72,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.BloomFilterFactory; @@ -1055,7 +1060,19 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h if (null == path) { return null; } - Path regionDir = regionFs.commitDaughterRegion(hri); + List splitFiles = new ArrayList<>(); + splitFiles.add(path); + MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class); + MasterServices mockServices = mock(MasterServices.class); + when(mockEnv.getMasterServices()).thenReturn(mockServices); + when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration()); + TableDescriptors mockTblDescs = mock(TableDescriptors.class); + when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs); + TableDescriptor mockTblDesc = mock(TableDescriptor.class); + when(mockTblDescs.get(any())).thenReturn(mockTblDesc); + ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class); + when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc); + Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); return new Path(new Path(regionDir, family), path.getName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java new file mode 100644 index 000000000000..c6205cb18493 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory. + TRACK_IMPL; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + + +@Category({RegionServerTests.class, LargeTests.class}) +public class TestMergesSplitsAddToTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMergesSplitsAddToTracker.class); + + private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + + public static final byte[] FAMILY_NAME = Bytes.toBytes("info"); + + @Rule + public TestName name = new TestName(); + + @BeforeClass + public static void setupClass() throws Exception { + TEST_UTIL.getConfiguration().set(TRACK_IMPL, TestStoreFileTracker.class.getName()); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup(){ + TestStoreFileTracker.trackedFiles = new HashMap<>(); + } + + @Test + public void testCommitDaughterRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //first put some data in order to have a store file created + putThreeRowsAndFlush(table); + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0); + HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem(); + RegionInfo daughterA = + RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey()). + setEndKey(Bytes.toBytes("002")).setSplit(false). + setRegionId(region.getRegionInfo().getRegionId() + + EnvironmentEdgeManager.currentTime()). + build(); + RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002")) + .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false) + .setRegionId(region.getRegionInfo().getRegionId()).build(); + HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + List splitFilesA = new ArrayList<>(); + splitFilesA.add(regionFS + .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, + Bytes.toBytes("002"), false, region.getSplitPolicy())); + List splitFilesB = new ArrayList<>(); + splitFilesB.add(regionFS + .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, + Bytes.toBytes("002"), true, region.getSplitPolicy())); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env); + Path resultB = regionFS.commitDaughterRegion(daughterB, splitFilesB, env); + FileSystem fs = regionFS.getFileSystem(); + verifyFilesAreTracked(resultA, fs); + verifyFilesAreTracked(resultB, fs); + } + + @Test + public void testCommitMergedRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //splitting the table first + TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002")); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + HRegion second = regions.get(1); + HRegionFileSystem regionFS = first.getRegionFileSystem(); + + RegionInfo mergeResult = + RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey()) + .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false) + .setRegionId(first.getRegionInfo().getRegionId() + + EnvironmentEdgeManager.currentTime()).build(); + + HRegionFileSystem mergeFS = HRegionFileSystem.createRegionOnFileSystem( + TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), + regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult); + + List mergedFiles = new ArrayList<>(); + //merge file from first region + mergedFiles.add(mergeFileFromRegion(first, mergeFS)); + //merge file from second region + mergedFiles.add(mergeFileFromRegion(second, mergeFS)); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + mergeFS.commitMergedRegion(mergedFiles, env); + //validate + FileSystem fs = first.getRegionFileSystem().getFileSystem(); + Path finalMergeDir = new Path(first.getRegionFileSystem().getTableDir(), + mergeResult.getEncodedName()); + verifyFilesAreTracked(finalMergeDir, fs); + } + + @Test + public void testSplitLoadsFromTracker() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0); + Pair copyResult = copyFileInTheStoreDir(region); + StoreFileInfo fileInfo = copyResult.getFirst(); + String copyName = copyResult.getSecond(); + //Now splits the region + TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002")); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + validateDaughterRegionsFiles(first, fileInfo.getActiveFileName(), copyName); + HRegion second = regions.get(1); + validateDaughterRegionsFiles(second, fileInfo.getActiveFileName(), copyName); + } + + @Test + public void testMergeLoadsFromTracker() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, new byte[][]{FAMILY_NAME}, + new byte[][]{Bytes.toBytes("002")}); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + Pair copyResult = copyFileInTheStoreDir(first); + StoreFileInfo fileInfo = copyResult.getFirst(); + String copyName = copyResult.getSecond(); + //Now merges the first two regions + TEST_UTIL.getAdmin().mergeRegionsAsync(new byte[][]{ + first.getRegionInfo().getEncodedNameAsBytes(), + regions.get(1).getRegionInfo().getEncodedNameAsBytes() + }, true).get(10, TimeUnit.SECONDS); + regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion merged = regions.get(0); + validateDaughterRegionsFiles(merged, fileInfo.getActiveFileName(), copyName); + } + + private Pair copyFileInTheStoreDir(HRegion region) throws IOException { + Path storeDir = region.getRegionFileSystem().getStoreDir("info"); + //gets the single file + StoreFileInfo fileInfo = region.getRegionFileSystem().getStoreFiles("info").get(0); + //make a copy of the valid file staight into the store dir, so that it's not tracked. + String copyName = UUID.randomUUID().toString().replaceAll("-", ""); + Path copy = new Path(storeDir, copyName); + FileUtil.copy(region.getFilesystem(), fileInfo.getFileStatus(), region.getFilesystem(), + copy , false, false, TEST_UTIL.getConfiguration()); + return new Pair<>(fileInfo, copyName); + } + + private void validateDaughterRegionsFiles(HRegion region, String orignalFileName, + String untrackedFile) throws IOException { + //verify there's no link for the untracked, copied file in first region + List infos = region.getRegionFileSystem().getStoreFiles("info"); + final MutableBoolean foundLink = new MutableBoolean(false); + infos.stream().forEach(i -> { + i.getActiveFileName().contains(orignalFileName); + if(i.getActiveFileName().contains(untrackedFile)){ + fail(); + } + if(i.getActiveFileName().contains(orignalFileName)){ + foundLink.setTrue(); + } + }); + assertTrue(foundLink.booleanValue()); + } + + private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Exception { + String storeId = regionDir.getName() + "-info"; + for(FileStatus f : fs.listStatus(new Path(regionDir, Bytes.toString(FAMILY_NAME)))){ + assertTrue(TestStoreFileTracker.trackedFiles.get(storeId).stream().filter(s -> + s.getPath().equals(f.getPath())).findFirst().isPresent()); + } + } + + private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS) + throws IOException { + HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); + } + + private void putThreeRowsAndFlush(TableName table) throws IOException { + Table tbl = TEST_UTIL.getConnection().getTable(table); + Put put = new Put(Bytes.toBytes("001")); + byte[] qualifier = Bytes.toBytes("1"); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1)); + tbl.put(put); + put = new Put(Bytes.toBytes("002")); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2)); + tbl.put(put); + put = new Put(Bytes.toBytes("003")); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2)); + tbl.put(put); + TEST_UTIL.flush(table); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index eb0b1c1ca694..80012dfcd461 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -120,6 +120,7 @@ private static HStoreFile createFile() throws Exception { private static TestStoreEngine createEngine(Configuration conf) throws Exception { HRegion region = mock(HRegion.class); HStore store = mock(HStore.class); + store.conf = conf; when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); when(store.getHRegion()).thenReturn(region); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java new file mode 100644 index 000000000000..05ca1fcb419b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestStoreFileTracker extends DefaultStoreFileTracker { + + private static final Logger LOG = LoggerFactory.getLogger(TestStoreFileTracker.class); + public static Map> trackedFiles = new HashMap<>(); + private String storeId; + + public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); + LOG.info("created storeId: {}", storeId); + trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + LOG.info("adding to storeId: {}", storeId); + trackedFiles.get(storeId).addAll(newFiles); + } + + @Override + public List load() throws IOException { + return trackedFiles.get(storeId); + } +} From e6979ce7465c156082122aac208276807e8c4635 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 9 Sep 2021 12:11:25 +0800 Subject: [PATCH 12/37] HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656) Signed-off-by: Wellington Chevreuil --- .../DefaultStoreFileTracker.java | 10 +- .../FileBasedStoreFileTracker.java | 15 +- .../MigrationStoreFileTracker.java | 88 ++++++++ .../storefiletracker/StoreFileListFile.java | 6 +- .../StoreFileTrackerBase.java | 12 +- .../StoreFileTrackerFactory.java | 40 ++-- .../TestMigrationStoreFileTracker.java | 193 ++++++++++++++++++ 7 files changed, 343 insertions(+), 21 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index 22e05132bf91..a13b75b0dcc4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -39,7 +40,9 @@ public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, Sto @Override public List load() throws IOException { - return ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + List files = + ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + return files != null ? files : Collections.emptyList(); } @Override @@ -57,4 +60,9 @@ protected void doAddCompactionResults(Collection compactedFiles, Collection newFiles) throws IOException { // NOOP } + + @Override + void set(List files) { + // NOOP + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index de28b0eb9996..c370b87c1154 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -48,7 +48,7 @@ * storages. */ @InterfaceAudience.Private -public class FileBasedStoreFileTracker extends StoreFileTrackerBase { +class FileBasedStoreFileTracker extends StoreFileTrackerBase { private final StoreFileListFile backedFile; @@ -139,4 +139,17 @@ protected void doAddCompactionResults(Collection compactedFiles, } } } + + @Override + void set(List files) throws IOException { + synchronized (storefiles) { + storefiles.clear(); + StoreFileList.Builder builder = StoreFileList.newBuilder(); + for (StoreFileInfo info : files) { + storefiles.put(info.getPath().getName(), info); + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java new file mode 100644 index 000000000000..e486e6d563af --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + +/** + * A store file tracker used for migrating between store file tracker implementations. + */ +@InterfaceAudience.Private +class MigrationStoreFileTracker extends StoreFileTrackerBase { + + public static final String SRC_IMPL = "hbase.store.file-tracker.migration.src.impl"; + + public static final String DST_IMPL = "hbase.store.file-tracker.migration.dst.impl"; + + private final StoreFileTrackerBase src; + + private final StoreFileTrackerBase dst; + + public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + this.src = StoreFileTrackerFactory.create(conf, SRC_IMPL, isPrimaryReplica, ctx); + this.dst = StoreFileTrackerFactory.create(conf, DST_IMPL, isPrimaryReplica, ctx); + Preconditions.checkArgument(!src.getClass().equals(dst.getClass()), + "src and dst is the same: %s", src.getClass()); + } + + @Override + public List load() throws IOException { + List files = src.load(); + dst.set(files); + return files; + } + + @Override + protected boolean requireWritingToTmpDirFirst() { + // Returns true if either of the two StoreFileTracker returns true. + // For example, if we want to migrate from a tracker implementation which can ignore the broken + // files under data directory to a tracker implementation which can not, if we still allow + // writing in tmp directory directly, we may have some broken files under the data directory and + // then after we finally change the implementation which can not ignore the broken files, we + // will be in trouble. + return src.requireWritingToTmpDirFirst() || dst.requireWritingToTmpDirFirst(); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + src.doAddNewStoreFiles(newFiles); + dst.doAddNewStoreFiles(newFiles); + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + src.doAddCompactionResults(compactedFiles, newFiles); + dst.doAddCompactionResults(compactedFiles, newFiles); + } + + @Override + void set(List files) { + throw new UnsupportedOperationException( + "Should not call this method on " + getClass().getSimpleName()); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java index c778bfc51deb..ffb3647e6259 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; @@ -121,7 +120,10 @@ StoreFileList load() throws IOException { * We will set the timestamp in this method so just pass the builder in */ void update(StoreFileList.Builder builder) throws IOException { - Preconditions.checkState(nextTrackFile >= 0, "should call load first before calling update"); + if (nextTrackFile < 0) { + // we need to call load first to load the prevTimestamp and also the next file + load(); + } FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); long timestamp = Math.max(prevTimestamp + 1, EnvironmentEdgeManager.currentTime()); try (FSDataOutputStream out = fs.create(trackFiles[nextTrackFile], true)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 92c699278c2e..d860f8e3812d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -95,8 +96,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, } @Override - public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) - throws IOException { + public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException { if (!isPrimaryReplica) { throw new IllegalStateException("Should not call create writer on secondary replicas"); } @@ -170,4 +170,12 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) protected abstract void doAddCompactionResults(Collection compactedFiles, Collection newFiles) throws IOException; + + /** + * used to mirror the store file list after loading when migration. + *

+ * Do not add this method to the {@link StoreFileTracker} interface since we do not need this + * method in upper layer. + */ + abstract void set(List files) throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index c446d5ae9a31..6eb269ee2b40 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,18 +18,20 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CompoundConfiguration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + /** * Factory method for creating store file tracker. */ @@ -39,7 +41,7 @@ public final class StoreFileTrackerFactory { private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + StoreContext ctx) { Class tracker = conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); @@ -47,22 +49,30 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli } public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, - HRegionFileSystem regionFs) { + HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); - StoreContext ctx = StoreContext.getBuilder(). - withColumnFamilyDescriptor(fDescBuilder.build()). - withRegionFileSystem(regionFs). - build(); - return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) + .withRegionFileSystem(regionFs).build(); + return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx); + } + + public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, + ColumnFamilyDescriptor family) { + return StoreUtils.createStoreConfiguration(global, table, family); } - public static Configuration mergeConfigurations(Configuration global, - TableDescriptor table, ColumnFamilyDescriptor family) { - return new CompoundConfiguration() - .add(global) - .addBytesMap(table.getValues()) - .addStringMap(family.getConfiguration()) - .addBytesMap(family.getValues()); + static StoreFileTrackerBase create(Configuration conf, String configName, + boolean isPrimaryReplica, StoreContext ctx) { + String className = + Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); + Class tracker; + try { + tracker = Class.forName(className).asSubclass(StoreFileTrackerBase.class); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); + return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java new file mode 100644 index 000000000000..e9ebb4c792e4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.ChunkCreator; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WAL; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.io.Closeables; + +@RunWith(Parameterized.class) +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestMigrationStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMigrationStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final TableDescriptor TD = + TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build(); + + @Rule + public TestName name = new TestName(); + + @Parameter(0) + public Class srcImplClass; + + @Parameter(1) + public Class dstImplClass; + + private HRegion region; + + private Path rootDir; + + private WAL wal; + + @Parameters(name = "{index}: src={0}, dst={1}") + public static List params() { + List> impls = + Arrays.asList(DefaultStoreFileTracker.class, FileBasedStoreFileTracker.class); + List params = new ArrayList<>(); + for (Class src : impls) { + for (Class dst : impls) { + if (src.equals(dst)) { + continue; + } + params.add(new Object[] { src, dst }); + } + } + return params; + } + + @BeforeClass + public static void setUpBeforeClass() { + ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, + MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); + } + + @Before + public void setUp() throws IOException { + Configuration conf = UTIL.getConfiguration(); + conf.setClass(MigrationStoreFileTracker.SRC_IMPL, srcImplClass, StoreFileTrackerBase.class); + conf.setClass(MigrationStoreFileTracker.DST_IMPL, dstImplClass, StoreFileTrackerBase.class); + rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_")); + wal = HBaseTestingUtil.createWal(conf, rootDir, RI); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + region.close(); + } + Closeables.close(wal, true); + UTIL.cleanupTestDir(); + } + + private List getStoreFiles() { + return Iterables.getOnlyElement(region.getStores()).getStorefiles().stream() + .map(s -> s.getFileInfo().getPath().getName()).collect(Collectors.toList()); + } + + private HRegion createRegion(Class trackerImplClass) + throws IOException { + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true); + } + + private void reopenRegion(Class trackerImplClass) + throws IOException { + region.flush(true); + List before = getStoreFiles(); + region.close(); + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + region = HRegion.openHRegion(rootDir, RI, TD, wal, conf); + List after = getStoreFiles(); + assertEquals(before.size(), after.size()); + assertThat(after, hasItems(before.toArray(new String[0]))); + } + + private void putData(int start, int end) throws IOException { + for (int i = start; i < end; i++) { + region.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + if (i % 30 == 0) { + region.flush(true); + } + } + } + + private void verifyData(int start, int end) throws IOException { + for (int i = start; i < end; i++) { + Result result = region.get(new Get(Bytes.toBytes(i))); + assertEquals(i, Bytes.toInt(result.getValue(CF, CQ))); + } + } + + @Test + public void testMigration() throws IOException { + region = createRegion(srcImplClass); + putData(0, 100); + verifyData(0, 100); + reopenRegion(MigrationStoreFileTracker.class); + verifyData(0, 100); + region.compact(true); + putData(100, 200); + reopenRegion(dstImplClass); + verifyData(0, 200); + } +} From 6c712ec3400b8880abd41ad09ac38d665a173300 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Sun, 12 Sep 2021 14:14:03 +0100 Subject: [PATCH 13/37] HBASE-26246 Persist the StoreFileTracker configurations to TableDescriptor when creating table (#3666) Signed-off-by: Duo Zhang --- .../procedure/CreateTableProcedure.java | 7 ++++++ .../hbase/regionserver/HRegionFileSystem.java | 2 +- .../MigrationStoreFileTracker.java | 8 ++++++ .../storefiletracker/StoreFileTracker.java | 8 ++++++ .../StoreFileTrackerBase.java | 13 ++++++++++ .../StoreFileTrackerFactory.java | 25 +++++++++++++------ .../apache/hadoop/hbase/client/TestAdmin.java | 6 +++++ .../hadoop/hbase/client/TestAdmin3.java | 6 +++++ .../hbase/client/TestAsyncTableAdminApi.java | 6 +++++ .../hbase/client/TestAsyncTableAdminApi3.java | 6 +++++ .../MasterProcedureTestingUtility.java | 7 ++++++ .../procedure/TestCreateTableProcedure.java | 17 +++++++++++++ .../TestStoreFileTracker.java | 14 ++++++++--- 13 files changed, 113 insertions(+), 12 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 2313e70f75bb..dccea5512c1f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -33,10 +34,12 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -285,6 +288,10 @@ private void preCreate(final MasterProcedureEnv env) getTableName(), (newRegions != null ? newRegions.size() : 0)); } + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); + StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder); + tableDescriptor = builder.build(); + final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { final RegionInfo[] regions = newRegions == null ? null : diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 3b053a349c2c..40a42ab10383 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -607,7 +607,7 @@ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegi writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false); - insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); + insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); } return regionDir; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index e486e6d563af..483a240baded 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -85,4 +86,11 @@ void set(List files) { throw new UnsupportedOperationException( "Should not call this method on " + getClass().getSimpleName()); } + + @Override + public void persistConfiguration(TableDescriptorBuilder builder) { + super.persistConfiguration(builder); + builder.setValue(SRC_IMPL, src.getClass().getName()); + builder.setValue(DST_IMPL, dst.getClass().getName()); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 0a85abb00a4e..81fa1a9be5b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.Collection; import java.util.List; + +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -71,4 +73,10 @@ void replace(Collection compactedFiles, Collection * @return Writer for a new StoreFile */ StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; + + /** + * Saves StoreFileTracker implementations specific configs into the table descriptors. + * @param builder The table descriptor builder for the given table. + */ + void persistConfiguration(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index d860f8e3812d..83ebbc78ab35 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -17,18 +17,22 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; + import java.io.IOException; import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -78,6 +82,15 @@ public final void replace(Collection compactedFiles, } } + @Override + public void persistConfiguration(TableDescriptorBuilder builder) { + if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) { + String trackerImpl = StoreFileTrackerFactory. + getStoreFileTrackerImpl(conf).getName(); + builder.setValue(TRACK_IMPL, trackerImpl).build(); + } + } + private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { if (compression == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 6eb269ee2b40..b9ec713cf235 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -6,9 +6,7 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +19,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreUtils; @@ -35,15 +34,17 @@ /** * Factory method for creating store file tracker. */ -@InterfaceAudience.Private -public final class StoreFileTrackerFactory { +@InterfaceAudience.Private public final class StoreFileTrackerFactory { public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); + public static Class getStoreFileTrackerImpl(Configuration conf) { + return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + } + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = - conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + Class tracker = getStoreFileTrackerImpl(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } @@ -54,7 +55,7 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx); + return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); } public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, @@ -75,4 +76,12 @@ static StoreFileTrackerBase create(Configuration conf, String configName, LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + + public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) { + TableDescriptor tableDescriptor = builder.build(); + ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; + StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); + tracker.persistConfiguration(builder); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 22302629dc70..10ab4961a5b4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -423,6 +425,10 @@ private void testCloneTableSchema(final TableName tableName, final TableName new assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + // HBASE-26246 introduced persist of store file tracker into table descriptor + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); if (preserveSplits) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java index e7be08e41edf..7d40fd12bda5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -236,6 +238,10 @@ public void testGetTableDescriptor() throws IOException { ADMIN.createTable(htd); Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); TableDescriptor confirmedHtd = table.getDescriptor(); + //HBASE-26246 introduced persist of store file tracker into table descriptor + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); table.close(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 572a1d5a6226..a1832f49956a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -372,6 +374,10 @@ private void testCloneTableSchema(final TableName tableName, assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + //HBASE-26246 introduced persist of store file tracker into table descriptor + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); if (preserveSplits) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index 4a71baf44a13..cb89bf564720 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -35,6 +36,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -147,6 +149,10 @@ public void testGetTableDescriptor() throws Exception { TableDescriptor desc = builder.build(); admin.createTable(desc).join(); TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get(); + //HBASE-26246 introduced persist of store file tracker into table descriptor + desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index a3206432b810..2ceec5f241c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -60,6 +61,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -226,6 +228,11 @@ public static void validateTableCreation(final HMaster master, final TableName t assertTrue("family not found " + family[i], htd.getColumnFamily(Bytes.toBytes(family[i])) != null); } assertEquals(family.length, htd.getColumnFamilyCount()); + + // checks store file tracker impl has been properly set in htd + String storeFileTrackerImpl = + StoreFileTrackerFactory.getStoreFileTrackerImpl(master.getConfiguration()).getName(); + assertEquals(storeFileTrackerImpl, htd.getValue(TRACK_IMPL)); } public static void validateTableDeletion( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 3b57ed56744b..0bc77f0ef8d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -88,6 +90,21 @@ private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) thr MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); } + @Test + public void testCreateWithTrackImpl() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + ProcedureExecutor procExec = getMasterProcedureExecutor(); + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); + String trackerName = TestStoreFileTracker.class.getName(); + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, trackerName).build(); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + htd = getMaster().getTableDescriptors().get(tableName); + assertEquals(trackerName, htd.getValue(TRACK_IMPL)); + } + @Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor procExec = getMasterProcedureExecutor(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java index 05ca1fcb419b..1dc9c4e53fc5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,15 +40,21 @@ public class TestStoreFileTracker extends DefaultStoreFileTracker { public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); - LOG.info("created storeId: {}", storeId); - trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + if (ctx.getRegionFileSystem() != null) { + this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); + LOG.info("created storeId: {}", storeId); + trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + } else { + LOG.info("ctx.getRegionFileSystem() returned null. Leaving storeId null."); + } + } @Override protected void doAddNewStoreFiles(Collection newFiles) throws IOException { LOG.info("adding to storeId: {}", storeId); trackedFiles.get(storeId).addAll(newFiles); + trackedFiles.putIfAbsent(storeId, (List)newFiles); } @Override From ae31799c414ec676c95b6278679d19f5411916c3 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 14 Sep 2021 16:28:21 +0800 Subject: [PATCH 14/37] HBASE-26248 Should find a suitable way to let users specify the store file tracker implementation (#3665) Signed-off-by: Wellington Chevreuil --- .../MigrationStoreFileTracker.java | 13 ++- .../storefiletracker/StoreFileTracker.java | 7 +- .../StoreFileTrackerBase.java | 12 ++- .../StoreFileTrackerFactory.java | 97 +++++++++++++++++-- .../apache/hadoop/hbase/client/TestAdmin.java | 6 +- .../hadoop/hbase/client/TestAdmin3.java | 6 +- .../hbase/client/TestAsyncTableAdminApi.java | 6 +- .../hbase/client/TestAsyncTableAdminApi3.java | 6 +- .../MasterProcedureTestingUtility.java | 6 +- .../procedure/TestCreateTableProcedure.java | 6 +- .../TestMergesSplitsAddToTracker.java | 4 +- .../TestMigrationStoreFileTracker.java | 27 +++--- ...stRegionWithFileBasedStoreFileTracker.java | 3 +- .../TestStoreFileTrackerFactory.java | 58 +++++++++++ 14 files changed, 202 insertions(+), 55 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 483a240baded..3eeef9000576 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -44,8 +45,8 @@ class MigrationStoreFileTracker extends StoreFileTrackerBase { public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - this.src = StoreFileTrackerFactory.create(conf, SRC_IMPL, isPrimaryReplica, ctx); - this.dst = StoreFileTrackerFactory.create(conf, DST_IMPL, isPrimaryReplica, ctx); + this.src = StoreFileTrackerFactory.createForMigration(conf, SRC_IMPL, isPrimaryReplica, ctx); + this.dst = StoreFileTrackerFactory.createForMigration(conf, DST_IMPL, isPrimaryReplica, ctx); Preconditions.checkArgument(!src.getClass().equals(dst.getClass()), "src and dst is the same: %s", src.getClass()); } @@ -90,7 +91,11 @@ void set(List files) { @Override public void persistConfiguration(TableDescriptorBuilder builder) { super.persistConfiguration(builder); - builder.setValue(SRC_IMPL, src.getClass().getName()); - builder.setValue(DST_IMPL, dst.getClass().getName()); + if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { + builder.setValue(SRC_IMPL, src.getTrackerName()); + } + if (StringUtils.isEmpty(builder.getValue(DST_IMPL))) { + builder.setValue(DST_IMPL, dst.getTrackerName()); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 81fa1a9be5b2..59fe7ef52f96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -75,7 +75,12 @@ void replace(Collection compactedFiles, Collection StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; /** - * Saves StoreFileTracker implementations specific configs into the table descriptors. + * Saves StoreFileTracker implementations specific configurations into the table descriptors. + *

+ * This is used to avoid accidentally data loss when changing the cluster level store file tracker + * implementation, and also possible misconfiguration between master and region servers. + *

+ * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ void persistConfiguration(TableDescriptorBuilder builder); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 83ebbc78ab35..a786add49b21 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import java.io.IOException; import java.util.Collection; @@ -84,13 +84,15 @@ public final void replace(Collection compactedFiles, @Override public void persistConfiguration(TableDescriptorBuilder builder) { - if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) { - String trackerImpl = StoreFileTrackerFactory. - getStoreFileTrackerImpl(conf).getName(); - builder.setValue(TRACK_IMPL, trackerImpl).build(); + if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { + builder.setValue(TRACKER_IMPL, getTrackerName()); } } + protected final String getTrackerName() { + return StoreFileTrackerFactory.getStoreFileTrackerName(getClass()); + } + private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { if (compression == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index b9ec713cf235..9be19ec15ed8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -15,6 +15,9 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -33,22 +36,81 @@ /** * Factory method for creating store file tracker. + *

+ * The current implementations are: + *

    + *
  • default: DefaultStoreFileTracker, see {@link DefaultStoreFileTracker}.
    • + *
  • file:FileBasedStoreFileTracker, see {@link FileBasedStoreFileTracker}.
    • + *
  • migration:MigrationStoreFileTracker, see {@link MigrationStoreFileTracker}.
    • + *
+ * @see DefaultStoreFileTracker + * @see FileBasedStoreFileTracker + * @see MigrationStoreFileTracker */ -@InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; +@InterfaceAudience.Private +public final class StoreFileTrackerFactory { + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); - public static Class getStoreFileTrackerImpl(Configuration conf) { - return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + public static final String TRACKER_IMPL = "hbase.store.file-tracker.impl"; + + /** + * Maps between configuration names for trackers and implementation classes. + */ + public enum Trackers { + DEFAULT(DefaultStoreFileTracker.class), FILE(FileBasedStoreFileTracker.class), + MIGRATION(MigrationStoreFileTracker.class); + + final Class clazz; + + Trackers(Class clazz) { + this.clazz = clazz; + } + } + + private static final Map, Trackers> CLASS_TO_ENUM = reverse(); + + private static Map, Trackers> reverse() { + Map, Trackers> map = new HashMap<>(); + for (Trackers tracker : Trackers.values()) { + map.put(tracker.clazz, tracker); + } + return Collections.unmodifiableMap(map); + } + + private StoreFileTrackerFactory() { + } + + public static String getStoreFileTrackerName(Configuration conf) { + return conf.get(TRACKER_IMPL, Trackers.DEFAULT.name()); + } + + static String getStoreFileTrackerName(Class clazz) { + Trackers name = CLASS_TO_ENUM.get(clazz); + return name != null ? name.name() : clazz.getName(); + } + + private static Class getTrackerClass(Configuration conf) { + try { + Trackers tracker = Trackers.valueOf(getStoreFileTrackerName(conf).toUpperCase()); + return tracker.clazz; + } catch (IllegalArgumentException e) { + // Fall back to them specifying a class name + return conf.getClass(TRACKER_IMPL, Trackers.DEFAULT.clazz, StoreFileTracker.class); + } } public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = getStoreFileTrackerImpl(conf); + Class tracker = getTrackerClass(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + /** + * Used at master side when splitting/merging regions, as we do not have a Store, thus no + * StoreContext at master side. + */ public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = @@ -63,15 +125,30 @@ public static Configuration mergeConfigurations(Configuration global, TableDescr return StoreUtils.createStoreConfiguration(global, table, family); } - static StoreFileTrackerBase create(Configuration conf, String configName, + /** + * Create store file tracker to be used as source or destination for + * {@link MigrationStoreFileTracker}. + */ + static StoreFileTrackerBase createForMigration(Configuration conf, String configName, boolean isPrimaryReplica, StoreContext ctx) { - String className = + String trackerName = Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); Class tracker; try { - tracker = Class.forName(className).asSubclass(StoreFileTrackerBase.class); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); + tracker = + Trackers.valueOf(trackerName.toUpperCase()).clazz.asSubclass(StoreFileTrackerBase.class); + } catch (IllegalArgumentException e) { + // Fall back to them specifying a class name + try { + tracker = Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException(cnfe); + } + } + // prevent nest of MigrationStoreFileTracker, it will cause infinite recursion. + if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { + throw new IllegalArgumentException("Should not specify " + configName + " as " + + Trackers.MIGRATION + " because it can not be nested"); } LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 10ab4961a5b4..ba843fe43460 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -426,8 +426,8 @@ private void testCloneTableSchema(final TableName tableName, final TableName new assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); // HBASE-26246 introduced persist of store file tracker into table descriptor - tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java index 7d40fd12bda5..c2de0fbd3555 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -239,8 +239,8 @@ public void testGetTableDescriptor() throws IOException { Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); TableDescriptor confirmedHtd = table.getDescriptor(); //HBASE-26246 introduced persist of store file tracker into table descriptor - htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index a1832f49956a..147630b53a38 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -375,8 +375,8 @@ private void testCloneTableSchema(final TableName tableName, assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); //HBASE-26246 introduced persist of store file tracker into table descriptor - tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index cb89bf564720..67377b07c180 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -150,8 +150,8 @@ public void testGetTableDescriptor() throws Exception { admin.createTable(desc).join(); TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get(); //HBASE-26246 introduced persist of store file tracker into table descriptor - desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index 2ceec5f241c6..726e117ef1ac 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -231,8 +231,8 @@ public static void validateTableCreation(final HMaster master, final TableName t // checks store file tracker impl has been properly set in htd String storeFileTrackerImpl = - StoreFileTrackerFactory.getStoreFileTrackerImpl(master.getConfiguration()).getName(); - assertEquals(storeFileTrackerImpl, htd.getValue(TRACK_IMPL)); + StoreFileTrackerFactory.getStoreFileTrackerName(master.getConfiguration()); + assertEquals(storeFileTrackerImpl, htd.getValue(TRACKER_IMPL)); } public static void validateTableDeletion( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 0bc77f0ef8d9..f432c8060d3d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -96,13 +96,13 @@ public void testCreateWithTrackImpl() throws Exception { ProcedureExecutor procExec = getMasterProcedureExecutor(); TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); String trackerName = TestStoreFileTracker.class.getName(); - htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, trackerName).build(); + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build(); RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); long procId = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); htd = getMaster().getTableDescriptors().get(tableName); - assertEquals(trackerName, htd.getValue(TRACK_IMPL)); + assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java index c6205cb18493..68fc444493c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory. - TRACK_IMPL; + TRACKER_IMPL; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -74,7 +74,7 @@ public class TestMergesSplitsAddToTracker { @BeforeClass public static void setupClass() throws Exception { - TEST_UTIL.getConfiguration().set(TRACK_IMPL, TestStoreFileTracker.class.getName()); + TEST_UTIL.getConfiguration().set(TRACKER_IMPL, TestStoreFileTracker.class.getName()); TEST_UTIL.startMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java index e9ebb4c792e4..119f8fb553f8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -86,10 +85,10 @@ public class TestMigrationStoreFileTracker { public TestName name = new TestName(); @Parameter(0) - public Class srcImplClass; + public StoreFileTrackerFactory.Trackers srcImpl; @Parameter(1) - public Class dstImplClass; + public StoreFileTrackerFactory.Trackers dstImpl; private HRegion region; @@ -99,11 +98,13 @@ public class TestMigrationStoreFileTracker { @Parameters(name = "{index}: src={0}, dst={1}") public static List params() { - List> impls = - Arrays.asList(DefaultStoreFileTracker.class, FileBasedStoreFileTracker.class); List params = new ArrayList<>(); - for (Class src : impls) { - for (Class dst : impls) { + for (StoreFileTrackerFactory.Trackers src : StoreFileTrackerFactory.Trackers.values()) { + for (StoreFileTrackerFactory.Trackers dst : StoreFileTrackerFactory.Trackers.values()) { + if (src == StoreFileTrackerFactory.Trackers.MIGRATION + || dst == StoreFileTrackerFactory.Trackers.MIGRATION) { + continue; + } if (src.equals(dst)) { continue; } @@ -122,8 +123,8 @@ public static void setUpBeforeClass() { @Before public void setUp() throws IOException { Configuration conf = UTIL.getConfiguration(); - conf.setClass(MigrationStoreFileTracker.SRC_IMPL, srcImplClass, StoreFileTrackerBase.class); - conf.setClass(MigrationStoreFileTracker.DST_IMPL, dstImplClass, StoreFileTrackerBase.class); + conf.set(MigrationStoreFileTracker.SRC_IMPL, srcImpl.name().toLowerCase()); + conf.set(MigrationStoreFileTracker.DST_IMPL, dstImpl.name().toLowerCase()); rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_")); wal = HBaseTestingUtil.createWal(conf, rootDir, RI); } @@ -145,7 +146,7 @@ private List getStoreFiles() { private HRegion createRegion(Class trackerImplClass) throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class); return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true); } @@ -155,7 +156,7 @@ private void reopenRegion(Class trackerImplClass List before = getStoreFiles(); region.close(); Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class); region = HRegion.openHRegion(rootDir, RI, TD, wal, conf); List after = getStoreFiles(); assertEquals(before.size(), after.size()); @@ -180,14 +181,14 @@ private void verifyData(int start, int end) throws IOException { @Test public void testMigration() throws IOException { - region = createRegion(srcImplClass); + region = createRegion(srcImpl.clazz.asSubclass(StoreFileTrackerBase.class)); putData(0, 100); verifyData(0, 100); reopenRegion(MigrationStoreFileTracker.class); verifyData(0, 100); region.compact(true); putData(100, 200); - reopenRegion(dstImplClass); + reopenRegion(dstImpl.clazz.asSubclass(StoreFileTrackerBase.class)); verifyData(0, 200); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java index 3bc60d13fd94..77cd3c448fa0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java @@ -71,8 +71,7 @@ public class TestRegionWithFileBasedStoreFileTracker { @Before public void setUp() throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, FileBasedStoreFileTracker.class, - StoreFileTracker.class); + conf.set(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()); region = HBaseTestingUtil.createRegionAndWAL(RI, UTIL.getDataTestDir(name.getMethodName()), conf, TD); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java new file mode 100644 index 000000000000..41f2afdfa421 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertThrows; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestStoreFileTrackerFactory { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStoreFileTrackerFactory.class); + + @Test + public void testCreateForMigration() { + Configuration conf = HBaseConfiguration.create(); + String configName = "config"; + + // no config + assertThrows(NullPointerException.class, () -> StoreFileTrackerFactory.createForMigration(conf, + configName, false, StoreContext.getBuilder().build())); + + // class not found + conf.set(configName, "config"); + assertThrows(RuntimeException.class, () -> StoreFileTrackerFactory.createForMigration(conf, + configName, false, StoreContext.getBuilder().build())); + + // nested MigrationStoreFileTracker + conf.setClass(configName, MigrationStoreFileTracker.class, StoreFileTrackerBase.class); + assertThrows(IllegalArgumentException.class, () -> StoreFileTrackerFactory + .createForMigration(conf, configName, false, StoreContext.getBuilder().build())); + } +} From 150d995c58a9edf01f0769e23e807e4ca0bbdc52 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Wed, 15 Sep 2021 23:00:03 +0800 Subject: [PATCH 15/37] HBASE-26264 Add more checks to prevent misconfiguration on store file tracker (#3681) Signed-off-by: Josh Elser --- .../MergeTableRegionsProcedure.java | 3 +- .../assignment/SplitTableRegionProcedure.java | 3 +- .../procedure/CreateTableProcedure.java | 8 +- .../procedure/ModifyTableProcedure.java | 5 + .../hbase/regionserver/HRegionFileSystem.java | 2 +- .../MigrationStoreFileTracker.java | 8 + .../StoreFileTrackerFactory.java | 173 ++++++++++++- .../TestChangeStoreFileTracker.java | 242 ++++++++++++++++++ 8 files changed, 422 insertions(+), 22 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 4b25a28a6055..9a00fe26a8f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -614,8 +614,7 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg String family = hcd.getNameAsString(); Configuration trackerConfig = StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, - family, regionFs); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 4d53df4bc670..44136dc7cbee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -668,8 +668,7 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en String family = cfd.getNameAsString(); Configuration trackerConfig = StoreFileTrackerFactory. mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, - family, regionFs); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); Collection sfis = tracker.load(); if (sfis == null) { continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index dccea5512c1f..ee8e51ffe86b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -277,15 +277,17 @@ private boolean prepareCreate(final MasterProcedureEnv env) throws IOException { MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom); } + // check for store file tracker configurations + StoreFileTrackerFactory.checkForCreateTable(env.getMasterConfiguration(), tableDescriptor); + return true; } private void preCreate(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { - ProcedureSyncWait.getMasterQuotaManager(env) - .checkNamespaceTableAndRegionQuota( - getTableName(), (newRegions != null ? newRegions.size() : 0)); + ProcedureSyncWait.getMasterQuotaManager(env).checkNamespaceTableAndRegionQuota(getTableName(), + (newRegions != null ? newRegions.size() : 0)); } TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index 247dd9c202f4..1640644328f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.Bytes; @@ -325,6 +326,10 @@ private void prepareModify(final MasterProcedureEnv env) throws IOException { modifiedTableDescriptor.getRegionServerGroup(), forWhom); MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom); } + + // check for store file tracker configurations + StoreFileTrackerFactory.checkForModifyTable(env.getMasterConfiguration(), + unmodifiedTableDescriptor, modifiedTableDescriptor); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 40a42ab10383..db18d4e50fd6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -626,7 +626,7 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, tblDesc.getColumnFamily(Bytes.toBytes(familyName))); return StoreFileTrackerFactory. - create(config, true, familyName, regionFs); + create(config, familyName, regionFs); }); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 3eeef9000576..1946d4bf7eef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -98,4 +98,12 @@ public void persistConfiguration(TableDescriptorBuilder builder) { builder.setValue(DST_IMPL, dst.getTrackerName()); } } + + static Class getSrcTrackerClass(Configuration conf) { + return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, SRC_IMPL); + } + + static Class getDstTrackerClass(Configuration conf) { + return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, DST_IMPL); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 9be19ec15ed8..90704fe528a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -15,10 +15,12 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -111,13 +113,13 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. */ - public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, + public static StoreFileTracker create(Configuration conf, String family, HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + return StoreFileTrackerFactory.create(conf, true, ctx); } public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, @@ -125,30 +127,35 @@ public static Configuration mergeConfigurations(Configuration global, TableDescr return StoreUtils.createStoreConfiguration(global, table, family); } - /** - * Create store file tracker to be used as source or destination for - * {@link MigrationStoreFileTracker}. - */ - static StoreFileTrackerBase createForMigration(Configuration conf, String configName, - boolean isPrimaryReplica, StoreContext ctx) { + static Class + getStoreFileTrackerClassForMigration(Configuration conf, String configName) { String trackerName = Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); - Class tracker; try { - tracker = - Trackers.valueOf(trackerName.toUpperCase()).clazz.asSubclass(StoreFileTrackerBase.class); + return Trackers.valueOf(trackerName.toUpperCase()).clazz + .asSubclass(StoreFileTrackerBase.class); } catch (IllegalArgumentException e) { // Fall back to them specifying a class name try { - tracker = Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); + return Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } } + } + + /** + * Create store file tracker to be used as source or destination for + * {@link MigrationStoreFileTracker}. + */ + static StoreFileTrackerBase createForMigration(Configuration conf, String configName, + boolean isPrimaryReplica, StoreContext ctx) { + Class tracker = + getStoreFileTrackerClassForMigration(conf, configName); // prevent nest of MigrationStoreFileTracker, it will cause infinite recursion. if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { - throw new IllegalArgumentException("Should not specify " + configName + " as " + - Trackers.MIGRATION + " because it can not be nested"); + throw new IllegalArgumentException("Should not specify " + configName + " as " + + Trackers.MIGRATION + " because it can not be nested"); } LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); @@ -161,4 +168,142 @@ public static void persistTrackerConfig(Configuration conf, TableDescriptorBuild StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); tracker.persistConfiguration(builder); } + + // should not use MigrationStoreFileTracker for new family + private static void checkForNewFamily(Configuration conf, TableDescriptor table, + ColumnFamilyDescriptor family) throws IOException { + Configuration mergedConf = mergeConfigurations(conf, table, family); + Class tracker = getTrackerClass(mergedConf); + if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { + throw new DoNotRetryIOException( + "Should not use " + Trackers.MIGRATION + " as store file tracker for new family " + + family.getNameAsString() + " of table " + table.getTableName()); + } + } + + /** + * Pre check when creating a new table. + *

+ * For now, only make sure that we do not use {@link Trackers#MIGRATION} for newly created tables. + * @throws IOException when there are check errors, the upper layer should fail the + * {@code CreateTableProcedure}. + */ + public static void checkForCreateTable(Configuration conf, TableDescriptor table) + throws IOException { + for (ColumnFamilyDescriptor family : table.getColumnFamilies()) { + checkForNewFamily(conf, table, family); + } + } + + + /** + * Pre check when modifying a table. + *

+ * The basic idea is when you want to change the store file tracker implementation, you should use + * {@link Trackers#MIGRATION} first and then change to the destination store file tracker + * implementation. + *

+ * There are several rules: + *

    + *
  • For newly added family, you should not use {@link Trackers#MIGRATION}.
    • + *
  • For modifying a family: + *
      + *
    • If old tracker is {@link Trackers#MIGRATION}, then: + *
        + *
      • The new tracker is also {@link Trackers#MIGRATION}, then they must have the same src and + * dst tracker.
        • + *
      • The new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the dst + * tracker of the old tracker.
        • + *
      + *
      • + *
    • If the old tracker is not {@link Trackers#MIGRATION}, then: + *
        + *
      • If the new tracker is {@link Trackers#MIGRATION}, then the old tracker must be the src + * tracker of the new tracker.
        • + *
      • If the new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the same + * with old tracker.
        • + *
      + *
      • + *
    + *
    • + *
+ * @throws IOException when there are check errors, the upper layer should fail the + * {@code ModifyTableProcedure}. + */ + public static void checkForModifyTable(Configuration conf, TableDescriptor oldTable, + TableDescriptor newTable) throws IOException { + for (ColumnFamilyDescriptor newFamily : newTable.getColumnFamilies()) { + ColumnFamilyDescriptor oldFamily = oldTable.getColumnFamily(newFamily.getName()); + if (oldFamily == null) { + checkForNewFamily(conf, newTable, newFamily); + continue; + } + Configuration oldConf = mergeConfigurations(conf, oldTable, oldFamily); + Configuration newConf = mergeConfigurations(conf, newTable, newFamily); + + Class oldTracker = getTrackerClass(oldConf); + Class newTracker = getTrackerClass(newConf); + + if (MigrationStoreFileTracker.class.isAssignableFrom(oldTracker)) { + Class oldSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(oldConf); + Class oldDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(oldConf); + if (oldTracker.equals(newTracker)) { + // confirm that we have the same src tracker and dst tracker + Class newSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(newConf); + if (!oldSrcTracker.equals(newSrcTracker)) { + throw new DoNotRetryIOException( + "The src tracker has been changed from " + getStoreFileTrackerName(oldSrcTracker) + + " to " + getStoreFileTrackerName(newSrcTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + Class newDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(newConf); + if (!oldDstTracker.equals(newDstTracker)) { + throw new DoNotRetryIOException( + "The dst tracker has been changed from " + getStoreFileTrackerName(oldDstTracker) + + " to " + getStoreFileTrackerName(newDstTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + } else { + // we can only change to the dst tracker + if (!newTracker.equals(oldDstTracker)) { + throw new DoNotRetryIOException( + "Should migrate tracker to " + getStoreFileTrackerName(oldDstTracker) + " but got " + + getStoreFileTrackerName(newTracker) + " for family " + newFamily.getNameAsString() + + " of table " + newTable.getTableName()); + } + } + } else { + if (!oldTracker.equals(newTracker)) { + // can only change to MigrationStoreFileTracker and the src tracker should be the old + // tracker + if (!MigrationStoreFileTracker.class.isAssignableFrom(newTracker)) { + throw new DoNotRetryIOException("Should change to " + Trackers.MIGRATION + + " first when migrating from " + getStoreFileTrackerName(oldTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + Class newSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(newConf); + if (!oldTracker.equals(newSrcTracker)) { + throw new DoNotRetryIOException( + "Should use src tracker " + getStoreFileTrackerName(oldTracker) + " first but got " + + getStoreFileTrackerName(newSrcTracker) + " when migrating from " + + getStoreFileTrackerName(oldTracker) + " for family " + newFamily.getNameAsString() + + " of table " + newTable.getTableName()); + } + Class newDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(newConf); + // the src and dst tracker should not be the same + if (newSrcTracker.equals(newDstTracker)) { + throw new DoNotRetryIOException("The src tracker and dst tracker are both " + + getStoreFileTrackerName(newSrcTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + } + } + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java new file mode 100644 index 000000000000..70f62c02ed28 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNameTestRule; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; + +/** + * Test changing store file tracker implementation by altering table. + */ +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestChangeStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestChangeStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + @Rule + public final TableNameTestRule tableName = new TableNameTestRule(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws IOException { + UTIL.shutdownMiniCluster(); + } + + @Test(expected = DoNotRetryIOException.class) + public void testCreateError() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().createTable(td); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError1() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError2() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError3() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + // return the TableDescriptor for creating table + private TableDescriptor createTableAndChangeToMigrationTracker() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + return td; + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError4() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError5() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError6() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = + TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.DEFAULT.name()).build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError7() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("family1")) + .setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .build()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + // actually a NPE as we do not specify the src and dst impl for migration store file tracker + @Test(expected = IOException.class) + public void testModifyError8() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = + TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()).build(); + UTIL.getAdmin().modifyTable(newTd); + } + + private String getStoreFileName(TableName table, byte[] family) { + return Iterables + .getOnlyElement(Iterables.getOnlyElement(UTIL.getMiniHBaseCluster().getRegions(table)) + .getStore(family).getStorefiles()) + .getPath().getName(); + } + + @Test + public void testModify() throws IOException { + TableName tn = tableName.getTableName(); + byte[] row = Bytes.toBytes("row"); + byte[] family = Bytes.toBytes("family"); + byte[] qualifier = Bytes.toBytes("qualifier"); + byte[] value = Bytes.toBytes("value"); + TableDescriptor td = TableDescriptorBuilder.newBuilder(tn) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); + UTIL.getAdmin().createTable(td); + try (Table table = UTIL.getConnection().getTable(tn)) { + table.put(new Put(row).addColumn(family, qualifier, value)); + } + UTIL.flush(tn); + String fileName = getStoreFileName(tn, family); + + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + assertEquals(fileName, getStoreFileName(tn, family)); + try (Table table = UTIL.getConnection().getTable(tn)) { + assertArrayEquals(value, table.get(new Get(row)).getValue(family, qualifier)); + } + + TableDescriptor newTd2 = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd2); + assertEquals(fileName, getStoreFileName(tn, family)); + try (Table table = UTIL.getConnection().getTable(tn)) { + assertArrayEquals(value, table.get(new Get(row)).getValue(family, qualifier)); + } + } +} From c4d7d28911748af0cf1076ee49a7b07533ea3380 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 17 Sep 2021 09:40:44 +0800 Subject: [PATCH 16/37] HBASE-26280 Use store file tracker when snapshoting (#3685) Signed-off-by: Wellington Chevreuil Reviewed-by: Josh Elser --- .../MergeTableRegionsProcedure.java | 7 +- .../assignment/SplitTableRegionProcedure.java | 5 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +-- .../StoreFileTrackerFactory.java | 17 ++--- .../hbase/snapshot/SnapshotManifest.java | 42 +++++------ ...apshotFromClientCloneLinksAfterDelete.java | 4 +- .../client/TestMobSnapshotFromClient.java | 7 +- .../hbase/client/TestSnapshotFromClient.java | 30 ++++++-- .../hbase/regionserver/TestHStoreFile.java | 6 +- .../snapshot/MobSnapshotTestingUtils.java | 74 ++++++++++--------- .../hbase/snapshot/SnapshotTestingUtils.java | 16 ++-- 11 files changed, 107 insertions(+), 111 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 9a00fe26a8f0..85a433af81e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -24,8 +24,6 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; - -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -612,9 +610,8 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg List mergedFiles = new ArrayList<>(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); - Configuration trackerConfig = - StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 44136dc7cbee..60ff48985f42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -666,9 +666,8 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en new HashMap>(htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); - Configuration trackerConfig = StoreFileTrackerFactory. - mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, cfd, regionFs); Collection sfis = tracker.load(); if (sfis == null) { continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index db18d4e50fd6..03a7c5c8baaf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -595,7 +595,6 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * to the proper location in the filesystem. * * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} - * @throws IOException */ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, MasterProcedureEnv env) throws IOException { @@ -622,12 +621,8 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced Map> fileInfoMap = new HashMap<>(); for(Path file : allFiles) { String familyName = file.getParent().getName(); - trackerMap.computeIfAbsent(familyName, t -> { - Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, - tblDesc.getColumnFamily(Bytes.toBytes(familyName))); - return StoreFileTrackerFactory. - create(config, familyName, regionFs); - }); + trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc, + tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs)); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); infos.add(new StoreFileInfo(conf, fs, file, true)); @@ -673,7 +668,6 @@ public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws I * this method is invoked on the Master side, then the RegionSplitPolicy will * NOT have a reference to a Region. * @return Path to created reference. - * @throws IOException */ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, boolean top, RegionSplitPolicy splitPolicy) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 90704fe528a0..b586027f8333 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -22,13 +22,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreUtils; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -113,16 +111,15 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. */ - public static StoreFileTracker create(Configuration conf, String family, - HRegionFileSystem regionFs) { - ColumnFamilyDescriptorBuilder fDescBuilder = - ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); - StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) - .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, true, ctx); + public static StoreFileTracker create(Configuration conf, TableDescriptor td, + ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs) { + StoreContext ctx = + StoreContext.getBuilder().withColumnFamilyDescriptor(cfd).withRegionFileSystem(regionFs) + .withFamilyStoreDirectoryPath(regionFs.getStoreDir(cfd.getNameAsString())).build(); + return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), true, ctx); } - public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, + private static Configuration mergeConfigurations(Configuration global, TableDescriptor table, ColumnFamilyDescriptor family) { return StoreUtils.createStoreConfiguration(global, table, family); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 5e82cad6b494..f154aa92cd6e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -47,7 +47,8 @@ import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.Threads; @@ -291,8 +292,8 @@ public void addRegion(final Path tableDir, final RegionInfo regionInfo) throws I addRegion(tableDir, regionInfo, visitor); } - protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor) - throws IOException { + protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor) + throws IOException { boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo); try { Path baseDir = tableDir; @@ -300,8 +301,8 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio if (isMobRegion) { baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable()); } - HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, - baseDir, regionInfo, true); + HRegionFileSystem regionFs = + HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true); monitor.rethrowException(); // 1. dump region meta info into the snapshot directory @@ -317,26 +318,19 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio // in batches and may miss files being added/deleted. This could be more robust (iteratively // checking to see if we have all the files until we are sure), but the limit is currently // 1000 files/batch, far more than the number of store files under a single column family. - Collection familyNames = regionFs.getFamilies(); - if (familyNames != null) { - for (String familyName: familyNames) { - Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName)); - monitor.rethrowException(); - - Collection storeFiles = regionFs.getStoreFiles(familyName); - if (storeFiles == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("No files under family: " + familyName); - } - continue; - } - - // 2.1. build the snapshot reference for the store - // iterate through all the store's files and create "references". - addReferenceFiles(visitor, regionData, familyData, storeFiles, false); - - visitor.familyClose(regionData, familyData); + for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { + Object familyData = visitor.familyOpen(regionData, cfd.getName()); + monitor.rethrowException(); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs); + List storeFiles = tracker.load(); + if (storeFiles.isEmpty()) { + LOG.debug("No files under family: {}", cfd.getNameAsString()); + continue; } + // 2.1. build the snapshot reference for the store + // iterate through all the store's files and create "references". + addReferenceFiles(visitor, regionData, familyData, storeFiles, false); + visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); } catch (IOException e) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java index c2087a935198..e352303f76ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; @@ -93,7 +94,8 @@ public static void setUpBeforeClass() throws Exception { @Override protected void createTable() throws IOException, InterruptedException { MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, - SnapshotTestingUtils.getSplitKeys(), getNumReplicas(), DelayFlushCoprocessor.class.getName(), + SnapshotTestingUtils.getSplitKeys(), getNumReplicas(), + StoreFileTrackerFactory.Trackers.DEFAULT.name(), DelayFlushCoprocessor.class.getName(), FAMILY); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java index cdc41b01e87d..0695be12d0c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java @@ -26,8 +26,6 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Test create/using/deleting snapshots from the client @@ -41,8 +39,6 @@ public class TestMobSnapshotFromClient extends TestSnapshotFromClient { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestMobSnapshotFromClient.class); - private static final Logger LOG = LoggerFactory.getLogger(TestMobSnapshotFromClient.class); - /** * Setup the config for the cluster * @throws Exception on failure @@ -60,6 +56,7 @@ protected static void setupConf(Configuration conf) { @Override protected void createTable() throws Exception { - MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), TEST_FAM); + MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), trackerImpl.name(), + TEST_FAM); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java index 08e33ac78d05..56a48c122a14 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; @@ -32,9 +33,11 @@ import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNameTestRule; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1; @@ -51,7 +54,10 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +70,8 @@ *

* This is an end-to-end test for the snapshot utility */ -@Category({LargeTests.class, ClientTests.class}) +@RunWith(Parameterized.class) +@Category({ LargeTests.class, ClientTests.class }) public class TestSnapshotFromClient { @ClassRule @@ -82,7 +89,16 @@ public class TestSnapshotFromClient { private static final Pattern MATCH_ALL = Pattern.compile(".*"); @Rule - public TestName name = new TestName(); + public TableNameTestRule name = new TableNameTestRule(); + + @Parameter + public StoreFileTrackerFactory.Trackers trackerImpl; + + @Parameters(name = "{index}: tracker={0}") + public static List params() { + return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT }, + new Object[] { StoreFileTrackerFactory.Trackers.FILE }); + } /** * Setup the config for the cluster @@ -109,7 +125,6 @@ protected static void setupConf(Configuration conf) { conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); - } @Before @@ -119,7 +134,8 @@ public void setup() throws Exception { protected void createTable() throws Exception { TableDescriptor htd = - TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()).build(); + TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build(); UTIL.createTable(htd, new byte[][] { TEST_FAM }, null); } @@ -316,7 +332,7 @@ public void testOfflineTableSnapshotWithEmptyRegions() throws Exception { @Test public void testListTableSnapshots() throws Exception { Admin admin = null; - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = name.getTableName(); try { admin = UTIL.getAdmin(); @@ -401,7 +417,7 @@ public void testListTableSnapshotsWithRegex() throws Exception { @Test public void testDeleteTableSnapshots() throws Exception { Admin admin = null; - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = name.getTableName(); try { admin = UTIL.getAdmin(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index c7203a928aa7..b4f628b358b5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -1068,10 +1069,9 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration()); TableDescriptors mockTblDescs = mock(TableDescriptors.class); when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs); - TableDescriptor mockTblDesc = mock(TableDescriptor.class); + TableDescriptor mockTblDesc = TableDescriptorBuilder.newBuilder(hri.getTable()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); when(mockTblDescs.get(any())).thenReturn(mockTblDesc); - ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class); - when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc); Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); return new Path(new Path(regionDir, family), path.getName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index fea4fb4ba583..7523ae8fa0ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; @@ -45,29 +46,40 @@ public class MobSnapshotTestingUtils { /** * Create the Mob Table. */ - public static void createMobTable(final HBaseTestingUtil util, - final TableName tableName, int regionReplication, - final byte[]... families) throws IOException, InterruptedException { - createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), - regionReplication, families); + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + int regionReplication, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), regionReplication, + StoreFileTrackerFactory.Trackers.DEFAULT.name(), families); + } + + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + int regionReplication, String storeFileTracker, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), regionReplication, + storeFileTracker, families); } - public static void createPreSplitMobTable(final HBaseTestingUtil util, - final TableName tableName, int nRegions, final byte[]... families) - throws IOException, InterruptedException { - createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), - 1, families); + public static void createPreSplitMobTable(final HBaseTestingUtil util, final TableName tableName, + int nRegions, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), 1, families); + } + + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + final byte[][] splitKeys, int regionReplication, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, splitKeys, regionReplication, + StoreFileTrackerFactory.Trackers.DEFAULT.name(), families); } public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, - final byte[][] splitKeys, int regionReplication, final byte[]... families) - throws IOException, InterruptedException { - createMobTable(util, tableName, splitKeys, regionReplication, null, families); + final byte[][] splitKeys, int regionReplication, String storeFileTracker, + final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, splitKeys, regionReplication, storeFileTracker, null, families); } - public static void createMobTable(HBaseTestingUtil util, TableName tableName, - byte[][] splitKeys, int regionReplication, String cpClassName, byte[]... families) - throws IOException, InterruptedException { + public static void createMobTable(HBaseTestingUtil util, TableName tableName, byte[][] splitKeys, + int regionReplication, String storeFileTracker, String cpClassName, byte[]... families) + throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); for (byte[] family : families) { @@ -77,6 +89,7 @@ public static void createMobTable(HBaseTestingUtil util, TableName tableName, if (!StringUtils.isBlank(cpClassName)) { builder.setCoprocessor(cpClassName); } + builder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, storeFileTracker); util.getAdmin().createTable(builder.build(), splitKeys); SnapshotTestingUtils.waitForTableToBeOnline(util, tableName); assertEquals((splitKeys.length + 1) * regionReplication, @@ -85,15 +98,10 @@ public static void createMobTable(HBaseTestingUtil util, TableName tableName, /** * Create a Mob table. - * - * @param util - * @param tableName - * @param families * @return An Table instance for the created table. - * @throws IOException */ - public static Table createMobTable(final HBaseTestingUtil util, - final TableName tableName, final byte[]... families) throws IOException { + public static Table createMobTable(final HBaseTestingUtil util, final TableName tableName, + final byte[]... families) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); for (byte[] family : families) { // Disable blooms (they are on by default as of 0.95) but we disable them @@ -102,10 +110,7 @@ public static Table createMobTable(final HBaseTestingUtil util, // and blooms being // on is interfering. builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family) - .setBloomFilterType(BloomType.NONE) - .setMobEnabled(true) - .setMobThreshold(0L) - .build()); + .setBloomFilterType(BloomType.NONE).setMobEnabled(true).setMobThreshold(0L).build()); } util.getAdmin().createTable(builder.build()); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait @@ -135,8 +140,8 @@ public static int countMobRows(final Table table, final byte[]... families) thro } } - public static void verifyMobRowCount(final HBaseTestingUtil util, - final TableName tableName, long expectedRows) throws IOException { + public static void verifyMobRowCount(final HBaseTestingUtil util, final TableName tableName, + long expectedRows) throws IOException { Table table = ConnectionFactory.createConnection(util.getConfiguration()).getTable(tableName); try { @@ -156,13 +161,10 @@ public SnapshotMock(final Configuration conf, final FileSystem fs, final Path ro @Override public TableDescriptor createHtd(final String tableName) { - return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)) - .setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(Bytes.toBytes(TEST_FAMILY)) - .setMobEnabled(true) - .setMobThreshold(0L) - .build()) - .build(); + return TableDescriptorBuilder + .newBuilder(TableName.valueOf(tableName)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(Bytes.toBytes(TEST_FAMILY)).setMobEnabled(true).setMobThreshold(0L).build()) + .build(); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index d8d2a5eed0b1..00d2e84a464b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -480,9 +480,8 @@ public SnapshotBuilder(final Configuration conf, final FileSystem fs, this.desc = desc; this.tableRegions = tableRegions; this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); - new FSTableDescriptors(conf) - .createTableDescriptorForTableDirectory(this.snapshotDir.getFileSystem(conf), - snapshotDir, htd, false); + FSTableDescriptors.createTableDescriptorForTableDirectory( + this.snapshotDir.getFileSystem(conf), snapshotDir, htd, false); } public TableDescriptor getTableDescriptor() { @@ -502,15 +501,13 @@ public Path[] addRegion() throws IOException { } public Path[] addRegionV1() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) - .build()); + return addRegion( + desc.toBuilder().setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION).build()); } public Path[] addRegionV2() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) - .build()); + return addRegion( + desc.toBuilder().setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION).build()); } private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { @@ -521,6 +518,7 @@ private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws I RegionData regionData = tableRegions[this.snapshotted++]; ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); + manifest.addTableDescriptor(htd); manifest.addRegion(regionData.tableDir, regionData.hri); return regionData.files; } From 1727115404f0536390efd6c0c1d5cf48def7e465 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 1 Oct 2021 21:16:59 +0100 Subject: [PATCH 17/37] HBASE-26326 CreateTableProcedure fails when FileBasedStoreFileTracker is set in global config Removed init() method from StoreFileTrakcerBase. Added check in FileBasedStoreFileTracker constructor to ignore null StoreContext or StoreContext instances that don't have store dir info. --- .../master/procedure/CreateTableProcedure.java | 2 +- .../FileBasedStoreFileTracker.java | 6 ++++-- .../MigrationStoreFileTracker.java | 4 ++-- .../storefiletracker/StoreFileTracker.java | 4 ++-- .../storefiletracker/StoreFileTrackerBase.java | 2 +- .../StoreFileTrackerFactory.java | 14 ++++++++++---- .../procedure/TestCreateTableProcedure.java | 16 ++++++++++++++++ 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index ee8e51ffe86b..95395104463c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -291,7 +291,7 @@ private void preCreate(final MasterProcedureEnv env) } TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); - StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder); + StoreFileTrackerFactory.updateDescriptor(env.getMasterConfiguration(), builder); tableDescriptor = builder.build(); final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index c370b87c1154..1a2bdceabddb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -50,13 +50,15 @@ @InterfaceAudience.Private class FileBasedStoreFileTracker extends StoreFileTrackerBase { - private final StoreFileListFile backedFile; + private StoreFileListFile backedFile; private final Map storefiles = new HashMap<>(); public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - backedFile = new StoreFileListFile(ctx); + if (ctx != null && ctx.getFamilyStoreDirectoryPath() != null) { + backedFile = new StoreFileListFile(ctx); + } } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 1946d4bf7eef..b043a6254e63 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -89,8 +89,8 @@ void set(List files) { } @Override - public void persistConfiguration(TableDescriptorBuilder builder) { - super.persistConfiguration(builder); + public void updateDescriptor(TableDescriptorBuilder builder) { + super.updateDescriptor(builder); if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { builder.setValue(SRC_IMPL, src.getTrackerName()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 59fe7ef52f96..a82e3e90e649 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -75,7 +75,7 @@ void replace(Collection compactedFiles, Collection StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; /** - * Saves StoreFileTracker implementations specific configurations into the table descriptors. + * Adds StoreFileTracker implementations specific configurations into the table descriptor. *

* This is used to avoid accidentally data loss when changing the cluster level store file tracker * implementation, and also possible misconfiguration between master and region servers. @@ -83,5 +83,5 @@ void replace(Collection compactedFiles, Collection * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ - void persistConfiguration(TableDescriptorBuilder builder); + void updateDescriptor(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index a786add49b21..82b6373be45d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -83,7 +83,7 @@ public final void replace(Collection compactedFiles, } @Override - public void persistConfiguration(TableDescriptorBuilder builder) { + public void updateDescriptor(TableDescriptorBuilder builder) { if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { builder.setValue(TRACKER_IMPL, getTrackerName()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 974238cb971e..18834ce5c916 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -100,13 +100,19 @@ private static Class getTrackerClass(Configuration c } } - public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, + private static StoreFileTracker getInstance(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { Class tracker = getTrackerClass(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, + StoreContext ctx) { + StoreFileTracker trackerImpl = getInstance(conf, isPrimaryReplica, ctx); + return trackerImpl; + } + /** * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. @@ -159,12 +165,12 @@ static StoreFileTrackerBase createForMigration(Configuration conf, String config return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } - public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) { + public static void updateDescriptor(Configuration conf, TableDescriptorBuilder builder) { TableDescriptor tableDescriptor = builder.build(); ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); - StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); - tracker.persistConfiguration(builder); + StoreFileTracker tracker = StoreFileTrackerFactory.getInstance(conf, true, context); + tracker.updateDescriptor(builder); } // should not use MigrationStoreFileTracker for new family diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index f432c8060d3d..51ea9f58248a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -105,6 +106,21 @@ public void testCreateWithTrackImpl() throws Exception { assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); } + @Test + public void testCreateWithFileBasedStoreTrackerImpl() throws Exception { + ProcedureExecutor procExec = getMasterProcedureExecutor(); + procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.FILE.name()); + final TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + htd = getMaster().getTableDescriptors().get(tableName); + assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL)); + } + @Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor procExec = getMasterProcedureExecutor(); From 27235c1ee1838179a6762e5c0ade36d88bc1617e Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 8 Oct 2021 11:36:27 +0100 Subject: [PATCH 18/37] Changed method name --- .../storefiletracker/MigrationStoreFileTracker.java | 5 ++--- .../regionserver/storefiletracker/StoreFileTracker.java | 2 +- .../regionserver/storefiletracker/StoreFileTrackerBase.java | 2 +- .../storefiletracker/StoreFileTrackerFactory.java | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 8e608a88be67..85af7f119e5e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -89,9 +89,8 @@ void set(List files) { } @Override - public void updateDescriptor(TableDescriptorBuilder builder) { - super.updateDescriptor(builder); - + public void updateWithTrackerConfigs(TableDescriptorBuilder builder) { + super.updateWithTrackerConfigs(builder); if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { builder.setValue(SRC_IMPL, src.getTrackerName()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index a82e3e90e649..23f9f6f5778a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -83,5 +83,5 @@ void replace(Collection compactedFiles, Collection * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ - void updateDescriptor(TableDescriptorBuilder builder); + void updateWithTrackerConfigs(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 82b6373be45d..b1b893071e45 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -83,7 +83,7 @@ public final void replace(Collection compactedFiles, } @Override - public void updateDescriptor(TableDescriptorBuilder builder) { + public void updateWithTrackerConfigs(TableDescriptorBuilder builder) { if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { builder.setValue(TRACKER_IMPL, getTrackerName()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 378272414753..fc7016bab705 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -171,7 +171,7 @@ public static void updateDescriptor(Configuration conf, TableDescriptorBuilder b ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); StoreFileTracker tracker = StoreFileTrackerFactory.getInstance(conf, true, context); - tracker.updateDescriptor(builder); + tracker.updateWithTrackerConfigs(builder); } // should not use MigrationStoreFileTracker for new family From 2e5a49684c5d7bce71d2313c550fd2f9e37918a6 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 8 Oct 2021 12:55:39 +0100 Subject: [PATCH 19/37] Addressing latest review commentts --- .../FileBasedStoreFileTracker.java | 4 +++- .../storefiletracker/StoreFileTrackerFactory.java | 14 ++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index 1a2bdceabddb..8b4b09c9150f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -50,7 +50,7 @@ @InterfaceAudience.Private class FileBasedStoreFileTracker extends StoreFileTrackerBase { - private StoreFileListFile backedFile; + private final StoreFileListFile backedFile; private final Map storefiles = new HashMap<>(); @@ -58,6 +58,8 @@ public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, S super(conf, isPrimaryReplica, ctx); if (ctx != null && ctx.getFamilyStoreDirectoryPath() != null) { backedFile = new StoreFileListFile(ctx); + } else { + backedFile = null; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index fc7016bab705..5f3e43fb4981 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CompoundConfiguration; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -102,19 +101,13 @@ private static Class getTrackerClass(Configuration c } } - private static StoreFileTracker getInstance(Configuration conf, boolean isPrimaryReplica, + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { Class tracker = getTrackerClass(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } - public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { - StoreFileTracker trackerImpl = getInstance(conf, isPrimaryReplica, ctx); - return trackerImpl; - } - /** * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. @@ -167,10 +160,7 @@ static StoreFileTrackerBase createForMigration(Configuration conf, String config } public static void updateDescriptor(Configuration conf, TableDescriptorBuilder builder) { - TableDescriptor tableDescriptor = builder.build(); - ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; - StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); - StoreFileTracker tracker = StoreFileTrackerFactory.getInstance(conf, true, context); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, null); tracker.updateWithTrackerConfigs(builder); } From 7d285ffae8f1b867ecb3c58f7e82821321c44108 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 8 Oct 2021 13:00:33 +0100 Subject: [PATCH 20/37] additional comments --- .../storefiletracker/FileBasedStoreFileTracker.java | 5 ++++- .../storefiletracker/StoreFileTrackerFactory.java | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index 8b4b09c9150f..4da7911bdded 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -56,7 +56,10 @@ class FileBasedStoreFileTracker extends StoreFileTrackerBase { public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - if (ctx != null && ctx.getFamilyStoreDirectoryPath() != null) { + //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table + //descriptors with the SFT impl specific configs. By the time this happens, the table has no + //regions nor stores yet, so it can't create a proper StoreContext. + if (ctx != null) { backedFile = new StoreFileListFile(ctx); } else { backedFile = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 5f3e43fb4981..64010955f68a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -160,6 +160,9 @@ static StoreFileTrackerBase createForMigration(Configuration conf, String config } public static void updateDescriptor(Configuration conf, TableDescriptorBuilder builder) { + //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table + //descriptors with the SFT impl specific configs. By the time this happens, the table has no + //regions nor stores yet, so it can't create a proper StoreContext. StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, null); tracker.updateWithTrackerConfigs(builder); } From a120c93554afa4ab5789457f28de0aed8d8dd5b1 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 8 Oct 2021 17:03:40 +0100 Subject: [PATCH 21/37] addressing latest review suggestions --- .../hbase/master/procedure/CreateTableProcedure.java | 5 +++-- .../storefiletracker/MigrationStoreFileTracker.java | 11 +++++++---- .../storefiletracker/StoreFileTracker.java | 6 +++--- .../storefiletracker/StoreFileTrackerBase.java | 8 ++++++-- .../storefiletracker/StoreFileTrackerFactory.java | 4 ++-- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 95395104463c..62493cda5c12 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -291,8 +291,9 @@ private void preCreate(final MasterProcedureEnv env) } TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); - StoreFileTrackerFactory.updateDescriptor(env.getMasterConfiguration(), builder); - tableDescriptor = builder.build(); + tableDescriptor = StoreFileTrackerFactory.updateDescriptor(env.getMasterConfiguration(), + builder.build()); + final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 85af7f119e5e..c2841fb55964 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.StoreContext; @@ -89,14 +90,16 @@ void set(List files) { } @Override - public void updateWithTrackerConfigs(TableDescriptorBuilder builder) { - super.updateWithTrackerConfigs(builder); - if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { + public TableDescriptor updateWithTrackerConfigs(TableDescriptor descriptor) { + descriptor = super.updateWithTrackerConfigs(descriptor); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(descriptor); + if (StringUtils.isEmpty(descriptor.getValue(SRC_IMPL))) { builder.setValue(SRC_IMPL, src.getTrackerName()); } - if (StringUtils.isEmpty(builder.getValue(DST_IMPL))) { + if (StringUtils.isEmpty(descriptor.getValue(DST_IMPL))) { builder.setValue(DST_IMPL, dst.getTrackerName()); } + return builder.build(); } static Class getSrcTrackerClass(Configuration conf) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 23f9f6f5778a..d5ba1c959049 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -21,7 +21,7 @@ import java.util.Collection; import java.util.List; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -81,7 +81,7 @@ void replace(Collection compactedFiles, Collection * implementation, and also possible misconfiguration between master and region servers. *

* See HBASE-26246 for more details. - * @param builder The table descriptor builder for the given table. + * @param descriptor The table descriptor for the given table. */ - void updateWithTrackerConfigs(TableDescriptorBuilder builder); + TableDescriptor updateWithTrackerConfigs(TableDescriptor descriptor); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index b1b893071e45..375eac7dc452 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -83,10 +84,13 @@ public final void replace(Collection compactedFiles, } @Override - public void updateWithTrackerConfigs(TableDescriptorBuilder builder) { - if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { + public TableDescriptor updateWithTrackerConfigs(TableDescriptor descriptor) { + if (StringUtils.isEmpty(descriptor.getValue(TRACKER_IMPL))) { + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(descriptor); builder.setValue(TRACKER_IMPL, getTrackerName()); + descriptor = builder.build(); } + return descriptor; } protected final String getTrackerName() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 64010955f68a..5c397805e6cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -159,12 +159,12 @@ static StoreFileTrackerBase createForMigration(Configuration conf, String config return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } - public static void updateDescriptor(Configuration conf, TableDescriptorBuilder builder) { + public static TableDescriptor updateDescriptor(Configuration conf, TableDescriptor descriptor) { //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table //descriptors with the SFT impl specific configs. By the time this happens, the table has no //regions nor stores yet, so it can't create a proper StoreContext. StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, null); - tracker.updateWithTrackerConfigs(builder); + return tracker.updateWithTrackerConfigs(descriptor); } // should not use MigrationStoreFileTracker for new family From c8566cf21fd14509dee8f3fee0752cd6617b67ff Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 8 Oct 2021 21:38:23 +0100 Subject: [PATCH 22/37] Fixing UT failure --- .../regionserver/storefiletracker/StoreFileTrackerFactory.java | 1 - .../regionserver/storefiletracker/TestStoreFileTracker.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 5c397805e6cd..4218784b8526 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java index 1dc9c4e53fc5..b30ca47772cb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -40,7 +40,7 @@ public class TestStoreFileTracker extends DefaultStoreFileTracker { public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - if (ctx.getRegionFileSystem() != null) { + if (ctx != null && ctx.getRegionFileSystem() != null) { this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); LOG.info("created storeId: {}", storeId); trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); From 7b03fd2452f77106ca15e8c2712c2a88ab5fd667 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 8 Oct 2021 23:24:33 +0100 Subject: [PATCH 23/37] HBASE-26328 Clone snapshot doesn't load reference files into FILE SFT impl Change-Id: I58ce5ef6bad9af6605c5f6cd041c85f48d36169a adding UT Change-Id: Ic0a0f8540a12dd7bcde5663c770e2e0d49c61fb6 --- .../org/apache/hadoop/hbase/io/HFileLink.java | 34 ++--- .../procedure/CloneSnapshotProcedure.java | 50 ++----- .../MigrationStoreFileTracker.java | 4 +- .../SnapshotStoreFileTracker.java | 77 +++++++++++ .../hbase/snapshot/RestoreSnapshotHelper.java | 90 ++++++++++--- ...estCloneSnapshotProcedureFileBasedSFT.java | 124 ++++++++++++++++++ 6 files changed, 296 insertions(+), 83 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index e2c80b24d325..c0345c118254 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -326,7 +326,7 @@ public static String createHFileLinkName(final TableName tableName, * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ - public static boolean create(final Configuration conf, final FileSystem fs, + public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName) throws IOException { return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true); @@ -347,7 +347,7 @@ public static boolean create(final Configuration conf, final FileSystem fs, * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ - public static boolean create(final Configuration conf, final FileSystem fs, + public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName, final boolean createBackRef) throws IOException { TableName linkedTable = hfileRegionInfo.getTable(); @@ -370,7 +370,7 @@ public static boolean create(final Configuration conf, final FileSystem fs, * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ - public static boolean create(final Configuration conf, final FileSystem fs, + public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, final String hfileName) throws IOException { return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true); @@ -392,7 +392,7 @@ public static boolean create(final Configuration conf, final FileSystem fs, * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ - public static boolean create(final Configuration conf, final FileSystem fs, + public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, final String hfileName, final boolean createBackRef) throws IOException { String familyName = dstFamilyPath.getName(); @@ -420,7 +420,9 @@ public static boolean create(final Configuration conf, final FileSystem fs, } try { // Create the link - return fs.createNewFile(new Path(dstFamilyPath, name)); + if(fs.createNewFile(new Path(dstFamilyPath, name))){ + return name; + } } catch (IOException e) { LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); // Revert the reference if the link creation failed @@ -429,25 +431,7 @@ public static boolean create(final Configuration conf, final FileSystem fs, } throw e; } - } - - /** - * Create a new HFileLink starting from a hfileLink name - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param hfileLinkName - HFileLink name (it contains hfile-region-table) - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final String hfileLinkName) - throws IOException { - return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true); + throw new IOException("File link=" + name + " already exists."); } /** @@ -464,7 +448,7 @@ public static boolean createFromHFileLink(final Configuration conf, final FileSy * @return true if the file is created, otherwise the file exists. * @throws IOException on file or parent directory creation failure */ - public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, + public static String createFromHFileLink(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef) throws IOException { Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index dae7b94134fe..b8001b4db7a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -453,56 +453,24 @@ private List createFsLayout( List newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); - final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first - final Path tempTableDir = CommonFSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); - if (CommonFSUtils.isExists(mfs.getFileSystem(), tempTableDir)) { + final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); + if (CommonFSUtils.isExists(mfs.getFileSystem(), tableDir)) { // if the region dirs exist, will cause exception and unlimited retry (see HBASE-24546) - LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tempTableDir); - CommonFSUtils.deleteDirectory(mfs.getFileSystem(), tempTableDir); + LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tableDir); + CommonFSUtils.deleteDirectory(mfs.getFileSystem(), tableDir); } - ((FSTableDescriptors) (env.getMasterServices().getTableDescriptors())) - .createTableDescriptorForTableDirectory(tempTableDir, - TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); + ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) + .createTableDescriptorForTableDirectory(tableDir, + TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions( - env, tempdir, tableDescriptor.getTableName(), newRegions); - - // 3. Move Table temp directory to the hbase root location - CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); - // Move Table temp mob directory to the hbase root location - Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName()); - if (mfs.getFileSystem().exists(tempMobTableDir)) { - moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir); - } - return newRegions; - } + env, mfs.getRootDir(), tableDescriptor.getTableName(), newRegions); - /** - * Move table temp mob directory to the hbase root location - * @param mfs The master file system - * @param tableDescriptor The table to operate on - * @param tempMobTableDir The temp mob directory of table - * @throws IOException If failed to move temp mob dir to hbase root dir - */ - private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs, - final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException { - FileSystem fs = mfs.getFileSystem(); - final Path tableMobDir = - MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); - if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) { - throw new IOException("Couldn't delete mob table " + tableMobDir); - } - if (!fs.exists(tableMobDir.getParent())) { - fs.mkdirs(tableMobDir.getParent()); - } - if (!fs.rename(tempMobTableDir, tableMobDir)) { - throw new IOException("Unable to move mob table from temp=" + tempMobTableDir - + " to hbase root=" + tableMobDir); - } + return newRegions; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index c2841fb55964..ec85fce4ce62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -40,9 +40,9 @@ class MigrationStoreFileTracker extends StoreFileTrackerBase { public static final String DST_IMPL = "hbase.store.file-tracker.migration.dst.impl"; - private final StoreFileTrackerBase src; + final StoreFileTrackerBase src; - private final StoreFileTrackerBase dst; + final StoreFileTrackerBase dst; public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java new file mode 100644 index 000000000000..f884121cdaf2 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.yetus.audience.InterfaceAudience; + +import java.io.IOException; +import java.util.List; + +/** + * Extends MigrationStoreFileTracker for Snapshot restore/clone specific case. + * When restoring/cloning snapshots, new regions are created with reference files to the + * original regions files. This work is done in snapshot specific classes. We need to somehow + * initialize these reference files in the configured StoreFileTracker. Once snapshot logic has + * cloned the store dir and created the references, it should set the list of reference files in + * SourceTracker.setReferenceFiles then invoke load method. + *

+ */ +@InterfaceAudience.Private +public class SnapshotStoreFileTracker extends MigrationStoreFileTracker { + + protected SnapshotStoreFileTracker(Configuration conf, boolean isPrimaryReplica, + StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + Preconditions.checkArgument(src instanceof SourceTracker, + "src for SnapshotStoreFileTracker should always be a SourceTracker!"); + } + + public SourceTracker getSourceTracker(){ + return (SourceTracker)this.src; + } + + /** + * The SFT impl to be set as source for SnapshotStoreFileTracker. + */ + public static class SourceTracker extends DefaultStoreFileTracker { + + private List files; + + public SourceTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + } + + public void setReferenceFiles(List files) { + this.files = files; + } + /** + * Overrides MigrationStoreFileTracker to simply call set on destination + * SFT implementation, passing the list of reference files. + * @return + * @throws IOException + */ + @Override + public List load() throws IOException { + return files; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index bd5388ca4dde..8d0bee16b12a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.snapshot; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -53,7 +55,10 @@ import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.SnapshotStoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.security.access.AccessControlClient; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil; @@ -153,7 +158,7 @@ public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final ForeignExceptionDispatcher monitor, final MonitoredTask status, final boolean createBackRefs) { this.fs = fs; - this.conf = conf; + this.conf = new Configuration(conf); this.snapshotManifest = manifest; this.snapshotDesc = manifest.getSnapshotDescription(); this.snapshotTable = TableName.valueOf(snapshotDesc.getTable()); @@ -163,6 +168,13 @@ public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, this.monitor = monitor; this.status = status; this.createBackRefs = createBackRefs; + this.conf.set(TRACKER_IMPL, SnapshotStoreFileTracker.class.getName()); + this.conf.set("hbase.store.file-tracker.migration.src.impl", + SnapshotStoreFileTracker.SourceTracker.class.getName()); + this.conf.set("hbase.store.file-tracker.migration.dst.impl", + this.tableDesc.getValue(TRACKER_IMPL) != null ? + this.tableDesc.getValue(TRACKER_IMPL) : + StoreFileTrackerFactory.Trackers.DEFAULT.name()); } /** @@ -500,18 +512,25 @@ private void restoreRegion(final RegionInfo regionInfo, String tableName = tableDesc.getTableName().getNameAsString(); final String snapshotName = snapshotDesc.getName(); + HRegionFileSystem regionFS = HRegionFileSystem.openRegionFromFileSystem(conf, fs, + tableDir, regionInfo, false); // Restore families present in the table for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { byte[] family = Bytes.toBytes(familyDir.getName()); + Set familyFiles = getTableRegionFamilyFiles(familyDir); List snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); + List filesToTrack = new ArrayList<>(); if (snapshotFamilyFiles != null) { List hfilesToAdd = new ArrayList<>(); for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) { if (familyFiles.contains(storeFile.getName())) { // HFile already present familyFiles.remove(storeFile.getName()); + //no need to restore already present files, but we need to add those to tracker + filesToTrack.add(new StoreFileInfo(conf, fs, + new Path(familyDir, storeFile.getName()), true)); } else { // HFile missing hfilesToAdd.add(storeFile); @@ -521,9 +540,11 @@ private void restoreRegion(final RegionInfo regionInfo, // Remove hfiles not present in the snapshot for (String hfileName: familyFiles) { Path hfile = new Path(familyDir, hfileName); - LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName+ - " from region=" + regionInfo.getEncodedName() + " table=" + tableName); - HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile); + if (!fs.getFileStatus(hfile).isDirectory()) { + LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); + HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile); + } } // Restore Missing files @@ -531,7 +552,10 @@ private void restoreRegion(final RegionInfo regionInfo, LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName+ " to region=" + regionInfo.getEncodedName() + " table=" + tableName); - restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); + String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); + //mark the reference file to be added to tracker + filesToTrack.add(new StoreFileInfo(conf, fs, + new Path(familyDir, fileName), true)); } } else { // Family doesn't exists in the snapshot @@ -540,12 +564,27 @@ private void restoreRegion(final RegionInfo regionInfo, HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family); fs.delete(familyDir, true); } + + SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) + StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). + withRegionFileSystem(regionFS).build()); + + //simply reset list of tracked files with the matching files + // and the extra one present in the snapshot + tracker.getSourceTracker().setReferenceFiles(filesToTrack); + tracker.load(); } // Add families not present in the table for (Map.Entry> familyEntry: snapshotFiles.entrySet()) { Path familyDir = new Path(regionDir, familyEntry.getKey()); + SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) + StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). + withRegionFileSystem(regionFS).build()); + List files = new ArrayList<>(); if (!fs.mkdirs(familyDir)) { throw new IOException("Unable to create familyDir=" + familyDir); } @@ -553,8 +592,11 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) { LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName); - restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); + String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); + files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); } + tracker.getSourceTracker().setReferenceFiles(files); + tracker.load(); } } @@ -627,7 +669,8 @@ private void cloneHdfsMobRegion(final Map region final RegionInfo region) throws IOException { // clone region info (change embedded tableName with the new one) Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName()); - cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName())); + cloneRegion(MobUtils.getMobRegionInfo(tableDesc.getTableName()), + clonedRegionPath, region, regionManifests.get(region.getEncodedName())); } /** @@ -641,18 +684,31 @@ private void cloneHdfsMobRegion(final Map region * @param regionDir {@link Path} cloned dir * @param snapshotRegionInfo */ - private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo, + private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final String tableName = tableDesc.getTableName().getNameAsString(); final String snapshotName = snapshotDesc.getName(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); + List clonedFiles = new ArrayList<>(); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName); - restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); + String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); + clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); } + HRegionFileSystem regionFS = HRegionFileSystem.openRegionFromFileSystem(conf, fs, + tableDir, newRegionInfo, false); + SnapshotStoreFileTracker tracker = + (SnapshotStoreFileTracker) StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). + withRegionFileSystem(regionFS).build()); + //since we are adding the reference files for the first time, + // we do tracker.load, rather than add + tracker.getSourceTracker().setReferenceFiles(clonedFiles); + tracker.load(); } + } /** @@ -668,7 +724,9 @@ private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionIn */ private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { - cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo, + cloneRegion(region.getRegionInfo(), + new Path(tableDir, region.getRegionInfo().getEncodedName()), + snapshotRegionInfo, manifest); } @@ -685,16 +743,16 @@ private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionIn * @param createBackRef - Whether back reference should be created. Defaults to true. * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ - private void restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, + private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { - HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); + return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); } else if (StoreFileInfo.isReference(hfileName)) { - restoreReferenceFile(familyDir, regionInfo, storeFile); + return restoreReferenceFile(familyDir, regionInfo, storeFile); } else { - HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); + return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); } } @@ -716,7 +774,7 @@ private void restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, * @param regionInfo destination region info for the table * @param storeFile reference file name */ - private void restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, + private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String hfileName = storeFile.getName(); @@ -760,6 +818,7 @@ private void restoreReferenceFile(final Path familyDir, final RegionInfo regionI IOUtils.copyBytes(in, out, conf); } + // Add the daughter region to the map String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes())); if (regionName == null) { @@ -777,6 +836,7 @@ private void restoreReferenceFile(final Path familyDir, final RegionInfo regionI daughters.setSecond(regionName); } } + return outPath.getName(); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java new file mode 100644 index 000000000000..a04129d615e0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.Trackers.FILE; +import static org.junit.Assert.assertTrue; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Stream; + +@Category({MasterTests.class, MediumTests.class}) +public class TestCloneSnapshotProcedureFileBasedSFT extends TestCloneSnapshotProcedure { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCloneSnapshotProcedureFileBasedSFT.class); + + @BeforeClass + public static void setupCluster() throws Exception { + UTIL.getConfiguration().set(TRACKER_IMPL, FILE.name()); + UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + UTIL.startMiniCluster(1); + } + + @Test + public void testCloneSnapshot() throws Exception { + super.testCloneSnapshot(); + } + + @Test + public void testCloneSnapshotToSameTableFileSFT() throws Exception { + super.testCloneSnapshotToSameTable(); + } + +// @Test +// public void testRecoveryAndDoubleExecution() throws Exception { +// final ProcedureExecutor procExec = getMasterProcedureExecutor(); +// final TableName clonedTableName = TableName.valueOf("testRecoveryAndDoubleExecution"); +// final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); +// +// // take the snapshot +// SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); +// +// // Here if you enable this then we will enter an infinite loop, as we will fail either after +// // TRSP.openRegion or after OpenRegionProcedure.execute, so we can never finish the TRSP... +// ProcedureTestingUtility.setKillIfHasParent(procExec, false); +// ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); +// +// // Start the Clone snapshot procedure && kill the executor +// long procId = procExec.submitProcedure( +// new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); +// +// // Restart the executor and execute the step twice +// MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); +// +// MasterProcedureTestingUtility.validateTableIsEnabled( +// UTIL.getHBaseCluster().getMaster(), +// clonedTableName); +// } +// +// @Test +// public void testRollbackAndDoubleExecution() throws Exception { +// final ProcedureExecutor procExec = getMasterProcedureExecutor(); +// final TableName clonedTableName = TableName.valueOf("testRollbackAndDoubleExecution"); +// final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); +// +// // take the snapshot +// SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); +// +// ProcedureTestingUtility.waitNoProcedureRunning(procExec); +// ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); +// +// // Start the Clone snapshot procedure && kill the executor +// long procId = procExec.submitProcedure( +// new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); +// +// int lastStep = 2; // failing before CLONE_SNAPSHOT_WRITE_FS_LAYOUT +// MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); +// +// MasterProcedureTestingUtility.validateTableDeletion( +// UTIL.getHBaseCluster().getMaster(), clonedTableName); +// } +} From ba9600c2fba0e0c291fdc3102ec78af4b57b01b8 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 13 Oct 2021 10:46:47 +0100 Subject: [PATCH 24/37] Additional fixes Change-Id: Ia4482881e46b2521683d23a1a2ea84d7a7368109 --- .../org/apache/hadoop/hbase/io/HFileLink.java | 4 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 28 +++++++-- ...estCloneSnapshotProcedureFileBasedSFT.java | 59 ------------------- 3 files changed, 24 insertions(+), 67 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index c0345c118254..9d5db7282573 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -202,7 +203,6 @@ public static boolean isHFileLink(final Path path) { return isHFileLink(path.getName()); } - /** * @param fileName File name to check. * @return True if the path is a HFileLink. @@ -431,7 +431,7 @@ public static String create(final Configuration conf, final FileSystem fs, } throw e; } - throw new IOException("File link=" + name + " already exists."); + throw new IOException("File link=" + name + " already exists under " + dstFamilyPath + " folder."); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 8d0bee16b12a..6f41b5654440 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -512,8 +512,11 @@ private void restoreRegion(final RegionInfo regionInfo, String tableName = tableDesc.getTableName().getNameAsString(); final String snapshotName = snapshotDesc.getName(); - HRegionFileSystem regionFS = HRegionFileSystem.openRegionFromFileSystem(conf, fs, - tableDir, regionInfo, false); + Path regionPath = new Path(tableDir, regionInfo.getEncodedName()); + HRegionFileSystem regionFS = (fs.exists(regionPath)) ? + HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) : + HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo); + // Restore families present in the table for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { byte[] family = Bytes.toBytes(familyDir.getName()); @@ -694,11 +697,24 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, f for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName); - String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); - clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); + if (MobUtils.isMobRegionInfo(newRegionInfo)) { + String mobFileName = HFileLink.createHFileLinkName(snapshotRegionInfo, + storeFile.getName()); + Path mobPath = new Path(familyDir, mobFileName); + if (fs.exists(mobPath)) { + fs.delete(mobPath, true); + } + restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); + } else { + String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); + clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); + } } - HRegionFileSystem regionFS = HRegionFileSystem.openRegionFromFileSystem(conf, fs, - tableDir, newRegionInfo, false); + Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); + HRegionFileSystem regionFS = (fs.exists(regionPath)) ? + HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : + HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); + SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java index a04129d615e0..12a50bdb3501 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java @@ -62,63 +62,4 @@ public static void setupCluster() throws Exception { UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); UTIL.startMiniCluster(1); } - - @Test - public void testCloneSnapshot() throws Exception { - super.testCloneSnapshot(); - } - - @Test - public void testCloneSnapshotToSameTableFileSFT() throws Exception { - super.testCloneSnapshotToSameTable(); - } - -// @Test -// public void testRecoveryAndDoubleExecution() throws Exception { -// final ProcedureExecutor procExec = getMasterProcedureExecutor(); -// final TableName clonedTableName = TableName.valueOf("testRecoveryAndDoubleExecution"); -// final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); -// -// // take the snapshot -// SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); -// -// // Here if you enable this then we will enter an infinite loop, as we will fail either after -// // TRSP.openRegion or after OpenRegionProcedure.execute, so we can never finish the TRSP... -// ProcedureTestingUtility.setKillIfHasParent(procExec, false); -// ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); -// -// // Start the Clone snapshot procedure && kill the executor -// long procId = procExec.submitProcedure( -// new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); -// -// // Restart the executor and execute the step twice -// MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); -// -// MasterProcedureTestingUtility.validateTableIsEnabled( -// UTIL.getHBaseCluster().getMaster(), -// clonedTableName); -// } -// -// @Test -// public void testRollbackAndDoubleExecution() throws Exception { -// final ProcedureExecutor procExec = getMasterProcedureExecutor(); -// final TableName clonedTableName = TableName.valueOf("testRollbackAndDoubleExecution"); -// final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); -// -// // take the snapshot -// SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); -// -// ProcedureTestingUtility.waitNoProcedureRunning(procExec); -// ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); -// -// // Start the Clone snapshot procedure && kill the executor -// long procId = procExec.submitProcedure( -// new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); -// -// int lastStep = 2; // failing before CLONE_SNAPSHOT_WRITE_FS_LAYOUT -// MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); -// -// MasterProcedureTestingUtility.validateTableDeletion( -// UTIL.getHBaseCluster().getMaster(), clonedTableName); -// } } From 9c9789bbd25dffbac244802b1d2182cd5131d879 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 13 Oct 2021 15:48:13 +0100 Subject: [PATCH 25/37] =?UTF-8?q?HBASE-26326=20CreateTableProcedure=20fail?= =?UTF-8?q?s=20when=20FileBasedStoreFileTracker=E2=80=A6=20(#3721)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Duo Zhang Signed-off-by: Josh Elser --- .../procedure/CreateTableProcedure.java | 6 ++---- .../FileBasedStoreFileTracker.java | 9 ++++++++- .../MigrationStoreFileTracker.java | 12 +---------- .../storefiletracker/StoreFileTracker.java | 5 +++-- .../StoreFileTrackerBase.java | 9 ++++----- .../StoreFileTrackerFactory.java | 20 +++++++++++++------ .../procedure/TestCreateTableProcedure.java | 16 +++++++++++++++ .../TestStoreFileTracker.java | 2 +- 8 files changed, 49 insertions(+), 30 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index ee8e51ffe86b..0a6a469328a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; @@ -290,9 +289,8 @@ private void preCreate(final MasterProcedureEnv env) (newRegions != null ? newRegions.size() : 0)); } - TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); - StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder); - tableDescriptor = builder.build(); + tableDescriptor = StoreFileTrackerFactory.updateWithTrackerConfigs(env.getMasterConfiguration(), + tableDescriptor); final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index c370b87c1154..4da7911bdded 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -56,7 +56,14 @@ class FileBasedStoreFileTracker extends StoreFileTrackerBase { public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - backedFile = new StoreFileListFile(ctx); + //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table + //descriptors with the SFT impl specific configs. By the time this happens, the table has no + //regions nor stores yet, so it can't create a proper StoreContext. + if (ctx != null) { + backedFile = new StoreFileListFile(ctx); + } else { + backedFile = null; + } } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 1946d4bf7eef..230c1ec1b7a8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.StoreContext; @@ -88,17 +89,6 @@ void set(List files) { "Should not call this method on " + getClass().getSimpleName()); } - @Override - public void persistConfiguration(TableDescriptorBuilder builder) { - super.persistConfiguration(builder); - if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { - builder.setValue(SRC_IMPL, src.getTrackerName()); - } - if (StringUtils.isEmpty(builder.getValue(DST_IMPL))) { - builder.setValue(DST_IMPL, dst.getTrackerName()); - } - } - static Class getSrcTrackerClass(Configuration conf) { return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, SRC_IMPL); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 59fe7ef52f96..fd8f7c99092d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -75,7 +76,7 @@ void replace(Collection compactedFiles, Collection StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; /** - * Saves StoreFileTracker implementations specific configurations into the table descriptors. + * Adds StoreFileTracker implementations specific configurations into the table descriptor. *

* This is used to avoid accidentally data loss when changing the cluster level store file tracker * implementation, and also possible misconfiguration between master and region servers. @@ -83,5 +84,5 @@ void replace(Collection compactedFiles, Collection * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ - void persistConfiguration(TableDescriptorBuilder builder); + TableDescriptorBuilder updateWithTrackerConfigs(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index a786add49b21..edbaace40c26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -32,7 +33,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -83,10 +83,9 @@ public final void replace(Collection compactedFiles, } @Override - public void persistConfiguration(TableDescriptorBuilder builder) { - if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { - builder.setValue(TRACKER_IMPL, getTrackerName()); - } + public TableDescriptorBuilder updateWithTrackerConfigs(TableDescriptorBuilder builder) { + builder.setValue(TRACKER_IMPL, getTrackerName()); + return builder; } protected final String getTrackerName() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index b586027f8333..1c683ae3de62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -24,8 +24,10 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; + import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -158,12 +160,18 @@ static StoreFileTrackerBase createForMigration(Configuration conf, String config return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } - public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) { - TableDescriptor tableDescriptor = builder.build(); - ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; - StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); - StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); - tracker.persistConfiguration(builder); + public static TableDescriptor updateWithTrackerConfigs(Configuration conf, + TableDescriptor descriptor) { + //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table + //descriptors with the SFT impl specific configs. By the time this happens, the table has no + //regions nor stores yet, so it can't create a proper StoreContext. + if (StringUtils.isEmpty(descriptor.getValue(TRACKER_IMPL))) { + StoreFileTracker tracker = + StoreFileTrackerFactory.create(conf, true, null); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(descriptor); + return tracker.updateWithTrackerConfigs(builder).build(); + } + return descriptor; } // should not use MigrationStoreFileTracker for new family diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index f432c8060d3d..51ea9f58248a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -105,6 +106,21 @@ public void testCreateWithTrackImpl() throws Exception { assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); } + @Test + public void testCreateWithFileBasedStoreTrackerImpl() throws Exception { + ProcedureExecutor procExec = getMasterProcedureExecutor(); + procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.FILE.name()); + final TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + htd = getMaster().getTableDescriptors().get(tableName); + assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL)); + } + @Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor procExec = getMasterProcedureExecutor(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java index 1dc9c4e53fc5..b30ca47772cb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -40,7 +40,7 @@ public class TestStoreFileTracker extends DefaultStoreFileTracker { public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - if (ctx.getRegionFileSystem() != null) { + if (ctx != null && ctx.getRegionFileSystem() != null) { this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); LOG.info("created storeId: {}", storeId); trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); From d75b0cec7b216006396cc14c14e5d920ff3fe313 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 29 Jul 2021 18:35:19 +0800 Subject: [PATCH 26/37] HBASE-26064 Introduce a StoreFileTracker to abstract the store file tracking logic Signed-off-by: Wellington Chevreuil --- .../hbase/mob/DefaultMobStoreCompactor.java | 22 +- .../hbase/mob/DefaultMobStoreFlusher.java | 4 +- .../CreateStoreFileWriterParams.java | 134 ++++ .../regionserver/DateTieredStoreEngine.java | 5 +- .../regionserver/DefaultStoreEngine.java | 5 +- .../regionserver/DefaultStoreFlusher.java | 11 +- .../hadoop/hbase/regionserver/HMobStore.java | 3 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +- .../hadoop/hbase/regionserver/HStore.java | 699 ++++-------------- .../hbase/regionserver/StoreContext.java | 9 + .../hbase/regionserver/StoreEngine.java | 438 ++++++++++- .../hbase/regionserver/StoreFileManager.java | 9 + .../hbase/regionserver/StoreFlusher.java | 9 +- .../hadoop/hbase/regionserver/StoreUtils.java | 37 +- .../hbase/regionserver/StripeStoreEngine.java | 9 +- .../regionserver/StripeStoreFlusher.java | 9 +- .../AbstractMultiOutputCompactor.java | 7 +- .../regionserver/compactions/Compactor.java | 36 +- .../compactions/DefaultCompactor.java | 16 +- .../DefaultStoreFileTracker.java | 61 ++ .../storefiletracker/StoreFileTracker.java | 75 ++ .../StoreFileTrackerBase.java | 178 +++++ .../StoreFileTrackerFactory.java | 35 + .../compaction/MajorCompactionRequest.java | 1 - .../apache/hadoop/hbase/TestIOFencing.java | 12 +- .../TestCacheOnWriteInSchema.java | 6 +- .../regionserver/TestDefaultStoreEngine.java | 5 +- .../hbase/regionserver/TestHRegion.java | 4 +- .../hadoop/hbase/regionserver/TestHStore.java | 28 +- .../TestRegionMergeTransactionOnCluster.java | 6 +- .../TestStoreFileRefresherChore.java | 3 +- .../regionserver/TestStoreScannerClosure.java | 6 +- .../regionserver/TestStripeStoreEngine.java | 2 + .../compactions/TestDateTieredCompactor.java | 12 +- .../TestStripeCompactionPolicy.java | 12 +- .../compactions/TestStripeCompactor.java | 12 +- 36 files changed, 1225 insertions(+), 705 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java index c45fdff7ca16..01fe0005f048 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map.Entry; import java.util.Optional; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -144,17 +143,16 @@ public InternalScanner createScanner(ScanInfo scanInfo, List s }; private final CellSinkFactory writerFactory = - new CellSinkFactory() { - @Override - public StoreFileWriter createWriter(InternalScanner scanner, - org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, - boolean shouldDropBehind, boolean major) throws IOException { - // make this writer with tags always because of possible new cells with tags. - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, true, true, shouldDropBehind); - } - }; + new CellSinkFactory() { + @Override + public StoreFileWriter createWriter(InternalScanner scanner, + org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, + boolean shouldDropBehind, boolean major) throws IOException { + // make this writer with tags always because of possible new cells with tags. + return store.getStoreEngine().createWriter( + createParams(fd, shouldDropBehind, major).includeMVCCReadpoint(true).includesTag(true)); + } + }; public DefaultMobStoreCompactor(Configuration conf, HStore store) { super(conf, store); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java index 480b85c58dfe..4a1dc7b33a51 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java @@ -25,7 +25,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -127,8 +126,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId, synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk - writer = store.createWriterInTmp(cellsCount, store.getColumnFamilyDescriptor().getCompressionType(), - false, true, true, false); + writer = createWriter(snapshot, true); IOException e = null; try { // It's a mob store, flush the cells in a mob way. This is the difference of flushing diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java new file mode 100644 index 000000000000..10cd9f009e4a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateStoreFileWriterParams.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public final class CreateStoreFileWriterParams { + + private long maxKeyCount; + + private Compression.Algorithm compression; + + private boolean isCompaction; + + private boolean includeMVCCReadpoint; + + private boolean includesTag; + + private boolean shouldDropBehind; + + private long totalCompactedFilesSize = -1; + + private String fileStoragePolicy = HConstants.EMPTY_STRING; + + private CreateStoreFileWriterParams() { + } + + public long maxKeyCount() { + return maxKeyCount; + } + + public CreateStoreFileWriterParams maxKeyCount(long maxKeyCount) { + this.maxKeyCount = maxKeyCount; + return this; + } + + public Compression.Algorithm compression() { + return compression; + } + + /** + * Set the compression algorithm to use + */ + public CreateStoreFileWriterParams compression(Compression.Algorithm compression) { + this.compression = compression; + return this; + } + + public boolean isCompaction() { + return isCompaction; + } + + /** + * Whether we are creating a new file in a compaction + */ + public CreateStoreFileWriterParams isCompaction(boolean isCompaction) { + this.isCompaction = isCompaction; + return this; + } + + public boolean includeMVCCReadpoint() { + return includeMVCCReadpoint; + } + + /** + * Whether to include MVCC or not + */ + public CreateStoreFileWriterParams includeMVCCReadpoint(boolean includeMVCCReadpoint) { + this.includeMVCCReadpoint = includeMVCCReadpoint; + return this; + } + + public boolean includesTag() { + return includesTag; + } + + /** + * Whether to includesTag or not + */ + public CreateStoreFileWriterParams includesTag(boolean includesTag) { + this.includesTag = includesTag; + return this; + } + + public boolean shouldDropBehind() { + return shouldDropBehind; + } + + public CreateStoreFileWriterParams shouldDropBehind(boolean shouldDropBehind) { + this.shouldDropBehind = shouldDropBehind; + return this; + } + + public long totalCompactedFilesSize() { + return totalCompactedFilesSize; + } + + public CreateStoreFileWriterParams totalCompactedFilesSize(long totalCompactedFilesSize) { + this.totalCompactedFilesSize = totalCompactedFilesSize; + return this; + } + + public String fileStoragePolicy() { + return fileStoragePolicy; + } + + public CreateStoreFileWriterParams fileStoragePolicy(String fileStoragePolicy) { + this.fileStoragePolicy = fileStoragePolicy; + return this; + } + + public static CreateStoreFileWriterParams create() { + return new CreateStoreFileWriterParams(); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index 1df953d93c96..7422d9112eab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -19,18 +19,17 @@ import java.io.IOException; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.yetus.audience.InterfaceAudience; /** * HBASE-15400 This store engine allows us to store data in date tiered layout with exponential diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java index 58f8bbbb6ac5..693b9c93b9fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; @@ -39,8 +38,8 @@ * their derivatives. */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class DefaultStoreEngine extends StoreEngine< - DefaultStoreFlusher, RatioBasedCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> { +public class DefaultStoreEngine extends StoreEngine { public static final String DEFAULT_STORE_FLUSHER_CLASS_KEY = "hbase.hstore.defaultengine.storeflusher.class"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java index a7d7fb1f3d56..306760d7ce6a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java @@ -21,15 +21,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.util.StringUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Default implementation of StoreFlusher. @@ -60,9 +59,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId, synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk - writer = store.createWriterInTmp(cellsCount, - store.getColumnFamilyDescriptor().getCompressionType(), false, true, - snapshot.isTagsPresent(), false); + writer = createWriter(snapshot, false); IOException e = null; try { performFlush(scanner, writer, throughputController); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index 7ce7f0310c7d..b00a50c522fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -28,7 +28,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -158,7 +157,7 @@ protected KeyValueScanner createScanner(Scan scan, ScanInfo scanInfo, protected StoreEngine createStoreEngine(HStore store, Configuration conf, CellComparator cellComparator) throws IOException { MobStoreEngine engine = new MobStoreEngine(); - engine.createComponents(conf, store, cellComparator); + engine.createComponentsOnce(conf, store, cellComparator); return engine; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 5b26f9aa8ab7..6ab9b0fc3fe8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -142,7 +142,7 @@ public Path getRegionDir() { // Temp Helpers // =========================================================================== /** @return {@link Path} to the region's temp directory, used for file creations */ - Path getTempDir() { + public Path getTempDir() { return new Path(getRegionDir(), REGION_TEMP_DIR); } @@ -237,11 +237,7 @@ public String getStoragePolicyName(String familyName) { * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ - public Collection getStoreFiles(final byte[] familyName) throws IOException { - return getStoreFiles(Bytes.toString(familyName)); - } - - public Collection getStoreFiles(final String familyName) throws IOException { + public List getStoreFiles(final String familyName) throws IOException { return getStoreFiles(familyName, true); } @@ -251,7 +247,7 @@ public Collection getStoreFiles(final String familyName) throws I * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ - public Collection getStoreFiles(final String familyName, final boolean validate) + public List getStoreFiles(final String familyName, final boolean validate) throws IOException { Path familyDir = getStoreDir(familyName); FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index e78e74ce0890..253605ab48e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; @@ -47,8 +48,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -72,17 +71,12 @@ import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; import org.apache.hadoop.hbase.coprocessor.ReadOnlyConfiguration; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContext; -import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.InvalidHFileException; -import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.quotas.RegionSizeStore; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; @@ -112,7 +106,6 @@ import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; import org.apache.hbase.thirdparty.org.apache.commons.collections4.IterableUtils; @@ -167,16 +160,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, private boolean cacheOnWriteLogged; - /** - * RWLock for store operations. - * Locked in shared mode when the list of component stores is looked at: - * - all reads/writes to table data - * - checking for split - * Locked in exclusive mode when the list of component stores is modified: - * - closing - * - completing a compaction - */ - final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); /** * Lock specific to archiving compacted store files. This avoids races around * the combination of retrieving the list of compacted files and moving them to @@ -285,14 +268,8 @@ protected HStore(final HRegion region, final ColumnFamilyDescriptor family, } this.storeEngine = createStoreEngine(this, this.conf, region.getCellComparator()); - List hStoreFiles = loadStoreFiles(warmup); - // Move the storeSize calculation out of loadStoreFiles() method, because the secondary read - // replica's refreshStoreFiles() will also use loadStoreFiles() to refresh its store files and - // update the storeSize in the refreshStoreSizeAndTotalBytes() finally (just like compaction) , so - // no need calculate the storeSize twice. - this.storeSize.addAndGet(getStorefilesSize(hStoreFiles, sf -> true)); - this.totalUncompressedBytes.addAndGet(getTotalUncompressedBytes(hStoreFiles)); - this.storeEngine.getStoreFileManager().loadFiles(hStoreFiles); + storeEngine.initialize(warmup); + refreshStoreSizeAndTotalBytes(); flushRetriesNumber = conf.getInt( "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER); @@ -508,102 +485,18 @@ void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) { this.dataBlockEncoder = blockEncoder; } - /** - * Creates an unsorted list of StoreFile loaded in parallel - * from the given directory. - */ - private List loadStoreFiles(boolean warmup) throws IOException { - Collection files = getRegionFileSystem().getStoreFiles(getColumnFamilyName()); - return openStoreFiles(files, warmup); - } - - private List openStoreFiles(Collection files, boolean warmup) - throws IOException { - if (CollectionUtils.isEmpty(files)) { - return Collections.emptyList(); - } - // initialize the thread pool for opening store files in parallel.. - ThreadPoolExecutor storeFileOpenerThreadPool = - this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-" - + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - CompletionService completionService = - new ExecutorCompletionService<>(storeFileOpenerThreadPool); - - int totalValidStoreFile = 0; - for (StoreFileInfo storeFileInfo : files) { - // open each store file in parallel - completionService.submit(() -> this.createStoreFileAndReader(storeFileInfo)); - totalValidStoreFile++; - } - - Set compactedStoreFiles = new HashSet<>(); - ArrayList results = new ArrayList<>(files.size()); - IOException ioe = null; - try { - for (int i = 0; i < totalValidStoreFile; i++) { - try { - HStoreFile storeFile = completionService.take().get(); - if (storeFile != null) { - LOG.debug("loaded {}", storeFile); - results.add(storeFile); - compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); - } - } catch (InterruptedException e) { - if (ioe == null) { - ioe = new InterruptedIOException(e.getMessage()); - } - } catch (ExecutionException e) { - if (ioe == null) { - ioe = new IOException(e.getCause()); - } - } - } - } finally { - storeFileOpenerThreadPool.shutdownNow(); - } - if (ioe != null) { - // close StoreFile readers - boolean evictOnClose = - getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; - for (HStoreFile file : results) { - try { - if (file != null) { - file.closeStoreFile(evictOnClose); - } - } catch (IOException e) { - LOG.warn("Could not close store file {}", file, e); - } - } - throw ioe; - } - - // Should not archive the compacted store files when region warmup. See HBASE-22163. - if (!warmup) { - // Remove the compacted files from result - List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); - for (HStoreFile storeFile : results) { - if (compactedStoreFiles.contains(storeFile.getPath().getName())) { - LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); - storeFile.getReader().close(storeFile.getCacheConf() != null ? - storeFile.getCacheConf().shouldEvictOnClose() : true); - filesToRemove.add(storeFile); - } - } - results.removeAll(filesToRemove); - if (!filesToRemove.isEmpty() && this.isPrimaryReplicaStore()) { - LOG.debug("Moving the files {} to archive", filesToRemove); - getRegionFileSystem().removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), - filesToRemove); - } - } - - return results; + private void postRefreshStoreFiles() throws IOException { + // Advance the memstore read point to be at least the new store files seqIds so that + // readers might pick it up. This assumes that the store is not getting any writes (otherwise + // in-flight transactions might be made visible) + getMaxSequenceId().ifPresent(region.getMVCC()::advanceTo); + refreshStoreSizeAndTotalBytes(); } @Override public void refreshStoreFiles() throws IOException { - Collection newFiles = getRegionFileSystem().getStoreFiles(getColumnFamilyName()); - refreshStoreFilesInternal(newFiles); + storeEngine.refreshStoreFiles(); + postRefreshStoreFiles(); } /** @@ -611,89 +504,8 @@ public void refreshStoreFiles() throws IOException { * region replicas to keep up to date with the primary region files. */ public void refreshStoreFiles(Collection newFiles) throws IOException { - List storeFiles = new ArrayList<>(newFiles.size()); - for (String file : newFiles) { - storeFiles.add(getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file)); - } - refreshStoreFilesInternal(storeFiles); - } - - /** - * Checks the underlying store files, and opens the files that have not - * been opened, and removes the store file readers for store files no longer - * available. Mainly used by secondary region replicas to keep up to date with - * the primary region files. - */ - private void refreshStoreFilesInternal(Collection newFiles) throws IOException { - StoreFileManager sfm = storeEngine.getStoreFileManager(); - Collection currentFiles = sfm.getStorefiles(); - Collection compactedFiles = sfm.getCompactedfiles(); - if (currentFiles == null) { - currentFiles = Collections.emptySet(); - } - if (newFiles == null) { - newFiles = Collections.emptySet(); - } - if (compactedFiles == null) { - compactedFiles = Collections.emptySet(); - } - - HashMap currentFilesSet = new HashMap<>(currentFiles.size()); - for (HStoreFile sf : currentFiles) { - currentFilesSet.put(sf.getFileInfo(), sf); - } - HashMap compactedFilesSet = new HashMap<>(compactedFiles.size()); - for (HStoreFile sf : compactedFiles) { - compactedFilesSet.put(sf.getFileInfo(), sf); - } - - Set newFilesSet = new HashSet(newFiles); - // Exclude the files that have already been compacted - newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); - Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); - Set toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet); - - if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) { - return; - } - - LOG.info("Refreshing store files for " + this + " files to add: " - + toBeAddedFiles + " files to remove: " + toBeRemovedFiles); - - Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); - for (StoreFileInfo sfi : toBeRemovedFiles) { - toBeRemovedStoreFiles.add(currentFilesSet.get(sfi)); - } - - // try to open the files - List openedFiles = openStoreFiles(toBeAddedFiles, false); - - // propogate the file changes to the underlying store file manager - replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception - - // Advance the memstore read point to be at least the new store files seqIds so that - // readers might pick it up. This assumes that the store is not getting any writes (otherwise - // in-flight transactions might be made visible) - if (!toBeAddedFiles.isEmpty()) { - // we must have the max sequence id here as we do have several store files - region.getMVCC().advanceTo(this.getMaxSequenceId().getAsLong()); - } - - refreshStoreSizeAndTotalBytes(); - } - - protected HStoreFile createStoreFileAndReader(final Path p) throws IOException { - StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), - p, isPrimaryReplicaStore()); - return createStoreFileAndReader(info); - } - - private HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { - info.setRegionCoprocessorHost(this.region.getCoprocessorHost()); - HStoreFile storeFile = new HStoreFile(info, getColumnFamilyDescriptor().getBloomFilterType(), - getCacheConfig()); - storeFile.initReader(); - return storeFile; + storeEngine.refreshStoreFiles(newFiles); + postRefreshStoreFiles(); } /** @@ -716,7 +528,7 @@ public void stopReplayingFromWAL(){ * Adds a value to the memstore */ public void add(final Cell cell, MemStoreSizing memstoreSizing) { - lock.readLock().lock(); + storeEngine.readLock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace("tableName={}, encodedName={}, columnFamilyName={} is too busy!", @@ -724,7 +536,7 @@ public void add(final Cell cell, MemStoreSizing memstoreSizing) { } this.memstore.add(cell, memstoreSizing); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); currentParallelPutCount.decrementAndGet(); } } @@ -733,7 +545,7 @@ public void add(final Cell cell, MemStoreSizing memstoreSizing) { * Adds the specified value to the memstore */ public void add(final Iterable cells, MemStoreSizing memstoreSizing) { - lock.readLock().lock(); + storeEngine.readLock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace("tableName={}, encodedName={}, columnFamilyName={} is too busy!", @@ -741,7 +553,7 @@ public void add(final Iterable cells, MemStoreSizing memstoreSizing) { } memstore.add(cells, memstoreSizing); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); currentParallelPutCount.decrementAndGet(); } } @@ -864,17 +676,16 @@ public Path bulkLoadHFile(byte[] family, String srcPathStr, Path dstPath) throws LOG.info("Loaded HFile " + srcPath + " into " + this + " as " + dstPath + " - updating store file list."); - HStoreFile sf = createStoreFileAndReader(dstPath); + HStoreFile sf = storeEngine.createStoreFileAndReader(dstPath); bulkLoadHFile(sf); - LOG.info("Successfully loaded {} into {} (new location: {})", - srcPath, this, dstPath); + LOG.info("Successfully loaded {} into {} (new location: {})", srcPath, this, dstPath); return dstPath; } public void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException { - HStoreFile sf = createStoreFileAndReader(fileInfo); + HStoreFile sf = storeEngine.createStoreFileAndReader(fileInfo); bulkLoadHFile(sf); } @@ -882,28 +693,74 @@ private void bulkLoadHFile(HStoreFile sf) throws IOException { StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); - - // Append the new storefile into the list - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); - } finally { - // We need the lock, as long as we are updating the storeFiles - // or changing the memstore. Let us release it before calling - // notifyChangeReadersObservers. See HBASE-4485 for a possible - // deadlock scenario that could have happened if continue to hold - // the lock. - this.lock.writeLock().unlock(); - } + storeEngine.addStoreFiles(Lists.newArrayList(sf)); LOG.info("Loaded HFile " + sf.getFileInfo() + " into " + this); if (LOG.isTraceEnabled()) { - String traceMessage = "BULK LOAD time,size,store size,store files [" - + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize - + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; + String traceMessage = "BULK LOAD time,size,store size,store files [" + + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize + "," + + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage); } } + private ImmutableCollection closeWithoutLock() throws IOException { + // Clear so metrics doesn't find them. + ImmutableCollection result = storeEngine.getStoreFileManager().clearFiles(); + Collection compactedfiles = storeEngine.getStoreFileManager().clearCompactedFiles(); + // clear the compacted files + if (CollectionUtils.isNotEmpty(compactedfiles)) { + removeCompactedfiles(compactedfiles, + getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true); + } + if (!result.isEmpty()) { + // initialize the thread pool for closing store files in parallel. + ThreadPoolExecutor storeFileCloserThreadPool = + this.region.getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" + + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); + + // close each store file in parallel + CompletionService completionService = + new ExecutorCompletionService<>(storeFileCloserThreadPool); + for (HStoreFile f : result) { + completionService.submit(new Callable() { + @Override + public Void call() throws IOException { + boolean evictOnClose = + getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; + f.closeStoreFile(evictOnClose); + return null; + } + }); + } + + IOException ioe = null; + try { + for (int i = 0; i < result.size(); i++) { + try { + Future future = completionService.take(); + future.get(); + } catch (InterruptedException e) { + if (ioe == null) { + ioe = new InterruptedIOException(); + ioe.initCause(e); + } + } catch (ExecutionException e) { + if (ioe == null) { + ioe = new IOException(e.getCause()); + } + } + } + } finally { + storeFileCloserThreadPool.shutdownNow(); + } + if (ioe != null) { + throw ioe; + } + } + LOG.trace("Closed {}", this); + return result; + } + /** * Close all the readers We don't need to worry about subsequent requests because the Region holds * a write lock that will prevent any more reads or writes. @@ -911,67 +768,18 @@ private void bulkLoadHFile(HStoreFile sf) throws IOException { * @throws IOException on failure */ public ImmutableCollection close() throws IOException { + // findbugs can not recognize storeEngine.writeLock is just a lock operation so it will report + // UL_UNRELEASED_LOCK_EXCEPTION_PATH, so here we have to use two try finally... + // Change later if findbugs becomes smarter in the future. this.archiveLock.lock(); - this.lock.writeLock().lock(); try { - // Clear so metrics doesn't find them. - ImmutableCollection result = storeEngine.getStoreFileManager().clearFiles(); - Collection compactedfiles = - storeEngine.getStoreFileManager().clearCompactedFiles(); - // clear the compacted files - if (CollectionUtils.isNotEmpty(compactedfiles)) { - removeCompactedfiles(compactedfiles, getCacheConfig() != null ? - getCacheConfig().shouldEvictOnClose() : true); - } - if (!result.isEmpty()) { - // initialize the thread pool for closing store files in parallel. - ThreadPoolExecutor storeFileCloserThreadPool = this.region - .getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" - + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - - // close each store file in parallel - CompletionService completionService = - new ExecutorCompletionService<>(storeFileCloserThreadPool); - for (HStoreFile f : result) { - completionService.submit(new Callable() { - @Override - public Void call() throws IOException { - boolean evictOnClose = - getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; - f.closeStoreFile(evictOnClose); - return null; - } - }); - } - - IOException ioe = null; - try { - for (int i = 0; i < result.size(); i++) { - try { - Future future = completionService.take(); - future.get(); - } catch (InterruptedException e) { - if (ioe == null) { - ioe = new InterruptedIOException(); - ioe.initCause(e); - } - } catch (ExecutionException e) { - if (ioe == null) { - ioe = new IOException(e.getCause()); - } - } - } - } finally { - storeFileCloserThreadPool.shutdownNow(); - } - if (ioe != null) { - throw ioe; - } + this.storeEngine.writeLock(); + try { + return closeWithoutLock(); + } finally { + this.storeEngine.writeUnlock(); } - LOG.trace("Closed {}", this); - return result; } finally { - this.lock.writeLock().unlock(); this.archiveLock.unlock(); } } @@ -1001,7 +809,7 @@ protected List flushCache(final long logCacheFlushId, MemStoreSnapshot sna try { for (Path pathName : pathNames) { lastPathName = pathName; - validateStoreFile(pathName); + storeEngine.validateStoreFile(pathName); } return pathNames; } catch (Exception e) { @@ -1047,167 +855,18 @@ public HStoreFile tryCommitRecoveredHFile(Path path) throws IOException { } Path dstPath = getRegionFileSystem().commitStoreFile(getColumnFamilyName(), path); - HStoreFile sf = createStoreFileAndReader(dstPath); + HStoreFile sf = storeEngine.createStoreFileAndReader(dstPath); StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); - } finally { - this.lock.writeLock().unlock(); - } + storeEngine.addStoreFiles(Lists.newArrayList(sf)); LOG.info("Loaded recovered hfile to {}, entries={}, sequenceid={}, filesize={}", sf, r.getEntries(), r.getSequenceID(), TraditionalBinaryPrefix.long2String(r.length(), "B", 1)); return sf; } - /** - * Commit the given {@code files}. - *

- * We will move the file into data directory, and open it. - * @param files the files want to commit - * @param validate whether to validate the store files - * @return the committed store files - */ - private List commitStoreFiles(List files, boolean validate) throws IOException { - List committedFiles = new ArrayList<>(files.size()); - HRegionFileSystem hfs = getRegionFileSystem(); - String familyName = getColumnFamilyName(); - for (Path file : files) { - try { - if (validate) { - validateStoreFile(file); - } - Path committedPath = hfs.commitStoreFile(familyName, file); - HStoreFile sf = createStoreFileAndReader(committedPath); - committedFiles.add(sf); - } catch (IOException e) { - LOG.error("Failed to commit store file {}", file, e); - // Try to delete the files we have committed before. - // It is OK to fail when deleting as leaving the file there does not cause any data - // corruption problem. It just introduces some duplicated data which may impact read - // performance a little when reading before compaction. - for (HStoreFile sf : committedFiles) { - Path pathToDelete = sf.getPath(); - try { - sf.deleteStoreFile(); - } catch (IOException deleteEx) { - LOG.warn(HBaseMarkers.FATAL, "Failed to delete committed store file {}", pathToDelete, - deleteEx); - } - } - throw new IOException("Failed to commit the flush", e); - } - } - return committedFiles; - } - - public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, - boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, - boolean shouldDropBehind) throws IOException { - return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint, - includesTag, shouldDropBehind, -1, HConstants.EMPTY_STRING); - } - - /** - * @param compression Compression algorithm to use - * @param isCompaction whether we are creating a new file in a compaction - * @param includeMVCCReadpoint - whether to include MVCC or not - * @param includesTag - includesTag or not - * @return Writer for a new StoreFile in the tmp dir. - */ - // TODO : allow the Writer factory to create Writers of ShipperListener type only in case of - // compaction - public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, - boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, - boolean shouldDropBehind, long totalCompactedFilesSize, String fileStoragePolicy) - throws IOException { - // creating new cache config for each new writer - final CacheConfig cacheConf = getCacheConfig(); - final CacheConfig writerCacheConf = new CacheConfig(cacheConf); - if (isCompaction) { - // Don't cache data on write on compactions, unless specifically configured to do so - // Cache only when total file size remains lower than configured threshold - final boolean cacheCompactedBlocksOnWrite = - getCacheConfig().shouldCacheCompactedBlocksOnWrite(); - // if data blocks are to be cached on write - // during compaction, we should forcefully - // cache index and bloom blocks as well - if (cacheCompactedBlocksOnWrite && totalCompactedFilesSize <= cacheConf - .getCacheCompactedBlocksOnWriteThreshold()) { - writerCacheConf.enableCacheOnWrite(); - if (!cacheOnWriteLogged) { - LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " + - "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this); - cacheOnWriteLogged = true; - } - } else { - writerCacheConf.setCacheDataOnWrite(false); - if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { - // checking condition once again for logging - LOG.debug( - "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted " - + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}", - this, totalCompactedFilesSize, - cacheConf.getCacheCompactedBlocksOnWriteThreshold()); - } - } - } else { - final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite(); - if (shouldCacheDataOnWrite) { - writerCacheConf.enableCacheOnWrite(); - if (!cacheOnWriteLogged) { - LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " + - "Index blocks and Bloom filter blocks", this); - cacheOnWriteLogged = true; - } - } - } - Encryption.Context encryptionContext = storeContext.getEncryptionContext(); - HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag, - encryptionContext); - Path familyTempDir = new Path(getRegionFileSystem().getTempDir(), getColumnFamilyName()); - StoreFileWriter.Builder builder = - new StoreFileWriter.Builder(conf, writerCacheConf, getFileSystem()) - .withOutputDir(familyTempDir) - .withBloomType(storeContext.getBloomFilterType()) - .withMaxKeyCount(maxKeyCount) - .withFavoredNodes(storeContext.getFavoredNodes()) - .withFileContext(hFileContext) - .withShouldDropCacheBehind(shouldDropBehind) - .withCompactedFilesSupplier(storeContext.getCompactedFilesSupplier()) - .withFileStoragePolicy(fileStoragePolicy); - return builder.build(); - } - - private HFileContext createFileContext(Compression.Algorithm compression, - boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { - if (compression == null) { - compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; - } - ColumnFamilyDescriptor family = getColumnFamilyDescriptor(); - HFileContext hFileContext = new HFileContextBuilder() - .withIncludesMvcc(includeMVCCReadpoint) - .withIncludesTags(includesTag) - .withCompression(compression) - .withCompressTags(family.isCompressTags()) - .withChecksumType(StoreUtils.getChecksumType(conf)) - .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)) - .withBlockSize(family.getBlocksize()) - .withHBaseCheckSum(true) - .withDataBlockEncoding(family.getDataBlockEncoding()) - .withEncryptionContext(encryptionContext) - .withCreateTime(EnvironmentEdgeManager.currentTime()) - .withColumnFamily(getColumnFamilyDescriptor().getName()) - .withTableName(getTableName().getName()) - .withCellComparator(getComparator()) - .build(); - return hFileContext; - } - private long getTotalSize(Collection sfs) { return sfs.stream().mapToLong(sf -> sf.getReader().length()).sum(); } @@ -1217,18 +876,8 @@ private long getTotalSize(Collection sfs) { * @param sfs Store files * @return Whether compaction is required. */ - private boolean updateStorefiles(List sfs, long snapshotId) throws IOException { - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().insertNewFiles(sfs); - } finally { - // We need the lock, as long as we are updating the storeFiles - // or changing the memstore. Let us release it before calling - // notifyChangeReadersObservers. See HBASE-4485 for a possible - // deadlock scenario that could have happened if continue to hold - // the lock. - this.lock.writeLock().unlock(); - } + private boolean completeFlush(List sfs, long snapshotId) throws IOException { + storeEngine.addStoreFiles(sfs); // We do not need to call clearSnapshot method inside the write lock. // The clearSnapshot itself is thread safe, which can be called at the same time with other // memstore operations expect snapshot and clearSnapshot. And for these two methods, in HRegion @@ -1254,11 +903,11 @@ private boolean updateStorefiles(List sfs, long snapshotId) throws I private void notifyChangedReadersObservers(List sfs) throws IOException { for (ChangedReadersObserver o : this.changedReaderObservers) { List memStoreScanners; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { memStoreScanners = this.memstore.getScanners(o.getReadPoint()); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } o.updateReaders(sfs, memStoreScanners); } @@ -1300,13 +949,13 @@ public List getScanners(boolean cacheBlocks, boolean usePread, byte[] stopRow, boolean includeStopRow, long readPt) throws IOException { Collection storeFilesToScan; List memStoreScanners; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { storeFilesToScan = this.storeEngine.getStoreFileManager().getFilesForScan(startRow, includeStartRow, stopRow, includeStopRow); memStoreScanners = this.memstore.getScanners(readPt); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } try { @@ -1383,11 +1032,11 @@ public List getScanners(List files, boolean cacheBl boolean includeMemstoreScanner) throws IOException { List memStoreScanners = null; if (includeMemstoreScanner) { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { memStoreScanners = this.memstore.getScanners(readPt); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } try { @@ -1503,14 +1152,13 @@ protected List doCompaction(CompactionRequestImpl cr, List newFiles) throws IOException { // Do the steps necessary to complete the compaction. setStoragePolicyFromFileName(newFiles); - List sfs = commitStoreFiles(newFiles, true); + List sfs = storeEngine.commitStoreFiles(newFiles, true); if (this.getCoprocessorHost() != null) { for (HStoreFile sf : sfs) { getCoprocessorHost().postCompact(this, sf, cr.getTracker(), cr, user); } } - writeCompactionWalRecord(filesToCompact, sfs); - replaceStoreFiles(filesToCompact, sfs); + replaceStoreFiles(filesToCompact, sfs, true); if (cr.isMajor()) { majorCompactedCellsCount.addAndGet(getCompactionProgress().getTotalCompactingKVs()); majorCompactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize); @@ -1574,25 +1222,24 @@ private void writeCompactionWalRecord(Collection filesCompacted, this.region.getRegionInfo(), compactionDescriptor, this.region.getMVCC()); } - void replaceStoreFiles(Collection compactedFiles, Collection result) - throws IOException { - this.lock.writeLock().lock(); - try { - this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result); - synchronized (filesCompacting) { - filesCompacting.removeAll(compactedFiles); - } - - // These may be null when the RS is shutting down. The space quota Chores will fix the Region - // sizes later so it's not super-critical if we miss these. - RegionServerServices rsServices = region.getRegionServerServices(); - if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { - updateSpaceQuotaAfterFileReplacement( - rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), - compactedFiles, result); - } - } finally { - this.lock.writeLock().unlock(); + @RestrictedApi(explanation = "Should only be called in TestHStore", link = "", + allowedOnPath = ".*/(HStore|TestHStore).java") + void replaceStoreFiles(Collection compactedFiles, Collection result, + boolean writeCompactionMarker) throws IOException { + storeEngine.replaceStoreFiles(compactedFiles, result); + if (writeCompactionMarker) { + writeCompactionWalRecord(compactedFiles, result); + } + synchronized (filesCompacting) { + filesCompacting.removeAll(compactedFiles); + } + // These may be null when the RS is shutting down. The space quota Chores will fix the Region + // sizes later so it's not super-critical if we miss these. + RegionServerServices rsServices = region.getRegionServerServices(); + if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { + updateSpaceQuotaAfterFileReplacement( + rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), + compactedFiles, result); } } @@ -1715,7 +1362,7 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick for (String compactionOutput : compactionOutputs) { StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput); - HStoreFile storeFile = createStoreFileAndReader(storeFileInfo); + HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); outputStoreFiles.add(storeFile); } } @@ -1723,7 +1370,7 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) { LOG.info("Replaying compaction marker, replacing input files: " + inputStoreFiles + " with output files : " + outputStoreFiles); - this.replaceStoreFiles(inputStoreFiles, outputStoreFiles); + this.replaceStoreFiles(inputStoreFiles, outputStoreFiles, false); this.refreshStoreSizeAndTotalBytes(); } } @@ -1732,14 +1379,14 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick public boolean hasReferences() { // Grab the read lock here, because we need to ensure that: only when the atomic // replaceStoreFiles(..) finished, we can get all the complete store file list. - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { // Merge the current store files with compacted files here due to HBASE-20940. Collection allStoreFiles = new ArrayList<>(getStorefiles()); allStoreFiles.addAll(getCompactedFiles()); return StoreUtils.hasReferences(allStoreFiles); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -1779,7 +1426,7 @@ public Optional requestCompaction(int priority, final CompactionContext compaction = storeEngine.createCompaction(); CompactionRequestImpl request = null; - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { synchronized (filesCompacting) { // First, see if coprocessor would want to override selection. @@ -1852,7 +1499,7 @@ public Optional requestCompaction(int priority, request.setTracker(tracker); } } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } if (LOG.isDebugEnabled()) { @@ -1885,7 +1532,7 @@ private void removeUnneededFiles() throws IOException { this, getColumnFamilyDescriptor().getMinVersions()); return; } - this.lock.readLock().lock(); + this.storeEngine.readLock(); Collection delSfs = null; try { synchronized (filesCompacting) { @@ -1897,7 +1544,7 @@ private void removeUnneededFiles() throws IOException { } } } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } if (CollectionUtils.isEmpty(delSfs)) { @@ -1905,8 +1552,7 @@ private void removeUnneededFiles() throws IOException { } Collection newFiles = Collections.emptyList(); // No new files. - writeCompactionWalRecord(delSfs, newFiles); - replaceStoreFiles(delSfs, newFiles); + replaceStoreFiles(delSfs, newFiles, true); refreshStoreSizeAndTotalBytes(); LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in " + this + "; total size is " @@ -1928,25 +1574,6 @@ protected void finishCompactionRequest(CompactionRequestImpl cr) { } } - /** - * Validates a store file by opening and closing it. In HFileV2 this should not be an expensive - * operation. - * @param path the path to the store file - */ - private void validateStoreFile(Path path) throws IOException { - HStoreFile storeFile = null; - try { - storeFile = createStoreFileAndReader(path); - } catch (IOException e) { - LOG.error("Failed to open store file : {}, keeping it in tmp location", path, e); - throw e; - } finally { - if (storeFile != null) { - storeFile.closeStoreFile(false); - } - } - } - /** * Update counts. */ @@ -1992,7 +1619,7 @@ public boolean canSplit() { * Determines if Store should be split. */ public Optional getSplitPoint() { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { // Should already be enforced by the split policy! assert !this.getRegionInfo().isMetaRegion(); @@ -2005,7 +1632,7 @@ public Optional getSplitPoint() { } catch(IOException e) { LOG.warn("Failed getting store size for {}", this, e); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } return Optional.empty(); } @@ -2038,7 +1665,7 @@ public void triggerMajorCompaction() { */ public KeyValueScanner getScanner(Scan scan, final NavigableSet targetCols, long readPt) throws IOException { - lock.readLock().lock(); + storeEngine.readLock(); try { ScanInfo scanInfo; if (this.getCoprocessorHost() != null) { @@ -2048,7 +1675,7 @@ public KeyValueScanner getScanner(Scan scan, final NavigableSet targetCo } return createScanner(scan, scanInfo, targetCols, readPt); } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); } } @@ -2078,7 +1705,7 @@ public List recreateScanners(List currentFileS boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, boolean includeStartRow, byte[] stopRow, boolean includeStopRow, long readPt, boolean includeMemstoreScanner) throws IOException { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { Map name2File = new HashMap<>(getStorefilesCount() + getCompactedFilesCount()); @@ -2103,7 +1730,7 @@ public List recreateScanners(List currentFileS return getScanners(filesToReopen, cacheBlocks, false, false, matcher, startRow, includeStartRow, stopRow, includeStopRow, readPt, false); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -2169,41 +1796,20 @@ public long getStoreSizeUncompressed() { @Override public long getStorefilesSize() { // Include all StoreFiles - return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), sf -> true); + return StoreUtils.getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), + sf -> true); } @Override public long getHFilesSize() { // Include only StoreFiles which are HFiles - return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), + return StoreUtils.getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), HStoreFile::isHFile); } - private long getTotalUncompressedBytes(List files) { - return files.stream() - .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) - .sum(); - } - - private long getStorefilesSize(Collection files, Predicate predicate) { - return files.stream().filter(predicate) - .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::length)).sum(); - } - - private long getStorefileFieldSize(HStoreFile file, ToLongFunction f) { - if (file == null) { - return 0L; - } - StoreFileReader reader = file.getReader(); - if (reader == null) { - return 0L; - } - return f.applyAsLong(reader); - } - private long getStorefilesFieldSize(ToLongFunction f) { return this.storeEngine.getStoreFileManager().getStorefiles().stream() - .mapToLong(file -> getStorefileFieldSize(file, f)).sum(); + .mapToLong(file -> StoreUtils.getStorefileFieldSize(file, f)).sum(); } @Override @@ -2274,11 +1880,11 @@ public long getSmallestReadPoint() { */ public void upsert(Iterable cells, long readpoint, MemStoreSizing memstoreSizing) throws IOException { - this.lock.readLock().lock(); + this.storeEngine.readLock(); try { this.memstore.upsert(cells, readpoint, memstoreSizing); } finally { - this.lock.readLock().unlock(); + this.storeEngine.readUnlock(); } } @@ -2331,7 +1937,7 @@ public boolean commit(MonitoredTask status) throws IOException { return false; } status.setStatus("Flushing " + this + ": reopening flushed file"); - List storeFiles = commitStoreFiles(tempFiles, false); + List storeFiles = storeEngine.commitStoreFiles(tempFiles, false); for (HStoreFile sf : storeFiles) { StoreFileReader r = sf.getReader(); if (LOG.isInfoEnabled()) { @@ -2354,7 +1960,7 @@ public boolean commit(MonitoredTask status) throws IOException { } } // Add new file to store files. Clear snapshot too while we have the Store write lock. - return updateStorefiles(storeFiles, snapshot.getId()); + return completeFlush(storeFiles, snapshot.getId()); } @Override @@ -2382,7 +1988,7 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) // open the file as a store file (hfile link, etc) StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file); - HStoreFile storeFile = createStoreFileAndReader(storeFileInfo); + HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); storeFiles.add(storeFile); HStore.this.storeSize.addAndGet(storeFile.getReader().length()); HStore.this.totalUncompressedBytes @@ -2399,7 +2005,7 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) snapshotId = snapshot.getId(); snapshot.close(); } - HStore.this.updateStorefiles(storeFiles, snapshotId); + HStore.this.completeFlush(storeFiles, snapshotId); } /** @@ -2412,7 +2018,7 @@ public void abort() throws IOException { //won't be closed. If we are using MSLAB, the chunk referenced by those scanners //can't be released, thus memory leak snapshot.close(); - HStore.this.updateStorefiles(Collections.emptyList(), snapshot.getId()); + HStore.this.completeFlush(Collections.emptyList(), snapshot.getId()); } } } @@ -2574,7 +2180,7 @@ public synchronized void closeAndArchiveCompactedFiles() throws IOException { // ensure other threads do not attempt to archive the same files on close() archiveLock.lock(); try { - lock.readLock().lock(); + storeEngine.readLock(); Collection copyCompactedfiles = null; try { Collection compactedfiles = @@ -2586,7 +2192,7 @@ public synchronized void closeAndArchiveCompactedFiles() throws IOException { LOG.trace("No compacted files to archive"); } } finally { - lock.readLock().unlock(); + storeEngine.readUnlock(); } if (CollectionUtils.isNotEmpty(copyCompactedfiles)) { removeCompactedfiles(copyCompactedfiles, true); @@ -2721,12 +2327,7 @@ public boolean isSloppyMemStore() { private void clearCompactedfiles(List filesToRemove) throws IOException { LOG.trace("Clearing the compacted file {} from this store", filesToRemove); - try { - lock.writeLock().lock(); - this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove); - } finally { - lock.writeLock().unlock(); - } + storeEngine.removeCompactedFiles(filesToRemove); } void reportArchivedFilesForQuota(List archivedFiles, List fileSizes) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java index 26233505db73..2a9f96859361 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -108,6 +109,14 @@ public RegionCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + public RegionInfo getRegionInfo() { + return regionFileSystem.getRegionInfo(); + } + + public boolean isPrimaryReplicaStore() { + return getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID; + } + public static Builder getBuilder() { return new Builder(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index 60b3c3d0d20f..f9d6c294f779 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -20,37 +20,129 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; - +import java.util.Set; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.Compactor; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; /** - * StoreEngine is a factory that can create the objects necessary for HStore to operate. - * Since not all compaction policies, compactors and store file managers are compatible, - * they are tied together and replaced together via StoreEngine-s. + * StoreEngine is a factory that can create the objects necessary for HStore to operate. Since not + * all compaction policies, compactors and store file managers are compatible, they are tied + * together and replaced together via StoreEngine-s. + *

+ * We expose read write lock methods to upper layer for store operations:
+ *

    + *
  • Locked in shared mode when the list of component stores is looked at: + *
      + *
    • all reads/writes to table data
      • + *
    • checking for split
      • + *
    + *
    • + *
  • Locked in exclusive mode when the list of component stores is modified: + *
      + *
    • closing
      • + *
    • completing a compaction
      • + *
    + *
    • + *
+ *

+ * It is a bit confusing that we have a StoreFileManager(SFM) and then a StoreFileTracker(SFT). As + * its name says, SFT is used to track the store files list. The reason why we have a SFT beside SFM + * is that, when introducing stripe compaction, we introduced the StoreEngine and also the SFM, but + * actually, the SFM here is not a general 'Manager', it is only designed to manage the in memory + * 'stripes', so we can select different store files when scanning or compacting. The 'tracking' of + * store files is actually done in {@link org.apache.hadoop.hbase.regionserver.HRegionFileSystem} + * and {@link HStore} before we have SFT. And since SFM is designed to only holds in memory states, + * we will hold write lock when updating it, the lock is also used to protect the normal read/write + * requests. This means we'd better not add IO operations to SFM. And also, no matter what the in + * memory state is, stripe or not, it does not effect how we track the store files. So consider all + * these facts, here we introduce a separated SFT to track the store files. + *

+ * Here, since we always need to update SFM and SFT almost at the same time, we introduce methods in + * StoreEngine directly to update them both, so upper layer just need to update StoreEngine once, to + * reduce the possible misuse. */ @InterfaceAudience.Private -public abstract class StoreEngine { +public abstract class StoreEngine { + + private static final Logger LOG = LoggerFactory.getLogger(StoreEngine.class); + protected SF storeFlusher; protected CP compactionPolicy; protected C compactor; protected SFM storeFileManager; + private Configuration conf; + private StoreContext ctx; + private RegionCoprocessorHost coprocessorHost; + private Function openStoreFileThreadPoolCreator; + private StoreFileTracker storeFileTracker; + + private final ReadWriteLock storeLock = new ReentrantReadWriteLock(); /** - * The name of the configuration parameter that specifies the class of - * a store engine that is used to manage and compact HBase store files. + * The name of the configuration parameter that specifies the class of a store engine that is used + * to manage and compact HBase store files. */ public static final String STORE_ENGINE_CLASS_KEY = "hbase.hstore.engine.class"; - private static final Class> - DEFAULT_STORE_ENGINE_CLASS = DefaultStoreEngine.class; + private static final Class> DEFAULT_STORE_ENGINE_CLASS = + DefaultStoreEngine.class; + + /** + * Acquire read lock of this store. + */ + public void readLock() { + storeLock.readLock().lock(); + } + + /** + * Release read lock of this store. + */ + public void readUnlock() { + storeLock.readLock().unlock(); + } + + /** + * Acquire write lock of this store. + */ + public void writeLock() { + storeLock.writeLock().lock(); + } + + /** + * Release write lock of this store. + */ + public void writeUnlock() { + storeLock.writeLock().unlock(); + } /** * @return Compaction policy to use. @@ -80,6 +172,11 @@ public StoreFlusher getStoreFlusher() { return this.storeFlusher; } + private StoreFileTracker createStoreFileTracker(HStore store) { + return StoreFileTrackerFactory.create(store.conf, store.getRegionInfo().getTable(), + store.isPrimaryReplicaStore(), store.getStoreContext()); + } + /** * @param filesCompacting Files currently compacting * @return whether a compaction selection is possible @@ -87,8 +184,8 @@ public StoreFlusher getStoreFlusher() { public abstract boolean needsCompaction(List filesCompacting); /** - * Creates an instance of a compaction context specific to this engine. - * Doesn't actually select or start a compaction. See CompactionContext class comment. + * Creates an instance of a compaction context specific to this engine. Doesn't actually select or + * start a compaction. See CompactionContext class comment. * @return New CompactionContext object. */ public abstract CompactionContext createCompaction() throws IOException; @@ -96,32 +193,321 @@ public StoreFlusher getStoreFlusher() { /** * Create the StoreEngine's components. */ - protected abstract void createComponents( - Configuration conf, HStore store, CellComparator cellComparator) throws IOException; + protected abstract void createComponents(Configuration conf, HStore store, + CellComparator cellComparator) throws IOException; - private void createComponentsOnce( - Configuration conf, HStore store, CellComparator cellComparator) throws IOException { - assert compactor == null && compactionPolicy == null - && storeFileManager == null && storeFlusher == null; + protected final void createComponentsOnce(Configuration conf, HStore store, + CellComparator cellComparator) throws IOException { + assert compactor == null && compactionPolicy == null && storeFileManager == null && + storeFlusher == null && storeFileTracker == null; createComponents(conf, store, cellComparator); - assert compactor != null && compactionPolicy != null - && storeFileManager != null && storeFlusher != null; + this.conf = conf; + this.ctx = store.getStoreContext(); + this.coprocessorHost = store.getHRegion().getCoprocessorHost(); + this.openStoreFileThreadPoolCreator = store.getHRegion()::getStoreFileOpenAndCloseThreadPool; + this.storeFileTracker = createStoreFileTracker(store); + assert compactor != null && compactionPolicy != null && storeFileManager != null && + storeFlusher != null && storeFileTracker != null; + } + + /** + * Create a writer for writing new store files. + * @return Writer for a new StoreFile + */ + public StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException { + return storeFileTracker.createWriter(params); + } + + public HStoreFile createStoreFileAndReader(Path p) throws IOException { + StoreFileInfo info = new StoreFileInfo(conf, ctx.getRegionFileSystem().getFileSystem(), p, + ctx.isPrimaryReplicaStore()); + return createStoreFileAndReader(info); + } + + public HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { + info.setRegionCoprocessorHost(coprocessorHost); + HStoreFile storeFile = + new HStoreFile(info, ctx.getFamily().getBloomFilterType(), ctx.getCacheConf()); + storeFile.initReader(); + return storeFile; + } + + /** + * Validates a store file by opening and closing it. In HFileV2 this should not be an expensive + * operation. + * @param path the path to the store file + */ + public void validateStoreFile(Path path) throws IOException { + HStoreFile storeFile = null; + try { + storeFile = createStoreFileAndReader(path); + } catch (IOException e) { + LOG.error("Failed to open store file : {}, keeping it in tmp location", path, e); + throw e; + } finally { + if (storeFile != null) { + storeFile.closeStoreFile(false); + } + } + } + + private List openStoreFiles(Collection files, boolean warmup) + throws IOException { + if (CollectionUtils.isEmpty(files)) { + return Collections.emptyList(); + } + // initialize the thread pool for opening store files in parallel.. + ExecutorService storeFileOpenerThreadPool = + openStoreFileThreadPoolCreator.apply("StoreFileOpener-" + + ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString()); + CompletionService completionService = + new ExecutorCompletionService<>(storeFileOpenerThreadPool); + + int totalValidStoreFile = 0; + for (StoreFileInfo storeFileInfo : files) { + // open each store file in parallel + completionService.submit(() -> createStoreFileAndReader(storeFileInfo)); + totalValidStoreFile++; + } + + Set compactedStoreFiles = new HashSet<>(); + ArrayList results = new ArrayList<>(files.size()); + IOException ioe = null; + try { + for (int i = 0; i < totalValidStoreFile; i++) { + try { + HStoreFile storeFile = completionService.take().get(); + if (storeFile != null) { + LOG.debug("loaded {}", storeFile); + results.add(storeFile); + compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); + } + } catch (InterruptedException e) { + if (ioe == null) { + ioe = new InterruptedIOException(e.getMessage()); + } + } catch (ExecutionException e) { + if (ioe == null) { + ioe = new IOException(e.getCause()); + } + } + } + } finally { + storeFileOpenerThreadPool.shutdownNow(); + } + if (ioe != null) { + // close StoreFile readers + boolean evictOnClose = + ctx.getCacheConf() != null ? ctx.getCacheConf().shouldEvictOnClose() : true; + for (HStoreFile file : results) { + try { + if (file != null) { + file.closeStoreFile(evictOnClose); + } + } catch (IOException e) { + LOG.warn("Could not close store file {}", file, e); + } + } + throw ioe; + } + + // Should not archive the compacted store files when region warmup. See HBASE-22163. + if (!warmup) { + // Remove the compacted files from result + List filesToRemove = new ArrayList<>(compactedStoreFiles.size()); + for (HStoreFile storeFile : results) { + if (compactedStoreFiles.contains(storeFile.getPath().getName())) { + LOG.warn("Clearing the compacted storefile {} from {}", storeFile, this); + storeFile.getReader().close( + storeFile.getCacheConf() != null ? storeFile.getCacheConf().shouldEvictOnClose() : + true); + filesToRemove.add(storeFile); + } + } + results.removeAll(filesToRemove); + if (!filesToRemove.isEmpty() && ctx.isPrimaryReplicaStore()) { + LOG.debug("Moving the files {} to archive", filesToRemove); + ctx.getRegionFileSystem().removeStoreFiles(ctx.getFamily().getNameAsString(), + filesToRemove); + } + } + + return results; + } + + public void initialize(boolean warmup) throws IOException { + List fileInfos = storeFileTracker.load(); + List files = openStoreFiles(fileInfos, warmup); + storeFileManager.loadFiles(files); + } + + public void refreshStoreFiles() throws IOException { + List fileInfos = storeFileTracker.load(); + refreshStoreFilesInternal(fileInfos); + } + + public void refreshStoreFiles(Collection newFiles) throws IOException { + List storeFiles = new ArrayList<>(newFiles.size()); + for (String file : newFiles) { + storeFiles + .add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), file)); + } + refreshStoreFilesInternal(storeFiles); + } + + /** + * Checks the underlying store files, and opens the files that have not been opened, and removes + * the store file readers for store files no longer available. Mainly used by secondary region + * replicas to keep up to date with the primary region files. + */ + private void refreshStoreFilesInternal(Collection newFiles) throws IOException { + Collection currentFiles = storeFileManager.getStorefiles(); + Collection compactedFiles = storeFileManager.getCompactedfiles(); + if (currentFiles == null) { + currentFiles = Collections.emptySet(); + } + if (newFiles == null) { + newFiles = Collections.emptySet(); + } + if (compactedFiles == null) { + compactedFiles = Collections.emptySet(); + } + + HashMap currentFilesSet = new HashMap<>(currentFiles.size()); + for (HStoreFile sf : currentFiles) { + currentFilesSet.put(sf.getFileInfo(), sf); + } + HashMap compactedFilesSet = new HashMap<>(compactedFiles.size()); + for (HStoreFile sf : compactedFiles) { + compactedFilesSet.put(sf.getFileInfo(), sf); + } + + Set newFilesSet = new HashSet(newFiles); + // Exclude the files that have already been compacted + newFilesSet = Sets.difference(newFilesSet, compactedFilesSet.keySet()); + Set toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet()); + Set toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet); + + if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) { + return; + } + + LOG.info("Refreshing store files for " + this + " files to add: " + toBeAddedFiles + + " files to remove: " + toBeRemovedFiles); + + Set toBeRemovedStoreFiles = new HashSet<>(toBeRemovedFiles.size()); + for (StoreFileInfo sfi : toBeRemovedFiles) { + toBeRemovedStoreFiles.add(currentFilesSet.get(sfi)); + } + + // try to open the files + List openedFiles = openStoreFiles(toBeAddedFiles, false); + + // propogate the file changes to the underlying store file manager + replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); // won't throw an exception + } + + /** + * Commit the given {@code files}. + *

+ * We will move the file into data directory, and open it. + * @param files the files want to commit + * @param validate whether to validate the store files + * @return the committed store files + */ + public List commitStoreFiles(List files, boolean validate) throws IOException { + List committedFiles = new ArrayList<>(files.size()); + HRegionFileSystem hfs = ctx.getRegionFileSystem(); + String familyName = ctx.getFamily().getNameAsString(); + Path storeDir = hfs.getStoreDir(familyName); + for (Path file : files) { + try { + if (validate) { + validateStoreFile(file); + } + Path committedPath; + // As we want to support writing to data directory directly, here we need to check whether + // the store file is already in the right place + if (file.getParent() != null && file.getParent().equals(storeDir)) { + // already in the right place, skip renmaing + committedPath = file; + } else { + // Write-out finished successfully, move into the right spot + committedPath = hfs.commitStoreFile(familyName, file); + } + HStoreFile sf = createStoreFileAndReader(committedPath); + committedFiles.add(sf); + } catch (IOException e) { + LOG.error("Failed to commit store file {}", file, e); + // Try to delete the files we have committed before. + // It is OK to fail when deleting as leaving the file there does not cause any data + // corruption problem. It just introduces some duplicated data which may impact read + // performance a little when reading before compaction. + for (HStoreFile sf : committedFiles) { + Path pathToDelete = sf.getPath(); + try { + sf.deleteStoreFile(); + } catch (IOException deleteEx) { + LOG.warn(HBaseMarkers.FATAL, "Failed to delete committed store file {}", pathToDelete, + deleteEx); + } + } + throw new IOException("Failed to commit the flush", e); + } + } + return committedFiles; + } + + public void addStoreFiles(Collection storeFiles) throws IOException { + storeFileTracker.add(StoreUtils.toStoreFileInfo(storeFiles)); + writeLock(); + try { + storeFileManager.insertNewFiles(storeFiles); + } finally { + // We need the lock, as long as we are updating the storeFiles + // or changing the memstore. Let us release it before calling + // notifyChangeReadersObservers. See HBASE-4485 for a possible + // deadlock scenario that could have happened if continue to hold + // the lock. + writeUnlock(); + } + } + + public void replaceStoreFiles(Collection compactedFiles, + Collection newFiles) throws IOException { + storeFileTracker.replace(StoreUtils.toStoreFileInfo(compactedFiles), + StoreUtils.toStoreFileInfo(newFiles)); + writeLock(); + try { + storeFileManager.addCompactionResults(compactedFiles, newFiles); + } finally { + writeUnlock(); + } + } + + public void removeCompactedFiles(Collection compactedFiles) { + writeLock(); + try { + storeFileManager.removeCompactedFiles(compactedFiles); + } finally { + writeUnlock(); + } } /** * Create the StoreEngine configured for the given Store. - * @param store The store. An unfortunate dependency needed due to it - * being passed to coprocessors via the compactor. + * @param store The store. An unfortunate dependency needed due to it being passed to coprocessors + * via the compactor. * @param conf Store configuration. * @param cellComparator CellComparator for storeFileManager. * @return StoreEngine to use. */ - public static StoreEngine create( - HStore store, Configuration conf, CellComparator cellComparator) throws IOException { + public static StoreEngine create(HStore store, Configuration conf, + CellComparator cellComparator) throws IOException { String className = conf.get(STORE_ENGINE_CLASS_KEY, DEFAULT_STORE_ENGINE_CLASS.getName()); try { - StoreEngine se = ReflectionUtils.instantiateWithCustomCtor( - className, new Class[] { }, new Object[] { }); + StoreEngine se = + ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {}, new Object[] {}); se.createComponentsOnce(conf, store, cellComparator); return se; } catch (Exception e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java index 27127f3a6c64..a40b209c6ebb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.util.Collection; import java.util.Comparator; @@ -49,12 +50,16 @@ public interface StoreFileManager { * Loads the initial store files into empty StoreFileManager. * @param storeFiles The files to load. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void loadFiles(List storeFiles); /** * Adds new files, either for from MemStore flush or bulk insert, into the structure. * @param sfs New store files. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void insertNewFiles(Collection sfs); /** @@ -62,12 +67,16 @@ public interface StoreFileManager { * @param compactedFiles The input files for the compaction. * @param results The resulting files for the compaction. */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void addCompactionResults(Collection compactedFiles, Collection results); /** * Remove the compacted files * @param compactedFiles the list of compacted files */ + @RestrictedApi(explanation = "Should only be called in StoreEngine", link = "", + allowedOnPath = ".*(/org/apache/hadoop/hbase/regionserver/StoreEngine.java|/src/test/.*)") void removeCompactedFiles(Collection compactedFiles); /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java index 1064b6c70547..58031288f751 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFlusher.java @@ -70,10 +70,17 @@ protected void finalizeWriter(StoreFileWriter writer, long cacheFlushSeqNum, writer.close(); } + protected final StoreFileWriter createWriter(MemStoreSnapshot snapshot, boolean alwaysIncludesTag) + throws IOException { + return store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(snapshot.getCellsCount()) + .compression(store.getColumnFamilyDescriptor().getCompressionType()).isCompaction(false) + .includeMVCCReadpoint(true).includesTag(alwaysIncludesTag || snapshot.isTagsPresent()) + .shouldDropBehind(false)); + } /** * Creates the scanner for flushing snapshot. Also calls coprocessors. - * @param snapshotScanners * @return The scanner; null if coprocessor is canceling the flush. */ protected final InternalScanner createScanner(List snapshotScanners, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java index 454b244fb512..10a9330f8326 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java @@ -20,10 +20,13 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; - +import java.util.function.Predicate; +import java.util.function.ToLongFunction; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; @@ -42,10 +45,13 @@ * Utility functions for region server storage layer. */ @InterfaceAudience.Private -public class StoreUtils { +public final class StoreUtils { private static final Logger LOG = LoggerFactory.getLogger(StoreUtils.class); + private StoreUtils() { + } + /** * Creates a deterministic hash code for store file collection. */ @@ -171,4 +177,31 @@ public static Configuration createStoreConfiguration(Configuration conf, TableDe return new CompoundConfiguration().add(conf).addBytesMap(td.getValues()) .addStringMap(cfd.getConfiguration()).addBytesMap(cfd.getValues()); } + + public static List toStoreFileInfo(Collection storefiles) { + return storefiles.stream().map(HStoreFile::getFileInfo).collect(Collectors.toList()); + } + + public static long getTotalUncompressedBytes(List files) { + return files.stream() + .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::getTotalUncompressedBytes)) + .sum(); + } + + public static long getStorefilesSize(Collection files, + Predicate predicate) { + return files.stream().filter(predicate) + .mapToLong(file -> getStorefileFieldSize(file, StoreFileReader::length)).sum(); + } + + public static long getStorefileFieldSize(HStoreFile file, ToLongFunction f) { + if (file == null) { + return 0L; + } + StoreFileReader reader = file.getReader(); + if (reader == null) { + return 0L; + } + return f.applyAsLong(reader); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java index 14863a69a9b1..bfb3f649ff27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java @@ -20,20 +20,19 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.StripeCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java index 1560aef5f6b3..f8183b7645a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java @@ -70,7 +70,7 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushSeqNum StripeMultiFileWriter mw = null; try { mw = req.createWriter(); // Writer according to the policy. - StripeMultiFileWriter.WriterFactory factory = createWriterFactory(cellsCount); + StripeMultiFileWriter.WriterFactory factory = createWriterFactory(snapshot); StoreScanner storeScanner = (scanner instanceof StoreScanner) ? (StoreScanner)scanner : null; mw.init(storeScanner, factory); @@ -98,13 +98,12 @@ public List flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushSeqNum return result; } - private StripeMultiFileWriter.WriterFactory createWriterFactory(final long kvCount) { + private StripeMultiFileWriter.WriterFactory createWriterFactory(MemStoreSnapshot snapshot) { return new StripeMultiFileWriter.WriterFactory() { @Override public StoreFileWriter createWriter() throws IOException { - StoreFileWriter writer = store.createWriterInTmp(kvCount, - store.getColumnFamilyDescriptor().getCompressionType(), false, true, true, false); - return writer; + // XXX: it used to always pass true for includesTag, re-consider? + return StripeStoreFlusher.this.createWriter(snapshot, true); } }; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java index 42841bfee531..533be176e7a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/AbstractMultiOutputCompactor.java @@ -51,13 +51,14 @@ protected void initMultiWriter(AbstractMultiFileWriter writer, InternalScanner s WriterFactory writerFactory = new WriterFactory() { @Override public StoreFileWriter createWriter() throws IOException { - return createTmpWriter(fd, shouldDropBehind, major); + return AbstractMultiOutputCompactor.this.createWriter(fd, shouldDropBehind, major); } @Override public StoreFileWriter createWriterWithStoragePolicy(String fileStoragePolicy) - throws IOException { - return createTmpWriter(fd, shouldDropBehind, fileStoragePolicy, major); + throws IOException { + return AbstractMultiOutputCompactor.this.createWriter(fd, shouldDropBehind, + fileStoragePolicy, major); } }; // Prepare multi-writer, and perform the compaction using scanner and writer. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index e524f7dfd5fd..47ef0f290251 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -28,7 +28,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -39,6 +38,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.regionserver.CellSink; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -61,6 +61,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.io.Closeables; /** @@ -261,29 +262,32 @@ public InternalScanner createScanner(ScanInfo scanInfo, List s } }; + protected final CreateStoreFileWriterParams createParams(FileDetails fd, boolean shouldDropBehind, + boolean major) { + return CreateStoreFileWriterParams.create().maxKeyCount(fd.maxKeyCount) + .compression(major ? majorCompactionCompression : minorCompactionCompression) + .isCompaction(true).includeMVCCReadpoint(fd.maxMVCCReadpoint > 0) + .includesTag(fd.maxTagsLength > 0).shouldDropBehind(shouldDropBehind) + .totalCompactedFilesSize(fd.totalCompactedFilesSize); + } + /** - * Creates a writer for a new file in a temporary directory. + * Creates a writer for a new file. * @param fd The file details. - * @return Writer for a new StoreFile in the tmp dir. + * @return Writer for a new StoreFile * @throws IOException if creation failed */ - protected final StoreFileWriter createTmpWriter(FileDetails fd, boolean shouldDropBehind, boolean major) - throws IOException { + protected final StoreFileWriter createWriter(FileDetails fd, boolean shouldDropBehind, + boolean major) throws IOException { // When all MVCC readpoints are 0, don't write them. // See HBASE-8166, HBASE-12600, and HBASE-13389. - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, fd.maxMVCCReadpoint > 0, - fd.maxTagsLength > 0, shouldDropBehind, fd.totalCompactedFilesSize, - HConstants.EMPTY_STRING); + return store.getStoreEngine().createWriter(createParams(fd, shouldDropBehind, major)); } - protected final StoreFileWriter createTmpWriter(FileDetails fd, boolean shouldDropBehind, - String fileStoragePolicy, boolean major) throws IOException { - return store.createWriterInTmp(fd.maxKeyCount, - major ? majorCompactionCompression : minorCompactionCompression, - true, fd.maxMVCCReadpoint > 0, - fd.maxTagsLength > 0, shouldDropBehind, fd.totalCompactedFilesSize, fileStoragePolicy); + protected final StoreFileWriter createWriter(FileDetails fd, boolean shouldDropBehind, + String fileStoragePolicy, boolean major) throws IOException { + return store.getStoreEngine() + .createWriter(createParams(fd, shouldDropBehind, major).fileStoragePolicy(fileStoragePolicy)); } private ScanInfo preCompactScannerOpen(CompactionRequestImpl request, ScanType scanType, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java index 49d3e8ee01e7..afa2429cb6e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java @@ -45,14 +45,14 @@ public DefaultCompactor(Configuration conf, HStore store) { } private final CellSinkFactory writerFactory = - new CellSinkFactory() { - @Override - public StoreFileWriter createWriter(InternalScanner scanner, - org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, - boolean shouldDropBehind, boolean major) throws IOException { - return createTmpWriter(fd, shouldDropBehind, major); - } - }; + new CellSinkFactory() { + @Override + public StoreFileWriter createWriter(InternalScanner scanner, + org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd, + boolean shouldDropBehind, boolean major) throws IOException { + return DefaultCompactor.this.createWriter(fd, shouldDropBehind, major); + } + }; /** * Do a minor/major compaction on an explicit set of storefiles from a Store. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java new file mode 100644 index 000000000000..d4c9a868eb48 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * The default implementation for store file tracker, where we do not persist the store file list, + * and use listing when loading store files. + */ +@InterfaceAudience.Private +class DefaultStoreFileTracker extends StoreFileTrackerBase { + + public DefaultStoreFileTracker(Configuration conf, TableName tableName, boolean isPrimaryReplica, + StoreContext ctx) { + super(conf, tableName, isPrimaryReplica, ctx); + } + + @Override + public List load() throws IOException { + return ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + } + + @Override + public boolean requireWritingToTmpDirFirst() { + return true; + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + // NOOP + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + // NOOP + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java new file mode 100644 index 000000000000..aadedc8ef727 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * An interface to define how we track the store files for a give store. + *

+ * In the old time, we will write store to a tmp directory first, and then rename it to the actual + * data file. And once a store file is under data directory, we will consider it as 'committed'. And + * we need to do listing when loading store files. + *

+ * When cloud age is coming, now we want to store the store files on object storage, where rename + * and list are not as cheap as on HDFS, especially rename. Although introducing a metadata + * management layer for object storage could solve the problem, but we still want HBase to run on + * pure object storage, so here we introduce this interface to abstract how we track the store + * files. For the old implementation, we just persist nothing here, and do listing to load store + * files. When running on object storage, we could persist the store file list in a system region, + * or in a file on the object storage, to make it possible to write directly into the data directory + * to avoid renaming, and also avoid listing when loading store files. + *

+ * The implementation requires to be thread safe as flush and compaction may occur as the same time, + * and we could also do multiple compactions at the same time. As the implementation may choose to + * persist the store file list to external storage, which could be slow, it is the duty for the + * callers to not call it inside a lock which may block normal read/write requests. + */ +@InterfaceAudience.Private +public interface StoreFileTracker { + + /** + * Load the store files list when opening a region. + */ + List load() throws IOException; + + /** + * Add new store files. + *

+ * Used for flush and bulk load. + */ + void add(Collection newFiles) throws IOException; + + /** + * Add new store files and remove compacted store files after compaction. + */ + void replace(Collection compactedFiles, Collection newFiles) + throws IOException; + + /** + * Create a writer for writing new store files. + * @return Writer for a new StoreFile + */ + StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java new file mode 100644 index 000000000000..2451f45bdc7b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for all store file tracker. + *

+ * Mainly used to place the common logic to skip persistent for secondary replicas. + */ +@InterfaceAudience.Private +abstract class StoreFileTrackerBase implements StoreFileTracker { + + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerBase.class); + + protected final Configuration conf; + + protected final TableName tableName; + + protected final boolean isPrimaryReplica; + + protected final StoreContext ctx; + + private volatile boolean cacheOnWriteLogged; + + protected StoreFileTrackerBase(Configuration conf, TableName tableName, boolean isPrimaryReplica, + StoreContext ctx) { + this.conf = conf; + this.tableName = tableName; + this.isPrimaryReplica = isPrimaryReplica; + this.ctx = ctx; + } + + @Override + public final void add(Collection newFiles) throws IOException { + if (isPrimaryReplica) { + doAddNewStoreFiles(newFiles); + } + } + + @Override + public final void replace(Collection compactedFiles, + Collection newFiles) throws IOException { + if (isPrimaryReplica) { + doAddCompactionResults(compactedFiles, newFiles); + } + } + + private HFileContext createFileContext(Compression.Algorithm compression, + boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { + if (compression == null) { + compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; + } + ColumnFamilyDescriptor family = ctx.getFamily(); + HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(includeMVCCReadpoint) + .withIncludesTags(includesTag).withCompression(compression) + .withCompressTags(family.isCompressTags()).withChecksumType(StoreUtils.getChecksumType(conf)) + .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)) + .withBlockSize(family.getBlocksize()).withHBaseCheckSum(true) + .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(encryptionContext) + .withCreateTime(EnvironmentEdgeManager.currentTime()).withColumnFamily(family.getName()) + .withTableName(tableName.getName()).withCellComparator(ctx.getComparator()).build(); + return hFileContext; + } + + @Override + public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) + throws IOException { + if (!isPrimaryReplica) { + throw new IllegalStateException("Should not call create writer on secondary replicas"); + } + // creating new cache config for each new writer + final CacheConfig cacheConf = ctx.getCacheConf(); + final CacheConfig writerCacheConf = new CacheConfig(cacheConf); + long totalCompactedFilesSize = params.totalCompactedFilesSize(); + if (params.isCompaction()) { + // Don't cache data on write on compactions, unless specifically configured to do so + // Cache only when total file size remains lower than configured threshold + final boolean cacheCompactedBlocksOnWrite = cacheConf.shouldCacheCompactedBlocksOnWrite(); + // if data blocks are to be cached on write + // during compaction, we should forcefully + // cache index and bloom blocks as well + if (cacheCompactedBlocksOnWrite && + totalCompactedFilesSize <= cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { + writerCacheConf.enableCacheOnWrite(); + if (!cacheOnWriteLogged) { + LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " + + "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this); + cacheOnWriteLogged = true; + } + } else { + writerCacheConf.setCacheDataOnWrite(false); + if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) { + // checking condition once again for logging + LOG.debug( + "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted " + + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}", + this, totalCompactedFilesSize, cacheConf.getCacheCompactedBlocksOnWriteThreshold()); + } + } + } else { + final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite(); + if (shouldCacheDataOnWrite) { + writerCacheConf.enableCacheOnWrite(); + if (!cacheOnWriteLogged) { + LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " + + "Index blocks and Bloom filter blocks", this); + cacheOnWriteLogged = true; + } + } + } + Encryption.Context encryptionContext = ctx.getEncryptionContext(); + HFileContext hFileContext = createFileContext(params.compression(), + params.includeMVCCReadpoint(), params.includesTag(), encryptionContext); + Path outputDir; + if (requireWritingToTmpDirFirst()) { + outputDir = + new Path(ctx.getRegionFileSystem().getTempDir(), ctx.getFamily().getNameAsString()); + } else { + throw new UnsupportedOperationException("not supported yet"); + } + StoreFileWriter.Builder builder = + new StoreFileWriter.Builder(conf, writerCacheConf, ctx.getRegionFileSystem().getFileSystem()) + .withOutputDir(outputDir).withBloomType(ctx.getBloomFilterType()) + .withMaxKeyCount(params.maxKeyCount()).withFavoredNodes(ctx.getFavoredNodes()) + .withFileContext(hFileContext).withShouldDropCacheBehind(params.shouldDropBehind()) + .withCompactedFilesSupplier(ctx.getCompactedFilesSupplier()) + .withFileStoragePolicy(params.fileStoragePolicy()); + return builder.build(); + } + + /** + * Whether the implementation of this tracker requires you to write to temp directory first, i.e, + * does not allow broken store files under the actual data directory. + */ + protected abstract boolean requireWritingToTmpDirFirst(); + + protected abstract void doAddNewStoreFiles(Collection newFiles) throws IOException; + + protected abstract void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java new file mode 100644 index 000000000000..4f7231bc3b9a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Factory method for creating store file tracker. + */ +@InterfaceAudience.Private +public final class StoreFileTrackerFactory { + + public static StoreFileTracker create(Configuration conf, TableName tableName, + boolean isPrimaryReplica, StoreContext ctx) { + return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java index 22ec6cb89ec4..291b909f69bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java @@ -105,7 +105,6 @@ Set getStoresRequiringCompaction(Set requestedStores, long times boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts) throws IOException { - // do we have any store files? Collection storeFiles = fileSystem.getStoreFiles(family); if (storeFiles == null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index 3c2bc3f0cb12..9314d7e9827f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -211,11 +211,13 @@ protected BlockCompactionsInCompletionHStore(HRegion region, ColumnFamilyDescrip @Override protected void refreshStoreSizeAndTotalBytes() throws IOException { - try { - r.compactionsWaiting.countDown(); - r.compactionsBlocked.await(); - } catch (InterruptedException ex) { - throw new IOException(ex); + if (r != null) { + try { + r.compactionsWaiting.countDown(); + r.compactionsBlocked.await(); + } catch (InterruptedException ex) { + throw new IOException(ex); + } } super.refreshStoreSizeAndTotalBytes(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index cd83dc8c2494..0a6b37bbba4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -215,8 +215,10 @@ public void tearDown() throws IOException { @Test public void testCacheOnWriteInSchema() throws IOException { // Write some random data into the store - StoreFileWriter writer = store.createWriterInTmp(Integer.MAX_VALUE, - HFile.DEFAULT_COMPRESSION_ALGORITHM, false, true, false, false); + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(Integer.MAX_VALUE) + .compression(HFile.DEFAULT_COMPRESSION_ALGORITHM).isCompaction(false) + .includeMVCCReadpoint(true).includesTag(false).shouldDropBehind(false)); writeStoreFile(writer); writer.close(); // Verify the block types of interest were cached on write diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java index e832c47aac81..3784876a59f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java @@ -65,9 +65,12 @@ public void testCustomParts() throws Exception { DummyCompactionPolicy.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, DummyStoreFlusher.class.getName()); + HRegion mockRegion = Mockito.mock(HRegion.class); HStore mockStore = Mockito.mock(HStore.class); Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); - StoreEngine se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); + Mockito.when(mockStore.getHRegion()).thenReturn(mockRegion); + StoreEngine se = + StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); Assert.assertTrue(se instanceof DefaultStoreEngine); Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy); Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index a5584ff32599..7fb57686c35d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -5744,7 +5744,7 @@ public void testCompactionFromPrimary() throws IOException { Collection storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); Collection storeFileInfos = primaryRegion.getRegionFileSystem() - .getStoreFiles(families[0]); + .getStoreFiles(Bytes.toString(families[0])); Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); verifyData(secondaryRegion, 0, 1000, cq, families); @@ -7648,7 +7648,7 @@ protected List doCompaction(CompactionRequestImpl cr, getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true; for (Path newFile : newFiles) { // Create storefile around what we wrote with a reader on it. - HStoreFile sf = createStoreFileAndReader(newFile); + HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); sf.closeStoreFile(evictOnClose); sfs.add(sf); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index b64f52192740..8e4c4a385de3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -310,7 +310,7 @@ public Object run() throws Exception { /** * Verify that compression and data block encoding are respected by the - * Store.createWriterInTmp() method, used on store flush. + * createWriter method, used on store flush. */ @Test public void testCreateWriter() throws Exception { @@ -322,9 +322,11 @@ public void testCreateWriter() throws Exception { .build(); init(name.getMethodName(), conf, hcd); - // Test createWriterInTmp() - StoreFileWriter writer = - store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false); + // Test createWriter + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(4) + .compression(hcd.getCompressionType()).isCompaction(false).includeMVCCReadpoint(true) + .includesTag(false).shouldDropBehind(false)); Path path = writer.getPath(); writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); @@ -1021,19 +1023,19 @@ public void testRefreshStoreFilesNotChanged() throws IOException { // add one more file addStoreFile(); - HStore spiedStore = spy(store); + StoreEngine spiedStoreEngine = spy(store.getStoreEngine()); // call first time after files changed - spiedStore.refreshStoreFiles(); + spiedStoreEngine.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); - verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); + verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any()); // call second time - spiedStore.refreshStoreFiles(); + spiedStoreEngine.refreshStoreFiles(); // ensure that replaceStoreFiles is not called, i.e, the times does not change, if files are not // refreshed, - verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); + verify(spiedStoreEngine, times(1)).replaceStoreFiles(any(), any()); } private long countMemStoreScanner(StoreScanner scanner) { @@ -1644,7 +1646,7 @@ public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger() throws // Do compaction MyThread thread = new MyThread(storeScanner); thread.start(); - store.replaceStoreFiles(actualStorefiles, actualStorefiles1); + store.replaceStoreFiles(actualStorefiles, actualStorefiles1, false); thread.join(); KeyValueHeap heap2 = thread.getHeap(); assertFalse(heap.equals(heap2)); @@ -1710,8 +1712,10 @@ public void testSpaceQuotaChangeAfterReplacement() throws IOException { @Test public void testHFileContextSetWithCFAndTable() throws Exception { init(this.name.getMethodName()); - StoreFileWriter writer = store.createWriterInTmp(10000L, - Compression.Algorithm.NONE, false, true, false, true); + StoreFileWriter writer = store.getStoreEngine() + .createWriter(CreateStoreFileWriterParams.create().maxKeyCount(10000L) + .compression(Compression.Algorithm.NONE).isCompaction(true).includeMVCCReadpoint(true) + .includesTag(false).shouldDropBehind(true)); HFileContext hFileContext = writer.getHFileWriter().getFileContext(); assertArrayEquals(family, hFileContext.getColumnFamily()); assertArrayEquals(table, hFileContext.getTableName()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 60f65be66c47..1dbf37aa8063 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -245,7 +245,7 @@ public void testCleanMergeReference() throws Exception { TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo); int count = 0; for(ColumnFamilyDescriptor colFamily : columnFamilies) { - count += hrfs.getStoreFiles(colFamily.getName()).size(); + count += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } ADMIN.compactRegion(mergedRegionInfo.getRegionName()); // clean up the merged region store files @@ -254,7 +254,7 @@ public void testCleanMergeReference() throws Exception { int newcount = 0; while (EnvironmentEdgeManager.currentTime() < timeout) { for(ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount += hrfs.getStoreFiles(colFamily.getName()).size(); + newcount += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } if(newcount > count) { break; @@ -273,7 +273,7 @@ public void testCleanMergeReference() throws Exception { while (EnvironmentEdgeManager.currentTime() < timeout) { int newcount1 = 0; for(ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount1 += hrfs.getStoreFiles(colFamily.getName()).size(); + newcount1 += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); } if(newcount1 <= 1) { break; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java index 9141327d26d4..2fab050446ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -98,7 +97,7 @@ static class FailingHRegionFileSystem extends HRegionFileSystem { } @Override - public Collection getStoreFiles(String familyName) throws IOException { + public List getStoreFiles(String familyName) throws IOException { if (fail) { throw new IOException("simulating FS failure"); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java index 4cb023f9b9d6..121f8301751f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java @@ -29,7 +29,6 @@ import java.util.Random; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -128,13 +127,12 @@ public void testScannerCloseAndUpdateReadersWithMemstoreScanner() throws Excepti p.addColumn(fam, Bytes.toBytes("q1"), Bytes.toBytes("val")); region.put(p); HStore store = region.getStore(fam); - ReentrantReadWriteLock lock = store.lock; // use the lock to manually get a new memstore scanner. this is what // HStore#notifyChangedReadersObservers does under the lock.(lock is not needed here //since it is just a testcase). - lock.readLock().lock(); + store.getStoreEngine().readLock(); final List memScanners = store.memstore.getScanners(Long.MAX_VALUE); - lock.readLock().unlock(); + store.getStoreEngine().readUnlock(); Thread closeThread = new Thread() { public void run() { // close should be completed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index f5330f6faa42..eb0b1c1ca694 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -118,8 +118,10 @@ private static HStoreFile createFile() throws Exception { } private static TestStoreEngine createEngine(Configuration conf) throws Exception { + HRegion region = mock(HRegion.class); HStore store = mock(HStore.class); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); + when(store.getHRegion()).thenReturn(region); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); return (TestStoreEngine) StoreEngine.create(store, conf, kvComparator); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java index d7b7ba760156..0ea82c57be1a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestDateTieredCompactor.java @@ -22,9 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,11 +42,13 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.Scanner; @@ -110,10 +109,9 @@ private DateTieredCompactor createCompactor(StoreFileWritersCapture writers, when(store.areWritesEnabled()).thenReturn(true); when(store.getFileSystem()).thenReturn(mock(FileSystem.class)); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.newBuilder(TABLE_NAME).build()); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); when(store.getComparator()).thenReturn(CellComparatorImpl.COMPARATOR); OptionalLong maxSequenceId = StoreUtils.getMaxSequenceIdInList(storefiles); when(store.getMaxSequenceId()).thenReturn(maxSequenceId); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java index 2ba15d1be18e..bdab20ead8a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java @@ -30,7 +30,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; @@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StripeMultiFileWriter; @@ -864,12 +865,9 @@ private StripeCompactor createCompactor() throws Exception { when(info.getRegionNameAsString()).thenReturn("testRegion"); when(store.getColumnFamilyDescriptor()).thenReturn(familyDescriptor); when(store.getRegionInfo()).thenReturn(info); - when( - store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when( - store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); Configuration conf = HBaseConfiguration.create(); conf.setBoolean("hbase.regionserver.compaction.private.readers", usePrivateReaders); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java index e49174e6afe3..ae59c74bad95 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactor.java @@ -21,9 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.createDummyRequest; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,10 +40,12 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.Scanner; import org.apache.hadoop.hbase.regionserver.compactions.TestCompactor.StoreFileWritersCapture; @@ -209,10 +208,9 @@ private StripeCompactor createCompactor(StoreFileWritersCapture writers, KeyValu when(store.areWritesEnabled()).thenReturn(true); when(store.getFileSystem()).thenReturn(mock(FileSystem.class)); when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.newBuilder(TABLE_NAME).build()); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean())).thenAnswer(writers); - when(store.createWriterInTmp(anyLong(), any(), anyBoolean(), - anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyString())).thenAnswer(writers); + StoreEngine storeEngine = mock(StoreEngine.class); + when(storeEngine.createWriter(any(CreateStoreFileWriterParams.class))).thenAnswer(writers); + when(store.getStoreEngine()).thenReturn(storeEngine); when(store.getComparator()).thenReturn(CellComparatorImpl.COMPARATOR); return new StripeCompactor(conf, store) { From cab6140e471e97cdb0bdc8a62b09dc0e1509c7af Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 26 Aug 2021 18:51:12 +0800 Subject: [PATCH 27/37] HBASE-25988 Store the store file list by a file (#3578) Signed-off-by: Wellington Chevreuil --- .../server/region/StoreFileTracker.proto | 36 +++++ .../hbase/regionserver/StoreContext.java | 5 + .../hbase/regionserver/StoreEngine.java | 8 +- .../DefaultStoreFileTracker.java | 5 +- .../FileBasedStoreFileTracker.java | 142 ++++++++++++++++++ .../storefiletracker/StoreFileListFile.java | 142 ++++++++++++++++++ .../StoreFileTrackerBase.java | 11 +- .../StoreFileTrackerFactory.java | 12 +- ...stRegionWithFileBasedStoreFileTracker.java | 109 ++++++++++++++ 9 files changed, 451 insertions(+), 19 deletions(-) create mode 100644 hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java diff --git a/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto new file mode 100644 index 000000000000..2a269ea4ac4e --- /dev/null +++ b/hbase-protocol-shaded/src/main/protobuf/server/region/StoreFileTracker.proto @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; +// This file contains protocol buffers that are used for store file tracker. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "StoreFileTrackerProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message StoreFileEntry { + required string name = 1; + required uint64 size = 2; +} + +message StoreFileList { + required uint64 timestamp = 1; + repeated StoreFileEntry store_file = 2; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java index 2a9f96859361..588f8f4027a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreContext.java @@ -22,6 +22,7 @@ import java.util.function.Supplier; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.io.HeapSize; @@ -109,6 +110,10 @@ public RegionCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + public TableName getTableName() { + return getRegionInfo().getTable(); + } + public RegionInfo getRegionInfo() { return regionFileSystem.getRegionInfo(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index f9d6c294f779..61b7dfa66848 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -172,9 +172,9 @@ public StoreFlusher getStoreFlusher() { return this.storeFlusher; } - private StoreFileTracker createStoreFileTracker(HStore store) { - return StoreFileTrackerFactory.create(store.conf, store.getRegionInfo().getTable(), - store.isPrimaryReplicaStore(), store.getStoreContext()); + private StoreFileTracker createStoreFileTracker(Configuration conf, HStore store) { + return StoreFileTrackerFactory.create(conf, store.isPrimaryReplicaStore(), + store.getStoreContext()); } /** @@ -205,7 +205,7 @@ protected final void createComponentsOnce(Configuration conf, HStore store, this.ctx = store.getStoreContext(); this.coprocessorHost = store.getHRegion().getCoprocessorHost(); this.openStoreFileThreadPoolCreator = store.getHRegion()::getStoreFileOpenAndCloseThreadPool; - this.storeFileTracker = createStoreFileTracker(store); + this.storeFileTracker = createStoreFileTracker(conf, store); assert compactor != null && compactionPolicy != null && storeFileManager != null && storeFlusher != null && storeFileTracker != null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index d4c9a868eb48..fa044818336f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -21,7 +21,6 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -33,9 +32,9 @@ @InterfaceAudience.Private class DefaultStoreFileTracker extends StoreFileTrackerBase { - public DefaultStoreFileTracker(Configuration conf, TableName tableName, boolean isPrimaryReplica, + public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - super(conf, tableName, isPrimaryReplica, ctx); + super(conf, isPrimaryReplica, ctx); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java new file mode 100644 index 000000000000..de28b0eb9996 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry; +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; + +/** + * A file based store file tracker. + *

+ * For this tracking way, the store file list will be persistent into a file, so we can write the + * new store files directly to the final data directory, as we will not load the broken files. This + * will greatly reduce the time for flush and compaction on some object storages as a rename is + * actual a copy on them. And it also avoid listing when loading store file list, which could also + * speed up the loading of store files as listing is also not a fast operation on most object + * storages. + */ +@InterfaceAudience.Private +public class FileBasedStoreFileTracker extends StoreFileTrackerBase { + + private final StoreFileListFile backedFile; + + private final Map storefiles = new HashMap<>(); + + public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + backedFile = new StoreFileListFile(ctx); + } + + @Override + public List load() throws IOException { + StoreFileList list = backedFile.load(); + if (list == null) { + return Collections.emptyList(); + } + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + List infos = new ArrayList<>(); + for (StoreFileEntry entry : list.getStoreFileList()) { + infos.add(ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, ctx.getRegionInfo(), + ctx.getRegionFileSystem().getRegionInfoForFS(), ctx.getFamily().getNameAsString(), + new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()))); + } + // In general, for primary replica, the load method should only be called once when + // initialization, so we do not need synchronized here. And for secondary replicas, though the + // load method could be called multiple times, we will never call other methods so no + // synchronized is also fine. + // But we have a refreshStoreFiles method in the Region interface, which can be called by CPs, + // and we have a RefreshHFilesEndpoint example to expose the refreshStoreFiles method as RPC, so + // for safety, let's still keep the synchronized here. + synchronized (storefiles) { + for (StoreFileInfo info : infos) { + storefiles.put(info.getPath().getName(), info); + } + } + return infos; + } + + @Override + protected boolean requireWritingToTmpDirFirst() { + return false; + } + + private StoreFileEntry toStoreFileEntry(StoreFileInfo info) { + return StoreFileEntry.newBuilder().setName(info.getPath().getName()).setSize(info.getSize()) + .build(); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + synchronized (storefiles) { + StoreFileList.Builder builder = StoreFileList.newBuilder(); + for (StoreFileInfo info : storefiles.values()) { + builder.addStoreFile(toStoreFileEntry(info)); + } + for (StoreFileInfo info : newFiles) { + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + for (StoreFileInfo info : newFiles) { + storefiles.put(info.getPath().getName(), info); + } + } + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + Set compactedFileNames = + compactedFiles.stream().map(info -> info.getPath().getName()).collect(Collectors.toSet()); + synchronized (storefiles) { + StoreFileList.Builder builder = StoreFileList.newBuilder(); + storefiles.forEach((name, info) -> { + if (compactedFileNames.contains(name)) { + return; + } + builder.addStoreFile(toStoreFileEntry(info)); + }); + for (StoreFileInfo info : newFiles) { + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + for (String name : compactedFileNames) { + storefiles.remove(name); + } + for (StoreFileInfo info : newFiles) { + storefiles.put(info.getPath().getName(), info); + } + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java new file mode 100644 index 000000000000..c778bfc51deb --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.FileNotFoundException; +import java.io.IOException; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; +import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; + +/** + * To fully avoid listing, here we use two files for tracking. When loading, we will try to read + * both the two files, if only one exists, we will trust this one, if both exist, we will compare + * the timestamp to see which one is newer and trust that one. And we will record in memory that + * which one is trusted by us, and when we need to update the store file list, we will write to the + * other file. + *

+ * So in this way, we could avoid listing when we want to load the store file list file. + */ +@InterfaceAudience.Private +class StoreFileListFile { + + private static final Logger LOG = LoggerFactory.getLogger(StoreFileListFile.class); + + private static final String TRACK_FILE_DIR = ".filelist"; + + private static final String TRACK_FILE = "f1"; + + private static final String TRACK_FILE_ROTATE = "f2"; + + private final StoreContext ctx; + + private final Path trackFileDir; + + private final Path[] trackFiles = new Path[2]; + + // this is used to make sure that we do not go backwards + private long prevTimestamp = -1; + + private int nextTrackFile = -1; + + StoreFileListFile(StoreContext ctx) { + this.ctx = ctx; + trackFileDir = new Path(ctx.getFamilyStoreDirectoryPath(), TRACK_FILE_DIR); + trackFiles[0] = new Path(trackFileDir, TRACK_FILE); + trackFiles[1] = new Path(trackFileDir, TRACK_FILE_ROTATE); + } + + private StoreFileList load(Path path) throws IOException { + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + byte[] bytes; + try (FSDataInputStream in = fs.open(path)) { + bytes = ByteStreams.toByteArray(in); + } + // Read all the bytes and then parse it, so we will only throw InvalidProtocolBufferException + // here. This is very important for upper layer to determine whether this is the normal case, + // where the file does not exist or is incomplete. If there is another type of exception, the + // upper layer should throw it out instead of just ignoring it, otherwise it will lead to data + // loss. + return StoreFileList.parseFrom(bytes); + } + + private int select(StoreFileList[] lists) { + if (lists[0] == null) { + return 1; + } + if (lists[1] == null) { + return 0; + } + return lists[0].getTimestamp() >= lists[1].getTimestamp() ? 0 : 1; + } + + StoreFileList load() throws IOException { + StoreFileList[] lists = new StoreFileList[2]; + for (int i = 0; i < 2; i++) { + try { + lists[i] = load(trackFiles[i]); + } catch (FileNotFoundException | InvalidProtocolBufferException e) { + // this is normal case, so use info and do not log stacktrace + LOG.info("Failed to load track file {}: {}", trackFiles[i], e); + } + } + int winnerIndex = select(lists); + if (lists[winnerIndex] != null) { + nextTrackFile = 1 - winnerIndex; + prevTimestamp = lists[winnerIndex].getTimestamp(); + } else { + nextTrackFile = 0; + } + return lists[winnerIndex]; + } + + /** + * We will set the timestamp in this method so just pass the builder in + */ + void update(StoreFileList.Builder builder) throws IOException { + Preconditions.checkState(nextTrackFile >= 0, "should call load first before calling update"); + FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); + long timestamp = Math.max(prevTimestamp + 1, EnvironmentEdgeManager.currentTime()); + try (FSDataOutputStream out = fs.create(trackFiles[nextTrackFile], true)) { + builder.setTimestamp(timestamp).build().writeTo(out); + } + // record timestamp + prevTimestamp = timestamp; + // rotate the file + nextTrackFile = 1 - nextTrackFile; + try { + fs.delete(trackFiles[nextTrackFile], false); + } catch (IOException e) { + // we will create new file with overwrite = true, so not a big deal here, only for speed up + // loading as we do not need to read this file when loading(we will hit FileNotFoundException) + LOG.debug("failed to delete old track file {}, not a big deal, just ignore", e); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 2451f45bdc7b..92c699278c2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -21,7 +21,6 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -51,18 +50,14 @@ abstract class StoreFileTrackerBase implements StoreFileTracker { protected final Configuration conf; - protected final TableName tableName; - protected final boolean isPrimaryReplica; protected final StoreContext ctx; private volatile boolean cacheOnWriteLogged; - protected StoreFileTrackerBase(Configuration conf, TableName tableName, boolean isPrimaryReplica, - StoreContext ctx) { + protected StoreFileTrackerBase(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { this.conf = conf; - this.tableName = tableName; this.isPrimaryReplica = isPrimaryReplica; this.ctx = ctx; } @@ -95,7 +90,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, .withBlockSize(family.getBlocksize()).withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(encryptionContext) .withCreateTime(EnvironmentEdgeManager.currentTime()).withColumnFamily(family.getName()) - .withTableName(tableName.getName()).withCellComparator(ctx.getComparator()).build(); + .withTableName(ctx.getTableName().getName()).withCellComparator(ctx.getComparator()).build(); return hFileContext; } @@ -153,7 +148,7 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) outputDir = new Path(ctx.getRegionFileSystem().getTempDir(), ctx.getFamily().getNameAsString()); } else { - throw new UnsupportedOperationException("not supported yet"); + outputDir = ctx.getFamilyStoreDirectoryPath(); } StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf, ctx.getRegionFileSystem().getFileSystem()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 4f7231bc3b9a..6cdfaf4a0fe1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; /** @@ -28,8 +28,12 @@ @InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static StoreFileTracker create(Configuration conf, TableName tableName, - boolean isPrimaryReplica, StoreContext ctx) { - return new DefaultStoreFileTracker(conf, tableName, isPrimaryReplica, ctx); + public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; + + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, + StoreContext ctx) { + Class tracker = + conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java new file mode 100644 index 000000000000..3bc60d13fd94 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestRegionWithFileBasedStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionWithFileBasedStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final TableDescriptor TD = + TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build(); + + @Rule + public TestName name = new TestName(); + + private HRegion region; + + @Before + public void setUp() throws IOException { + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, FileBasedStoreFileTracker.class, + StoreFileTracker.class); + region = + HBaseTestingUtil.createRegionAndWAL(RI, UTIL.getDataTestDir(name.getMethodName()), conf, TD); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + HBaseTestingUtil.closeRegionAndWAL(region); + } + UTIL.cleanupTestDir(); + } + + @Test + public void testFlushAndCompaction() throws IOException { + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int v = i * 10 + j; + region.put(new Put(Bytes.toBytes(v)).addColumn(CF, CQ, Bytes.toBytes(v))); + } + region.flush(true); + if (i % 3 == 2) { + region.compact(true); + } + } + // reopen the region, make sure the store file tracker works, i.e, we can get all the records + // back + region.close(); + region = HRegion.openHRegion(region, null); + for (int i = 0; i < 100; i++) { + Result result = region.get(new Get(Bytes.toBytes(i))); + assertEquals(i, Bytes.toInt(result.getValue(CF, CQ))); + } + } +} From 708b7c143df74d3ffe21623295ca1f27cbc9b8ed Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 8 Sep 2021 10:31:49 +0100 Subject: [PATCH 28/37] HBASE-26079 Use StoreFileTracker when splitting and merging (#3617) Signed-off-by: Duo Zhang --- .../MergeTableRegionsProcedure.java | 23 +- .../assignment/SplitTableRegionProcedure.java | 41 +-- .../hbase/regionserver/HRegionFileSystem.java | 42 ++- .../DefaultStoreFileTracker.java | 4 +- .../storefiletracker/StoreFileTracker.java | 1 - .../StoreFileTrackerFactory.java | 33 ++- .../regionserver/TestDefaultStoreEngine.java | 1 + .../TestDirectStoreSplitsMerges.java | 32 ++- .../hbase/regionserver/TestHStoreFile.java | 19 +- .../TestMergesSplitsAddToTracker.java | 262 ++++++++++++++++++ .../regionserver/TestStripeStoreEngine.java | 1 + .../TestStoreFileTracker.java | 56 ++++ 12 files changed, 473 insertions(+), 42 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index fb57cb92dc2f..4b25a28a6055 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -24,6 +24,8 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -53,6 +55,8 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WALSplitUtil; @@ -584,40 +588,47 @@ private void createMergedRegion(final MasterProcedureEnv env) throws IOException final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); - + List mergedFiles = new ArrayList<>(); HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem( env.getMasterConfiguration(), fs, tableDir, mergedRegion); for (RegionInfo ri: this.regionsToMerge) { HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tableDir, ri, false); - mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion); + mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion)); } assert mergeRegionFs != null; - mergeRegionFs.commitMergedRegion(); + mergeRegionFs.commitMergedRegion(mergedFiles, env); // Prepare to create merged regions env.getAssignmentManager().getRegionStates(). getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); } - private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, + private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException { final TableDescriptor htd = env.getMasterServices().getTableDescriptors() .get(mergedRegion.getTable()); + List mergedFiles = new ArrayList<>(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); - final Collection storeFiles = regionFs.getStoreFiles(family); + Configuration trackerConfig = + StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, + family, regionFs); + final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { // Create reference file(s) to parent region file here in mergedDir. // As this procedure is running on master, use CacheConfig.DISABLED means // don't cache any block. - mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, + Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + mergedFiles.add(refFile); } } } + return mergedFiles; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 0a15e36a16af..4d53df4bc670 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -64,6 +64,8 @@ import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.RegionSplitRestriction; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -618,21 +620,20 @@ public void createDaughterRegions(final MasterProcedureEnv env) throws IOExcepti final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); - regionFs.createSplitsDir(daughterOneRI, daughterTwoRI); - Pair expectedReferences = splitStoreFiles(env, regionFs); + Pair, List> expectedReferences = splitStoreFiles(env, regionFs); - assertReferenceFileCount(fs, expectedReferences.getFirst(), + assertReferenceFileCount(fs, expectedReferences.getFirst().size(), regionFs.getSplitsDir(daughterOneRI)); - regionFs.commitDaughterRegion(daughterOneRI); - assertReferenceFileCount(fs, expectedReferences.getFirst(), + regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst(), env); + assertReferenceFileCount(fs, expectedReferences.getFirst().size(), new Path(tabledir, daughterOneRI.getEncodedName())); - assertReferenceFileCount(fs, expectedReferences.getSecond(), + assertReferenceFileCount(fs, expectedReferences.getSecond().size(), regionFs.getSplitsDir(daughterTwoRI)); - regionFs.commitDaughterRegion(daughterTwoRI); - assertReferenceFileCount(fs, expectedReferences.getSecond(), + regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond(), env); + assertReferenceFileCount(fs, expectedReferences.getSecond().size(), new Path(tabledir, daughterTwoRI.getEncodedName())); } @@ -649,7 +650,7 @@ private void deleteDaughterRegions(final MasterProcedureEnv env) throws IOExcept * Create Split directory * @param env MasterProcedureEnv */ - private Pair splitStoreFiles(final MasterProcedureEnv env, + private Pair, List> splitStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws IOException { final Configuration conf = env.getMasterConfiguration(); TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); @@ -665,7 +666,11 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, new HashMap>(htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); - Collection sfis = regionFs.getStoreFiles(family); + Configuration trackerConfig = StoreFileTrackerFactory. + mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, + family, regionFs); + Collection sfis = tracker.load(); if (sfis == null) { continue; } @@ -691,7 +696,7 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, } if (nbFiles == 0) { // no file needs to be splitted. - return new Pair(0, 0); + return new Pair<>(Collections.emptyList(), Collections.emptyList()); } // Max #threads is the smaller of the number of storefiles or the default max determined above. int maxThreads = Math.min( @@ -744,14 +749,18 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, throw (InterruptedIOException) new InterruptedIOException().initCause(e); } - int daughterA = 0; - int daughterB = 0; + List daughterA = new ArrayList<>(); + List daughterB = new ArrayList<>(); // Look for any exception for (Future> future : futures) { try { Pair p = future.get(); - daughterA += p.getFirst() != null ? 1 : 0; - daughterB += p.getSecond() != null ? 1 : 0; + if(p.getFirst() != null){ + daughterA.add(p.getFirst()); + } + if(p.getSecond() != null){ + daughterB.add(p.getSecond()); + } } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } catch (ExecutionException e) { @@ -764,7 +773,7 @@ private Pair splitStoreFiles(final MasterProcedureEnv env, getParentRegion().getShortNameToLog() + " Daughter A: " + daughterA + " storefiles, Daughter B: " + daughterB + " storefiles."); } - return new Pair(daughterA, daughterB); + return new Pair<>(daughterA, daughterB); } private void assertReferenceFileCount(final FileSystem fs, final int expectedReferenceFileCount, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 6ab9b0fc3fe8..3b053a349c2c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -23,7 +23,9 @@ import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -45,6 +47,9 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; @@ -592,19 +597,46 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} * @throws IOException */ - public Path commitDaughterRegion(final RegionInfo regionInfo) - throws IOException { + public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, + MasterProcedureEnv env) throws IOException { Path regionDir = this.getSplitsDir(regionInfo); if (fs.exists(regionDir)) { // Write HRI to a file in case we need to recover hbase:meta Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); + HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( + env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false); + insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); } - return regionDir; } + private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProcedureEnv env, + HRegionFileSystem regionFs) throws IOException { + TableDescriptor tblDesc = env.getMasterServices().getTableDescriptors(). + get(regionInfo.getTable()); + //we need to map trackers per store + Map trackerMap = new HashMap<>(); + //we need to map store files per store + Map> fileInfoMap = new HashMap<>(); + for(Path file : allFiles) { + String familyName = file.getParent().getName(); + trackerMap.computeIfAbsent(familyName, t -> { + Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, + tblDesc.getColumnFamily(Bytes.toBytes(familyName))); + return StoreFileTrackerFactory. + create(config, true, familyName, regionFs); + }); + fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); + List infos = fileInfoMap.get(familyName); + infos.add(new StoreFileInfo(conf, fs, file, true)); + } + for(Map.Entry entry : trackerMap.entrySet()) { + entry.getValue().add(fileInfoMap.get(entry.getKey())); + } + } + /** * Creates region split daughter directories under the table dir. If the daughter regions already * exist, for example, in the case of a recovery from a previous failed split procedure, this @@ -755,13 +787,15 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi * Commit a merged region, making it ready for use. * @throws IOException */ - public void commitMergedRegion() throws IOException { + public void commitMergedRegion(List allMergedFiles, MasterProcedureEnv env) + throws IOException { Path regionDir = getMergesDir(regionInfoForFs); if (regionDir != null && fs.exists(regionDir)) { // Write HRI to a file in case we need to recover hbase:meta Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); + insertRegionFilesIntoStoreTracker(allMergedFiles, env, this); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index fa044818336f..22e05132bf91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -32,8 +33,7 @@ @InterfaceAudience.Private class DefaultStoreFileTracker extends StoreFileTrackerBase { - public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index aadedc8ef727..0a85abb00a4e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -48,7 +48,6 @@ */ @InterfaceAudience.Private public interface StoreFileTracker { - /** * Load the store files list when opening a region. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 6cdfaf4a0fe1..c446d5ae9a31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,22 +18,51 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompoundConfiguration; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Factory method for creating store file tracker. */ @InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + StoreContext ctx) { Class tracker = conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, + HRegionFileSystem regionFs) { + ColumnFamilyDescriptorBuilder fDescBuilder = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); + StoreContext ctx = StoreContext.getBuilder(). + withColumnFamilyDescriptor(fDescBuilder.build()). + withRegionFileSystem(regionFs). + build(); + return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + } + + public static Configuration mergeConfigurations(Configuration global, + TableDescriptor table, ColumnFamilyDescriptor family) { + return new CompoundConfiguration() + .add(global) + .addBytesMap(table.getValues()) + .addStringMap(family.getConfiguration()) + .addBytesMap(family.getValues()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java index 3784876a59f3..523f27782362 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java @@ -67,6 +67,7 @@ public void testCustomParts() throws Exception { DummyStoreFlusher.class.getName()); HRegion mockRegion = Mockito.mock(HRegion.class); HStore mockStore = Mockito.mock(HStore.class); + mockStore.conf = conf; Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); Mockito.when(mockStore.getHRegion()).thenReturn(mockRegion); StoreEngine se = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java index bd24f1b22293..0eba8aa541ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.Path; @@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -139,7 +141,9 @@ public void testCommitDaughterRegionNoFiles() throws Exception { setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime()).build(); Path splitDir = regionFS.getSplitsDir(daughterA); - Path result = regionFS.commitDaughterRegion(daughterA); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path result = regionFS.commitDaughterRegion(daughterA, new ArrayList<>(), env); assertEquals(splitDir, result); } @@ -162,14 +166,18 @@ public void testCommitDaughterRegionWithFiles() throws Exception { Path splitDirA = regionFS.getSplitsDir(daughterA); Path splitDirB = regionFS.getSplitsDir(daughterB); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - regionFS + List filesA = new ArrayList<>(); + filesA.add(regionFS .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy()); - regionFS + Bytes.toBytes("002"), false, region.getSplitPolicy())); + List filesB = new ArrayList<>(); + filesB.add(regionFS .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), true, region.getSplitPolicy()); - Path resultA = regionFS.commitDaughterRegion(daughterA); - Path resultB = regionFS.commitDaughterRegion(daughterB); + Bytes.toBytes("002"), true, region.getSplitPolicy())); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path resultA = regionFS.commitDaughterRegion(daughterA, filesA, env); + Path resultB = regionFS.commitDaughterRegion(daughterB, filesB, env); assertEquals(splitDirA, resultA); assertEquals(splitDirB, resultB); } @@ -203,8 +211,11 @@ public void testCommitMergedRegion() throws Exception { mergeFileFromRegion(mergeRegionFs, first, file); //merge file from second region file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - mergeFileFromRegion(mergeRegionFs, second, file); - mergeRegionFs.commitMergedRegion(); + List mergedFiles = new ArrayList<>(); + mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file)); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + mergeRegionFs.commitMergedRegion(mergedFiles, env); } private void waitForSplitProcComplete(int attempts, int waitTime) throws Exception { @@ -223,11 +234,12 @@ private void waitForSplitProcComplete(int attempts, int waitTime) throws Excepti } } - private void mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, + private Path mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, HStoreFile file) throws IOException { Path mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile); + return mergedFile; } private void validateResultingFile(String originalRegion, Path result){ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index e6f0357204fc..c7203a928aa7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -49,12 +50,14 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -69,6 +72,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.BloomFilterFactory; @@ -1055,7 +1060,19 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h if (null == path) { return null; } - Path regionDir = regionFs.commitDaughterRegion(hri); + List splitFiles = new ArrayList<>(); + splitFiles.add(path); + MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class); + MasterServices mockServices = mock(MasterServices.class); + when(mockEnv.getMasterServices()).thenReturn(mockServices); + when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration()); + TableDescriptors mockTblDescs = mock(TableDescriptors.class); + when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs); + TableDescriptor mockTblDesc = mock(TableDescriptor.class); + when(mockTblDescs.get(any())).thenReturn(mockTblDesc); + ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class); + when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc); + Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); return new Path(new Path(regionDir, family), path.getName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java new file mode 100644 index 000000000000..c6205cb18493 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory. + TRACK_IMPL; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + + +@Category({RegionServerTests.class, LargeTests.class}) +public class TestMergesSplitsAddToTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMergesSplitsAddToTracker.class); + + private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + + public static final byte[] FAMILY_NAME = Bytes.toBytes("info"); + + @Rule + public TestName name = new TestName(); + + @BeforeClass + public static void setupClass() throws Exception { + TEST_UTIL.getConfiguration().set(TRACK_IMPL, TestStoreFileTracker.class.getName()); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup(){ + TestStoreFileTracker.trackedFiles = new HashMap<>(); + } + + @Test + public void testCommitDaughterRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //first put some data in order to have a store file created + putThreeRowsAndFlush(table); + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0); + HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem(); + RegionInfo daughterA = + RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey()). + setEndKey(Bytes.toBytes("002")).setSplit(false). + setRegionId(region.getRegionInfo().getRegionId() + + EnvironmentEdgeManager.currentTime()). + build(); + RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002")) + .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false) + .setRegionId(region.getRegionInfo().getRegionId()).build(); + HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + List splitFilesA = new ArrayList<>(); + splitFilesA.add(regionFS + .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, + Bytes.toBytes("002"), false, region.getSplitPolicy())); + List splitFilesB = new ArrayList<>(); + splitFilesB.add(regionFS + .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, + Bytes.toBytes("002"), true, region.getSplitPolicy())); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env); + Path resultB = regionFS.commitDaughterRegion(daughterB, splitFilesB, env); + FileSystem fs = regionFS.getFileSystem(); + verifyFilesAreTracked(resultA, fs); + verifyFilesAreTracked(resultB, fs); + } + + @Test + public void testCommitMergedRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //splitting the table first + TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002")); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + HRegion second = regions.get(1); + HRegionFileSystem regionFS = first.getRegionFileSystem(); + + RegionInfo mergeResult = + RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey()) + .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false) + .setRegionId(first.getRegionInfo().getRegionId() + + EnvironmentEdgeManager.currentTime()).build(); + + HRegionFileSystem mergeFS = HRegionFileSystem.createRegionOnFileSystem( + TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), + regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult); + + List mergedFiles = new ArrayList<>(); + //merge file from first region + mergedFiles.add(mergeFileFromRegion(first, mergeFS)); + //merge file from second region + mergedFiles.add(mergeFileFromRegion(second, mergeFS)); + MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterProcedureExecutor().getEnvironment(); + mergeFS.commitMergedRegion(mergedFiles, env); + //validate + FileSystem fs = first.getRegionFileSystem().getFileSystem(); + Path finalMergeDir = new Path(first.getRegionFileSystem().getTableDir(), + mergeResult.getEncodedName()); + verifyFilesAreTracked(finalMergeDir, fs); + } + + @Test + public void testSplitLoadsFromTracker() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, FAMILY_NAME); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0); + Pair copyResult = copyFileInTheStoreDir(region); + StoreFileInfo fileInfo = copyResult.getFirst(); + String copyName = copyResult.getSecond(); + //Now splits the region + TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002")); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + validateDaughterRegionsFiles(first, fileInfo.getActiveFileName(), copyName); + HRegion second = regions.get(1); + validateDaughterRegionsFiles(second, fileInfo.getActiveFileName(), copyName); + } + + @Test + public void testMergeLoadsFromTracker() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + TEST_UTIL.createTable(table, new byte[][]{FAMILY_NAME}, + new byte[][]{Bytes.toBytes("002")}); + //Add data and flush to create files in the two different regions + putThreeRowsAndFlush(table); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion first = regions.get(0); + Pair copyResult = copyFileInTheStoreDir(first); + StoreFileInfo fileInfo = copyResult.getFirst(); + String copyName = copyResult.getSecond(); + //Now merges the first two regions + TEST_UTIL.getAdmin().mergeRegionsAsync(new byte[][]{ + first.getRegionInfo().getEncodedNameAsBytes(), + regions.get(1).getRegionInfo().getEncodedNameAsBytes() + }, true).get(10, TimeUnit.SECONDS); + regions = TEST_UTIL.getHBaseCluster().getRegions(table); + HRegion merged = regions.get(0); + validateDaughterRegionsFiles(merged, fileInfo.getActiveFileName(), copyName); + } + + private Pair copyFileInTheStoreDir(HRegion region) throws IOException { + Path storeDir = region.getRegionFileSystem().getStoreDir("info"); + //gets the single file + StoreFileInfo fileInfo = region.getRegionFileSystem().getStoreFiles("info").get(0); + //make a copy of the valid file staight into the store dir, so that it's not tracked. + String copyName = UUID.randomUUID().toString().replaceAll("-", ""); + Path copy = new Path(storeDir, copyName); + FileUtil.copy(region.getFilesystem(), fileInfo.getFileStatus(), region.getFilesystem(), + copy , false, false, TEST_UTIL.getConfiguration()); + return new Pair<>(fileInfo, copyName); + } + + private void validateDaughterRegionsFiles(HRegion region, String orignalFileName, + String untrackedFile) throws IOException { + //verify there's no link for the untracked, copied file in first region + List infos = region.getRegionFileSystem().getStoreFiles("info"); + final MutableBoolean foundLink = new MutableBoolean(false); + infos.stream().forEach(i -> { + i.getActiveFileName().contains(orignalFileName); + if(i.getActiveFileName().contains(untrackedFile)){ + fail(); + } + if(i.getActiveFileName().contains(orignalFileName)){ + foundLink.setTrue(); + } + }); + assertTrue(foundLink.booleanValue()); + } + + private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Exception { + String storeId = regionDir.getName() + "-info"; + for(FileStatus f : fs.listStatus(new Path(regionDir, Bytes.toString(FAMILY_NAME)))){ + assertTrue(TestStoreFileTracker.trackedFiles.get(storeId).stream().filter(s -> + s.getPath().equals(f.getPath())).findFirst().isPresent()); + } + } + + private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS) + throws IOException { + HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); + } + + private void putThreeRowsAndFlush(TableName table) throws IOException { + Table tbl = TEST_UTIL.getConnection().getTable(table); + Put put = new Put(Bytes.toBytes("001")); + byte[] qualifier = Bytes.toBytes("1"); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1)); + tbl.put(put); + put = new Put(Bytes.toBytes("002")); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2)); + tbl.put(put); + put = new Put(Bytes.toBytes("003")); + put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2)); + tbl.put(put); + TEST_UTIL.flush(table); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java index eb0b1c1ca694..80012dfcd461 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreEngine.java @@ -120,6 +120,7 @@ private static HStoreFile createFile() throws Exception { private static TestStoreEngine createEngine(Configuration conf) throws Exception { HRegion region = mock(HRegion.class); HStore store = mock(HStore.class); + store.conf = conf; when(store.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO); when(store.getHRegion()).thenReturn(region); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java new file mode 100644 index 000000000000..05ca1fcb419b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestStoreFileTracker extends DefaultStoreFileTracker { + + private static final Logger LOG = LoggerFactory.getLogger(TestStoreFileTracker.class); + public static Map> trackedFiles = new HashMap<>(); + private String storeId; + + public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); + LOG.info("created storeId: {}", storeId); + trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + LOG.info("adding to storeId: {}", storeId); + trackedFiles.get(storeId).addAll(newFiles); + } + + @Override + public List load() throws IOException { + return trackedFiles.get(storeId); + } +} From 41ed7e279cef41e064f0a25a7bc7eb4ae8949007 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 9 Sep 2021 12:11:25 +0800 Subject: [PATCH 29/37] HBASE-26224 Introduce a MigrationStoreFileTracker to support migrating from different store file tracker implementations (#3656) Signed-off-by: Wellington Chevreuil --- .../DefaultStoreFileTracker.java | 10 +- .../FileBasedStoreFileTracker.java | 15 +- .../MigrationStoreFileTracker.java | 88 ++++++++ .../storefiletracker/StoreFileListFile.java | 6 +- .../StoreFileTrackerBase.java | 12 +- .../StoreFileTrackerFactory.java | 40 ++-- .../TestMigrationStoreFileTracker.java | 193 ++++++++++++++++++ 7 files changed, 343 insertions(+), 21 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index 22e05132bf91..a13b75b0dcc4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -39,7 +40,9 @@ public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, Sto @Override public List load() throws IOException { - return ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + List files = + ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + return files != null ? files : Collections.emptyList(); } @Override @@ -57,4 +60,9 @@ protected void doAddCompactionResults(Collection compactedFiles, Collection newFiles) throws IOException { // NOOP } + + @Override + void set(List files) { + // NOOP + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index de28b0eb9996..c370b87c1154 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -48,7 +48,7 @@ * storages. */ @InterfaceAudience.Private -public class FileBasedStoreFileTracker extends StoreFileTrackerBase { +class FileBasedStoreFileTracker extends StoreFileTrackerBase { private final StoreFileListFile backedFile; @@ -139,4 +139,17 @@ protected void doAddCompactionResults(Collection compactedFiles, } } } + + @Override + void set(List files) throws IOException { + synchronized (storefiles) { + storefiles.clear(); + StoreFileList.Builder builder = StoreFileList.newBuilder(); + for (StoreFileInfo info : files) { + storefiles.put(info.getPath().getName(), info); + builder.addStoreFile(toStoreFileEntry(info)); + } + backedFile.update(builder); + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java new file mode 100644 index 000000000000..e486e6d563af --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + +/** + * A store file tracker used for migrating between store file tracker implementations. + */ +@InterfaceAudience.Private +class MigrationStoreFileTracker extends StoreFileTrackerBase { + + public static final String SRC_IMPL = "hbase.store.file-tracker.migration.src.impl"; + + public static final String DST_IMPL = "hbase.store.file-tracker.migration.dst.impl"; + + private final StoreFileTrackerBase src; + + private final StoreFileTrackerBase dst; + + public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + this.src = StoreFileTrackerFactory.create(conf, SRC_IMPL, isPrimaryReplica, ctx); + this.dst = StoreFileTrackerFactory.create(conf, DST_IMPL, isPrimaryReplica, ctx); + Preconditions.checkArgument(!src.getClass().equals(dst.getClass()), + "src and dst is the same: %s", src.getClass()); + } + + @Override + public List load() throws IOException { + List files = src.load(); + dst.set(files); + return files; + } + + @Override + protected boolean requireWritingToTmpDirFirst() { + // Returns true if either of the two StoreFileTracker returns true. + // For example, if we want to migrate from a tracker implementation which can ignore the broken + // files under data directory to a tracker implementation which can not, if we still allow + // writing in tmp directory directly, we may have some broken files under the data directory and + // then after we finally change the implementation which can not ignore the broken files, we + // will be in trouble. + return src.requireWritingToTmpDirFirst() || dst.requireWritingToTmpDirFirst(); + } + + @Override + protected void doAddNewStoreFiles(Collection newFiles) throws IOException { + src.doAddNewStoreFiles(newFiles); + dst.doAddNewStoreFiles(newFiles); + } + + @Override + protected void doAddCompactionResults(Collection compactedFiles, + Collection newFiles) throws IOException { + src.doAddCompactionResults(compactedFiles, newFiles); + dst.doAddCompactionResults(compactedFiles, newFiles); + } + + @Override + void set(List files) { + throw new UnsupportedOperationException( + "Should not call this method on " + getClass().getSimpleName()); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java index c778bfc51deb..ffb3647e6259 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; @@ -121,7 +120,10 @@ StoreFileList load() throws IOException { * We will set the timestamp in this method so just pass the builder in */ void update(StoreFileList.Builder builder) throws IOException { - Preconditions.checkState(nextTrackFile >= 0, "should call load first before calling update"); + if (nextTrackFile < 0) { + // we need to call load first to load the prevTimestamp and also the next file + load(); + } FileSystem fs = ctx.getRegionFileSystem().getFileSystem(); long timestamp = Math.max(prevTimestamp + 1, EnvironmentEdgeManager.currentTime()); try (FSDataOutputStream out = fs.create(trackFiles[nextTrackFile], true)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 92c699278c2e..d860f8e3812d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -95,8 +96,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, } @Override - public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) - throws IOException { + public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException { if (!isPrimaryReplica) { throw new IllegalStateException("Should not call create writer on secondary replicas"); } @@ -170,4 +170,12 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) protected abstract void doAddCompactionResults(Collection compactedFiles, Collection newFiles) throws IOException; + + /** + * used to mirror the store file list after loading when migration. + *

+ * Do not add this method to the {@link StoreFileTracker} interface since we do not need this + * method in upper layer. + */ + abstract void set(List files) throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index c446d5ae9a31..6eb269ee2b40 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -18,18 +18,20 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CompoundConfiguration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; + /** * Factory method for creating store file tracker. */ @@ -39,7 +41,7 @@ public final class StoreFileTrackerFactory { private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { + StoreContext ctx) { Class tracker = conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); @@ -47,22 +49,30 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli } public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, - HRegionFileSystem regionFs) { + HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); - StoreContext ctx = StoreContext.getBuilder(). - withColumnFamilyDescriptor(fDescBuilder.build()). - withRegionFileSystem(regionFs). - build(); - return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) + .withRegionFileSystem(regionFs).build(); + return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx); + } + + public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, + ColumnFamilyDescriptor family) { + return StoreUtils.createStoreConfiguration(global, table, family); } - public static Configuration mergeConfigurations(Configuration global, - TableDescriptor table, ColumnFamilyDescriptor family) { - return new CompoundConfiguration() - .add(global) - .addBytesMap(table.getValues()) - .addStringMap(family.getConfiguration()) - .addBytesMap(family.getValues()); + static StoreFileTrackerBase create(Configuration conf, String configName, + boolean isPrimaryReplica, StoreContext ctx) { + String className = + Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); + Class tracker; + try { + tracker = Class.forName(className).asSubclass(StoreFileTrackerBase.class); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); + return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java new file mode 100644 index 000000000000..e9ebb4c792e4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.ChunkCreator; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WAL; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.io.Closeables; + +@RunWith(Parameterized.class) +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestMigrationStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMigrationStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final TableDescriptor TD = + TableDescriptorBuilder.newBuilder(TableName.valueOf("file_based_tracker")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TD.getTableName()).build(); + + @Rule + public TestName name = new TestName(); + + @Parameter(0) + public Class srcImplClass; + + @Parameter(1) + public Class dstImplClass; + + private HRegion region; + + private Path rootDir; + + private WAL wal; + + @Parameters(name = "{index}: src={0}, dst={1}") + public static List params() { + List> impls = + Arrays.asList(DefaultStoreFileTracker.class, FileBasedStoreFileTracker.class); + List params = new ArrayList<>(); + for (Class src : impls) { + for (Class dst : impls) { + if (src.equals(dst)) { + continue; + } + params.add(new Object[] { src, dst }); + } + } + return params; + } + + @BeforeClass + public static void setUpBeforeClass() { + ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, + MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); + } + + @Before + public void setUp() throws IOException { + Configuration conf = UTIL.getConfiguration(); + conf.setClass(MigrationStoreFileTracker.SRC_IMPL, srcImplClass, StoreFileTrackerBase.class); + conf.setClass(MigrationStoreFileTracker.DST_IMPL, dstImplClass, StoreFileTrackerBase.class); + rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_")); + wal = HBaseTestingUtil.createWal(conf, rootDir, RI); + } + + @After + public void tearDown() throws IOException { + if (region != null) { + region.close(); + } + Closeables.close(wal, true); + UTIL.cleanupTestDir(); + } + + private List getStoreFiles() { + return Iterables.getOnlyElement(region.getStores()).getStorefiles().stream() + .map(s -> s.getFileInfo().getPath().getName()).collect(Collectors.toList()); + } + + private HRegion createRegion(Class trackerImplClass) + throws IOException { + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true); + } + + private void reopenRegion(Class trackerImplClass) + throws IOException { + region.flush(true); + List before = getStoreFiles(); + region.close(); + Configuration conf = new Configuration(UTIL.getConfiguration()); + conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + region = HRegion.openHRegion(rootDir, RI, TD, wal, conf); + List after = getStoreFiles(); + assertEquals(before.size(), after.size()); + assertThat(after, hasItems(before.toArray(new String[0]))); + } + + private void putData(int start, int end) throws IOException { + for (int i = start; i < end; i++) { + region.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + if (i % 30 == 0) { + region.flush(true); + } + } + } + + private void verifyData(int start, int end) throws IOException { + for (int i = start; i < end; i++) { + Result result = region.get(new Get(Bytes.toBytes(i))); + assertEquals(i, Bytes.toInt(result.getValue(CF, CQ))); + } + } + + @Test + public void testMigration() throws IOException { + region = createRegion(srcImplClass); + putData(0, 100); + verifyData(0, 100); + reopenRegion(MigrationStoreFileTracker.class); + verifyData(0, 100); + region.compact(true); + putData(100, 200); + reopenRegion(dstImplClass); + verifyData(0, 200); + } +} From f2addb1099f84b13107dc5ee9a6689f9f739bf3f Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Sun, 12 Sep 2021 14:14:03 +0100 Subject: [PATCH 30/37] HBASE-26246 Persist the StoreFileTracker configurations to TableDescriptor when creating table (#3666) Signed-off-by: Duo Zhang --- .../procedure/CreateTableProcedure.java | 7 ++++++ .../hbase/regionserver/HRegionFileSystem.java | 2 +- .../MigrationStoreFileTracker.java | 8 ++++++ .../storefiletracker/StoreFileTracker.java | 8 ++++++ .../StoreFileTrackerBase.java | 13 ++++++++++ .../StoreFileTrackerFactory.java | 25 +++++++++++++------ .../apache/hadoop/hbase/client/TestAdmin.java | 6 +++++ .../hadoop/hbase/client/TestAdmin3.java | 6 +++++ .../hbase/client/TestAsyncTableAdminApi.java | 6 +++++ .../hbase/client/TestAsyncTableAdminApi3.java | 6 +++++ .../MasterProcedureTestingUtility.java | 7 ++++++ .../procedure/TestCreateTableProcedure.java | 17 +++++++++++++ .../TestStoreFileTracker.java | 14 ++++++++--- 13 files changed, 113 insertions(+), 12 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 2313e70f75bb..dccea5512c1f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -33,10 +34,12 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -285,6 +288,10 @@ private void preCreate(final MasterProcedureEnv env) getTableName(), (newRegions != null ? newRegions.size() : 0)); } + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); + StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder); + tableDescriptor = builder.build(); + final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { final RegionInfo[] regions = newRegions == null ? null : diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 3b053a349c2c..40a42ab10383 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -607,7 +607,7 @@ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegi writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false); - insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); + insertRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs); } return regionDir; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index e486e6d563af..483a240baded 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -85,4 +86,11 @@ void set(List files) { throw new UnsupportedOperationException( "Should not call this method on " + getClass().getSimpleName()); } + + @Override + public void persistConfiguration(TableDescriptorBuilder builder) { + super.persistConfiguration(builder); + builder.setValue(SRC_IMPL, src.getClass().getName()); + builder.setValue(DST_IMPL, dst.getClass().getName()); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 0a85abb00a4e..81fa1a9be5b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.Collection; import java.util.List; + +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -71,4 +73,10 @@ void replace(Collection compactedFiles, Collection * @return Writer for a new StoreFile */ StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; + + /** + * Saves StoreFileTracker implementations specific configs into the table descriptors. + * @param builder The table descriptor builder for the given table. + */ + void persistConfiguration(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index d860f8e3812d..83ebbc78ab35 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -17,18 +17,22 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; + import java.io.IOException; import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -78,6 +82,15 @@ public final void replace(Collection compactedFiles, } } + @Override + public void persistConfiguration(TableDescriptorBuilder builder) { + if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) { + String trackerImpl = StoreFileTrackerFactory. + getStoreFileTrackerImpl(conf).getName(); + builder.setValue(TRACK_IMPL, trackerImpl).build(); + } + } + private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { if (compression == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 6eb269ee2b40..b9ec713cf235 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -6,9 +6,7 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +19,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreUtils; @@ -35,15 +34,17 @@ /** * Factory method for creating store file tracker. */ -@InterfaceAudience.Private -public final class StoreFileTrackerFactory { +@InterfaceAudience.Private public final class StoreFileTrackerFactory { public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); + public static Class getStoreFileTrackerImpl(Configuration conf) { + return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + } + public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = - conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + Class tracker = getStoreFileTrackerImpl(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } @@ -54,7 +55,7 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, TRACK_IMPL, isPrimaryReplica, ctx); + return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); } public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, @@ -75,4 +76,12 @@ static StoreFileTrackerBase create(Configuration conf, String configName, LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + + public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) { + TableDescriptor tableDescriptor = builder.build(); + ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; + StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); + tracker.persistConfiguration(builder); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 22302629dc70..10ab4961a5b4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -423,6 +425,10 @@ private void testCloneTableSchema(final TableName tableName, final TableName new assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + // HBASE-26246 introduced persist of store file tracker into table descriptor + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); if (preserveSplits) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java index e7be08e41edf..7d40fd12bda5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -236,6 +238,10 @@ public void testGetTableDescriptor() throws IOException { ADMIN.createTable(htd); Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); TableDescriptor confirmedHtd = table.getDescriptor(); + //HBASE-26246 introduced persist of store file tracker into table descriptor + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); table.close(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 572a1d5a6226..a1832f49956a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -372,6 +374,10 @@ private void testCloneTableSchema(final TableName tableName, assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); + //HBASE-26246 introduced persist of store file tracker into table descriptor + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); if (preserveSplits) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index 4a71baf44a13..cb89bf564720 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -35,6 +36,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -147,6 +149,10 @@ public void testGetTableDescriptor() throws Exception { TableDescriptor desc = builder.build(); admin.createTable(desc).join(); TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get(); + //HBASE-26246 introduced persist of store file tracker into table descriptor + desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index a3206432b810..2ceec5f241c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -60,6 +61,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -226,6 +228,11 @@ public static void validateTableCreation(final HMaster master, final TableName t assertTrue("family not found " + family[i], htd.getColumnFamily(Bytes.toBytes(family[i])) != null); } assertEquals(family.length, htd.getColumnFamilyCount()); + + // checks store file tracker impl has been properly set in htd + String storeFileTrackerImpl = + StoreFileTrackerFactory.getStoreFileTrackerImpl(master.getConfiguration()).getName(); + assertEquals(storeFileTrackerImpl, htd.getValue(TRACK_IMPL)); } public static void validateTableDeletion( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 3b57ed56744b..0bc77f0ef8d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -88,6 +90,21 @@ private void testSimpleCreate(final TableName tableName, byte[][] splitKeys) thr MasterProcedureTestingUtility.validateTableCreation(getMaster(), tableName, regions, F1, F2); } + @Test + public void testCreateWithTrackImpl() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + ProcedureExecutor procExec = getMasterProcedureExecutor(); + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); + String trackerName = TestStoreFileTracker.class.getName(); + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, trackerName).build(); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + htd = getMaster().getTableDescriptors().get(tableName); + assertEquals(trackerName, htd.getValue(TRACK_IMPL)); + } + @Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor procExec = getMasterProcedureExecutor(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java index 05ca1fcb419b..1dc9c4e53fc5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,15 +40,21 @@ public class TestStoreFileTracker extends DefaultStoreFileTracker { public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); - LOG.info("created storeId: {}", storeId); - trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + if (ctx.getRegionFileSystem() != null) { + this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); + LOG.info("created storeId: {}", storeId); + trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); + } else { + LOG.info("ctx.getRegionFileSystem() returned null. Leaving storeId null."); + } + } @Override protected void doAddNewStoreFiles(Collection newFiles) throws IOException { LOG.info("adding to storeId: {}", storeId); trackedFiles.get(storeId).addAll(newFiles); + trackedFiles.putIfAbsent(storeId, (List)newFiles); } @Override From 5c357444ff48921b9e3cfc661c72440d8e8f4fda Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 14 Sep 2021 16:28:21 +0800 Subject: [PATCH 31/37] HBASE-26248 Should find a suitable way to let users specify the store file tracker implementation (#3665) Signed-off-by: Wellington Chevreuil --- .../MigrationStoreFileTracker.java | 13 ++- .../storefiletracker/StoreFileTracker.java | 7 +- .../StoreFileTrackerBase.java | 12 ++- .../StoreFileTrackerFactory.java | 97 +++++++++++++++++-- .../apache/hadoop/hbase/client/TestAdmin.java | 6 +- .../hadoop/hbase/client/TestAdmin3.java | 6 +- .../hbase/client/TestAsyncTableAdminApi.java | 6 +- .../hbase/client/TestAsyncTableAdminApi3.java | 6 +- .../MasterProcedureTestingUtility.java | 6 +- .../procedure/TestCreateTableProcedure.java | 6 +- .../TestMergesSplitsAddToTracker.java | 4 +- .../TestMigrationStoreFileTracker.java | 27 +++--- ...stRegionWithFileBasedStoreFileTracker.java | 3 +- .../TestStoreFileTrackerFactory.java | 58 +++++++++++ 14 files changed, 202 insertions(+), 55 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 483a240baded..3eeef9000576 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -44,8 +45,8 @@ class MigrationStoreFileTracker extends StoreFileTrackerBase { public MigrationStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - this.src = StoreFileTrackerFactory.create(conf, SRC_IMPL, isPrimaryReplica, ctx); - this.dst = StoreFileTrackerFactory.create(conf, DST_IMPL, isPrimaryReplica, ctx); + this.src = StoreFileTrackerFactory.createForMigration(conf, SRC_IMPL, isPrimaryReplica, ctx); + this.dst = StoreFileTrackerFactory.createForMigration(conf, DST_IMPL, isPrimaryReplica, ctx); Preconditions.checkArgument(!src.getClass().equals(dst.getClass()), "src and dst is the same: %s", src.getClass()); } @@ -90,7 +91,11 @@ void set(List files) { @Override public void persistConfiguration(TableDescriptorBuilder builder) { super.persistConfiguration(builder); - builder.setValue(SRC_IMPL, src.getClass().getName()); - builder.setValue(DST_IMPL, dst.getClass().getName()); + if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { + builder.setValue(SRC_IMPL, src.getTrackerName()); + } + if (StringUtils.isEmpty(builder.getValue(DST_IMPL))) { + builder.setValue(DST_IMPL, dst.getTrackerName()); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 81fa1a9be5b2..59fe7ef52f96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -75,7 +75,12 @@ void replace(Collection compactedFiles, Collection StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; /** - * Saves StoreFileTracker implementations specific configs into the table descriptors. + * Saves StoreFileTracker implementations specific configurations into the table descriptors. + *

+ * This is used to avoid accidentally data loss when changing the cluster level store file tracker + * implementation, and also possible misconfiguration between master and region servers. + *

+ * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ void persistConfiguration(TableDescriptorBuilder builder); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 83ebbc78ab35..a786add49b21 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import java.io.IOException; import java.util.Collection; @@ -84,13 +84,15 @@ public final void replace(Collection compactedFiles, @Override public void persistConfiguration(TableDescriptorBuilder builder) { - if (StringUtils.isEmpty(builder.getValue(TRACK_IMPL))) { - String trackerImpl = StoreFileTrackerFactory. - getStoreFileTrackerImpl(conf).getName(); - builder.setValue(TRACK_IMPL, trackerImpl).build(); + if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { + builder.setValue(TRACKER_IMPL, getTrackerName()); } } + protected final String getTrackerName() { + return StoreFileTrackerFactory.getStoreFileTrackerName(getClass()); + } + private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context encryptionContext) { if (compression == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index b9ec713cf235..9be19ec15ed8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -15,6 +15,9 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -33,22 +36,81 @@ /** * Factory method for creating store file tracker. + *

+ * The current implementations are: + *

    + *
  • default: DefaultStoreFileTracker, see {@link DefaultStoreFileTracker}.
    • + *
  • file:FileBasedStoreFileTracker, see {@link FileBasedStoreFileTracker}.
    • + *
  • migration:MigrationStoreFileTracker, see {@link MigrationStoreFileTracker}.
    • + *
+ * @see DefaultStoreFileTracker + * @see FileBasedStoreFileTracker + * @see MigrationStoreFileTracker */ -@InterfaceAudience.Private public final class StoreFileTrackerFactory { - public static final String TRACK_IMPL = "hbase.store.file-tracker.impl"; +@InterfaceAudience.Private +public final class StoreFileTrackerFactory { + private static final Logger LOG = LoggerFactory.getLogger(StoreFileTrackerFactory.class); - public static Class getStoreFileTrackerImpl(Configuration conf) { - return conf.getClass(TRACK_IMPL, DefaultStoreFileTracker.class, StoreFileTracker.class); + public static final String TRACKER_IMPL = "hbase.store.file-tracker.impl"; + + /** + * Maps between configuration names for trackers and implementation classes. + */ + public enum Trackers { + DEFAULT(DefaultStoreFileTracker.class), FILE(FileBasedStoreFileTracker.class), + MIGRATION(MigrationStoreFileTracker.class); + + final Class clazz; + + Trackers(Class clazz) { + this.clazz = clazz; + } + } + + private static final Map, Trackers> CLASS_TO_ENUM = reverse(); + + private static Map, Trackers> reverse() { + Map, Trackers> map = new HashMap<>(); + for (Trackers tracker : Trackers.values()) { + map.put(tracker.clazz, tracker); + } + return Collections.unmodifiableMap(map); + } + + private StoreFileTrackerFactory() { + } + + public static String getStoreFileTrackerName(Configuration conf) { + return conf.get(TRACKER_IMPL, Trackers.DEFAULT.name()); + } + + static String getStoreFileTrackerName(Class clazz) { + Trackers name = CLASS_TO_ENUM.get(clazz); + return name != null ? name.name() : clazz.getName(); + } + + private static Class getTrackerClass(Configuration conf) { + try { + Trackers tracker = Trackers.valueOf(getStoreFileTrackerName(conf).toUpperCase()); + return tracker.clazz; + } catch (IllegalArgumentException e) { + // Fall back to them specifying a class name + return conf.getClass(TRACKER_IMPL, Trackers.DEFAULT.clazz, StoreFileTracker.class); + } } public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - Class tracker = getStoreFileTrackerImpl(conf); + Class tracker = getTrackerClass(conf); LOG.info("instantiating StoreFileTracker impl {}", tracker.getName()); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } + /** + * Used at master side when splitting/merging regions, as we do not have a Store, thus no + * StoreContext at master side. + */ public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = @@ -63,15 +125,30 @@ public static Configuration mergeConfigurations(Configuration global, TableDescr return StoreUtils.createStoreConfiguration(global, table, family); } - static StoreFileTrackerBase create(Configuration conf, String configName, + /** + * Create store file tracker to be used as source or destination for + * {@link MigrationStoreFileTracker}. + */ + static StoreFileTrackerBase createForMigration(Configuration conf, String configName, boolean isPrimaryReplica, StoreContext ctx) { - String className = + String trackerName = Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); Class tracker; try { - tracker = Class.forName(className).asSubclass(StoreFileTrackerBase.class); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); + tracker = + Trackers.valueOf(trackerName.toUpperCase()).clazz.asSubclass(StoreFileTrackerBase.class); + } catch (IllegalArgumentException e) { + // Fall back to them specifying a class name + try { + tracker = Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException(cnfe); + } + } + // prevent nest of MigrationStoreFileTracker, it will cause infinite recursion. + if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { + throw new IllegalArgumentException("Should not specify " + configName + " as " + + Trackers.MIGRATION + " because it can not be nested"); } LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 10ab4961a5b4..ba843fe43460 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -426,8 +426,8 @@ private void testCloneTableSchema(final TableName tableName, final TableName new assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); // HBASE-26246 introduced persist of store file tracker into table descriptor - tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java index 7d40fd12bda5..c2de0fbd3555 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.client; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -239,8 +239,8 @@ public void testGetTableDescriptor() throws IOException { Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); TableDescriptor confirmedHtd = table.getDescriptor(); //HBASE-26246 introduced persist of store file tracker into table descriptor - htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index a1832f49956a..147630b53a38 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -375,8 +375,8 @@ private void testCloneTableSchema(final TableName tableName, assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); //HBASE-26246 introduced persist of store file tracker into table descriptor - tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index cb89bf564720..67377b07c180 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -150,8 +150,8 @@ public void testGetTableDescriptor() throws Exception { admin.createTable(desc).join(); TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get(); //HBASE-26246 introduced persist of store file tracker into table descriptor - desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACK_IMPL, - StoreFileTrackerFactory.getStoreFileTrackerImpl(TEST_UTIL.getConfiguration()).getName()). + desc = TableDescriptorBuilder.newBuilder(desc).setValue(TRACKER_IMPL, + StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())). build(); assertEquals(0, TableDescriptor.COMPARATOR.compare(desc, confirmedHtd)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index 2ceec5f241c6..726e117ef1ac 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -231,8 +231,8 @@ public static void validateTableCreation(final HMaster master, final TableName t // checks store file tracker impl has been properly set in htd String storeFileTrackerImpl = - StoreFileTrackerFactory.getStoreFileTrackerImpl(master.getConfiguration()).getName(); - assertEquals(storeFileTrackerImpl, htd.getValue(TRACK_IMPL)); + StoreFileTrackerFactory.getStoreFileTrackerName(master.getConfiguration()); + assertEquals(storeFileTrackerImpl, htd.getValue(TRACKER_IMPL)); } public static void validateTableDeletion( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 0bc77f0ef8d9..f432c8060d3d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; -import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACK_IMPL; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -96,13 +96,13 @@ public void testCreateWithTrackImpl() throws Exception { ProcedureExecutor procExec = getMasterProcedureExecutor(); TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); String trackerName = TestStoreFileTracker.class.getName(); - htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACK_IMPL, trackerName).build(); + htd = TableDescriptorBuilder.newBuilder(htd).setValue(TRACKER_IMPL, trackerName).build(); RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); long procId = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); htd = getMaster().getTableDescriptors().get(tableName); - assertEquals(trackerName, htd.getValue(TRACK_IMPL)); + assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java index c6205cb18493..68fc444493c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory. - TRACK_IMPL; + TRACKER_IMPL; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -74,7 +74,7 @@ public class TestMergesSplitsAddToTracker { @BeforeClass public static void setupClass() throws Exception { - TEST_UTIL.getConfiguration().set(TRACK_IMPL, TestStoreFileTracker.class.getName()); + TEST_UTIL.getConfiguration().set(TRACKER_IMPL, TestStoreFileTracker.class.getName()); TEST_UTIL.startMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java index e9ebb4c792e4..119f8fb553f8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestMigrationStoreFileTracker.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -86,10 +85,10 @@ public class TestMigrationStoreFileTracker { public TestName name = new TestName(); @Parameter(0) - public Class srcImplClass; + public StoreFileTrackerFactory.Trackers srcImpl; @Parameter(1) - public Class dstImplClass; + public StoreFileTrackerFactory.Trackers dstImpl; private HRegion region; @@ -99,11 +98,13 @@ public class TestMigrationStoreFileTracker { @Parameters(name = "{index}: src={0}, dst={1}") public static List params() { - List> impls = - Arrays.asList(DefaultStoreFileTracker.class, FileBasedStoreFileTracker.class); List params = new ArrayList<>(); - for (Class src : impls) { - for (Class dst : impls) { + for (StoreFileTrackerFactory.Trackers src : StoreFileTrackerFactory.Trackers.values()) { + for (StoreFileTrackerFactory.Trackers dst : StoreFileTrackerFactory.Trackers.values()) { + if (src == StoreFileTrackerFactory.Trackers.MIGRATION + || dst == StoreFileTrackerFactory.Trackers.MIGRATION) { + continue; + } if (src.equals(dst)) { continue; } @@ -122,8 +123,8 @@ public static void setUpBeforeClass() { @Before public void setUp() throws IOException { Configuration conf = UTIL.getConfiguration(); - conf.setClass(MigrationStoreFileTracker.SRC_IMPL, srcImplClass, StoreFileTrackerBase.class); - conf.setClass(MigrationStoreFileTracker.DST_IMPL, dstImplClass, StoreFileTrackerBase.class); + conf.set(MigrationStoreFileTracker.SRC_IMPL, srcImpl.name().toLowerCase()); + conf.set(MigrationStoreFileTracker.DST_IMPL, dstImpl.name().toLowerCase()); rootDir = UTIL.getDataTestDir(name.getMethodName().replaceAll("[=:\\[ ]", "_")); wal = HBaseTestingUtil.createWal(conf, rootDir, RI); } @@ -145,7 +146,7 @@ private List getStoreFiles() { private HRegion createRegion(Class trackerImplClass) throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class); return HRegion.createHRegion(RI, rootDir, conf, TD, wal, true); } @@ -155,7 +156,7 @@ private void reopenRegion(Class trackerImplClass List before = getStoreFiles(); region.close(); Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, trackerImplClass, StoreFileTracker.class); + conf.setClass(StoreFileTrackerFactory.TRACKER_IMPL, trackerImplClass, StoreFileTracker.class); region = HRegion.openHRegion(rootDir, RI, TD, wal, conf); List after = getStoreFiles(); assertEquals(before.size(), after.size()); @@ -180,14 +181,14 @@ private void verifyData(int start, int end) throws IOException { @Test public void testMigration() throws IOException { - region = createRegion(srcImplClass); + region = createRegion(srcImpl.clazz.asSubclass(StoreFileTrackerBase.class)); putData(0, 100); verifyData(0, 100); reopenRegion(MigrationStoreFileTracker.class); verifyData(0, 100); region.compact(true); putData(100, 200); - reopenRegion(dstImplClass); + reopenRegion(dstImpl.clazz.asSubclass(StoreFileTrackerBase.class)); verifyData(0, 200); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java index 3bc60d13fd94..77cd3c448fa0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestRegionWithFileBasedStoreFileTracker.java @@ -71,8 +71,7 @@ public class TestRegionWithFileBasedStoreFileTracker { @Before public void setUp() throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); - conf.setClass(StoreFileTrackerFactory.TRACK_IMPL, FileBasedStoreFileTracker.class, - StoreFileTracker.class); + conf.set(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()); region = HBaseTestingUtil.createRegionAndWAL(RI, UTIL.getDataTestDir(name.getMethodName()), conf, TD); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java new file mode 100644 index 000000000000..41f2afdfa421 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertThrows; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestStoreFileTrackerFactory { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStoreFileTrackerFactory.class); + + @Test + public void testCreateForMigration() { + Configuration conf = HBaseConfiguration.create(); + String configName = "config"; + + // no config + assertThrows(NullPointerException.class, () -> StoreFileTrackerFactory.createForMigration(conf, + configName, false, StoreContext.getBuilder().build())); + + // class not found + conf.set(configName, "config"); + assertThrows(RuntimeException.class, () -> StoreFileTrackerFactory.createForMigration(conf, + configName, false, StoreContext.getBuilder().build())); + + // nested MigrationStoreFileTracker + conf.setClass(configName, MigrationStoreFileTracker.class, StoreFileTrackerBase.class); + assertThrows(IllegalArgumentException.class, () -> StoreFileTrackerFactory + .createForMigration(conf, configName, false, StoreContext.getBuilder().build())); + } +} From 767a3e14af73a234db3fd3655f8736213fbe843b Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Wed, 15 Sep 2021 23:00:03 +0800 Subject: [PATCH 32/37] HBASE-26264 Add more checks to prevent misconfiguration on store file tracker (#3681) Signed-off-by: Josh Elser --- .../MergeTableRegionsProcedure.java | 3 +- .../assignment/SplitTableRegionProcedure.java | 3 +- .../procedure/CreateTableProcedure.java | 8 +- .../procedure/ModifyTableProcedure.java | 5 + .../hbase/regionserver/HRegionFileSystem.java | 2 +- .../MigrationStoreFileTracker.java | 8 + .../StoreFileTrackerFactory.java | 173 ++++++++++++- .../TestChangeStoreFileTracker.java | 242 ++++++++++++++++++ 8 files changed, 422 insertions(+), 22 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 4b25a28a6055..9a00fe26a8f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -614,8 +614,7 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg String family = hcd.getNameAsString(); Configuration trackerConfig = StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, - family, regionFs); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 4d53df4bc670..44136dc7cbee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -668,8 +668,7 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en String family = cfd.getNameAsString(); Configuration trackerConfig = StoreFileTrackerFactory. mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, true, - family, regionFs); + StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); Collection sfis = tracker.load(); if (sfis == null) { continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index dccea5512c1f..ee8e51ffe86b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -277,15 +277,17 @@ private boolean prepareCreate(final MasterProcedureEnv env) throws IOException { MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom); } + // check for store file tracker configurations + StoreFileTrackerFactory.checkForCreateTable(env.getMasterConfiguration(), tableDescriptor); + return true; } private void preCreate(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { - ProcedureSyncWait.getMasterQuotaManager(env) - .checkNamespaceTableAndRegionQuota( - getTableName(), (newRegions != null ? newRegions.size() : 0)); + ProcedureSyncWait.getMasterQuotaManager(env).checkNamespaceTableAndRegionQuota(getTableName(), + (newRegions != null ? newRegions.size() : 0)); } TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index 247dd9c202f4..1640644328f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.Bytes; @@ -325,6 +326,10 @@ private void prepareModify(final MasterProcedureEnv env) throws IOException { modifiedTableDescriptor.getRegionServerGroup(), forWhom); MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom); } + + // check for store file tracker configurations + StoreFileTrackerFactory.checkForModifyTable(env.getMasterConfiguration(), + unmodifiedTableDescriptor, modifiedTableDescriptor); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 40a42ab10383..db18d4e50fd6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -626,7 +626,7 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, tblDesc.getColumnFamily(Bytes.toBytes(familyName))); return StoreFileTrackerFactory. - create(config, true, familyName, regionFs); + create(config, familyName, regionFs); }); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 3eeef9000576..1946d4bf7eef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -98,4 +98,12 @@ public void persistConfiguration(TableDescriptorBuilder builder) { builder.setValue(DST_IMPL, dst.getTrackerName()); } } + + static Class getSrcTrackerClass(Configuration conf) { + return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, SRC_IMPL); + } + + static Class getDstTrackerClass(Configuration conf) { + return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, DST_IMPL); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 9be19ec15ed8..90704fe528a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -15,10 +15,12 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -111,13 +113,13 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. */ - public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica, String family, + public static StoreFileTracker create(Configuration conf, String family, HRegionFileSystem regionFs) { ColumnFamilyDescriptorBuilder fDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, isPrimaryReplica, ctx); + return StoreFileTrackerFactory.create(conf, true, ctx); } public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, @@ -125,30 +127,35 @@ public static Configuration mergeConfigurations(Configuration global, TableDescr return StoreUtils.createStoreConfiguration(global, table, family); } - /** - * Create store file tracker to be used as source or destination for - * {@link MigrationStoreFileTracker}. - */ - static StoreFileTrackerBase createForMigration(Configuration conf, String configName, - boolean isPrimaryReplica, StoreContext ctx) { + static Class + getStoreFileTrackerClassForMigration(Configuration conf, String configName) { String trackerName = Preconditions.checkNotNull(conf.get(configName), "config %s is not set", configName); - Class tracker; try { - tracker = - Trackers.valueOf(trackerName.toUpperCase()).clazz.asSubclass(StoreFileTrackerBase.class); + return Trackers.valueOf(trackerName.toUpperCase()).clazz + .asSubclass(StoreFileTrackerBase.class); } catch (IllegalArgumentException e) { // Fall back to them specifying a class name try { - tracker = Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); + return Class.forName(trackerName).asSubclass(StoreFileTrackerBase.class); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } } + } + + /** + * Create store file tracker to be used as source or destination for + * {@link MigrationStoreFileTracker}. + */ + static StoreFileTrackerBase createForMigration(Configuration conf, String configName, + boolean isPrimaryReplica, StoreContext ctx) { + Class tracker = + getStoreFileTrackerClassForMigration(conf, configName); // prevent nest of MigrationStoreFileTracker, it will cause infinite recursion. if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { - throw new IllegalArgumentException("Should not specify " + configName + " as " + - Trackers.MIGRATION + " because it can not be nested"); + throw new IllegalArgumentException("Should not specify " + configName + " as " + + Trackers.MIGRATION + " because it can not be nested"); } LOG.info("instantiating StoreFileTracker impl {} as {}", tracker.getName(), configName); return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); @@ -161,4 +168,142 @@ public static void persistTrackerConfig(Configuration conf, TableDescriptorBuild StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); tracker.persistConfiguration(builder); } + + // should not use MigrationStoreFileTracker for new family + private static void checkForNewFamily(Configuration conf, TableDescriptor table, + ColumnFamilyDescriptor family) throws IOException { + Configuration mergedConf = mergeConfigurations(conf, table, family); + Class tracker = getTrackerClass(mergedConf); + if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) { + throw new DoNotRetryIOException( + "Should not use " + Trackers.MIGRATION + " as store file tracker for new family " + + family.getNameAsString() + " of table " + table.getTableName()); + } + } + + /** + * Pre check when creating a new table. + *

+ * For now, only make sure that we do not use {@link Trackers#MIGRATION} for newly created tables. + * @throws IOException when there are check errors, the upper layer should fail the + * {@code CreateTableProcedure}. + */ + public static void checkForCreateTable(Configuration conf, TableDescriptor table) + throws IOException { + for (ColumnFamilyDescriptor family : table.getColumnFamilies()) { + checkForNewFamily(conf, table, family); + } + } + + + /** + * Pre check when modifying a table. + *

+ * The basic idea is when you want to change the store file tracker implementation, you should use + * {@link Trackers#MIGRATION} first and then change to the destination store file tracker + * implementation. + *

+ * There are several rules: + *

    + *
  • For newly added family, you should not use {@link Trackers#MIGRATION}.
    • + *
  • For modifying a family: + *
      + *
    • If old tracker is {@link Trackers#MIGRATION}, then: + *
        + *
      • The new tracker is also {@link Trackers#MIGRATION}, then they must have the same src and + * dst tracker.
        • + *
      • The new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the dst + * tracker of the old tracker.
        • + *
      + *
      • + *
    • If the old tracker is not {@link Trackers#MIGRATION}, then: + *
        + *
      • If the new tracker is {@link Trackers#MIGRATION}, then the old tracker must be the src + * tracker of the new tracker.
        • + *
      • If the new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the same + * with old tracker.
        • + *
      + *
      • + *
    + *
    • + *
+ * @throws IOException when there are check errors, the upper layer should fail the + * {@code ModifyTableProcedure}. + */ + public static void checkForModifyTable(Configuration conf, TableDescriptor oldTable, + TableDescriptor newTable) throws IOException { + for (ColumnFamilyDescriptor newFamily : newTable.getColumnFamilies()) { + ColumnFamilyDescriptor oldFamily = oldTable.getColumnFamily(newFamily.getName()); + if (oldFamily == null) { + checkForNewFamily(conf, newTable, newFamily); + continue; + } + Configuration oldConf = mergeConfigurations(conf, oldTable, oldFamily); + Configuration newConf = mergeConfigurations(conf, newTable, newFamily); + + Class oldTracker = getTrackerClass(oldConf); + Class newTracker = getTrackerClass(newConf); + + if (MigrationStoreFileTracker.class.isAssignableFrom(oldTracker)) { + Class oldSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(oldConf); + Class oldDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(oldConf); + if (oldTracker.equals(newTracker)) { + // confirm that we have the same src tracker and dst tracker + Class newSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(newConf); + if (!oldSrcTracker.equals(newSrcTracker)) { + throw new DoNotRetryIOException( + "The src tracker has been changed from " + getStoreFileTrackerName(oldSrcTracker) + + " to " + getStoreFileTrackerName(newSrcTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + Class newDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(newConf); + if (!oldDstTracker.equals(newDstTracker)) { + throw new DoNotRetryIOException( + "The dst tracker has been changed from " + getStoreFileTrackerName(oldDstTracker) + + " to " + getStoreFileTrackerName(newDstTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + } else { + // we can only change to the dst tracker + if (!newTracker.equals(oldDstTracker)) { + throw new DoNotRetryIOException( + "Should migrate tracker to " + getStoreFileTrackerName(oldDstTracker) + " but got " + + getStoreFileTrackerName(newTracker) + " for family " + newFamily.getNameAsString() + + " of table " + newTable.getTableName()); + } + } + } else { + if (!oldTracker.equals(newTracker)) { + // can only change to MigrationStoreFileTracker and the src tracker should be the old + // tracker + if (!MigrationStoreFileTracker.class.isAssignableFrom(newTracker)) { + throw new DoNotRetryIOException("Should change to " + Trackers.MIGRATION + + " first when migrating from " + getStoreFileTrackerName(oldTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + Class newSrcTracker = + MigrationStoreFileTracker.getSrcTrackerClass(newConf); + if (!oldTracker.equals(newSrcTracker)) { + throw new DoNotRetryIOException( + "Should use src tracker " + getStoreFileTrackerName(oldTracker) + " first but got " + + getStoreFileTrackerName(newSrcTracker) + " when migrating from " + + getStoreFileTrackerName(oldTracker) + " for family " + newFamily.getNameAsString() + + " of table " + newTable.getTableName()); + } + Class newDstTracker = + MigrationStoreFileTracker.getDstTrackerClass(newConf); + // the src and dst tracker should not be the same + if (newSrcTracker.equals(newDstTracker)) { + throw new DoNotRetryIOException("The src tracker and dst tracker are both " + + getStoreFileTrackerName(newSrcTracker) + " for family " + + newFamily.getNameAsString() + " of table " + newTable.getTableName()); + } + } + } + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java new file mode 100644 index 000000000000..70f62c02ed28 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestChangeStoreFileTracker.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNameTestRule; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; + +/** + * Test changing store file tracker implementation by altering table. + */ +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestChangeStoreFileTracker { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestChangeStoreFileTracker.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + @Rule + public final TableNameTestRule tableName = new TableNameTestRule(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws IOException { + UTIL.shutdownMiniCluster(); + } + + @Test(expected = DoNotRetryIOException.class) + public void testCreateError() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().createTable(td); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError1() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError2() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError3() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + // return the TableDescriptor for creating table + private TableDescriptor createTableAndChangeToMigrationTracker() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + return td; + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError4() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError5() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError6() throws IOException { + TableDescriptor td = createTableAndChangeToMigrationTracker(); + TableDescriptor newTd = + TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.DEFAULT.name()).build(); + UTIL.getAdmin().modifyTable(newTd); + } + + @Test(expected = DoNotRetryIOException.class) + public void testModifyError7() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("family1")) + .setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .build()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + } + + // actually a NPE as we do not specify the src and dst impl for migration store file tracker + @Test(expected = IOException.class) + public void testModifyError8() throws IOException { + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName.getTableName()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build(); + UTIL.getAdmin().createTable(td); + TableDescriptor newTd = + TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()).build(); + UTIL.getAdmin().modifyTable(newTd); + } + + private String getStoreFileName(TableName table, byte[] family) { + return Iterables + .getOnlyElement(Iterables.getOnlyElement(UTIL.getMiniHBaseCluster().getRegions(table)) + .getStore(family).getStorefiles()) + .getPath().getName(); + } + + @Test + public void testModify() throws IOException { + TableName tn = tableName.getTableName(); + byte[] row = Bytes.toBytes("row"); + byte[] family = Bytes.toBytes("family"); + byte[] qualifier = Bytes.toBytes("qualifier"); + byte[] value = Bytes.toBytes("value"); + TableDescriptor td = TableDescriptorBuilder.newBuilder(tn) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); + UTIL.getAdmin().createTable(td); + try (Table table = UTIL.getConnection().getTable(tn)) { + table.put(new Put(row).addColumn(family, qualifier, value)); + } + UTIL.flush(tn); + String fileName = getStoreFileName(tn, family); + + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.MIGRATION.name()) + .setValue(MigrationStoreFileTracker.SRC_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()) + .setValue(MigrationStoreFileTracker.DST_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd); + assertEquals(fileName, getStoreFileName(tn, family)); + try (Table table = UTIL.getConnection().getTable(tn)) { + assertArrayEquals(value, table.get(new Get(row)).getValue(family, qualifier)); + } + + TableDescriptor newTd2 = TableDescriptorBuilder.newBuilder(td) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.FILE.name()) + .build(); + UTIL.getAdmin().modifyTable(newTd2); + assertEquals(fileName, getStoreFileName(tn, family)); + try (Table table = UTIL.getConnection().getTable(tn)) { + assertArrayEquals(value, table.get(new Get(row)).getValue(family, qualifier)); + } + } +} From be5a148ec028847efc87247337cb3d36d31f43f7 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 17 Sep 2021 09:40:44 +0800 Subject: [PATCH 33/37] HBASE-26280 Use store file tracker when snapshoting (#3685) Signed-off-by: Wellington Chevreuil Reviewed-by: Josh Elser --- .../MergeTableRegionsProcedure.java | 7 +- .../assignment/SplitTableRegionProcedure.java | 5 +- .../hbase/regionserver/HRegionFileSystem.java | 10 +-- .../StoreFileTrackerFactory.java | 17 ++--- .../hbase/snapshot/SnapshotManifest.java | 42 +++++------ ...apshotFromClientCloneLinksAfterDelete.java | 4 +- .../client/TestMobSnapshotFromClient.java | 7 +- .../hbase/client/TestSnapshotFromClient.java | 30 ++++++-- .../hbase/regionserver/TestHStoreFile.java | 6 +- .../snapshot/MobSnapshotTestingUtils.java | 74 ++++++++++--------- .../hbase/snapshot/SnapshotTestingUtils.java | 16 ++-- 11 files changed, 107 insertions(+), 111 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 9a00fe26a8f0..85a433af81e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -24,8 +24,6 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; - -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -612,9 +610,8 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg List mergedFiles = new ArrayList<>(); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); - Configuration trackerConfig = - StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs); final Collection storeFiles = tracker.load(); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 44136dc7cbee..60ff48985f42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -666,9 +666,8 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en new HashMap>(htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); - Configuration trackerConfig = StoreFileTrackerFactory. - mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName())); - StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, family, regionFs); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, cfd, regionFs); Collection sfis = tracker.load(); if (sfis == null) { continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index db18d4e50fd6..03a7c5c8baaf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -595,7 +595,6 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException { * to the proper location in the filesystem. * * @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo} - * @throws IOException */ public Path commitDaughterRegion(final RegionInfo regionInfo, List allRegionFiles, MasterProcedureEnv env) throws IOException { @@ -622,12 +621,8 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced Map> fileInfoMap = new HashMap<>(); for(Path file : allFiles) { String familyName = file.getParent().getName(); - trackerMap.computeIfAbsent(familyName, t -> { - Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc, - tblDesc.getColumnFamily(Bytes.toBytes(familyName))); - return StoreFileTrackerFactory. - create(config, familyName, regionFs); - }); + trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc, + tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs)); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); infos.add(new StoreFileInfo(conf, fs, file, true)); @@ -673,7 +668,6 @@ public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws I * this method is invoked on the Master side, then the RegionSplitPolicy will * NOT have a reference to a Region. * @return Path to created reference. - * @throws IOException */ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, boolean top, RegionSplitPolicy splitPolicy) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 90704fe528a0..b586027f8333 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -22,13 +22,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreUtils; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -113,16 +111,15 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli * Used at master side when splitting/merging regions, as we do not have a Store, thus no * StoreContext at master side. */ - public static StoreFileTracker create(Configuration conf, String family, - HRegionFileSystem regionFs) { - ColumnFamilyDescriptorBuilder fDescBuilder = - ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)); - StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()) - .withRegionFileSystem(regionFs).build(); - return StoreFileTrackerFactory.create(conf, true, ctx); + public static StoreFileTracker create(Configuration conf, TableDescriptor td, + ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs) { + StoreContext ctx = + StoreContext.getBuilder().withColumnFamilyDescriptor(cfd).withRegionFileSystem(regionFs) + .withFamilyStoreDirectoryPath(regionFs.getStoreDir(cfd.getNameAsString())).build(); + return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), true, ctx); } - public static Configuration mergeConfigurations(Configuration global, TableDescriptor table, + private static Configuration mergeConfigurations(Configuration global, TableDescriptor table, ColumnFamilyDescriptor family) { return StoreUtils.createStoreConfiguration(global, table, family); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 5e82cad6b494..f154aa92cd6e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -47,7 +47,8 @@ import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.Threads; @@ -291,8 +292,8 @@ public void addRegion(final Path tableDir, final RegionInfo regionInfo) throws I addRegion(tableDir, regionInfo, visitor); } - protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor) - throws IOException { + protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor) + throws IOException { boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo); try { Path baseDir = tableDir; @@ -300,8 +301,8 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio if (isMobRegion) { baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable()); } - HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, - baseDir, regionInfo, true); + HRegionFileSystem regionFs = + HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true); monitor.rethrowException(); // 1. dump region meta info into the snapshot directory @@ -317,26 +318,19 @@ protected void addRegion(final Path tableDir, final RegionInfo regionInfo, Regio // in batches and may miss files being added/deleted. This could be more robust (iteratively // checking to see if we have all the files until we are sure), but the limit is currently // 1000 files/batch, far more than the number of store files under a single column family. - Collection familyNames = regionFs.getFamilies(); - if (familyNames != null) { - for (String familyName: familyNames) { - Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName)); - monitor.rethrowException(); - - Collection storeFiles = regionFs.getStoreFiles(familyName); - if (storeFiles == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("No files under family: " + familyName); - } - continue; - } - - // 2.1. build the snapshot reference for the store - // iterate through all the store's files and create "references". - addReferenceFiles(visitor, regionData, familyData, storeFiles, false); - - visitor.familyClose(regionData, familyData); + for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { + Object familyData = visitor.familyOpen(regionData, cfd.getName()); + monitor.rethrowException(); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs); + List storeFiles = tracker.load(); + if (storeFiles.isEmpty()) { + LOG.debug("No files under family: {}", cfd.getNameAsString()); + continue; } + // 2.1. build the snapshot reference for the store + // iterate through all the store's files and create "references". + addReferenceFiles(visitor, regionData, familyData, storeFiles, false); + visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); } catch (IOException e) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java index c2087a935198..e352303f76ec 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobCloneSnapshotFromClientCloneLinksAfterDelete.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; @@ -93,7 +94,8 @@ public static void setUpBeforeClass() throws Exception { @Override protected void createTable() throws IOException, InterruptedException { MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, - SnapshotTestingUtils.getSplitKeys(), getNumReplicas(), DelayFlushCoprocessor.class.getName(), + SnapshotTestingUtils.getSplitKeys(), getNumReplicas(), + StoreFileTrackerFactory.Trackers.DEFAULT.name(), DelayFlushCoprocessor.class.getName(), FAMILY); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java index cdc41b01e87d..0695be12d0c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java @@ -26,8 +26,6 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Test create/using/deleting snapshots from the client @@ -41,8 +39,6 @@ public class TestMobSnapshotFromClient extends TestSnapshotFromClient { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestMobSnapshotFromClient.class); - private static final Logger LOG = LoggerFactory.getLogger(TestMobSnapshotFromClient.class); - /** * Setup the config for the cluster * @throws Exception on failure @@ -60,6 +56,7 @@ protected static void setupConf(Configuration conf) { @Override protected void createTable() throws Exception { - MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), TEST_FAM); + MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, getNumReplicas(), trackerImpl.name(), + TEST_FAM); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java index 08e33ac78d05..56a48c122a14 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; @@ -32,9 +33,11 @@ import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNameTestRule; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1; @@ -51,7 +54,10 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +70,8 @@ *

* This is an end-to-end test for the snapshot utility */ -@Category({LargeTests.class, ClientTests.class}) +@RunWith(Parameterized.class) +@Category({ LargeTests.class, ClientTests.class }) public class TestSnapshotFromClient { @ClassRule @@ -82,7 +89,16 @@ public class TestSnapshotFromClient { private static final Pattern MATCH_ALL = Pattern.compile(".*"); @Rule - public TestName name = new TestName(); + public TableNameTestRule name = new TableNameTestRule(); + + @Parameter + public StoreFileTrackerFactory.Trackers trackerImpl; + + @Parameters(name = "{index}: tracker={0}") + public static List params() { + return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT }, + new Object[] { StoreFileTrackerFactory.Trackers.FILE }); + } /** * Setup the config for the cluster @@ -109,7 +125,6 @@ protected static void setupConf(Configuration conf) { conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); - } @Before @@ -119,7 +134,8 @@ public void setup() throws Exception { protected void createTable() throws Exception { TableDescriptor htd = - TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()).build(); + TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas()) + .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build(); UTIL.createTable(htd, new byte[][] { TEST_FAM }, null); } @@ -316,7 +332,7 @@ public void testOfflineTableSnapshotWithEmptyRegions() throws Exception { @Test public void testListTableSnapshots() throws Exception { Admin admin = null; - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = name.getTableName(); try { admin = UTIL.getAdmin(); @@ -401,7 +417,7 @@ public void testListTableSnapshotsWithRegex() throws Exception { @Test public void testDeleteTableSnapshots() throws Exception { Admin admin = null; - final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName tableName = name.getTableName(); try { admin = UTIL.getAdmin(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index c7203a928aa7..b4f628b358b5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -1068,10 +1069,9 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration()); TableDescriptors mockTblDescs = mock(TableDescriptors.class); when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs); - TableDescriptor mockTblDesc = mock(TableDescriptor.class); + TableDescriptor mockTblDesc = TableDescriptorBuilder.newBuilder(hri.getTable()) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); when(mockTblDescs.get(any())).thenReturn(mockTblDesc); - ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class); - when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc); Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv); return new Path(new Path(regionDir, family), path.getName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index fea4fb4ba583..7523ae8fa0ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; @@ -45,29 +46,40 @@ public class MobSnapshotTestingUtils { /** * Create the Mob Table. */ - public static void createMobTable(final HBaseTestingUtil util, - final TableName tableName, int regionReplication, - final byte[]... families) throws IOException, InterruptedException { - createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), - regionReplication, families); + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + int regionReplication, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), regionReplication, + StoreFileTrackerFactory.Trackers.DEFAULT.name(), families); + } + + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + int regionReplication, String storeFileTracker, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), regionReplication, + storeFileTracker, families); } - public static void createPreSplitMobTable(final HBaseTestingUtil util, - final TableName tableName, int nRegions, final byte[]... families) - throws IOException, InterruptedException { - createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), - 1, families); + public static void createPreSplitMobTable(final HBaseTestingUtil util, final TableName tableName, + int nRegions, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), 1, families); + } + + public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, + final byte[][] splitKeys, int regionReplication, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, splitKeys, regionReplication, + StoreFileTrackerFactory.Trackers.DEFAULT.name(), families); } public static void createMobTable(final HBaseTestingUtil util, final TableName tableName, - final byte[][] splitKeys, int regionReplication, final byte[]... families) - throws IOException, InterruptedException { - createMobTable(util, tableName, splitKeys, regionReplication, null, families); + final byte[][] splitKeys, int regionReplication, String storeFileTracker, + final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, splitKeys, regionReplication, storeFileTracker, null, families); } - public static void createMobTable(HBaseTestingUtil util, TableName tableName, - byte[][] splitKeys, int regionReplication, String cpClassName, byte[]... families) - throws IOException, InterruptedException { + public static void createMobTable(HBaseTestingUtil util, TableName tableName, byte[][] splitKeys, + int regionReplication, String storeFileTracker, String cpClassName, byte[]... families) + throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); for (byte[] family : families) { @@ -77,6 +89,7 @@ public static void createMobTable(HBaseTestingUtil util, TableName tableName, if (!StringUtils.isBlank(cpClassName)) { builder.setCoprocessor(cpClassName); } + builder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, storeFileTracker); util.getAdmin().createTable(builder.build(), splitKeys); SnapshotTestingUtils.waitForTableToBeOnline(util, tableName); assertEquals((splitKeys.length + 1) * regionReplication, @@ -85,15 +98,10 @@ public static void createMobTable(HBaseTestingUtil util, TableName tableName, /** * Create a Mob table. - * - * @param util - * @param tableName - * @param families * @return An Table instance for the created table. - * @throws IOException */ - public static Table createMobTable(final HBaseTestingUtil util, - final TableName tableName, final byte[]... families) throws IOException { + public static Table createMobTable(final HBaseTestingUtil util, final TableName tableName, + final byte[]... families) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); for (byte[] family : families) { // Disable blooms (they are on by default as of 0.95) but we disable them @@ -102,10 +110,7 @@ public static Table createMobTable(final HBaseTestingUtil util, // and blooms being // on is interfering. builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family) - .setBloomFilterType(BloomType.NONE) - .setMobEnabled(true) - .setMobThreshold(0L) - .build()); + .setBloomFilterType(BloomType.NONE).setMobEnabled(true).setMobThreshold(0L).build()); } util.getAdmin().createTable(builder.build()); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait @@ -135,8 +140,8 @@ public static int countMobRows(final Table table, final byte[]... families) thro } } - public static void verifyMobRowCount(final HBaseTestingUtil util, - final TableName tableName, long expectedRows) throws IOException { + public static void verifyMobRowCount(final HBaseTestingUtil util, final TableName tableName, + long expectedRows) throws IOException { Table table = ConnectionFactory.createConnection(util.getConfiguration()).getTable(tableName); try { @@ -156,13 +161,10 @@ public SnapshotMock(final Configuration conf, final FileSystem fs, final Path ro @Override public TableDescriptor createHtd(final String tableName) { - return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)) - .setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(Bytes.toBytes(TEST_FAMILY)) - .setMobEnabled(true) - .setMobThreshold(0L) - .build()) - .build(); + return TableDescriptorBuilder + .newBuilder(TableName.valueOf(tableName)).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(Bytes.toBytes(TEST_FAMILY)).setMobEnabled(true).setMobThreshold(0L).build()) + .build(); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index d8d2a5eed0b1..00d2e84a464b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -480,9 +480,8 @@ public SnapshotBuilder(final Configuration conf, final FileSystem fs, this.desc = desc; this.tableRegions = tableRegions; this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); - new FSTableDescriptors(conf) - .createTableDescriptorForTableDirectory(this.snapshotDir.getFileSystem(conf), - snapshotDir, htd, false); + FSTableDescriptors.createTableDescriptorForTableDirectory( + this.snapshotDir.getFileSystem(conf), snapshotDir, htd, false); } public TableDescriptor getTableDescriptor() { @@ -502,15 +501,13 @@ public Path[] addRegion() throws IOException { } public Path[] addRegionV1() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) - .build()); + return addRegion( + desc.toBuilder().setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION).build()); } public Path[] addRegionV2() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION) - .build()); + return addRegion( + desc.toBuilder().setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION).build()); } private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { @@ -521,6 +518,7 @@ private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws I RegionData regionData = tableRegions[this.snapshotted++]; ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); + manifest.addTableDescriptor(htd); manifest.addRegion(regionData.tableDir, regionData.hri); return regionData.files; } From d3556bf41f05196f22840790e21342bb3d634ef7 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 13 Oct 2021 15:48:13 +0100 Subject: [PATCH 34/37] =?UTF-8?q?HBASE-26326=20CreateTableProcedure=20fail?= =?UTF-8?q?s=20when=20FileBasedStoreFileTracker=E2=80=A6=20(#3721)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Duo Zhang Signed-off-by: Josh Elser --- .../procedure/CreateTableProcedure.java | 6 ++---- .../FileBasedStoreFileTracker.java | 9 ++++++++- .../MigrationStoreFileTracker.java | 12 +---------- .../storefiletracker/StoreFileTracker.java | 5 +++-- .../StoreFileTrackerBase.java | 9 ++++----- .../StoreFileTrackerFactory.java | 20 +++++++++++++------ .../procedure/TestCreateTableProcedure.java | 16 +++++++++++++++ .../TestStoreFileTracker.java | 2 +- 8 files changed, 49 insertions(+), 30 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index ee8e51ffe86b..0a6a469328a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; @@ -290,9 +289,8 @@ private void preCreate(final MasterProcedureEnv env) (newRegions != null ? newRegions.size() : 0)); } - TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); - StoreFileTrackerFactory.persistTrackerConfig(env.getMasterConfiguration(), builder); - tableDescriptor = builder.build(); + tableDescriptor = StoreFileTrackerFactory.updateWithTrackerConfigs(env.getMasterConfiguration(), + tableDescriptor); final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index c370b87c1154..4da7911bdded 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -56,7 +56,14 @@ class FileBasedStoreFileTracker extends StoreFileTrackerBase { public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - backedFile = new StoreFileListFile(ctx); + //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table + //descriptors with the SFT impl specific configs. By the time this happens, the table has no + //regions nor stores yet, so it can't create a proper StoreContext. + if (ctx != null) { + backedFile = new StoreFileListFile(ctx); + } else { + backedFile = null; + } } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java index 1946d4bf7eef..230c1ec1b7a8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/MigrationStoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.StoreContext; @@ -88,17 +89,6 @@ void set(List files) { "Should not call this method on " + getClass().getSimpleName()); } - @Override - public void persistConfiguration(TableDescriptorBuilder builder) { - super.persistConfiguration(builder); - if (StringUtils.isEmpty(builder.getValue(SRC_IMPL))) { - builder.setValue(SRC_IMPL, src.getTrackerName()); - } - if (StringUtils.isEmpty(builder.getValue(DST_IMPL))) { - builder.setValue(DST_IMPL, dst.getTrackerName()); - } - } - static Class getSrcTrackerClass(Configuration conf) { return StoreFileTrackerFactory.getStoreFileTrackerClassForMigration(conf, SRC_IMPL); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index 59fe7ef52f96..fd8f7c99092d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -75,7 +76,7 @@ void replace(Collection compactedFiles, Collection StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; /** - * Saves StoreFileTracker implementations specific configurations into the table descriptors. + * Adds StoreFileTracker implementations specific configurations into the table descriptor. *

* This is used to avoid accidentally data loss when changing the cluster level store file tracker * implementation, and also possible misconfiguration between master and region servers. @@ -83,5 +84,5 @@ void replace(Collection compactedFiles, Collection * See HBASE-26246 for more details. * @param builder The table descriptor builder for the given table. */ - void persistConfiguration(TableDescriptorBuilder builder); + TableDescriptorBuilder updateWithTrackerConfigs(TableDescriptorBuilder builder); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index a786add49b21..edbaace40c26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -32,7 +33,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -83,10 +83,9 @@ public final void replace(Collection compactedFiles, } @Override - public void persistConfiguration(TableDescriptorBuilder builder) { - if (StringUtils.isEmpty(builder.getValue(TRACKER_IMPL))) { - builder.setValue(TRACKER_IMPL, getTrackerName()); - } + public TableDescriptorBuilder updateWithTrackerConfigs(TableDescriptorBuilder builder) { + builder.setValue(TRACKER_IMPL, getTrackerName()); + return builder; } protected final String getTrackerName() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index b586027f8333..1c683ae3de62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -24,8 +24,10 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; + import org.apache.hadoop.hbase.regionserver.StoreUtils; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -158,12 +160,18 @@ static StoreFileTrackerBase createForMigration(Configuration conf, String config return ReflectionUtils.newInstance(tracker, conf, isPrimaryReplica, ctx); } - public static void persistTrackerConfig(Configuration conf, TableDescriptorBuilder builder) { - TableDescriptor tableDescriptor = builder.build(); - ColumnFamilyDescriptor cfDesc = tableDescriptor.getColumnFamilies()[0]; - StoreContext context = StoreContext.getBuilder().withColumnFamilyDescriptor(cfDesc).build(); - StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, context); - tracker.persistConfiguration(builder); + public static TableDescriptor updateWithTrackerConfigs(Configuration conf, + TableDescriptor descriptor) { + //CreateTableProcedure needs to instantiate the configured SFT impl, in order to update table + //descriptors with the SFT impl specific configs. By the time this happens, the table has no + //regions nor stores yet, so it can't create a proper StoreContext. + if (StringUtils.isEmpty(descriptor.getValue(TRACKER_IMPL))) { + StoreFileTracker tracker = + StoreFileTrackerFactory.create(conf, true, null); + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(descriptor); + return tracker.updateWithTrackerConfigs(builder).build(); + } + return descriptor; } // should not use MigrationStoreFileTracker for new family diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index f432c8060d3d..51ea9f58248a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -105,6 +106,21 @@ public void testCreateWithTrackImpl() throws Exception { assertEquals(trackerName, htd.getValue(TRACKER_IMPL)); } + @Test + public void testCreateWithFileBasedStoreTrackerImpl() throws Exception { + ProcedureExecutor procExec = getMasterProcedureExecutor(); + procExec.getEnvironment().getMasterConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL, + StoreFileTrackerFactory.Trackers.FILE.name()); + final TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, F1); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + htd = getMaster().getTableDescriptors().get(tableName); + assertEquals(StoreFileTrackerFactory.Trackers.FILE.name(), htd.getValue(TRACKER_IMPL)); + } + @Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor procExec = getMasterProcedureExecutor(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java index 1dc9c4e53fc5..b30ca47772cb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTracker.java @@ -40,7 +40,7 @@ public class TestStoreFileTracker extends DefaultStoreFileTracker { public TestStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); - if (ctx.getRegionFileSystem() != null) { + if (ctx != null && ctx.getRegionFileSystem() != null) { this.storeId = ctx.getRegionInfo().getEncodedName() + "-" + ctx.getFamily().getNameAsString(); LOG.info("created storeId: {}", storeId); trackedFiles.computeIfAbsent(storeId, v -> new ArrayList<>()); From 714e6a0f54d6ee15964281efc7cb2bb51309731c Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Tue, 19 Oct 2021 22:15:01 +0100 Subject: [PATCH 35/37] Fixing UTs, checkstyle and addressing review comments. Change-Id: Iff08397b54c2a4d5980aa2ce0e8cad07239128ff --- .../org/apache/hadoop/hbase/io/HFileLink.java | 24 +++++++------- .../procedure/CloneSnapshotProcedure.java | 4 +-- .../SnapshotStoreFileTracker.java | 10 +++--- .../hbase/snapshot/RestoreSnapshotHelper.java | 31 ++++++++++--------- ...estCloneSnapshotProcedureFileBasedSFT.java | 25 +-------------- 5 files changed, 37 insertions(+), 57 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index 9d5db7282573..93c7efffa58f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -323,8 +322,8 @@ public static String createHFileLinkName(final TableName tableName, * @param dstFamilyPath - Destination path (table/region/cf/) * @param hfileRegionInfo - Linked HFile Region Info * @param hfileName - Linked HFile name - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure + * @return the file link name. + * @throws IOException on file or parent directory creation failure. */ public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final RegionInfo hfileRegionInfo, @@ -344,8 +343,8 @@ public static String create(final Configuration conf, final FileSystem fs, * @param hfileRegionInfo - Linked HFile Region Info * @param hfileName - Linked HFile name * @param createBackRef - Whether back reference should be created. Defaults to true. - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure + * @return the file link name. + * @throws IOException on file or parent directory creation failure. */ public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final RegionInfo hfileRegionInfo, @@ -367,8 +366,8 @@ public static String create(final Configuration conf, final FileSystem fs, * @param linkedTable - Linked Table Name * @param linkedRegion - Linked Region Name * @param hfileName - Linked HFile name - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure + * @return the file link name. + * @throws IOException on file or parent directory creation failure. */ public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, @@ -389,8 +388,8 @@ public static String create(final Configuration conf, final FileSystem fs, * @param linkedRegion - Linked Region Name * @param hfileName - Linked HFile name * @param createBackRef - Whether back reference should be created. Defaults to true. - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure + * @return the file link name. + * @throws IOException on file or parent directory creation failure. */ public static String create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, @@ -431,7 +430,8 @@ public static String create(final Configuration conf, final FileSystem fs, } throw e; } - throw new IOException("File link=" + name + " already exists under " + dstFamilyPath + " folder."); + throw new IOException("File link=" + name + " already exists under " + + dstFamilyPath + " folder."); } /** @@ -445,8 +445,8 @@ public static String create(final Configuration conf, final FileSystem fs, * @param dstFamilyPath - Destination path (table/region/cf/) * @param hfileLinkName - HFileLink name (it contains hfile-region-table) * @param createBackRef - Whether back reference should be created. Defaults to true. - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure + * @return the file link name. + * @throws IOException on file or parent directory creation failure. */ public static String createFromHFileLink(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index b8001b4db7a6..31e7ec273a3b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions; -import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; @@ -456,7 +455,8 @@ private List createFsLayout( // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first - final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); + final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), + tableDescriptor.getTableName()); if (CommonFSUtils.isExists(mfs.getFileSystem(), tableDir)) { // if the region dirs exist, will cause exception and unlimited retry (see HBASE-24546) LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tableDir); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java index f884121cdaf2..be393df40527 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java @@ -17,15 +17,15 @@ */ package org.apache.hadoop.hbase.regionserver.storefiletracker; +import java.io.IOException; +import java.util.List; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.yetus.audience.InterfaceAudience; -import java.io.IOException; -import java.util.List; - /** * Extends MigrationStoreFileTracker for Snapshot restore/clone specific case. * When restoring/cloning snapshots, new regions are created with reference files to the @@ -64,8 +64,8 @@ public void setReferenceFiles(List files) { this.files = files; } /** - * Overrides MigrationStoreFileTracker to simply call set on destination - * SFT implementation, passing the list of reference files. + * Overrides MigrationStoreFileTracker to simply call set + * on destination SFT implementation, passing the list of reference files. * @return * @throws IOException */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 6f41b5654440..ce0ae3f71f9a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -687,8 +687,9 @@ private void cloneHdfsMobRegion(final Map region * @param regionDir {@link Path} cloned dir * @param snapshotRegionInfo */ - private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, final RegionInfo snapshotRegionInfo, - final SnapshotRegionManifest manifest) throws IOException { + private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, + final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) + throws IOException { final String tableName = tableDesc.getTableName().getNameAsString(); final String snapshotName = snapshotDesc.getName(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { @@ -710,19 +711,21 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, f clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); } } - Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); - HRegionFileSystem regionFS = (fs.exists(regionPath)) ? - HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : - HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); - - SnapshotStoreFileTracker tracker = - (SnapshotStoreFileTracker) StoreFileTrackerFactory.create(conf, true, - StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). + //we don't need to track regions under mobdir + if (!MobUtils.isMobRegionInfo(newRegionInfo)) { + Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); + HRegionFileSystem regionFS = (fs.exists(regionPath)) ? + HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : + HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); + + SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) StoreFileTrackerFactory + .create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). withRegionFileSystem(regionFS).build()); - //since we are adding the reference files for the first time, - // we do tracker.load, rather than add - tracker.getSourceTracker().setReferenceFiles(clonedFiles); - tracker.load(); + //since we are adding the reference files for the first time, + // we do tracker.load, rather than add + tracker.getSourceTracker().setReferenceFiles(clonedFiles); + tracker.load(); + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java index 12a50bdb3501..f3ae1283b48a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedureFileBasedSFT.java @@ -19,37 +19,14 @@ import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.Trackers.FILE; -import static org.junit.Assert.assertTrue; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.TableExistsException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.procedure2.Procedure; -import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; -import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.After; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.stream.Stream; - -@Category({MasterTests.class, MediumTests.class}) +@Category({ MasterTests.class, MediumTests.class}) public class TestCloneSnapshotProcedureFileBasedSFT extends TestCloneSnapshotProcedure { @ClassRule From 8e3400e7d2b8c31075f6b8d11d51a574d8616c7e Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Thu, 21 Oct 2021 16:03:03 +0100 Subject: [PATCH 36/37] adapting this PR to the changes from HBASE-26386 Change-Id: Idbebddc6e455ab3630942f1e982bc9da91cc6a31 --- .../SnapshotStoreFileTracker.java | 77 ------------------- .../hbase/snapshot/RestoreSnapshotHelper.java | 34 +++----- 2 files changed, 10 insertions(+), 101 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java deleted file mode 100644 index be393df40527..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/SnapshotStoreFileTracker.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver.storefiletracker; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.regionserver.StoreContext; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Extends MigrationStoreFileTracker for Snapshot restore/clone specific case. - * When restoring/cloning snapshots, new regions are created with reference files to the - * original regions files. This work is done in snapshot specific classes. We need to somehow - * initialize these reference files in the configured StoreFileTracker. Once snapshot logic has - * cloned the store dir and created the references, it should set the list of reference files in - * SourceTracker.setReferenceFiles then invoke load method. - *

- */ -@InterfaceAudience.Private -public class SnapshotStoreFileTracker extends MigrationStoreFileTracker { - - protected SnapshotStoreFileTracker(Configuration conf, boolean isPrimaryReplica, - StoreContext ctx) { - super(conf, isPrimaryReplica, ctx); - Preconditions.checkArgument(src instanceof SourceTracker, - "src for SnapshotStoreFileTracker should always be a SourceTracker!"); - } - - public SourceTracker getSourceTracker(){ - return (SourceTracker)this.src; - } - - /** - * The SFT impl to be set as source for SnapshotStoreFileTracker. - */ - public static class SourceTracker extends DefaultStoreFileTracker { - - private List files; - - public SourceTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { - super(conf, isPrimaryReplica, ctx); - } - - public void setReferenceFiles(List files) { - this.files = files; - } - /** - * Overrides MigrationStoreFileTracker to simply call set - * on destination SFT implementation, passing the list of reference files. - * @return - * @throws IOException - */ - @Override - public List load() throws IOException { - return files; - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index ce0ae3f71f9a..cb5bf1528ede 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -57,7 +57,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.regionserver.storefiletracker.SnapshotStoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.security.access.AccessControlClient; import org.apache.hadoop.hbase.security.access.Permission; @@ -158,7 +158,7 @@ public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final ForeignExceptionDispatcher monitor, final MonitoredTask status, final boolean createBackRefs) { this.fs = fs; - this.conf = new Configuration(conf); + this.conf = conf; this.snapshotManifest = manifest; this.snapshotDesc = manifest.getSnapshotDescription(); this.snapshotTable = TableName.valueOf(snapshotDesc.getTable()); @@ -168,13 +168,6 @@ public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, this.monitor = monitor; this.status = status; this.createBackRefs = createBackRefs; - this.conf.set(TRACKER_IMPL, SnapshotStoreFileTracker.class.getName()); - this.conf.set("hbase.store.file-tracker.migration.src.impl", - SnapshotStoreFileTracker.SourceTracker.class.getName()); - this.conf.set("hbase.store.file-tracker.migration.dst.impl", - this.tableDesc.getValue(TRACKER_IMPL) != null ? - this.tableDesc.getValue(TRACKER_IMPL) : - StoreFileTrackerFactory.Trackers.DEFAULT.name()); } /** @@ -568,23 +561,20 @@ private void restoreRegion(final RegionInfo regionInfo, fs.delete(familyDir, true); } - SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) - StoreFileTrackerFactory.create(conf, true, + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). withRegionFileSystem(regionFS).build()); //simply reset list of tracked files with the matching files - // and the extra one present in the snapshot - tracker.getSourceTracker().setReferenceFiles(filesToTrack); - tracker.load(); + //and the extra one present in the snapshot + tracker.set(filesToTrack); } // Add families not present in the table for (Map.Entry> familyEntry: snapshotFiles.entrySet()) { Path familyDir = new Path(regionDir, familyEntry.getKey()); - SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) - StoreFileTrackerFactory.create(conf, true, + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). withRegionFileSystem(regionFS).build()); List files = new ArrayList<>(); @@ -598,8 +588,7 @@ private void restoreRegion(final RegionInfo regionInfo, String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); } - tracker.getSourceTracker().setReferenceFiles(files); - tracker.load(); + tracker.set(files); } } @@ -718,13 +707,10 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); - SnapshotStoreFileTracker tracker = (SnapshotStoreFileTracker) StoreFileTrackerFactory - .create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir). withRegionFileSystem(regionFS).build()); - //since we are adding the reference files for the first time, - // we do tracker.load, rather than add - tracker.getSourceTracker().setReferenceFiles(clonedFiles); - tracker.load(); + tracker.set(clonedFiles); } } From 1adaf4277de723fb79be508f19d068f543700fe5 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 22 Oct 2021 16:55:14 +0100 Subject: [PATCH 37/37] Added/Fixed comments Change-Id: I08378c1254aaca82e732248826a5394835fba8b5 --- .../src/main/java/org/apache/hadoop/hbase/io/HFileLink.java | 3 ++- .../apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index 93c7efffa58f..fc158a549751 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -376,7 +376,8 @@ public static String create(final Configuration conf, final FileSystem fs, } /** - * Create a new HFileLink + * Create a new HFileLink. In the event of link creation failure, this method throws an + * IOException, so that the calling upper laying can decide on how to proceed with this. * *

It also adds a back-reference to the hfile back-reference directory * to simplify the reference-count and the cleaning process. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index cb5bf1528ede..7c75e4658305 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -700,7 +700,7 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); } } - //we don't need to track regions under mobdir + //we don't need to track files under mobdir if (!MobUtils.isMobRegionInfo(newRegionInfo)) { Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); HRegionFileSystem regionFS = (fs.exists(regionPath)) ?