diff --git a/.github/workflows/git.yml b/.github/workflows/git.yml index 9a01fbab..11cf5879 100644 --- a/.github/workflows/git.yml +++ b/.github/workflows/git.yml @@ -14,11 +14,12 @@ jobs: - name: Gen docs run: make docs - - name: Check for uncommitted changes + - name: Check for uncommitted documentation changes run: | if [[ -n $(git status -s) ]]; then echo "There are uncommitted changes after the task:" git status -s + echo "Update the documentation on the proper *Config.java and commit generated docs" exit 1 else echo "No changes detected." diff --git a/Makefile b/Makefile index 2ef484ea..112277d9 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,6 @@ all: clean build test clean: ./gradlew clean - rm -f docs/*.rst checkstyle: ./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest @@ -44,12 +43,8 @@ storage/azure/build/distributions/azure-$(VERSION).tgz: ./gradlew build :storage:azure:distTar -x test -x integrationTest -x e2e:test .PHONY: docs -docs: config.rst metrics.rst - -config.rst: - ./gradlew :docs:genConfigDocs - -metrics.rst: +docs: + ./gradlew :docs:genConfigsDocs ./gradlew :docs:genMetricsDocs test: build diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index db28947f..25ee31d9 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -39,5 +39,6 @@ + diff --git a/docs/build.gradle b/docs/build.gradle index 548e384e..589d8883 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -26,7 +26,7 @@ dependencies { implementation project(":storage:filesystem") } -tasks.register('genConfigDocs', JavaExec) { +tasks.register('genConfigsDocs', JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigsDocs' standardOutput = new File("docs/configs.rst").newOutputStream() @@ -36,4 +36,8 @@ tasks.register('genMetricsDocs', JavaExec) { classpath = sourceSets.main.runtimeClasspath mainClass = 'io.aiven.kafka.tieredstorage.misc.MetricsDocs' standardOutput = new File("docs/metrics.rst").newOutputStream() +} + +tasks.named('compileJava') { + finalizedBy 'genConfigsDocs', 'genMetricsDocs' } \ No newline at end of file diff --git a/docs/configs.rst b/docs/configs.rst index db1d7055..36a6cd9c 100644 --- a/docs/configs.rst +++ b/docs/configs.rst @@ -1,6 +1,8 @@ ================= Core components ================= +.. Generated from *Config.java classes by io.aiven.kafka.tieredstorage.misc.ConfigsDocs + ----------------- RemoteStorageManagerConfig ----------------- @@ -94,6 +96,7 @@ RemoteStorageManagerConfig * Importance: low + ----------------- SegmentManifestCacheConfig ----------------- @@ -132,6 +135,7 @@ Under ``fetch.manifest.cache.`` * Importance: low + ----------------- SegmentIndexesCacheConfig ----------------- @@ -170,6 +174,7 @@ Under ``fetch.indexes.cache.`` * Importance: low + ----------------- ChunkManagerFactoryConfig ----------------- @@ -182,6 +187,7 @@ ChunkManagerFactoryConfig * Importance: medium + ----------------- MemoryChunkCacheConfig ----------------- @@ -227,6 +233,7 @@ Under ``fetch.chunk.cache.`` * Importance: low + ----------------- DiskChunkCacheConfig ----------------- @@ -278,6 +285,7 @@ Under ``fetch.chunk.cache.`` * Importance: low + ================= Storage Backends ================= @@ -342,6 +350,7 @@ AzureBlobStorageStorageConfig * Importance: low + ----------------- AzureBlobStorageStorageConfig ----------------- @@ -392,6 +401,7 @@ AzureBlobStorageStorageConfig * Importance: low + ----------------- S3StorageConfig ----------------- @@ -486,10 +496,11 @@ S3StorageConfig * Importance: low + ----------------- FilesystemStorageConfig ----------------- -> Only for development/testing purposes +.. Only for development/testing purposes ``root`` Root directory @@ -504,3 +515,4 @@ FilesystemStorageConfig * Importance: medium + diff --git a/docs/metrics.rst b/docs/metrics.rst index 5110439c..c99aa231 100644 --- a/docs/metrics.rst +++ b/docs/metrics.rst @@ -1,6 +1,7 @@ ================= Core components metrics ================= +.. Generated from MetricRegistry classes by io.aiven.kafka.tieredstorage.misc.MetricsDocs ----------------- RemoteStorageManager metrics diff --git a/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigsDocs.java b/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigsDocs.java index ef7f588e..6c9e0b1b 100644 --- a/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigsDocs.java +++ b/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigsDocs.java @@ -33,6 +33,7 @@ import static io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig.FETCH_INDEXES_CACHE_PREFIX; import static io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig.SEGMENT_MANIFEST_CACHE_PREFIX; import static io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig.STORAGE_PREFIX; +import static java.lang.System.out; /** * Gather all config definitions across the project and generate a documentation page @@ -40,64 +41,76 @@ public class ConfigsDocs { public static void main(final String[] args) { printSectionTitle("Core components"); + out.println(".. Generated from *Config.java classes by " + ConfigsDocs.class.getCanonicalName()); + out.println(); printSubsectionTitle("RemoteStorageManagerConfig"); final var rsmConfigDef = RemoteStorageManagerConfig.configDef(); - System.out.println(rsmConfigDef.toEnrichedRst()); + out.println(rsmConfigDef.toEnrichedRst()); + out.println(); printSubsectionTitle("SegmentManifestCacheConfig"); - System.out.println("Under ``" + SEGMENT_MANIFEST_CACHE_PREFIX + "``\n"); + out.println("Under ``" + SEGMENT_MANIFEST_CACHE_PREFIX + "``\n"); final var segmentManifestCacheDef = MemorySegmentManifestCache.configDef(); - System.out.println(segmentManifestCacheDef.toEnrichedRst()); + out.println(segmentManifestCacheDef.toEnrichedRst()); + out.println(); printSubsectionTitle("SegmentIndexesCacheConfig"); - System.out.println("Under ``" + FETCH_INDEXES_CACHE_PREFIX + "``\n"); + out.println("Under ``" + FETCH_INDEXES_CACHE_PREFIX + "``\n"); final var segmentIndexesCacheDef = MemorySegmentIndexesCache.configDef(); - System.out.println(segmentIndexesCacheDef.toEnrichedRst()); + out.println(segmentIndexesCacheDef.toEnrichedRst()); + out.println(); printSubsectionTitle("ChunkManagerFactoryConfig"); final var chunkCacheFactoryDef = ChunkManagerFactoryConfig.configDef(); - System.out.println(chunkCacheFactoryDef.toEnrichedRst()); + out.println(chunkCacheFactoryDef.toEnrichedRst()); + out.println(); printSubsectionTitle("MemoryChunkCacheConfig"); - System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n"); + out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n"); final var memChunkCacheDef = ChunkCacheConfig.configDef(new ConfigDef()); - System.out.println(memChunkCacheDef.toEnrichedRst()); + out.println(memChunkCacheDef.toEnrichedRst()); + out.println(); printSubsectionTitle("DiskChunkCacheConfig"); - System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n"); + out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n"); final var diskChunkCacheDef = DiskChunkCacheConfig.configDef(); - System.out.println(diskChunkCacheDef.toEnrichedRst()); + out.println(diskChunkCacheDef.toEnrichedRst()); + out.println(); printSectionTitle("Storage Backends"); - System.out.println("Under ``" + STORAGE_PREFIX + "``\n"); + out.println("Under ``" + STORAGE_PREFIX + "``\n"); printSubsectionTitle("AzureBlobStorageStorageConfig"); final var azBlobStorageConfigDef = AzureBlobStorageConfig.configDef(); - System.out.println(azBlobStorageConfigDef.toEnrichedRst()); + out.println(azBlobStorageConfigDef.toEnrichedRst()); + out.println(); printSubsectionTitle("AzureBlobStorageStorageConfig"); final var googleCloudConfigDef = GcsStorageConfig.configDef(); - System.out.println(googleCloudConfigDef.toEnrichedRst()); + out.println(googleCloudConfigDef.toEnrichedRst()); + out.println(); printSubsectionTitle("S3StorageConfig"); final var s3StorageConfigDef = S3StorageConfig.configDef(); - System.out.println(s3StorageConfigDef.toEnrichedRst()); - + out.println(s3StorageConfigDef.toEnrichedRst()); + out.println(); + printSubsectionTitle("FilesystemStorageConfig"); - System.out.println("> Only for development/testing purposes"); + out.println(".. Only for development/testing purposes"); final var fsStorageConfigDef = FileSystemStorageConfig.configDef(); - System.out.println(fsStorageConfigDef.toEnrichedRst()); + out.println(fsStorageConfigDef.toEnrichedRst()); + out.println(); } static void printSectionTitle(final String title) { - System.out.println("=================\n" + out.println("=================\n" + title + "\n" + "================="); } static void printSubsectionTitle(final String title) { - System.out.println("-----------------\n" + out.println("-----------------\n" + title + "\n" + "-----------------"); } diff --git a/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricsDocs.java b/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricsDocs.java index c6549733..80d9f4ab 100644 --- a/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricsDocs.java +++ b/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricsDocs.java @@ -32,62 +32,65 @@ import io.aiven.kafka.tieredstorage.metrics.MetricsRegistry; import io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry; +import static java.lang.System.out; + public class MetricsDocs { public static void main(final String[] args) { printSectionTitle("Core components metrics"); - System.out.println(); + out.println(".. Generated from MetricRegistry classes by " + MetricsDocs.class.getCanonicalName()); + out.println(); printSubsectionTitle("RemoteStorageManager metrics"); - System.out.println(); - System.out.println(toRstTable(MetricsRegistry.METRIC_CONTEXT, new MetricsRegistry().all())); + out.println(); + out.println(toRstTable(MetricsRegistry.METRIC_CONTEXT, new MetricsRegistry().all())); - System.out.println(); + out.println(); printSubsectionTitle("SegmentManifestCache metrics"); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( CaffeineMetricsRegistry.METRIC_CONTEXT, new CaffeineMetricsRegistry(MemorySegmentManifestCache.METRIC_GROUP).all())); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG, new ThreadPoolMonitorMetricsRegistry(MemorySegmentManifestCache.THREAD_POOL_METRIC_GROUP).all())); - System.out.println(); + out.println(); printSubsectionTitle("SegmentIndexesCache metrics"); - System.out.println(toRstTable( + out.println(toRstTable( CaffeineMetricsRegistry.METRIC_CONTEXT, new CaffeineMetricsRegistry(MemorySegmentIndexesCache.METRIC_GROUP).all())); - System.out.println(toRstTable( + out.println(toRstTable( ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG, new ThreadPoolMonitorMetricsRegistry(MemorySegmentIndexesCache.THREAD_POOL_METRIC_GROUP).all())); - System.out.println(); + out.println(); printSubsectionTitle("ChunkCache metrics"); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( CaffeineMetricsRegistry.METRIC_CONTEXT, new CaffeineMetricsRegistry(ChunkCache.METRIC_GROUP).all())); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG, new ThreadPoolMonitorMetricsRegistry(ChunkCache.THREAD_POOL_METRIC_GROUP).all())); - System.out.println(); + out.println(); printSectionTitle("Storage Backend metrics"); - System.out.println(); + out.println(); printSubsectionTitle("AzureBlobStorage metrics"); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.METRIC_CONTEXT, new io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry().all())); - System.out.println(); + out.println(); printSubsectionTitle("GcsStorage metrics"); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT, new io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry().all())); - System.out.println(); + out.println(); printSubsectionTitle("S3Storage metrics"); - System.out.println(); - System.out.println(toRstTable( + out.println(); + out.println(toRstTable( io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.METRIC_CONTEXT, new io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry().all())); } @@ -181,13 +184,13 @@ static String getMBeanName(final String prefix, final MetricName metricName) { } static void printSectionTitle(final String title) { - System.out.println("=================\n" + out.println("=================\n" + title + "\n" + "================="); } static void printSubsectionTitle(final String title) { - System.out.println("-----------------\n" + out.println("-----------------\n" + title + "\n" + "-----------------"); } diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/GcsStorageConfig.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/GcsStorageConfig.java index 263d12cf..c8e0cdfd 100644 --- a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/GcsStorageConfig.java +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/GcsStorageConfig.java @@ -42,7 +42,9 @@ public class GcsStorageConfig extends AbstractConfig { private static final String GCS_RESUMABLE_UPLOAD_CHUNK_SIZE_DOC = "The chunk size for resumable upload. " + "Must be a multiple of 256 KiB (256 x 1024 bytes). " + "Larger chunk sizes typically make uploads faster, but requires bigger memory buffers. " - + "The recommended minimum is 8 MiB. The default is 15 MiB."; + + "The recommended minimum is 8 MiB. The default is 15 MiB, " + + "`dictated by the GCS SDK `_ " + + "when we set it to null."; static final String GCP_CREDENTIALS_JSON_CONFIG = "gcs.credentials.json"; static final String GCP_CREDENTIALS_PATH_CONFIG = "gcs.credentials.path";