Merge pull request #623 from Aiven-Open/jeqo/fix-make

chore: clarify docs generation

.github/workflows/git.yml

@@ -14,11 +14,12 @@ jobs:
     - name: Gen docs
       run: make docs
 
-    - name: Check for uncommitted changes
+    - name: Check for uncommitted documentation changes
       run: |
         if [[ -n $(git status -s) ]]; then
           echo "There are uncommitted changes after the task:"
           git status -s
+          echo "Update the documentation on the proper *Config.java and commit generated docs"
           exit 1
         else
           echo "No changes detected."

Makefile

@@ -24,7 +24,6 @@ all: clean build test
 
 clean:
 	./gradlew clean
-	rm -f docs/*.rst
 
 checkstyle:
 	./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest
@@ -44,12 +43,8 @@ storage/azure/build/distributions/azure-$(VERSION).tgz:
 	./gradlew build :storage:azure:distTar -x test -x integrationTest -x e2e:test
 
 .PHONY: docs
-docs: config.rst metrics.rst
-
-config.rst:
-	./gradlew :docs:genConfigDocs
-
-metrics.rst:
+docs:
+	./gradlew :docs:genConfigsDocs
 	./gradlew :docs:genMetricsDocs
 
 test: build

checkstyle/suppressions.xml

@@ -39,5 +39,6 @@
     <suppress checks="JavaNCSSCheck" files="MetricsRegistry.java"/>
     <suppress checks="JavaNCSSCheck" files="RemoteStorageManagerMetricsTest.java"/>
     <suppress checks="JavaNCSSCheck" files="SingleBrokerTest.java"/>
+    <suppress checks="JavaNCSSCheck" files=".*Docs\.java"/>
     <suppress checks="NPathComplexity" files="CredentialsBuilder.java"/>
 </suppressions>

docs/build.gradle

@@ -26,7 +26,7 @@ dependencies {
     implementation project(":storage:filesystem")
 }
 
-tasks.register('genConfigDocs', JavaExec) {
+tasks.register('genConfigsDocs', JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
     mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigsDocs'
     standardOutput = new File("docs/configs.rst").newOutputStream()
@@ -36,4 +36,8 @@ tasks.register('genMetricsDocs', JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
     mainClass = 'io.aiven.kafka.tieredstorage.misc.MetricsDocs'
     standardOutput = new File("docs/metrics.rst").newOutputStream()
+}
+
+tasks.named('compileJava') {
+    finalizedBy 'genConfigsDocs', 'genMetricsDocs'
 }

docs/configs.rst

@@ -1,6 +1,8 @@
 =================
 Core components
 =================
+.. Generated from *Config.java classes by io.aiven.kafka.tieredstorage.misc.ConfigsDocs
+
 -----------------
 RemoteStorageManagerConfig
 -----------------
@@ -94,6 +96,7 @@ RemoteStorageManagerConfig
   * Importance: low
 
 
+
 -----------------
 SegmentManifestCacheConfig
 -----------------
@@ -132,6 +135,7 @@ Under ``fetch.manifest.cache.``
   * Importance: low
 
 
+
 -----------------
 SegmentIndexesCacheConfig
 -----------------
@@ -170,6 +174,7 @@ Under ``fetch.indexes.cache.``
   * Importance: low
 
 
+
 -----------------
 ChunkManagerFactoryConfig
 -----------------
@@ -182,6 +187,7 @@ ChunkManagerFactoryConfig
   * Importance: medium
 
 
+
 -----------------
 MemoryChunkCacheConfig
 -----------------
@@ -227,6 +233,7 @@ Under ``fetch.chunk.cache.``
   * Importance: low
 
 
+
 -----------------
 DiskChunkCacheConfig
 -----------------
@@ -278,6 +285,7 @@ Under ``fetch.chunk.cache.``
   * Importance: low
 
 
+
 =================
 Storage Backends
 =================
@@ -342,6 +350,7 @@ AzureBlobStorageStorageConfig
   * Importance: low
 
 
+
 -----------------
 AzureBlobStorageStorageConfig
 -----------------
@@ -392,6 +401,7 @@ AzureBlobStorageStorageConfig
   * Importance: low
 
 
+
 -----------------
 S3StorageConfig
 -----------------
@@ -486,10 +496,11 @@ S3StorageConfig
   * Importance: low
 
 
+
 -----------------
 FilesystemStorageConfig
 -----------------
-> Only for development/testing purposes
+.. Only for development/testing purposes
 ``root``
   Root directory
 
@@ -504,3 +515,4 @@ FilesystemStorageConfig
   * Importance: medium
 
 
+

docs/metrics.rst

@@ -1,6 +1,7 @@
 =================
 Core components metrics
 =================
+.. Generated from MetricRegistry classes by io.aiven.kafka.tieredstorage.misc.MetricsDocs
 
 -----------------
 RemoteStorageManager metrics

docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigsDocs.java

@@ -33,71 +33,84 @@
 import static io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig.FETCH_INDEXES_CACHE_PREFIX;
 import static io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig.SEGMENT_MANIFEST_CACHE_PREFIX;
 import static io.aiven.kafka.tieredstorage.config.RemoteStorageManagerConfig.STORAGE_PREFIX;
+import static java.lang.System.out;
 
 /**
  * Gather all config definitions across the project and generate a documentation page
  **/
 public class ConfigsDocs {
     public static void main(final String[] args) {
         printSectionTitle("Core components");
+        out.println(".. Generated from *Config.java classes by " + ConfigsDocs.class.getCanonicalName());
+        out.println();
 
         printSubsectionTitle("RemoteStorageManagerConfig");
         final var rsmConfigDef = RemoteStorageManagerConfig.configDef();
-        System.out.println(rsmConfigDef.toEnrichedRst());
+        out.println(rsmConfigDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("SegmentManifestCacheConfig");
-        System.out.println("Under ``" + SEGMENT_MANIFEST_CACHE_PREFIX + "``\n");
+        out.println("Under ``" + SEGMENT_MANIFEST_CACHE_PREFIX + "``\n");
         final var segmentManifestCacheDef = MemorySegmentManifestCache.configDef();
-        System.out.println(segmentManifestCacheDef.toEnrichedRst());
+        out.println(segmentManifestCacheDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("SegmentIndexesCacheConfig");
-        System.out.println("Under ``" + FETCH_INDEXES_CACHE_PREFIX + "``\n");
+        out.println("Under ``" + FETCH_INDEXES_CACHE_PREFIX + "``\n");
         final var segmentIndexesCacheDef = MemorySegmentIndexesCache.configDef();
-        System.out.println(segmentIndexesCacheDef.toEnrichedRst());
+        out.println(segmentIndexesCacheDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("ChunkManagerFactoryConfig");
         final var chunkCacheFactoryDef = ChunkManagerFactoryConfig.configDef();
-        System.out.println(chunkCacheFactoryDef.toEnrichedRst());
+        out.println(chunkCacheFactoryDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("MemoryChunkCacheConfig");
-        System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
+        out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
         final var memChunkCacheDef = ChunkCacheConfig.configDef(new ConfigDef());
-        System.out.println(memChunkCacheDef.toEnrichedRst());
+        out.println(memChunkCacheDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("DiskChunkCacheConfig");
-        System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
+        out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
         final var diskChunkCacheDef = DiskChunkCacheConfig.configDef();
-        System.out.println(diskChunkCacheDef.toEnrichedRst());
+        out.println(diskChunkCacheDef.toEnrichedRst());
+        out.println();
 
         printSectionTitle("Storage Backends");
-        System.out.println("Under ``" + STORAGE_PREFIX + "``\n");
+        out.println("Under ``" + STORAGE_PREFIX + "``\n");
 
         printSubsectionTitle("AzureBlobStorageStorageConfig");
         final var azBlobStorageConfigDef = AzureBlobStorageConfig.configDef();
-        System.out.println(azBlobStorageConfigDef.toEnrichedRst());
+        out.println(azBlobStorageConfigDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("AzureBlobStorageStorageConfig");
         final var googleCloudConfigDef = GcsStorageConfig.configDef();
-        System.out.println(googleCloudConfigDef.toEnrichedRst());
+        out.println(googleCloudConfigDef.toEnrichedRst());
+        out.println();
 
         printSubsectionTitle("S3StorageConfig");
         final var s3StorageConfigDef = S3StorageConfig.configDef();
-        System.out.println(s3StorageConfigDef.toEnrichedRst());
-
+        out.println(s3StorageConfigDef.toEnrichedRst());
+        out.println();
+
         printSubsectionTitle("FilesystemStorageConfig");
-        System.out.println("> Only for development/testing purposes");
+        out.println(".. Only for development/testing purposes");
         final var fsStorageConfigDef = FileSystemStorageConfig.configDef();
-        System.out.println(fsStorageConfigDef.toEnrichedRst());
+        out.println(fsStorageConfigDef.toEnrichedRst());
+        out.println();
     }
 
     static void printSectionTitle(final String title) {
-        System.out.println("=================\n"
+        out.println("=================\n"
             + title + "\n"
             + "=================");
     }
 
     static void printSubsectionTitle(final String title) {
-        System.out.println("-----------------\n"
+        out.println("-----------------\n"
             + title + "\n"
             + "-----------------");
     }

docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricsDocs.java

@@ -32,62 +32,65 @@
 import io.aiven.kafka.tieredstorage.metrics.MetricsRegistry;
 import io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry;
 
+import static java.lang.System.out;
+
 public class MetricsDocs {
     public static void main(final String[] args) {
         printSectionTitle("Core components metrics");
-        System.out.println();
+        out.println(".. Generated from MetricRegistry classes by " + MetricsDocs.class.getCanonicalName());
+        out.println();
         printSubsectionTitle("RemoteStorageManager metrics");
-        System.out.println();
-        System.out.println(toRstTable(MetricsRegistry.METRIC_CONTEXT, new MetricsRegistry().all()));
+        out.println();
+        out.println(toRstTable(MetricsRegistry.METRIC_CONTEXT, new MetricsRegistry().all()));
 
-        System.out.println();
+        out.println();
         printSubsectionTitle("SegmentManifestCache metrics");
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             CaffeineMetricsRegistry.METRIC_CONTEXT,
             new CaffeineMetricsRegistry(MemorySegmentManifestCache.METRIC_GROUP).all()));
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG,
             new ThreadPoolMonitorMetricsRegistry(MemorySegmentManifestCache.THREAD_POOL_METRIC_GROUP).all()));
 
-        System.out.println();
+        out.println();
         printSubsectionTitle("SegmentIndexesCache metrics");
-        System.out.println(toRstTable(
+        out.println(toRstTable(
             CaffeineMetricsRegistry.METRIC_CONTEXT,
             new CaffeineMetricsRegistry(MemorySegmentIndexesCache.METRIC_GROUP).all()));
-        System.out.println(toRstTable(
+        out.println(toRstTable(
             ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG,
             new ThreadPoolMonitorMetricsRegistry(MemorySegmentIndexesCache.THREAD_POOL_METRIC_GROUP).all()));
-        System.out.println();
+        out.println();
         printSubsectionTitle("ChunkCache metrics");
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             CaffeineMetricsRegistry.METRIC_CONTEXT,
             new CaffeineMetricsRegistry(ChunkCache.METRIC_GROUP).all()));
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG,
             new ThreadPoolMonitorMetricsRegistry(ChunkCache.THREAD_POOL_METRIC_GROUP).all()));
 
-        System.out.println();
+        out.println();
         printSectionTitle("Storage Backend metrics");
-        System.out.println();
+        out.println();
         printSubsectionTitle("AzureBlobStorage metrics");
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.METRIC_CONTEXT,
             new io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry().all()));
-        System.out.println();
+        out.println();
         printSubsectionTitle("GcsStorage metrics");
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT,
             new io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry().all()));
-        System.out.println();
+        out.println();
         printSubsectionTitle("S3Storage metrics");
-        System.out.println();
-        System.out.println(toRstTable(
+        out.println();
+        out.println(toRstTable(
             io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.METRIC_CONTEXT,
             new io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry().all()));
     }
@@ -181,13 +184,13 @@ static String getMBeanName(final String prefix, final MetricName metricName) {
     }
 
     static void printSectionTitle(final String title) {
-        System.out.println("=================\n"
+        out.println("=================\n"
             + title + "\n"
             + "=================");
     }
 
     static void printSubsectionTitle(final String title) {
-        System.out.println("-----------------\n"
+        out.println("-----------------\n"
             + title + "\n"
             + "-----------------");
     }

storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/GcsStorageConfig.java

@@ -42,7 +42,9 @@ public class GcsStorageConfig extends AbstractConfig {
     private static final String GCS_RESUMABLE_UPLOAD_CHUNK_SIZE_DOC = "The chunk size for resumable upload. "
         + "Must be a multiple of 256 KiB (256 x 1024 bytes). "
         + "Larger chunk sizes typically make uploads faster, but requires bigger memory buffers. "
-        + "The recommended minimum is 8 MiB. The default is 15 MiB.";
+        + "The recommended minimum is 8 MiB. The default is 15 MiB, "
+        + "`dictated by the GCS SDK <https://cloud.google.com/storage/docs/resumable-uploads#java>`_ "
+        + "when we set it to null.";
 
     static final String GCP_CREDENTIALS_JSON_CONFIG = "gcs.credentials.json";
     static final String GCP_CREDENTIALS_PATH_CONFIG = "gcs.credentials.path";