From 005a018cb62f5ae69d8dbd3244a12d8b9e392e86 Mon Sep 17 00:00:00 2001 From: Pavlo Smahin Date: Thu, 2 Nov 2023 12:42:45 +0200 Subject: [PATCH] fix(domain-events): create env variable to control number of partitions for topics (#947) Closes: MODINVSTOR-1131 --- PULL_REQUEST_TEMPLATE.md | 23 ++++++++ README.MD | 54 +++++++++++++++++-- .../java/org/folio/InventoryKafkaTopic.java | 9 ++++ .../folio/rest/api/ReindexJobRunnerTest.java | 1 - .../org/folio/rest/api/StorageTestSuite.java | 1 + 5 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 PULL_REQUEST_TEMPLATE.md diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..a0d4e6778 --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,23 @@ +### Purpose +_Describe the purpose of this pull request. Why is this change necessary? What problem does it solve?_ + +### Approach +_How does this change fulfill the purpose? Provide a high-level overview of the technical approach taken to address the problem._ + +### Changes Checklist +- [ ] **API Changes**: List any API paths, methods, request or response bodies changed, added, or removed. +- [ ] **Database Schema Changes**: Specify any database schema changes and their impact. +- [ ] **Interface Version Changes**: Indicate any changes to interface versions. +- [ ] **Interface Dependencies**: Note added or removed dependencies. +- [ ] **Permissions**: Document any changes to permissions. +- [ ] **Logging**: Confirm that logging is appropriately handled. +- [ ] **NEWS**: Ensure that the `NEWS` file is updated with relevant information about the changes made in this pull request. + +### Related Issues +_List any Jira issues related to this pull request._ + +### Learning and Resources (if applicable) +_Discuss any research conducted during the development of this pull request. Include links to relevant blog posts, patterns, libraries, or addons that were used to solve the problem._ + +### Screenshots (if applicable) +_If this pull request involves any visual changes or new features, consider including screenshots or GIFs to illustrate the changes._ diff --git a/README.MD b/README.MD index 1e1a01d22..7f3310c75 100644 --- a/README.MD +++ b/README.MD @@ -5,6 +5,51 @@ Copyright (C) 2016-2023 The Open Library Foundation This software is distributed under the terms of the Apache License, Version 2.0. See the file "[LICENSE](LICENSE)" for more information. + +* [mod-inventory-storage](#mod-inventory-storage) +* [Goal](#goal) +* [Prerequisites](#prerequisites) +* [Preparation](#preparation) + * [Git Submodules](#git-submodules) + * [Postgres](#postgres) + * [Kafka](#kafka) +* [Building](#building) +* [Environment Variables](#environment-variables) +* [Local Deployment using Docker](#local-deployment-using-docker) + * [Preparation](#preparation-1) + * [Start the infrastructure](#start-the-infrastructure) + * [Start the Module](#start-the-module) + * [Stop the Module](#stop-the-module) + * [Stop the infrastructure](#stop-the-infrastructure) +* [Local Deployment using Homebrew on macOS](#local-deployment-using-homebrew-on-macos) +* [Running](#running) + * [Preparation](#preparation-2) + * [Running Okapi](#running-okapi) + * [Registration](#registration) + * [Tenant Initialization](#tenant-initialization) + * [Sample Data](#sample-data) +* [Making Requests](#making-requests) + * [Okapi Root Address](#okapi-root-address) +* [Operating System Support](#operating-system-support) +* [Additional Information](#additional-information) + * [Issue tracker](#issue-tracker) + * [ModuleDescriptor](#moduledescriptor) + * [API documentation](#api-documentation) + * [Code analysis](#code-analysis) + * [Download and configuration](#download-and-configuration) +* [Appendix 1 - Docker Information](#appendix-1---docker-information) + * [When Using the Modules as Docker Containers](#when-using-the-modules-as-docker-containers) + * [Finding a Routable Address](#finding-a-routable-address) +* [Batch interface](#batch-interface) +* [HRID Management](#hrid-management) +* [Inventory view endpoint](#inventory-view-endpoint) +* [Domain event pattern](#domain-event-pattern) + * [Domain events for items](#domain-events-for-items) + * [Domain events for delete all APIs](#domain-events-for-delete-all-apis) + * [Reindex of instances](#reindex-of-instances) + * [Iteration of instances](#iteration-of-instances) + + # Goal FOLIO compatible inventory storage module. @@ -46,18 +91,18 @@ Mod-inventory-storage implements domain event pattern and requires kafka to be l `KAFKA_HOST` environment variables. For production deployments it is also required to set `REPLICATION_FACTOR` -env variable, this property has following description: +env variable, this property has the following description: > The replication factor controls how many servers will replicate each message that is written. > If replication factor set to 3 then up to 2 servers can fail before access to the data will be lost. The default configuration for this property is `1` for production environments it is usually `3`. -There is another important property - `number of partitions` for a topic - it has following +There is another important property - `number of partitions` for a topic - it has the following description: > The partition count controls how many logs the topic will be sharded into. -This property has fixed value - `50`. +This property can be changed by setting `KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS` env variable. Default value - `50`. # Building @@ -71,6 +116,7 @@ These environment variables configure Kafka, for details see [Kafka](#kafka): * `KAFKA_PORT` * `KAFKA_HOST` * `REPLICATION_FACTOR` +* `KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS` mod-inventory-storage also supports all Raml Module Builder (RMB) environment variables, for details see [RMB](https://github.com/folio-org/raml-module-builder#environment-variables): @@ -288,7 +334,7 @@ Changing the start number to a number lower than the current number is not suppo likely lead to generation of duplicate HRIDs. If an inventory type is added that contains a duplicate HRID, the module will reject the submission. -# /inventory-view/instances +# Inventory view endpoint Running a query against the `/inventory-view/instances` API writes this log message: diff --git a/src/main/java/org/folio/InventoryKafkaTopic.java b/src/main/java/org/folio/InventoryKafkaTopic.java index bad72e54e..0998a0720 100644 --- a/src/main/java/org/folio/InventoryKafkaTopic.java +++ b/src/main/java/org/folio/InventoryKafkaTopic.java @@ -1,5 +1,6 @@ package org.folio; +import org.apache.commons.lang3.StringUtils; import org.folio.kafka.services.KafkaTopic; public enum InventoryKafkaTopic implements KafkaTopic { @@ -26,4 +27,12 @@ public String moduleName() { public String topicName() { return topic; } + + @Override + public int numPartitions() { + return Integer.parseInt(StringUtils.firstNonBlank( + System.getenv("KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS"), + System.getProperty("KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS"), + System.getProperty("kafka-domain-topic-num-partitions"), "50")); + } } diff --git a/src/test/java/org/folio/rest/api/ReindexJobRunnerTest.java b/src/test/java/org/folio/rest/api/ReindexJobRunnerTest.java index ef1cbb14c..88390ba3d 100644 --- a/src/test/java/org/folio/rest/api/ReindexJobRunnerTest.java +++ b/src/test/java/org/folio/rest/api/ReindexJobRunnerTest.java @@ -110,7 +110,6 @@ public void canGetAllInstancesReindexJobs() { var jobs = instanceReindex.getReindexJobs(); - assertThat(jobs.getReindexJobs().get(0).getPublished(), is(0)); assertThat(jobs.getReindexJobs().get(0).getJobStatus(), is(IDS_PUBLISHED)); assertThat(jobs.getTotalRecords(), notNullValue()); diff --git a/src/test/java/org/folio/rest/api/StorageTestSuite.java b/src/test/java/org/folio/rest/api/StorageTestSuite.java index 0400b4149..b280acaf9 100644 --- a/src/test/java/org/folio/rest/api/StorageTestSuite.java +++ b/src/test/java/org/folio/rest/api/StorageTestSuite.java @@ -114,6 +114,7 @@ public static void before() { // tests expect English error messages only, no Danish/German/... Locale.setDefault(Locale.US); + System.setProperty("KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS", "1"); PostgresClient.setPostgresTester(new PostgresTesterContainer()); startKafka();