From 602b2862cba0581ac909703d6d28c8307625f05c Mon Sep 17 00:00:00 2001 From: Carter Cundiff Date: Mon, 16 Sep 2024 11:43:11 -0400 Subject: [PATCH] #332 Generate Config Store by default - Update documentation to live under supporting components - #219 improve manual action message keys --- DRAFT_RELEASE_NOTES.md | 3 + docs/modules/ROOT/nav.adoc | 1 + .../ROOT/pages/configuration-store.adoc | 179 ++++++++++++++ .../guides/guides-configuration-store.adoc | 231 ------------------ .../README.md | 23 +- .../aissemble-jenkins-chart/values.yaml | 10 - .../foundation-configuration-store/README.md | 28 +-- .../mda/ManualActionNotificationService.java | 143 ++++++----- .../KubernetesConfigStoreGenerator.java | 31 --- .../mda/generator/KubernetesGenerator.java | 4 + .../ModelAgnosticResourcesGenerator.java | 7 + .../src/main/resources/targets.json | 8 +- .../v2/configuration-store.values-dev.yaml.vm | 7 +- .../v2/configuration-store.values.yaml.vm | 1 - .../notification.configuration.store.tilt.vm | 16 -- .../mda/generator/DataRecordsModuleStep.java | 14 +- 16 files changed, 314 insertions(+), 392 deletions(-) create mode 100644 docs/modules/ROOT/pages/configuration-store.adoc delete mode 100644 docs/modules/ROOT/pages/guides/guides-configuration-store.adoc delete mode 100644 extensions/extensions-helm/aissemble-jenkins-chart/values.yaml delete mode 100644 foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesConfigStoreGenerator.java delete mode 100644 foundation/foundation-mda/src/main/resources/templates/notifications/notification.configuration.store.tilt.vm diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md index bcfaa6d8d..f5e2d3132 100644 --- a/DRAFT_RELEASE_NOTES.md +++ b/DRAFT_RELEASE_NOTES.md @@ -1,5 +1,8 @@ # Major Additions +## Universal Configurations Store +The Configuration Store is a tool that enables the various configurations for a project to be centrally defined and managed. It then provides a standardized way of accessing them, allowing the environment specific configurations to be dynamically provided to the their respective resources within the project at runtime. See the [official documentation](https://boozallen.github.io/aissemble/aissemble/current-dev/configuration-store.html) for more details on leveraging the configuration store. + ## aiSSEMBLE Infrastructure Helm Chart Created a helm chart with the necessary infrastructure for deploying your aiSSEMBLE project. This chart includes support for [Argo CD](https://argo-cd.readthedocs.io/en/stable/), [Jenkins](https://www.jenkins.io/), and [Nginx Ingress](https://docs.nginx.com/nginx-ingress-controller/). See the [chart README](https://github.com/boozallen/aissemble/tree/dev/extensions/extensions-helm/aissemble-infrastructure-chart#readme) for more details. diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index cdf143536..ea57cfe53 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -19,6 +19,7 @@ * Supporting Components ** xref:alerting-details.adoc[] ** xref:bias-detection.adoc[] +** xref:configuration-store.adoc[] ** xref:data-access-details.adoc[] ** xref:data-encryption.adoc[] ** xref:data-profiling-details.adoc[] diff --git a/docs/modules/ROOT/pages/configuration-store.adoc b/docs/modules/ROOT/pages/configuration-store.adoc new file mode 100644 index 000000000..0b14b886b --- /dev/null +++ b/docs/modules/ROOT/pages/configuration-store.adoc @@ -0,0 +1,179 @@ += Universal Configuration Store + +== Overview +The Universal Configuration Store is a tool that enables the various configurations for a project to be centrally defined and managed. +It then provides a standardized way of accessing them, allowing the environment specific configurations to be dynamically provided to +their respective resources within the project at runtime. + +== Setup +To setup the Configuration Store one must <> through properties files, optionally creating <>. Then the Configuration Store is ready for <> to a Kubernetes cluster, using the <> +of choice. + +=== Defining Configuration Properties +The Configuration Store utilizes the opensource project https://github.com/TechnologyBrewery/krausening[Krausening,role=external,window=_blank] for managing its configurations, where each configuration is +represented by a key value pair property. The key values pairs are stored as string types using the standard `.properties` file +format. + +Here is an example properties file: +[source,properties] +---- +connector=smallrye-kafka +topic=baseTopic +---- +This file defines two configuration properties: `connector` and `topic`. + +TIP: For sensitive configuration values, Krausening supports https://github.com/TechnologyBrewery/krausening/blob/dev/krausening/README.md#krausening-in-four-pints-leveraging-jasypt-for-encryptingdecrypting-properties[encrypting and decrypting properties,role=external,window=_blank] + +==== Environment Specific Overrides +In addition to the properties file introduced above, one can optionally specify a secondary properties file that houses environment +specific configurations. These configurations will then override their respective base configurations. + +Continuing with the example above, suppose one defines an additional properties file like the following: +[source,properties] +---- +topic=ciTopic +auth-token=auth-secret +---- + +Then when these two properties files are reconciled within the store, it results in a configuration equivalent to the following: +[source,properties] +---- +connector=smallrye-kafka +topic=ciTopic +auth-token=auth-secret +---- + +NOTE: The `topic` property is overwritten when the environment specific value `ciTopic`. + +=== Deploying the Configuration Store +The https://github.com/boozallen/aissemble/blob/{git-tree}/extensions/extensions-helm/aissemble-configuration-store-chart/README.md[Configuration Store Helm chart,role=external,window=_blank] is used for easily deploying it to a Kubernetes cluster. As part of the deployment, one must +provide the store access to the custom configurations by <> to the Kubernetes resource. Additionally, +the <> can be adjusted to fit a project's needs. + +NOTE: When deploying a project locally, the Configuration Store must be manually deployed first before all other resources. Please +create a https://github.com/boozallen/aissemble/issues[Github Issue,role=external,window=_blank] if additional assistance is required. + +==== Mounting Properties Files +Properties files are attached to the local file system of the Configuration Store within Kubernetes through a mounted volume. To mount +a volume to the Configuration Store, one must set the Helm chart's https://github.com/boozallen/aissemble/blob/{git-tree}/extensions/extensions-helm/aissemble-configuration-store-chart/README.md#persistent-volume-properties[persistent volume properties,role=external,window=_blank]. Then the Helm chart's https://github.com/boozallen/aissemble/blob/{git-tree}/extensions/extensions-helm/aissemble-configuration-store-chart/README.md#environment-variables[environment +variables,role=external,window=_blank] must be updated with the corresponding location(s) of the properties files within that volume. It is required to specify +a base property location that houses the base/default property configurations. In addition to the base location, one can optionally specify a +secondary location for properties files that house the environment specific configurations. + +NOTE: Multiple configuration properties files can be stored at either location. + +=== Backend Storage Types +By default, the Configuration Store uses Krausening for it backend property storage. This provides a more light weight but still robust +storage option perfect for smaller applications. + +For enterprise applications that have more robust requirements, it is recommended to utilize https://developer.hashicorp.com/vault[Vault,role=external,window=_blank]. Additionally, the configuration +store can be set up to use in-memory storage — this is best for unit testing. + +To change it to use one of the other options, set the environment variable `STORAGE_CLASS` to the desired value. The values currently +available are `krausening`, `vault`, and `inMemory`. + +== Usage +After the configurations are ingested and reconciled within the store, the next step is to access them within a project using one of +the two available methods: the <> or <>. +Configuration properties are identified by using a group name and property name, where group name is determined by the name of the file +the property is defined in. + +Given an example properties file named `messaging.properties`: +[source,properties] +---- +connector=smallrye-kafka +---- + +The property name would be `connector` and group name would be `messaging`. + +=== REST Endpoint +The Configuration Store includes a Kubernetes service with the endpoints detailed below. The API endpoint provides a simple yet flexible way +to access any configurations from within a project. + + +==== GET/aissemble-properties/{groupName}/{propertyName} +Returns a JSON object containing the value of a property with the given group name and property name. + +*Parameters* + +|=== +|*Name* | *Description* +|`groupName` +|The name of the file the property is defined in (excluding `.properties`). + +|`propertyName` +|The name of the property + +|=== + +Example GET request using the properties files defined above: +[source,bash] +---- +curl http://configuration-store.config-store-ns.svc:8083/aissemble-properties/messaging/topic +---- + +Example JSON output: +[source,json] +---- +{ + "groupName": "messaging", + "name": "topic", + "value": "ciTopic" +} +---- + +NOTE: The name of the configuration service may vary depending on the project's Helm chart configuration, by default it is +`http://configuration-store.config-store-ns.svc:8083` + +==== GET/aissemble-properties/healthcheck +Returns http `200` and a plain text string used for validating the service is reachable. + + +=== Kubernetes Resource Injection (Recommended) +The Configuration Store provisions a https://Kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/[Kubernetes admission webhook,role=external,window=_blank] as part of its deployment. This webhook is used for injecting property values +from the Configuration Store directly into Kubernetes resources as they are deployed to the cluster. + +To insert a configuration value into a Kubernetes resource, the resource must include the following label within its metadata: +[source,yaml] +---- +metadata: + labels: + aissemble-configuration-store: enabled +---- + +Then anywhere within the resource definition, all instances of `$getConfigValue(groupName={groupName};propertyName={propertyName})` will +be injected with the respective value from the Configuration Store. + +Here is an example ConfigMap making use of resource injection with the properties files defined above. This is the resource definition before +it is deployed to the cluster: +[source,yaml] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: messaging-config-map + labels: + aissemble-configuration-store: enabled +data: + messaging.properties: |- + connector=$getConfigValue(groupName=messaging;propertyName=connector) + topic=$getConfigValue(groupName=messaging;propertyName=topic) + auth-token=$getConfigValue(groupName=messaging;propertyName=auth-token) +---- + +Resulting resource on the cluster: +[source,yaml] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: messaging-config-map + labels: + aissemble-configuration-store: enabled +data: + messaging.properties: |- + connector=smallrye-kafka + topic=ciTopic + auth-token=auth-secret +---- \ No newline at end of file diff --git a/docs/modules/ROOT/pages/guides/guides-configuration-store.adoc b/docs/modules/ROOT/pages/guides/guides-configuration-store.adoc deleted file mode 100644 index 04471a3a3..000000000 --- a/docs/modules/ROOT/pages/guides/guides-configuration-store.adoc +++ /dev/null @@ -1,231 +0,0 @@ -= Leveraging the Configuration Store - -== Overview -The Configuration Store is a tool that enables the various configurations for a project to be centrally defined and -managed, while also standardizing access to the configurations. The Configuration Store dynamically provides -environment-specific configurations, based on the project's development lifecycle phase by leverages the opensource -project https://github.com/TechnologyBrewery/krausening[Krausening]. Usage limitations and regeneration strategies can -also be set, allowing configurations to automatically refreshed, thereby bolstering security of sensitive properties. - -=== Setup -The Configuration Store tool is currently under development. Stay tuned for its release! - -In order for the configuration store to read your configs, they will need to be placed into a read-only PVC. It is -assumed a PVC will meet most projects needs and be the most widely accepted structured solution. The configuration -service will then consume these configs and place them into the configuration store. Please see the -https://github.com/boozallen/aissemble/blob/{git-tree}/extensions/extensions-helm/aissemble-configuration-store-chart/README.md[configuration-store,role=external,window=_blank] -helm chart documentation on creating the PVC. - -=== Usage -Via the configuration store's helm chart, project teams can specify environment variables with the URIs that house -the project's various Krausening properties. It is required to -specify a base property URI that houses the base/default -property configurations. Optionally, one can specify a base policy URI that houses the base/default policy configurations. - -In addition to the base URIs, one can optionally specify a secondary URI for both properties and/or policies that -houses environment-specific configurations that will override and/or augment the respective base configurations. -Multiple configuration files can be stored at any given URI. Further guidance is covered below. - -Since it is common practice to define separate Helm chart values for each environment of a project's development -lifecycle, it is encouraged to define one shared base URI and respective environment-specific URIs, each housing -the relevant overrides and augmentations. - -The following example Configuration Store Helm chart demonstrates a URI specifications for a CI deployment: -[source,yaml] ----- - env: - BASE_PROPERTY: - ENVIRONMENT_PROPERTY: - BASE_POLICY_URI: - ENVIRONMENT_POLICY_URI: ----- - -==== Properties -Properties represent key values pairs for storing your project configurations. The key values pairs are stored as -string types using the standard `.properties` file format. - -Suppose we defined the following property configurations file named `messaging.properties` at the `BASE_PROPERTY`: -[source,properties] ----- -connector=smallrye-kafka -topic=baseTopic ----- - -Next, suppose we defined the following property configurations file named `messaging.properties` at the -`ENVIRONMENT_PROPERTY`: -[source,properties] ----- -topic=ciTopic -auth-token=auth-secret ----- - -When these two configuration file are reconciled within the Configuration Store, it results in a configuration with -a group name of `messaging` equivalent to the following: -[source,properties] ----- -connector=smallrye-kafka -topic=ciTopic -auth-token=auth-secret ----- - -Such that the following calls to the tool would provide the following configurations: -[source,java] ----- -ConfigServiceClient client = new ConfigServiceClient(); -client.getProperty("messaging", "connector") ---> smallrye-kafka -client.getProperty("messaging", "topic") ---> ciTopic -client.getProperty("messaging", "auth-token") ---> auth-secret ----- - -For more information on property definition and usage, visit -https://github.com/TechnologyBrewery/krausening[Krausening]. - -==== Properties Storage Options -The universal config store offers robust production ready configuration storage using vault. This is an excellent -solution for large enterprise applications. For smaller applications looking for a more light weight but still robust -storage option, the universal config can be set to use Krausening. The universal config can also be set up to use in -memory storage if you need to run some quick local tests. - -The default property storage is Krausening. To change it to use one of the other options you can set the environment -variable STORAGE_CLASS. The values currently available are `krausening`, `vault`, and `inMemory` - -==== Property Regeneration Policies -Policies are used to define limitations and rules for when a given property should be regenerated. This allows the user -to outline which properties they would like to be regenerated, when they would like them to be regenerated, and how to -regenerate them. The policies are defined in `JSON` format using the options below. - - -.Property Regeneration Policy Options -[cols="2,1,5"] -|=== -| Element Name | Required | Use - -| `identifier` -| Yes -| Unique name for the policy. A policy with the same identifier in the environment specific policies will override the its respective - base policy. - -| `description` -| No -| A statement of what the policy is for. - -| `targets` -| Yes -| List of properties that the policy applies to. Can be one to many targets for a given policy. Multiple policies cannot share the same - target property (exception for targets being shared between an environment policy and its respective base policy). - -| `targets/retrieve_url` -| Yes -| Describes where the property lives in the format `/GROUP_NAME/PROPERTY_NAME`. - -| `rules` -| Yes -| Logic to be executed when any of the properties within `targets` are accessed. The rule will determine whether the target property(ies) - need to be regenerated or can continued being used. Can be one to many rules for a given policy. - -| `rules/className` -| Yes -| Java class that contains the necessary implementation for executing the rule. - -| `rules/configurations` -| No -| Additional configuration details to pass to rule implementation. Can be zero to many key value pairs for a given rule. - -| `regeneration_method` -| Yes -| Logic to be executed when any of the `rules` determine that the target property(ies) need to be regenerated. The regeneration method can provide - updated values for zero to all of the properties specified in `targets`, however it cannot update properties not explicitly specified in `targets`. - There is only one regeneration method for a given policy. - -| `regeneration_method/className` -| Yes -| Java class that contains the necessary implementation for executing the regeneration method. - -| `regeneration_method/configurations` -| No -| Additional configuration details to pass to regeneration method implementation. Can be zero to many key value pairs for a given - regeneration method. - -|=== - -Basic example of a policy configuration: -[source,json] ----- -[ - { - "identifier": "messaging-auth-token-policy", - "description": "Policy for updating the messaging auth token when its 5 days old", - "targets": [ - { - "retrieve_url": "/messaging/auth-token" - } - ], - "rules": [ - { - "className": "your.custom.Limitation", - "configurations":{ - "expirationDate": "5 Days" - } - } - ], - "regeneration_method": { - "className": "your.custom.AuthTokenRefresh", - "configurations":{ - "endpoint": "https://your-new-token-service.org/getNewToken?exp=5days" - } - } - } -] ----- -This policy is defining a rule and regeneration method for a targeted property with the group name `messaging` and property -name `auth-token`. When the property is accessed, the `Limitation` rule will be run to see if the property needs to be -regenerated. In the event that it does, then the `AuthTokenRefresh` regeneration method will be run and the respective value -will be updated in the Configuration Store. - -More complex example of a policy configuration: -[source,json] ----- -[ - { - "identifier":"aws-credentials-policy", - "description": "Policy for updating AWS creds when there old or accessed too many times", - "targets": [ - { - "retrieve_url":"/aws-access/AWS_ACCESS_KEY_ID" - }, - { - "retrieve_url":"/aws-access/AWS_SECRET_ACCESS_KEY" - } - ], - "rules": [ - { - "className":"com.boozallen.aissemble.configuration.UseLimitation", - "configurations":{ - "maxUses": "5" - } - }, - { - "className":"com.boozallen.aissemble.configuration.TimeLimitation", - "configurations":{ - "expirationDate": "5 Days" - } - } - ], - "regeneration_method": [ - { - "className":"com.example.AWSCredsRefresh", - "configurations":{ - "endpoint": "https://my-new-token-service.org/getNewToken?exp=5days" - } - } - ] - } -] ----- -In this example, both rules will be run when either property in `targets` is accessed. In the event that either rule determines the -property should be refreshed, then the regeneration method `AWSCredsRefresh` will be called. This method is responsible for returning -the updated values for whichever properties it deems necessary. - -//// -TODO: Add details on creating rules and regeneration_method classes -//// diff --git a/extensions/extensions-helm/aissemble-configuration-store-chart/README.md b/extensions/extensions-helm/aissemble-configuration-store-chart/README.md index 9dceedd09..dec20ebb9 100644 --- a/extensions/extensions-helm/aissemble-configuration-store-chart/README.md +++ b/extensions/extensions-helm/aissemble-configuration-store-chart/README.md @@ -36,9 +36,9 @@ The following properties are override from the base (Quarkus) | toggleableConfiguration.configMap.supplementalQuarkusConfig | The toggleable list of additional properties to provide to the Quarkus app | No | `- quarkus.http.ssl.certificate.file=/etc/webhook/cert/tls.crt`
`- quarkus.http.ssl.certificate.key-file=/etc/webhook/cert/tls.key` | ## Persistent Volume Properties - The following properties are used to create the Persistent Volume and Persistent Volume Claim used to mount the configs -onto the configuration service +onto the configuration service. Additionally you must define [environment variables](#environment-variables) specifying the location +of the configs to the service. | Property | Description | Required Override | Default | |----------------------------------------------|-----------------------------------------------------------------------------------------------|-------------------|---------------------| @@ -46,12 +46,31 @@ onto the configuration service | configurationVolume.name | The name used to create the pv and pvc | No | configuration-store | | configurationVolume.storageType | Determines mounting a local mount for tests or a custom mount
Options: local, custom | No | custom | | configurationVolume.storageClass | The PVCs class of storage. If storageType is custom then the storageClass needs to be defined | No* | "" | +| configurationVolume.volumePathOnNode | Defines the host machine filesystem path to mount | Yes* | "" | | configurationVolume.accessModes.ReadOnlyMany | The modes this PVC will support when mounting | No | ReadOnlyMany | | configurationVolume.size | The size of the PVC | No | 1Gi | \* If the storage class is left as default empty string then Kubernetes will try and find the Clusters default StorageClass +\*\* Should only be overwritten when using a local mount as the storage type + + +### Environment Variables +The following environment variables are used by the configuration service for locating the mounted configs. These environment +variables should specify their filepath location within the Persistent Volume. + +```yaml +aissemble-configuration-store-chart: + aissemble-quarkus-chart: + deployment: + env: + - name: BASE_PROPERTY + value: + - name: ENVIRONMENT_PROPERTY + value: +``` + ## Custom Properties The following properties are specific to the aiSSEMBLE Configuration Store chart and it's creation of a [Kubernetes mutating webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) for injecting config store values into kubernetes resources. When the webhook is enabled, this helm chart should be deployed in a separate namespace than your project resources to [avoid deadlocks with self hosted webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#avoiding-deadlocks-in-self-hosted-webhooks). It is recommended to keep the webhook enabled to ensure full functionality of the configuration store. diff --git a/extensions/extensions-helm/aissemble-jenkins-chart/values.yaml b/extensions/extensions-helm/aissemble-jenkins-chart/values.yaml deleted file mode 100644 index f91916fb6..000000000 --- a/extensions/extensions-helm/aissemble-jenkins-chart/values.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Default values for aissemble-jenkins. -jenkins: - controller: - image: ghcr.io/boozallen/aissemble-jenkins-controller - # note the official chart does not use the controller tag value for testing, so disabling here - testEnabled: false - tag: "1.9.0-SNAPSHOT" - agent: - image: ghcr.io/boozallen/aissemble-jenkins-agent - tag: "1.9.0-SNAPSHOT" diff --git a/foundation/foundation-configuration-store/README.md b/foundation/foundation-configuration-store/README.md index 4a0b7c12c..5d915f492 100644 --- a/foundation/foundation-configuration-store/README.md +++ b/foundation/foundation-configuration-store/README.md @@ -19,30 +19,4 @@ AWS_SECRET_ACCESS_KEY=base-secret-access-key ``` ### Handling Load Status -The `ConfigLoader` leverages `load-status` to detect whether properties were already fully loaded or only partially loaded due to an interrupted process. The code logic ensures that a `fully-loaded` status is set when properties are loaded successfully. This status is then used to skip loading if the properties were successfully loaded previously, thus avoiding unnecessary reloading during deployment refreshes. For details on this behavior, refer to the implementation code and comments in the `ConfigLoader` class. - -### Inject Configuration Value -To properly insert the configuration value into the kubernetes resource configuration, you need to place the configuration $getConfigValue() function with groupName and propertyName parameters specified (`;` is the delimiter of the parameters) - -An example of the format is -```text -$getConfigValue(groupName=groupNameValue;propertyName=propertyNameValue) -``` - -For example: -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: mytestconfigmap - labels: - aissemble-configuration-store: enabled -data: - mytest.properties: |- - AWS_ACCESS_KEY_ID=$getConfigValue(groupName=aws-credentials;propertyName=AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) - fully-loaded=$getConfigValue(groupName=load-status;propertyName=fully-loaded) -``` - -The configuration store should be up and running before other services. -To start, run `tilt up -f Tiltfile-config --port 10351` at the root directory of the project. \ No newline at end of file +The `ConfigLoader` leverages `load-status` to detect whether properties were already fully loaded or only partially loaded due to an interrupted process. The code logic ensures that a `fully-loaded` status is set when properties are loaded successfully. This status is then used to skip loading if the properties were successfully loaded previously, thus avoiding unnecessary reloading during deployment refreshes. For details on this behavior, refer to the implementation code and comments in the `ConfigLoader` class. \ No newline at end of file diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/ManualActionNotificationService.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/ManualActionNotificationService.java index 2a974cdfd..e71403619 100644 --- a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/ManualActionNotificationService.java +++ b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/ManualActionNotificationService.java @@ -58,13 +58,12 @@ public class ManualActionNotificationService { private static final Logger logger = LoggerFactory.getLogger(ManualActionNotificationService.class); private static final String EMPTY_LINE = "\n"; private static final String SUPPRESS_WARNINGS = "maven-suppress-warnings"; - private static final String SUPPRESS_TILT_WARNINGS_MSG = "* NOTE: If you do not want to see this message add '# " + SUPPRESS_WARNINGS + "' to the file\n"; public static final String GROUP_TILT = "tilt"; public void addSchemaElementDeprecationNotice(String illegalElement, String objectType) { final String SCHEMA_ELEMENT_DEPRECATION_KEY = "schema_element_deprecation"; - VelocityNotification notification = new VelocityNotification(SCHEMA_ELEMENT_DEPRECATION_KEY, "schemaelementdeprecation", new HashSet(), "templates/notifications/notification.schema.element.deprecation.vm"); + VelocityNotification notification = new VelocityNotification(getMessageKey(SCHEMA_ELEMENT_DEPRECATION_KEY, illegalElement), "schemaelementdeprecation", new HashSet(), "templates/notifications/notification.schema.element.deprecation.vm"); notification.addToVelocityContext("objectType", objectType); notification.addToVelocityContext("illegalElement", illegalElement); addManualAction(SCHEMA_ELEMENT_DEPRECATION_KEY, notification); @@ -96,33 +95,21 @@ public void addNoticeToAddPipelines(GenerationContext context) { * @param moduleType the module type */ public void addNoticeToAddModuleToParentBuild(GenerationContext context, String artifactId, String moduleType) { - - final String parentArtifactId = context.getArtifactId(); final String pomFilePath = context.getProjectDirectory() + File.separator + "pom.xml"; final String query = "" + artifactId + ""; boolean alreadyExists = existsInFile(pomFilePath, query); - File projectDirectory = context.getProjectDirectory(); - File executionRootDirectory = context.getExecutionRootDirectory(); - StringBuilder parentDirectory = new StringBuilder(); - - while (!executionRootDirectory.equals(projectDirectory)) { - - parentDirectory.insert(0, projectDirectory.getParentFile().getName() + File.separator); - projectDirectory = projectDirectory.getParentFile(); - } - - String displayPomFilePath = parentDirectory + parentArtifactId + File.separator + "pom.xml"; + String relativePomFilePath = getRelativePathToProjectRoot(context.getExecutionRootDirectory(), new File(pomFilePath)); if (!alreadyExists) { - final String key = getMessageKey(pomFilePath, "module"); + final String key = getMessageKey(relativePomFilePath, "module"); HashSet items = new HashSet(); items.add("" + (artifactId) + ""); VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.module.to.parent.vm"); notification.addToVelocityContext("moduleType", moduleType); - notification.addToVelocityContext("displayPomFilePath", displayPomFilePath); + notification.addToVelocityContext("displayPomFilePath", relativePomFilePath); notification.addToVelocityContext("artifactId", artifactId); addManualAction(pomFilePath, notification); } @@ -142,7 +129,8 @@ public void addNoticeToAddDependency(GenerationContext context, String artifactI boolean alreadyExists = existsInFile(pomFilePath, String.format("extensions-data-delivery-spark-%s", persistType)); if (!alreadyExists) { - final String key = getMessageKey(pomFilePath, "extensions-data-delivery-spark"); + String relativePomFilePath = getRelativePathToProjectRoot(context.getExecutionRootDirectory(), new File(pomFilePath)); + final String key = getMessageKey(relativePomFilePath, "extensions-data-delivery-spark", persistType); VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.dependency.vm"); notification.addToVelocityContext("dependencyArtifactId", String.format("extensions-data-delivery-spark-%s", persistType)); @@ -173,7 +161,7 @@ public void addDockerBuildWithLiveUpdateTiltFileMessage(final GenerationContext final String stepNameSnakeCase = PipelineUtils.deriveLowerSnakeCaseNameFromHyphenatedString(stepName); boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, dockerApplicationArtifactId); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - final String key = getMessageKey(tiltFilePath, "Tiltfile"); + final String key = getMessageKey("Tiltfile", "docker-build", dockerApplicationArtifactId); addLocalResourceTiltFileMessage(context, dockerArtifactId, dockerApplicationArtifactId, stepName, pipelineName + "/" + stepName, true); @@ -216,7 +204,8 @@ public void addDockerBuildTiltFileMessage(DockerBuildParams params) { boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, params.getDockerApplicationArtifactId()); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - VelocityNotification notification = new VelocityNotification("docker-build-" + params.getDeployedAppName(), + final String key = getMessageKey("Tilefile", "docker-build", params.getDockerApplicationArtifactId()); + VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet<>(), "templates/notifications/notification.docker.tilt.vm"); notification.addToVelocityContext("appNameTitle", params.getAppName()); notification.addToVelocityContext("dockerArtifactId", params.getDockerArtifactId()); @@ -239,7 +228,7 @@ public void addDockerBuildTiltFileMessage(DockerBuildParams params) { public void addHabushuRegexPluginInvocation(final GenerationContext context) { if(!existsInFile("pom.xml", "set-habushu-dist-artifact-version")) { VelocityNotification notification = new VelocityNotification( - "set-habushu-dist-artifact-version", + getMessageKey("pom.xml", "set-habushu-dist-artifact-version"), "root-plugins", new HashSet<>(), "templates/notifications/notification.root.habushu.regex.plugin.vm" @@ -255,14 +244,18 @@ public void addHabushuRegexPluginInvocation(final GenerationContext context) { * @param context generation context */ public void addCleanPluginNotification(final String deployArtifactId, final GenerationContext context) { - if(deployArtifactId != null && !existsInFile(deployArtifactId + "/pom.xml", "maven-clean-plugin")) { + final File rootDir = context.getExecutionRootDirectory(); + final String deployPom = rootDir.getAbsolutePath() + File.separator + deployArtifactId + File.separator + "pom.xml"; + + if(deployArtifactId != null && !existsInFile(deployPom, "maven-clean-plugin")) { + String relativePomFilePath = getRelativePathToProjectRoot(rootDir, new File(deployPom)); VelocityNotification notification = new VelocityNotification( - "clean-deploy-apps-targets", + getMessageKey(relativePomFilePath, "clean-deploy-apps-targets"), "deploy-plugins", new HashSet<>(), "templates/notifications/notification.deploy.clean.app.target.vm" ); - addManualAction(context.getRootArtifactId() + "/" + deployArtifactId, notification); + addManualAction(deployPom, notification); } } @@ -272,16 +265,20 @@ public void addCleanPluginNotification(final String deployArtifactId, final Gene * @param context generation context */ public void addPipelineInvocationServiceDeployment(final GenerationContext context) { - String deployArtifactId = MavenUtil.getDeployModuleName(context.getExecutionRootDirectory()); - if(deployArtifactId != null && !existsInFile(deployArtifactId + "/pom.xml", "mda-maven-plugin")) { + final File rootDir = context.getExecutionRootDirectory(); + String deployArtifactId = MavenUtil.getDeployModuleName(rootDir); + final String deployPom = rootDir.getAbsolutePath() + File.separator + deployArtifactId + File.separator + "pom.xml"; + + if(deployArtifactId != null && !existsInFile(deployPom, "mda-maven-plugin")) { + String relativePomFilePath = getRelativePathToProjectRoot(rootDir, new File(deployPom)); VelocityNotification notification = new VelocityNotification( - "pipeline-invocation-service-spark-apps", + getMessageKey(relativePomFilePath, "pipeline-invocation-service-spark-apps"), "deploy-plugins", new HashSet<>(), "templates/notifications/notification.mda.maven.pipeline.invocation.execution.vm" ); notification.addToExternalVelocityContextProperties("deployArtifactId", deployArtifactId); - addManualAction(context.getRootArtifactId() + "/" + deployArtifactId, notification); + addManualAction(deployPom, notification); addDeployPomMessage(context, "pipeline-invocation-service-v2", "pipeline-invocation-service"); } addCleanPluginNotification(deployArtifactId, context); @@ -313,7 +310,7 @@ public void addLocalResourceTiltFileMessage(final GenerationContext context, fin final String pipelinesArtifactId = dockerArtifactId.replace("-docker", "-pipelines"); boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, "compile-" + moduleName); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - final String key = getMessageKey(tiltFilePath, "Tiltfile"); + final String key = getMessageKey("Tiltfile", "local-resource", moduleName); VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.local.resource.tilt.vm"); notification.addToVelocityContext("srcModuleName", moduleName); @@ -371,25 +368,15 @@ public void addSparkWorkerDockerBuildTiltMessage(final GenerationContext context boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - VelocityNotification notification = new VelocityNotification("spark-worker", GROUP_TILT, new HashSet<>(), + final String key = getMessageKey("Tiltfile", "spark-worker-docker-build"); + + VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet<>(), "templates/notifications/notification.spark.worker.docker.build.tilt.vm"); addManualAction(tiltFilePath, notification); } } } - /** - * Adds a notification to update the Tiltfile. - * - * @param context the generation context - * @param appName the application name - * @param deployArtifactId the deploy artifact ID - */ - public void addHelmTiltFileMessage(final GenerationContext context, final String appName, - final String deployArtifactId) { - addHelmTiltFileMessage(context, appName, deployArtifactId, false); - } - /** * Adds a notification to update the Tiltfile. * @@ -399,7 +386,7 @@ public void addHelmTiltFileMessage(final GenerationContext context, final String * @param isConfigStore whether the deployment is the config store */ public void addHelmTiltFileMessage(final GenerationContext context, final String appName, - final String deployArtifactId, final Boolean isConfigStore) { + final String deployArtifactId) { final File rootDir = context.getExecutionRootDirectory(); if (!rootDir.exists() || !tiltFileFound(rootDir)) { @@ -410,12 +397,8 @@ public void addHelmTiltFileMessage(final GenerationContext context, final String boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - VelocityNotification notification; - if (isConfigStore) { - notification = new VelocityNotification("helm-" + appName, GROUP_TILT, new HashSet(), "templates/notifications/notification.configuration.store.tilt.vm"); - } else { - notification = new VelocityNotification("helm-" + appName, GROUP_TILT, new HashSet(), "templates/notifications/notification.helm.tilt.vm"); - } + final String key = getMessageKey("Tiltfile", "helm", appName); + VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet(), "templates/notifications/notification.helm.tilt.vm"); notification.addToVelocityContext("appName", appName); notification.addToVelocityContext("deployArtifactId", deployArtifactId); addManualAction(tiltFilePath, notification); @@ -445,7 +428,9 @@ public void addResourceDependenciesTiltFileMessage(GenerationContext context, St boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, item.toString().trim()); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - VelocityNotification notification = new VelocityNotification("resource-dependencies", new HashSet<>(), "templates/notifications/notification.resource.tilt.vm"); + final String key = getMessageKey("Tiltfile", "resource-dependencies", appName); + + VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.resource.tilt.vm"); notification.addToVelocityContext("appName", appName); notification.addToVelocityContext("formattedAppDependencies", formattedAppDependencies); addManualAction(tiltFilePath, notification); @@ -498,12 +483,12 @@ public void addYamlTiltFileMessage(final GenerationContext context, final String boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - final String key = getMessageKey(tiltFilePath, "Tiltfile"); + final String key = getMessageKey("Tiltfile", "yaml", appName); HashSet items = new HashSet(); items.add(yamlFilePath); - VelocityNotification notification = new VelocityNotification("yaml", GROUP_TILT, items, + VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, items, "templates/notifications/notification.yaml.tiltfile.vm"); addManualAction(tiltFilePath, notification); } @@ -537,7 +522,9 @@ public void addElasticsearchTiltResources(final GenerationContext context, final //add a manual action for any resources that may need to be added if (items.size() > 0) { - VelocityNotification notification = new VelocityNotification("elasticsearch", GROUP_TILT, items, + final String key = getMessageKey("Tiltfile", "elasticsearch"); + + VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, items, "templates/notifications/notification.elastic.search.tilt.vm"); addManualAction(tiltFilePath, notification); } @@ -566,7 +553,9 @@ public void addSparkWorkerTiltResources(final GenerationContext context, final S boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text); if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) { - VelocityNotification notification = new VelocityNotification("spark-worker", GROUP_TILT, new HashSet<>(), + final String key = getMessageKey("Tiltfile", "spark-application"); + + VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet<>(), "templates/notifications/notification.spark.worker.tilt.vm"); notification.addToVelocityContext("parentArtifactId", parentArtifactId); notification.addToVelocityContext("pipelineArtifactId", pipelineArtifactId); @@ -606,9 +595,12 @@ public void addDeployPomMessage(final GenerationContext context, final String pr NotificationParams params = configureNotification(rootDir, profile, appName, MavenUtil::getDeployModuleName); boolean hasAppDependencies = appDependencies != null; + String relativePomFilePath = getRelativePathToProjectRoot(rootDir, params.getPomFile()); + final String key = getMessageKey(relativePomFilePath, params.getKey()); + if (!params.isExistsInFileOrNotification()) { - VelocityNotification notification = new VelocityNotification(params.getKey(), "deploypom", new HashSet<>(), "templates/notifications/notification.deploy.pom.vm"); + VelocityNotification notification = new VelocityNotification(key, "deploypom", new HashSet<>(), "templates/notifications/notification.deploy.pom.vm"); notification.addToVelocityContext("appName", appName); notification.addToVelocityContext(("profile"), profile); notification.addToVelocityContext("basePackage", context.getBasePackage()); @@ -622,7 +614,7 @@ public void addDeployPomMessage(final GenerationContext context, final String pr } else if (hasAppDependencies && !propertyVariableExistsInPomFile(params.getPomFile(), appName, "appDependencies", String.join(",", appDependencies))) { params.setKey(getMessageKey(params.getPomFilePath(), "execution", "appDependencies", appName)); - VelocityNotification notification = new VelocityNotification(params.getKey(), new HashSet(), "templates/notifications/notification.deploy.pom.property.variables.vm"); + VelocityNotification notification = new VelocityNotification(key, new HashSet(), "templates/notifications/notification.deploy.pom.property.variables.vm"); notification.addToVelocityContext("profile", profile); notification.addToVelocityContext("deployArtifactId", params.getArtifactId()); notification.addToVelocityContext("appDependencies", appDependencies); @@ -646,8 +638,10 @@ public void addDockerPomMessage(final GenerationContext context, final String pr NotificationParams params = configureNotification(optionalRoot, profile, artifactId, MavenUtil::getDockerModuleName); if (StringUtils.isNotEmpty(profile)) { if (!params.isExistsInFileOrNotification()) { + String relativePomFilePath = getRelativePathToProjectRoot(optionalRoot, params.getPomFile()); + final String key = getMessageKey(relativePomFilePath, params.getKey()); - VelocityNotification notification = new VelocityNotification(params.getKey(), "dockerpom", new HashSet<>(), "templates/notifications/notification.docker.pom.vm"); + VelocityNotification notification = new VelocityNotification(key, "dockerpom", new HashSet<>(), "templates/notifications/notification.docker.pom.vm"); notification.addToVelocityContext(("profile"), profile); notification.addToVelocityContext("appName", artifactId); notification.addToVelocityContext("basePackage", context.getBasePackage()); @@ -676,7 +670,9 @@ public void addSagemakerDockerPomMessage(final GenerationContext context, final String pomFilePath = pomPath + File.separator + trainingDockerArtifactId; boolean registryUrlExists = existsInFile(pomFilePath,"ECR_REGISTRY_URL"); if (registryUrlExists) { - final String key = getMessageKey(pomFilePath, "pom"); + String relativePomFilePath = getRelativePathToProjectRoot(rootDir, pomRoot) + File.separator + trainingDockerArtifactId; + final String key = getMessageKey(relativePomFilePath, "sagemaker"); + VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.sagemaker.docker.pom.vm"); notification.addToVelocityContext("artifactId", artifactId); notification.addToVelocityContext("dockerArtifactId", trainingDockerArtifactId); @@ -753,7 +749,8 @@ public void addNoticeToAddPythonDependencies(GenerationContext context, Set items = new HashSet(); for (String dependency : dependenciesToAdd) { @@ -784,13 +781,15 @@ public void addNoticeToUpdateS3LocalConfig(final GenerationContext context, fina "apps", "s3-local", "values.yaml")).toAbsolutePath().normalize(); try { if (showWarnings(s3ValuesPath.toString())) { + String relativeValuesFilePath = getRelativePathToProjectRoot(rootDir, s3ValuesPath.toFile()); + if (!existsInFile(s3ValuesPath.toString(), "- name: " + bucketName)) { HashSet items = new HashSet(); for (String objectName : objectNames) { items.add(objectName); } - final String key = getMessageKey(s3ValuesPath.toString(), bucketName, "buckets"); + final String key = getMessageKey(relativeValuesFilePath, "buckets", bucketName); VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.s3.local.buckets.vm"); notification.addToVelocityContext("rootPath", rootPath.relativize(s3ValuesPath).toString()); notification.addToVelocityContext("bucketName", bucketName); @@ -806,7 +805,7 @@ public void addNoticeToUpdateS3LocalConfig(final GenerationContext context, fina } if (pathExistsInFile) { - final String key = getMessageKey(s3ValuesPath.toString(), bucketName, "bucketobjects"); + final String key = getMessageKey(relativeValuesFilePath, "bucketobjects", bucketName); VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.s3.local.bucketobjects.vm"); notification.addToVelocityContext("rootPath", rootPath.relativize(s3ValuesPath).toString()); @@ -839,7 +838,9 @@ public void addNoticeToUpdateKafkaConfig(final GenerationContext context, final if (showWarnings(kafkaValuesPath.toString()) && !existsInFile(kafkaValuesPath.toString(), topicName + ":") && existsInFile(kafkaValuesPath.toString(), "KAFKA_CREATE_TOPICS")) { - final String key = getMessageKey(kafkaValuesPath.toString(), "topics"); + + String relativeValuesFilePath = getRelativePathToProjectRoot(rootDir, kafkaValuesPath.toFile()); + final String key = getMessageKey(relativeValuesFilePath, "kafka-topics"); HashSet items = new HashSet(); items.add(topicName); @@ -866,6 +867,22 @@ private boolean existsInFile(final String filePath, final String text) { } } + /** + * Returns the path of the current project directory relative to the project root. + * Given: + * my/test/project + * my/test/project/submodule1/submodule2 + * Returns: + * submodule1/submodule2 + * + * @param projectRoot The root directory + * @param currentDirectory The current directory + * @return The relative path as a {@link String} + */ + private String getRelativePathToProjectRoot(File projectRoot, File currentDirectory) { + return projectRoot.toPath().relativize(currentDirectory.toPath()).toString(); + } + private String getMessageKey(final String... keyComponents) { if (keyComponents.length > 0) { return String.join("_", keyComponents); @@ -944,7 +961,7 @@ private NotificationParams configureNotification(File optionalRoot, String profi params.setPomFilePath(deployDir + File.separator + "pom.xml"); params.setProfileConfiguration("" + profile + ""); params.setPomFile(new File(params.getPomFilePath())); - params.setKey(getMessageKey(params.getPomFilePath(), "execution", appName)); + params.setKey(getMessageKey("execution", appName)); params.setExistsInFileOrNotification(executionAppExistsInPomFile(params.getPomFile(), appName) || executionAppExistsInNotification(params.getPomFilePath(), params.getKey(), appName)); return params; diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesConfigStoreGenerator.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesConfigStoreGenerator.java deleted file mode 100644 index ed2d01a65..000000000 --- a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesConfigStoreGenerator.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.boozallen.aiops.mda.generator; - -/*- - * #%L - * AIOps Foundation::AIOps MDA - * %% - * Copyright (C) 2021 Booz Allen - * %% - * This software package is licensed under the Booz Allen Public License. All Rights Reserved. - * #L% - */ - -import org.technologybrewery.fermenter.mda.generator.GenerationContext; - -public class KubernetesConfigStoreGenerator extends AbstractKubernetesGenerator { - /*--~-~-~~ - * Usages: - * | Target | Template | Generated File | - * |---------------------------------------------|---------------------------------------------------------------------------|----------------------------------| - * | aissembleConfigurationStoreArgoCDV2 | deployment/argocd/v2/configuration-store.yaml.vm | templates/${appName}.yaml | - * | aissembleConfigurationStoreHelmChartFileV2 | deployment/configuration-store/v2/configuration-store.chart.yaml.vm | apps/${appName}/Chart.yaml | - * | aissembleConfigurationStoreValuesDevFileV2 | deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm | apps/${appName}/values-dev.yaml | - * | aissembleConfigurationStoreValuesFileV2 | deployment/configuration-store/v2/configuration-store.values.yaml.vm | apps/${appName}/values.yaml | - */ - - @Override - protected void addTiltNotification(GenerationContext generationContext, String appName, String deployArtifactId) { - getNotificationService().addHelmTiltFileMessage(generationContext, appName, deployArtifactId, true); - } - -} diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesGenerator.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesGenerator.java index 86ce45ea6..7797da717 100644 --- a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesGenerator.java +++ b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/KubernetesGenerator.java @@ -57,6 +57,7 @@ public class KubernetesGenerator extends AbstractKubernetesGenerator { * | sparkInfrastructureArgoCD | deployment/argocd/spark-infrastructure.yaml.vm | templates/spark-infrastructure.yaml | * | sparkOperatorArgoCD | deployment/argocd/spark-operator.yaml.vm | templates/spark-operator.yaml | * | airflowArgoCDV2 | deployment/argocd/v2/airflow.yaml.vm | templates/airflow.yaml | + * | aissembleConfigurationStoreArgoCDV2 | deployment/argocd/v2/configuration-store.yaml.vm | templates/${appName}.yaml | * | dataAccessArgoCDV2 | deployment/argocd/v2/data-access.yaml.vm | templates/data-access.yaml | * | elasticsearchOperatorArgoCDV2 | deployment/argocd/v2/elasticsearch.operator.yaml.vm | templates/elasticsearch-operator.yaml | * | elasticsearchArgoCDV2 | deployment/argocd/v2/elasticsearch.yaml.vm | templates/elasticsearch.yaml | @@ -78,6 +79,9 @@ public class KubernetesGenerator extends AbstractKubernetesGenerator { * | zookeeperAlertArgoCD | deployment/argocd/zookeeper-alert.yaml.vm | templates/zookeeper-alert.yaml | * | bomValuesDevFile | deployment/bom/bom.values-dev.yaml.vm | apps/${appName}/values-dev.yaml | * | bomValuesFile | deployment/bom/bom.values.yaml.vm | apps/${appName}/values.yaml | + * | aissembleConfigurationStoreHelmChartFileV2 | deployment/configuration-store/v2/configuration-store.chart.yaml.vm | apps/${appName}/Chart.yaml | + * | aissembleConfigurationStoreValuesDevFileV2 | deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm | apps/${appName}/values-dev.yaml | + * | aissembleConfigurationStoreValuesFileV2 | deployment/configuration-store/v2/configuration-store.values.yaml.vm | apps/${appName}/values.yaml | * | dataAccessValuesDevFile | deployment/data-access/data.access.values-dev.yaml.vm | apps/${appName}/values-dev.yaml | * | dataAccessValuesFile | deployment/data-access/data.access.values.yaml.vm | apps/${appName}/values.yaml | * | dataAccessHelmChartFileV2 | deployment/data-access/v2/data.access.chart.yaml.vm | apps/${appName}/Chart.yaml | diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java index 78545d986..e40e2e356 100644 --- a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java +++ b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java @@ -13,6 +13,8 @@ import org.technologybrewery.fermenter.mda.generator.AbstractModelAgnosticGenerator; import org.technologybrewery.fermenter.mda.generator.GenerationContext; +import com.boozallen.aiops.mda.ManualActionNotificationService; + /** * Generates configuration code with no model interaction. This is often useful for * configuration files that must exist in some form or similar constructs. @@ -44,6 +46,8 @@ public class ModelAgnosticResourcesGenerator extends AbstractModelAgnosticGenera */ + protected ManualActionNotificationService manualActionNotificationService = new ManualActionNotificationService(); + @Override protected String getOutputSubFolder() { return "resources/"; @@ -52,6 +56,9 @@ protected String getOutputSubFolder() { @Override public void generate(GenerationContext context) { super.generate(context); + + // Add the config store manual action + manualActionNotificationService.addDeployPomMessage(context, "configuration-store-deploy-v2", "configuration-store"); } } diff --git a/foundation/foundation-mda/src/main/resources/targets.json b/foundation/foundation-mda/src/main/resources/targets.json index c93d17fd1..73fd3326f 100644 --- a/foundation/foundation-mda/src/main/resources/targets.json +++ b/foundation/foundation-mda/src/main/resources/targets.json @@ -3123,7 +3123,7 @@ "name": "aissembleConfigurationStoreHelmChartFileV2", "templateName": "templates/deployment/configuration-store/v2/configuration-store.chart.yaml.vm", "outputFile": "apps/${appName}/Chart.yaml", - "generator": "com.boozallen.aiops.mda.generator.KubernetesConfigStoreGenerator", + "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator", "metadataContext": "targeted", "overwritable": false }, @@ -3131,7 +3131,7 @@ "name": "aissembleConfigurationStoreValuesFileV2", "templateName": "templates/deployment/configuration-store/v2/configuration-store.values.yaml.vm", "outputFile": "apps/${appName}/values.yaml", - "generator": "com.boozallen.aiops.mda.generator.KubernetesConfigStoreGenerator", + "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator", "metadataContext": "targeted", "overwritable": false }, @@ -3139,7 +3139,7 @@ "name": "aissembleConfigurationStoreValuesDevFileV2", "templateName": "templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm", "outputFile": "apps/${appName}/values-dev.yaml", - "generator": "com.boozallen.aiops.mda.generator.KubernetesConfigStoreGenerator", + "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator", "metadataContext": "targeted", "overwritable": false }, @@ -3147,7 +3147,7 @@ "name": "aissembleConfigurationStoreArgoCDV2", "templateName": "templates/deployment/argocd/v2/configuration-store.yaml.vm", "outputFile": "templates/${appName}.yaml", - "generator": "com.boozallen.aiops.mda.generator.KubernetesConfigStoreGenerator", + "generator": "com.boozallen.aiops.mda.generator.KubernetesGenerator", "metadataContext": "targeted", "overwritable": false } diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm index 740697e58..27ddb2f5f 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm @@ -12,6 +12,11 @@ aissemble-configuration-store-chart: service: type: LoadBalancer + # Update 'volumePathOnNode' with the absolute path of the config files to be loaded into the configuration store. + # Example '/Users/MyUser/my-project/resources/configs' + # + # For WSL users, the configuration files need to be in an accessible path. + # Example: '/mnt/c' or '/mnt/wsl/rancher-desktop' configurationVolume: storageType: "local" - volumePathOnNode: "/local/path/set/in/tilt" + volumePathOnNode: "/update/this/path" diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values.yaml.vm index 9aad3c40b..c597c3c04 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values.yaml.vm @@ -21,7 +21,6 @@ aissemble-configuration-store-chart: supplementalVolumeMounts: - name: configurations mountPath: "/configurations" - readOnly: true configurationVolume: storageType: "custom" diff --git a/foundation/foundation-mda/src/main/resources/templates/notifications/notification.configuration.store.tilt.vm b/foundation/foundation-mda/src/main/resources/templates/notifications/notification.configuration.store.tilt.vm deleted file mode 100644 index 749c73d2e..000000000 --- a/foundation/foundation-mda/src/main/resources/templates/notifications/notification.configuration.store.tilt.vm +++ /dev/null @@ -1,16 +0,0 @@ -# For WSL users, the configuration files need to be in an accessible path. Update the project path to the root file system. Example: '/mnt/c' or '/mnt/wsl/rancher-desktop' -project_path = os.path.abspath('.') -# Update configuration_files_path to match the path of the config files to be loaded into the configuration store. Example 'my-project-deploy/src/main/resources/config' -configuration_files_path = '' - -load('ext://helm_resource', 'helm_resource') -helm_resource( - name='${appName}', - release_name='${appName}', - chart='${deployArtifactId}/src/main/resources/apps/${appName}', - namespace='config-store-ns', - flags=['--values=${deployArtifactId}/src/main/resources/apps/${appName}/values.yaml', - '--values=${deployArtifactId}/src/main/resources/apps/${appName}/values-dev.yaml', - '--set=aissemble-configuration-store-chart.configurationVolume.volumePathOnNode=' + project_path + '/' + configuration_files_path, - '--create-namespace'] -) diff --git a/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/DataRecordsModuleStep.java b/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/DataRecordsModuleStep.java index ab5a73f35..e4e915ec9 100644 --- a/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/DataRecordsModuleStep.java +++ b/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/DataRecordsModuleStep.java @@ -17,8 +17,6 @@ import com.boozallen.aiops.mda.metamodel.element.BasePipelineDecorator; import com.boozallen.aiops.mda.metamodel.element.Pipeline; import com.boozallen.aiops.mda.metamodel.element.python.PythonRecord; -import com.boozallen.aiops.mda.metamodel.element.python.PythonStep; -import com.boozallen.aiops.mda.util.TestMetamodelUtil; import io.cucumber.java.Before; import io.cucumber.java.Scenario; import io.cucumber.java.en.Given; @@ -136,7 +134,11 @@ public void the_module_generates_the_profile(String module, String profile) thro public void the_user_is_notified_that_the_module_must_be_added_to_the_parent_pom(String moduleName) { String moduleItem = "" + moduleName + ""; String file = projectDir.resolve("pom.xml").toString(); - Notification moduleNotification = getNotification(file, "module"); + + // Since Project Directory equals Execution Root Directory, the resulting relative path for the key is just pom.xml + String key = "pom.xml_module"; + + Notification moduleNotification = getNotification(file, key); assertTrue("Module notification missing item " + moduleItem, moduleNotification.getItems().contains(moduleItem)); } @@ -279,12 +281,12 @@ private String getPomPackaging(Path pom) throws Exception { return packaging; } - private Notification getNotification(String file, String notificationType) { + private Notification getNotification(String file, String key) { Map> notifications = NotificationCollector.getNotifications(); assertTrue("No notifications for file " + file, notifications.containsKey(file)); Map fileNotifications = notifications.get(file); - assertTrue("No notifications of type " + notificationType + " for " + file, fileNotifications.containsKey(file + "_" + notificationType)); - return fileNotifications.get(file + "_" + notificationType); + assertTrue("No notifications of type " + key + " for " + file, fileNotifications.containsKey(key)); + return fileNotifications.get(key); } /**