opensearch cluster cdk package

opensearch-cluster-cdk/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

opensearch-cluster-cdk/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

opensearch-cluster-cdk/README.md

@@ -0,0 +1,66 @@
+# CDK for deploying single-node and multi-node OpenSearch cluster with dashboards
+
+This project enables uses to deploy either a single-node or a multi-node OpenSearch cluster.
+There are two stacks that get deployed:
+1. OpenSearch-Network-Stack: Use this stack to either use an existing Vpc or create a new Vpc. This stack also creates a new security group to manage access.
+2. OpenSearch-Infra-Stack: Sets up EC2 ASG (installs opensearch and opensearch-dashboards using userdata), cloudwatch logging, load balancer. Check your cluster log in the log group created from your stack in the cloudwatch.
+
+## Getting Started
+
+- Requires [NPM](https://docs.npmjs.com/cli/v7/configuring-npm/install) to be installed
+- Install project dependencies using `npm install` from this project directory
+- Configure [aws credentials](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_prerequisites)
+
+## Deployment
+
+### Required context parameters
+
+In order to deploy both the stacks the user needs to provide a set of mandatory and optional parameters listed below:
+
+| Name                                                   | Type     | Description                                                                              |
+|--------------------------------------------------------|:---------|:-----------------------------------------------------------------------------------------|
+| distVersion (mandatory)                                | string   | The OpenSearch distribution version (released/un-released) the user wants to deploy      |
+| securityDisabled (mandatory)                           | boolean  | Enable or disable security plugin                                                        |
+| minDistribution (mandatory)                            | boolean  | Is it an un-released OpenSearch distribution with no plugins                             |
+| distributionUrl (mandatory)                            | string   | OpenSearch tar distribution url                                                          |
+| dashboardsUrl (mandatory)                              | string   | OpenSearch Dashboards tar distribution url                                               |
+| cpuArch (mandatory)                                    | string   | CPU platform for EC2, could be either `x64` or `arm64`                                   |
+| singleNodeCluster (mandatory)                          | boolean  | Set `true` for single-node cluster else `false` for multi-node                           |
+| vpcId (Optional)                                       | string   | Re-use existing vpc, provide vpc id                                                      |
+| securityGroupId (Optional)                             | boolean  | Re-use existing security group, provide security group id                                |
+| cidr (Optional)                                        | string   | User provided CIDR block for new Vpc, default is `10.0.0.0/16`                           |
+| managerNodeCount (Optional)                            | number   | Number of cluster manager nodes, default is 3                                            |
+| dateNodeCount (Optional)                               | number   | Number of data nodes, default is 2                                                       |
+| clientNodeCount (Optional)                             | number   | Number of dedicated client nodes, default is 0                                           |
+| ingestNodeCount (Optional)                             | number   | Number of dedicated ingest nodes, default is 0                                           |
+
+#### Sample command to setup multi-node cluster with security disabled on x64 AL2 machine
+
+```
+cdk deploy "*" --context securityDisabled=true \
+--context minDistribution=false --context distributionUrl='https://artifacts.opensearch.org/releases/bundle/opensearch/2.3.0/opensearch-2.3.0-linux-x64.tar.gz' \
+--context cpuArch='x64' --context singleNodeCluster=false --context dataNodeCount=3 \
+--context dashboardsUrl='https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.3.0/opensearch-dashboards-2.3.0-linux-x64.tar.gz' \
+--context distVersion=2.3.0
+```
+
+### Interacting with OpenSearch cluster
+
+After CDK Stack deployment the user will be returned a load-balancer url which they can use to interact with the cluster.
+
+#### Sample commands
+`curl -X GET "http://<load-balancer-url>/_cluster/health?pretty"` for OpenSearch
+
+To interact with dashboards use port `8443`. Type `http://<load-balancer-url>:8443` in your browser.
+
+For security enabled cluster run `curl -X GET https://<load-balancer-url> -u 'admin:admin' --insecure`
+The security enabled dashboard is accessible using `http` on port `8443`
+
+#### Please note the load-balancer url is internet facing and can be accessed by anyone.
+To restrict access please refer [Client IP Preservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-target-groups.html#client-ip-preservation) to restrict access on internet-facing network load balancer.
+You need to add the ip/prefix-list rule in the security group created in network stack.
+
+### Check logs
+
+The opensearch logs are available in cloudwatch logs log-group `opensearchLogGroup/opensearch.log` in the same region your stack is deployed.
+Each ec2 instance will create its own log-stream and the log-stream will be named after each instance-id.

opensearch-cluster-cdk/bin/app.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+/* Copyright OpenSearch Contributors
+SPDX-License-Identifier: Apache-2.0
+
+The OpenSearch Contributors require contributions made to
+this file be licensed under the Apache-2.0 license or a
+compatible open source license. */
+
+import 'source-map-support/register';
+import { App } from 'aws-cdk-lib';
+import { OsClusterEntrypoint } from '../lib/os-cluster-entrypoint';
+
+const app = new App();
+
+new OsClusterEntrypoint(app, {
+  env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
+});

opensearch-cluster-cdk/cdk.json

@@ -0,0 +1,42 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/app.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
+    "@aws-cdk/core:stackRelativeExports": true,
+    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
+    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ]
+  }
+}

opensearch-cluster-cdk/jest.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  },
+  testTimeout: 50000
+};

opensearch-cluster-cdk/lib/cloudwatch/agent-section.ts

@@ -0,0 +1,33 @@
+/* Copyright OpenSearch Contributors
+SPDX-License-Identifier: Apache-2.0
+
+The OpenSearch Contributors require contributions made to
+this file be licensed under the Apache-2.0 license or a
+compatible open source license. */
+
+/* eslint-disable max-len */
+interface EditableCloudwatchAgentSection {
+    // eslint-disable-next-line camelcase
+    metrics_collection_interval: number;
+    logfile: string;
+    // eslint-disable-next-line camelcase
+    omit_hostname: boolean;
+    debug: boolean;
+}
+
+/**
+ * Cloudwatch configuration - Agent Section
+ *
+ * See definition at https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html#CloudWatch-Agent-Configuration-File-Agentsection
+ *
+ * Example configuration:
+ * ```
+ * agent: {
+ *   metrics_collection_interval: 60, // seconds between collections
+ *   logfile: '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log',
+ *   omit_hostname: true,
+ *   debug: true,
+ * }
+ * ```
+ */
+export type CloudwatchAgentSection = Readonly<EditableCloudwatchAgentSection>;

opensearch-cluster-cdk/lib/cloudwatch/cloudwatch-agent.ts

@@ -0,0 +1,25 @@
+/* Copyright OpenSearch Contributors
+SPDX-License-Identifier: Apache-2.0
+
+The OpenSearch Contributors require contributions made to
+this file be licensed under the Apache-2.0 license or a
+compatible open source license. */
+
+import { InitFile, InitFileOptions } from 'aws-cdk-lib/aws-ec2';
+import { CloudwatchAgentSection } from './agent-section';
+import { CloudwatchLogsSection } from './logs-section';
+
+export interface CloudwatchAgentConfig {
+    agent: CloudwatchAgentSection,
+    logs: CloudwatchLogsSection
+}
+
+export class CloudwatchAgent {
+  /**
+     * Creates a cloudwatch agent config file as an InitFile that can be deployed onto an EC2 instance
+     */
+  public static asInitFile(filePath: string, config: CloudwatchAgentConfig, options?: InitFileOptions): InitFile {
+    const configAsString = JSON.stringify(config, undefined, 2);
+    return InitFile.fromString(filePath, configAsString, options);
+  }
+}

opensearch-cluster-cdk/lib/cloudwatch/logs-section.ts

@@ -0,0 +1,57 @@
+/* Copyright OpenSearch Contributors
+SPDX-License-Identifier: Apache-2.0
+
+The OpenSearch Contributors require contributions made to
+this file be licensed under the Apache-2.0 license or a
+compatible open source license. */
+
+/* eslint-disable max-len */
+interface FileCollectionDefinition {
+    // eslint-disable-next-line camelcase
+    file_path: string;
+    // eslint-disable-next-line camelcase
+    log_group_name: string;
+    // eslint-disable-next-line camelcase
+    auto_removal: boolean;
+    // eslint-disable-next-line camelcase
+    log_stream_name: string,
+}
+
+interface EditableLogsSection {
+    // eslint-disable-next-line camelcase
+    logs_collected: {
+        files: {
+            // eslint-disable-next-line camelcase
+            collect_list: FileCollectionDefinition[]
+        }
+    };
+    // eslint-disable-next-line camelcase
+    force_flush_interval: number;
+}
+
+/**
+ * Cloudwatch configuration - Logs Section
+ *
+ * See definition at https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html#CloudWatch-Agent-Configuration-File-Logssection
+ *
+ * Example configuration:
+ * ```
+ logs: {
+    logs_collected: {
+      files: {
+        collect_list: [
+          {
+            file_path: '/var/log/jenkins/jenkins.log',
+            log_group_name: 'JenkinsMainNode/jenkins',
+            auto_removal: true,
+            log_stream_name: 'jenkins.log',
+            timestamp_format: '%Y-%m-%d %H:%M:%S.%f%z',
+          },
+        ],
+      },
+    },
+    force_flush_interval: 15,
+  }
+ * ```
+ */
+export type CloudwatchLogsSection = Readonly<EditableLogsSection>;