[fleet] fix elastic-agent setup

* adapt setup to changes from elastic/beats#24220

scripts/modules/aux_services.py

@@ -25,7 +25,7 @@ def build_candidate_manifest(self):
 
     def _content(self):
         self.es_urls = ",".join(self.options.get(
-            "logstash_elasticsearch_urls") or [self.DEFAULT_ELASTICSEARCH_HOSTS])
+            "logstash_elasticsearch_urls") or [self.DEFAULT_ELASTICSEARCH_HOSTS_NO_TLS])
         if self.at_least_version("7.3") \
                 or self.options.get("apm_server_snapshot") \
                 or (not self.options.get("apm_server_version") is None and

scripts/modules/elastic_stack.py

@@ -696,34 +696,82 @@ def __init__(self, **options):
         self.depends_on = {"kibana": {"condition": "service_healthy"}} if options.get("enable_kibana", True) else {}
 
         # build environment
+        #
         # Environment variables consumed by the Elastic Agent entrypoint
-        # https://github.com/elastic/beats/blob/4f4a5536b72f4a25962d56262f31e3b8533b252e/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl
-        # FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll
-        # FLEET_ENROLL - if set to 1 enroll will be performed
-        # FLEET_ENROLL_INSECURE - if set to 1, agent will enroll with fleet using --insecure flag
-        # FLEET_SETUP - if  set to 1 fleet setup will be performed
-        # FLEET_TOKEN_NAME - token name for a token to be created
-        # KIBANA_HOST - actual kibana host [http://localhost:5601]
-        # KIBANA_PASSWORD - password for accessing kibana API [changeme]
-        # KIBANA_USERNAME - username for accessing kibana API [elastic]
+        # ---- Preparing Kibana for Fleet
+        # KIBANA_FLEET_SETUP - set to 1 enables this setup
+
+        # KIBANA_FLEET_HOST - kibana host to enable Fleet on, fallback: [$KIBANA_HOST]
+        # KIBANA_FLEET_USERNAME - kibana username to enable Fleet, fallback: [$KIBANA_USERNAME]
+        # KIBANA_FLEET_PASSWORD - kibana password to enable Fleet, fallback: [$KIBANA_PASSWORD]
+
+        # ---- Bootstrapping Fleet Server
+        # This bootstraps the Fleet Server to be run by this Elastic Agent. At least one Fleet Server is required in a Fleet
+        # deployment for other Elastic Agent to bootstrap.
+        # FLEET_SERVER_ENABLE - set to 1 enables bootstrapping of Fleet Server (forces FLEET_ENROLL enabled)
+        # FLEET_SERVER_ELASTICSEARCH_HOST - elasticsearch host for Fleet Server to communicate with [$ELASTICSEARCH_HOST]
+        # FLEET_SERVER_ELASTICSEARCH_USERNAME - elasticsearch username for Fleet Server [$ELASTICSEARCH_USERNAME]
+        # FLEET_SERVER_ELASTICSEARCH_PASSWORD - elasticsearch password for Fleet Server [$ELASTICSEARCH_PASSWORD]
+        # FLEET_SERVER_POLICY_NAME - name of policy for the Fleet Server to use for itself [$FLEET_TOKEN_POLICY_NAME]
+        # FLEET_SERVER_POLICY_ID - policy ID for Fleet Server to use for itself ("Default Fleet Server policy" used when undefined)
+        # FLEET_SERVER_HOST - binding host for Fleet Server HTTP (overrides the policy)
+        # FLEET_SERVER_PORT - binding port for Fleet Server HTTP (overrides the policy)
+        # FLEET_SERVER_CERT - path to certificate to use for HTTPS endpoint
+        # FLEET_SERVER_CERT_KEY - path to private key for certificate to use for HTTPS endpoint
+        # FLEET_SERVER_INSECURE_HTTP - expose Fleet Server over HTTP (not recommended; insecure)
+
+        # ---- Elastic Agent Fleet Enrollment
+        # This enrolls the Elastic Agent into a Fleet Server. It is also possible to have this create a new enrollment token
+        # for this specific Elastic Agent.
+        # FLEET_ENROLL - set to 1 for enrollment to occur
+        # FLEET_URL - URL of the Fleet Server to enroll into
+        # FLEET_ENROLLMENT_TOKEN - token to use for enrollment
+        # FLEET_TOKEN_NAME - token name to use for fetching token from Kibana
+        # FLEET_TOKEN_POLICY_NAME - token policy name to use for fetching token from Kibana
+        # FLEET_INSECURE - communicate with Fleet with either insecure HTTP or un-verified HTTPS
+        # KIBANA_FLEET_HOST - kibana host to enable create enrollment token on [$KIBANA_HOST]
+        # KIBANA_FLEET_USERNAME - kibana username to create enrollment token [$KIBANA_USERNAME]
+        # KIBANA_FLEET_PASSWORD - kibana password to create enrollment token [$KIBANA_PASSWORD]
+
+        # ELASTICSEARCH_HOST - elasticsearch host [http://elasticsearch:9200]
+        # ELASTICSEARCH_USERNAME - elasticsearch username [elastic]
+        # ELASTICSEARCH_PASSWORD - elasticsearch password [changeme]
+        # KIBANA_HOST - kibana host [http://kibana:5601]
+        # KIBANA_USERNAME - kibana username [$ELASTICSEARCH_USERNAME]
+        # KIBANA_PASSWORD - kibana password [$ELASTICSEARCH_PASSWORD]
+
+        # --------------
         kibana_url = options.get("elastic_agent_kibana_url")
         if not kibana_url:
             kibana_scheme = "https" if self.options.get("kibana_enable_tls", False) else "http"
             # TODO(gr): add default elastic-agent user
             kibana_url = kibana_scheme + "://admin:changeme@" + self.DEFAULT_KIBANA_HOST
-
         kibana_parsed_url = urlparse(kibana_url)
+
+        es_url = options.get("elastic_agent_elasticsearch_url")
+        if not es_url:
+            es_scheme = "https" if self.options.get("elasticsearch_enable_tls", False) else "http"
+            es_url = es_scheme + "://admin:changeme@" + self.DEFAULT_ELASTICSEARCH_HOST
+        es_parsed_url = urlparse(es_url)
+
         self.environment = {
+            "KIBANA_FLEET_SETUP": "1",
+            "FLEET_SERVER_ENABLE": "1",
             "FLEET_ENROLL": "1",
-            "FLEET_SETUP": "1",
+            "FLEET_SERVER_POLICY_NAME": "Default policy",  # TODO(simitt): make configurable
             "KIBANA_HOST": kibana_url,
+            "ELASTICSEARCH_HOST": es_url
         }
         if kibana_parsed_url.password:
             self.environment["KIBANA_PASSWORD"] = kibana_parsed_url.password
         if kibana_parsed_url.username:
             self.environment["KIBANA_USERNAME"] = kibana_parsed_url.username
         if not kibana_url.startswith("https://"):
-            self.environment["FLEET_ENROLL_INSECURE"] = 1
+            self.environment["FLEET_INSECURE"] = "1"
+        if es_parsed_url.password:
+            self.environment["ELASTICSEARCH_PASSWORD"] = es_parsed_url.password
+        if es_parsed_url.username:
+            self.environment["ELASTICSEARCH_USERNAME"] = es_parsed_url.username
 
         # set ports for defined integrations
         self.ports = []
@@ -752,6 +800,11 @@ def add_arguments(cls, parser):
             default="http://admin:changeme@" + cls.DEFAULT_KIBANA_HOST,
             help="Elastic Agent's Kibana URL, including username:password"
         )
+        parser.add_argument(
+            "--elastic-agent-elasticsearch-url",
+            default="http://admin:changeme@" + cls.DEFAULT_ELASTICSEARCH_HOST,
+            help="Elastic Agent's Elasticsearch URL, including username:password"
+        )
 
     def build_candidate_manifest(self):
         version = self.version

scripts/modules/opbeans.py

@@ -24,7 +24,8 @@ def __init__(self, **options):
         self.agent_local_repo = options.get(self.option_name() + "_agent_local_repo")
         self.opbeans_branch = options.get(self.option_name() + "_branch") or ""
         self.opbeans_repo = options.get(self.option_name() + "_repo") or ""
-        self.es_urls = ",".join(self.options.get("opbeans_elasticsearch_urls") or [self.DEFAULT_ELASTICSEARCH_HOSTS])
+        self.es_urls = ",".join(self.options.get("opbeans_elasticsearch_urls")
+                                or [self.DEFAULT_ELASTICSEARCH_HOSTS_NO_TLS])
         self.service_environment = \
             options.get(self.option_name() + "_service_environment") or self.DEFAULT_ELASTIC_APM_ENVIRONMENT
 

scripts/modules/service.py

@@ -15,10 +15,10 @@
 class Service(object):
     """encapsulate docker-compose service definition"""
 
-    DEFAULT_ELASTICSEARCH_HOSTS = "http://elasticsearch:9200"
-    DEFAULT_ELASTICSEARCH_HOSTS_TLS = "https://elasticsearch:9200"
     DEFAULT_KIBANA_HOST = "kibana:5601"
-
+    DEFAULT_ELASTICSEARCH_HOST = "elasticsearch:9200"
+    DEFAULT_ELASTICSEARCH_HOSTS_NO_TLS = "http://" + DEFAULT_ELASTICSEARCH_HOST
+    DEFAULT_ELASTICSEARCH_HOSTS_TLS = "https://" + DEFAULT_ELASTICSEARCH_HOST
     # is this a side car service for opbeans. If yes, it will automatically
     # start if any opbeans service starts
     opbeans_side_car = False
@@ -63,6 +63,9 @@ def __init__(self, **options):
                 print('ERROR: OSS distribution is ONLY supported in 7.11+/6.8.14+ for Kibana and Elasticsearch.')
                 sys.exit(1)
 
+        self._es_tls = options.get("elasticsearch_enable_tls", False)
+        self._kibana_tls = options.get("kibana_enable_tls", False)
+
     @property
     def bc(self):
         return self._bc
@@ -175,7 +178,7 @@ def default_elasticsearch_hosts(self, tls=False):
         if tls:
             return self.DEFAULT_ELASTICSEARCH_HOSTS_TLS
         else:
-            return self.DEFAULT_ELASTICSEARCH_HOSTS
+            return self.DEFAULT_ELASTICSEARCH_HOSTS_NO_TLS
 
     @abstractmethod
     def _content(self):

scripts/tests/service_tests.py

@@ -917,16 +917,21 @@ def test_debug(self):
 
 class ElasticAgentServiceTest(ServiceTest):
     def test_default(self):
-        ea = ElasticAgent(version="7.12.345",enable_apm_server=True,apm_server_managed=True).render()["elastic-agent"]
+        ea = ElasticAgent(version="7.12.345", enable_apm_server=True, apm_server_managed=True).render()["elastic-agent"]
         self.assertEqual(
             ea, {"container_name": "localtesting_7.12.345_elastic-agent",
                  "depends_on": {"kibana": {"condition": "service_healthy"}},
-                 "environment": {"FLEET_ENROLL": "1",
-                                 "FLEET_ENROLL_INSECURE": 1,
-                                 "FLEET_SETUP": "1",
-                                 "KIBANA_HOST": "http://admin:changeme@kibana:5601",
-                                 "KIBANA_PASSWORD": "changeme",
-                                 "KIBANA_USERNAME": "admin"},
+                 'environment': {'ELASTICSEARCH_HOST': 'http://admin:changeme@elasticsearch:9200',
+                                 'ELASTICSEARCH_PASSWORD': 'changeme',
+                                 'ELASTICSEARCH_USERNAME': 'admin',
+                                 'FLEET_ENROLL': '1',
+                                 'FLEET_SERVER_ENABLE': '1',
+                                 'FLEET_INSECURE': '1',
+                                 'FLEET_SERVER_POLICY_NAME': 'Default policy',
+                                 'KIBANA_FLEET_SETUP': '1',
+                                 'KIBANA_HOST': 'http://admin:changeme@kibana:5601',
+                                 'KIBANA_PASSWORD': 'changeme',
+                                 'KIBANA_USERNAME': 'admin'},
                  "healthcheck": {"test": ["CMD", "elastic-agent", "version"]},
                  "image": "docker.elastic.co/beats/elastic-agent:7.12.345-SNAPSHOT",
                  "labels": ["co.elastic.apm.stack-version=7.12.345"],