Merge branch 'main' into seanstory/bump-backport-for-8.17

Dockerfile.ftest.wolfi

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/wolfi/python:3.11-dev@sha256:b70abdb46f6a1ed5057fb50018924eea03345044b5224c260a3d0334a748eb96
+FROM docker.elastic.co/wolfi/python:3.11-dev@sha256:8d8b75a14cc043bee2f1b6c1a7815611acde442519fecdc2d27a23fbc0b0de26
 USER root
 COPY . /connectors
 WORKDIR /connectors

Dockerfile.wolfi

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/wolfi/python:3.11-dev@sha256:b70abdb46f6a1ed5057fb50018924eea03345044b5224c260a3d0334a748eb96
+FROM docker.elastic.co/wolfi/python:3.11-dev@sha256:8d8b75a14cc043bee2f1b6c1a7815611acde442519fecdc2d27a23fbc0b0de26
 USER root
 COPY . /app
 WORKDIR /app

connectors/sources/confluence.py

@@ -908,14 +908,22 @@ async def fetch_documents(self, api_query):
             doc = {
                 "_id": str(document["id"]),
                 "type": document["type"],
-                "_timestamp": document["history"]["lastUpdated"]["when"],
+                "_timestamp": nested_get_from_dict(
+                    document, ["history", "lastUpdated", "when"]
+                ),
                 "title": document.get("title"),
                 "ancestors": ancestor_title,
-                "space": document["space"]["name"],
-                "body": html_to_text(document["body"]["storage"]["value"]),
+                "space": nested_get_from_dict(document, ["space", "name"]),
+                "body": html_to_text(
+                    nested_get_from_dict(document, ["body", "storage", "value"])
+                ),
                 "url": document_url,
-                "author": document["history"]["createdBy"][self.authorkey],
-                "createdDate": document["history"]["createdDate"],
+                "author": nested_get_from_dict(
+                    document, ["history", "createdBy", self.authorkey]
+                ),
+                "createdDate": nested_get_from_dict(
+                    document, ["history", "createdDate"]
+                ),
             }
             if self.confluence_client.index_labels:
                 doc["labels"] = document["labels"]

connectors/sources/outlook.py

@@ -44,6 +44,7 @@
     html_to_text,
     iso_utc,
     retryable,
+    url_encode,
 )
 
 RETRIES = 3
@@ -410,7 +411,8 @@ async def _fetch_token(self):
     )
     async def get_users(self):
         access_token = await self._fetch_token()
-        url = f"https://graph.microsoft.com/v1.0/users?$top={TOP}"
+        filter_ = url_encode("accountEnabled eq true")
+        url = f"https://graph.microsoft.com/v1.0/users?$top={TOP}&$filter={filter_}"
         while True:
             try:
                 async with self._get_session.get(

pytest.ini

@@ -16,3 +16,5 @@ filterwarnings =
   ignore:Inheritance class RetryableAiohttpSession from ClientSession is discouraged:DeprecationWarning
 ; pytest may generate its own warnings in some situations, such as improper usage or deprecated features.
   ignore::pytest.PytestUnraisableExceptionWarning
+; this error comes from 8.16 Elasticsearch client: we use some non-GA features in our CI code
+  ignore::elasticsearch.exceptions.GeneralAvailabilityWarning

requirements/framework.txt

@@ -3,8 +3,8 @@ aiofiles==23.2.1
 aiomysql==0.1.1
 httpx==0.27.0
 httpx-ntlm==1.4.0
-elasticsearch[async]==8.15.0
-elastic-transport==8.15.0
+elasticsearch[async]==8.16.0
+elastic-transport==8.15.1
 pyyaml==6.0.1
 cffi==1.16.0
 envyaml==1.10.211231

scripts/stack/run-stack.sh

@@ -41,10 +41,10 @@ if [[ "${connectors_only}" != true ]]; then
   source $CURDIR/wait-for-elasticsearch.sh
 
   # Start Kibana
+  source $CURDIR/update-kibana-user-password.sh
   echo "Starting Kibana..."
   docker-compose -f $compose_file up --detach kibana
   source $CURDIR/wait-for-kibana.sh
-  source $CURDIR/update-kibana-user-password.sh
 fi
 
 source ./copy-config.sh

scripts/stack/update-kibana-user-password.sh

@@ -11,10 +11,6 @@ if [[ ${CURDIR:-} == "" ]]; then
     export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 fi
 
-if [[ "${ELASTIC_PASSWORD:-}" == "" ]]; then
-    source $CURDIR/read-env.sh $CURDIR/.env
-fi
-
 echo "Updating Kibana password in Elasticsearch running on $ELASTICSEARCH_URL"
 change_data="{ \"password\": \"${ELASTIC_PASSWORD}\" }"
 curl -u elastic:$ELASTIC_PASSWORD "$@" -X POST "${ELASTICSEARCH_URL}/_security/user/kibana_system/_password?pretty" -H 'Content-Type: application/json' -d"${change_data}"

tests/ftest.sh

@@ -14,13 +14,116 @@ VIRTUAL_ENV="$ROOT_DIR/.venv"
 PLATFORM='unknown'
 MAX_RSS="200M"
 MAX_DURATION=600
+CONNECTORS_VERSION=$(cat "$ROOT_DIR/connectors/VERSION")
+ARTIFACT_BASE_URL="https://artifacts-snapshot.elastic.co"
 
 export DOCKERFILE_FTEST_PATH=${DOCKERFILE_FTEST_PATH:-Dockerfile.ftest}
 export PERF8_TRACE=${PERF8_TRACE:-False}
 export REFRESH_RATE="${REFRESH_RATE:-5}"
 export DATA_SIZE="${DATA_SIZE:-medium}"
 export RUNNING_FTEST=True
-export VERSION='9.0.0-SNAPSHOT'
+export VERSION="${CONNECTORS_VERSION}-SNAPSHOT"
+
+# Download and load ES Docker images from DRA artifacts instead of relying on the snapshot image in the registry.
+# This is needed for the release process when the ES snapshot image may not yet be available.
+# Snapshot images are pushed to the registry by the unified release workflow.
+
+# Function to resolve the DRA manifest URL
+function resolve_dra_manifest {
+  DRA_ARTIFACT=$1
+  DRA_VERSION=$2
+
+  # Perform the curl request and capture the output and exit code
+  RESPONSE=$(curl -sS -f $ARTIFACT_BASE_URL/$DRA_ARTIFACT/latest/$DRA_VERSION.json 2>&1)
+  CURL_EXIT_CODE=$?
+
+  # Check if the curl command failed
+  if [ $CURL_EXIT_CODE -ne 0 ]; then
+    echo "Error: Failed to fetch DRA manifest for artifact $DRA_ARTIFACT and version $DRA_VERSION."
+    echo "Details: $RESPONSE"
+    exit 1
+  fi
+
+  # Extract the manifest_url using jq
+  MANIFEST_URL=$(echo "$RESPONSE" | jq -r '.manifest_url' 2>/dev/null)
+
+  # Check if the jq command succeeded and if manifest_url is non-empty
+  if [ -z "$MANIFEST_URL" ] || [ "$MANIFEST_URL" == "null" ]; then
+    echo "Error: No manifest_url found in the response for artifact $DRA_ARTIFACT and version $DRA_VERSION."
+    echo "Response: $RESPONSE"
+    exit 1
+  fi
+
+  # Output the manifest URL
+  echo "$MANIFEST_URL"
+}
+
+# Function to download the Docker image tarball
+function download_docker_tarball {
+  TAR_URL=$1
+  TAR_FILE=$2
+  MAX_RETRIES=3
+
+  echo "Downloading Docker image tarball from $TAR_URL..."
+  curl --retry $MAX_RETRIES --retry-connrefused -O "$TAR_URL"
+
+  if [ ! -f "$TAR_FILE" ]; then
+    echo "Error: Download failed. File $TAR_FILE not found."
+    exit 1
+  fi
+}
+
+# Function to load the Docker image directly from the tarball
+function load_docker_image {
+  TAR_FILE=$1
+  echo "Loading Docker image from $TAR_FILE..."
+  docker load < "$TAR_FILE"
+}
+
+# Determine system architecture
+ARCH=$(uname -m)
+
+# Normalize architecture name
+if [[ $ARCH == "arm64" ]]; then
+  ARCH="aarch64"
+fi
+
+# Select the appropriate Docker tarball name based on architecture
+case $ARCH in
+  x86_64)
+    DOCKER_TARBALL_NAME="elasticsearch-$VERSION-docker-image.tar.gz"
+    ;;
+  aarch64)
+    DOCKER_TARBALL_NAME="elasticsearch-$VERSION-docker-image-aarch64.tar.gz"
+    ;;
+  *)
+    echo "Error: Unsupported architecture $ARCH"
+    exit 1
+    ;;
+esac
+
+# Get the DRA manifest URL for Elasticsearch
+ELASTICSEARCH_DRA_MANIFEST=$(resolve_dra_manifest "elasticsearch" $VERSION)
+
+# Parse Docker image tarball information
+DOCKER_TARBALL_URL=$(curl -sS "$ELASTICSEARCH_DRA_MANIFEST" | jq -r ".projects.elasticsearch.packages.\"$DOCKER_TARBALL_NAME\".url")
+
+if [ -z "$DOCKER_TARBALL_URL" ] || [ "$DOCKER_TARBALL_URL" == "null" ]; then
+  echo "Error: Docker tarball URL not found in the manifest."
+  exit 1
+fi
+
+# Execute the functions to download and load the image
+download_docker_tarball "$DOCKER_TARBALL_URL" "$DOCKER_TARBALL_NAME"
+load_docker_image "$DOCKER_TARBALL_NAME"
+
+# Export image name following DRA conventions
+export ELASTICSEARCH_DRA_DOCKER_IMAGE="elasticsearch:$ARCH"
+
+echo "Docker image for Elasticsearch $VERSION loaded successfully. Image name: $ELASTICSEARCH_DRA_DOCKER_IMAGE"
+
+echo "Cleaning up the downloaded tarball..."
+rm -f "$DOCKER_TARBALL_NAME"
 
 if [ "$PERF8_TRACE" == true ]; then
     echo 'Tracing is enabled, memray stats will be delivered'

tests/sources/fixtures/azure_blob_storage/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.9'
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
+    image: ${ELASTICSEARCH_DRA_DOCKER_IMAGE}
     container_name: elasticsearch
     environment:
       - cluster.name=docker-cluster
@@ -24,60 +24,11 @@ services:
     networks:
       - esnet
 
-  kibana:
-    image: docker.elastic.co/kibana/kibana:${VERSION}
-    ports:
-      - 5601:5601
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    depends_on:
-        - "elasticsearch"
-    profiles:
-      - "enterprise-search"
-    environment:
-      ELASTICSEARCH_URL: http://host.docker.internal:9200
-      ELASTICSEARCH_HOSTS: http://host.docker.internal:9200
-      ENTERPRISESEARCH_HOST: http://host.docker.internal:3002
-      ELASTICSEARCH_USERNAME: kibana_system
-      ELASTICSEARCH_PASSWORD: changeme
-    networks:
-      - esnet
-
-  enterprise_search:
-    image: docker.elastic.co/enterprise-search/enterprise-search:${VERSION}
-    profiles:
-      - "enterprise-search"
-    depends_on:
-      - "elasticsearch"
-    environment:
-      - ENT_SEARCH_DEFAULT_PASSWORD=changeme
-      - elasticsearch.username=elastic
-      - elasticsearch.password=changeme
-      - elasticsearch.host=http://host.docker.internal:9200
-      - allow_es_settings_modification=true
-      - kibana.host=http://host.docker.internal:5601
-      - kibana.external_url=http://localhost:5601
-      - secret_management.encryption_keys=["4a2cd3f81d39bf28738c10db0ca782095ffac07279561809eecc722e0c20eb09"]
-      - JAVA_OPTS=-Xms2g -Xmx2g
-      - email.account.enabled=true
-      - email.account.smtp.auth=plain
-      - email.account.smtp.starttls.enable=false
-      - email.account.smtp.host=host.docker.internal
-      - email.account.smtp.port=1025
-      - [email protected]
-      - DEBUG=true
-    ports:
-      - 3002:3002
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    networks:
-      - esnet
-
   azureblobstorage:
     image: mcr.microsoft.com/azure-storage/azurite
     ports:
       - 10000:10000
-    command: ["azurite-blob","--blobHost","0.0.0.0","--blobPort","10000"]    
+    command: ["azurite-blob","--blobHost","0.0.0.0","--blobPort","10000"]
 
 networks:
   esnet:

tests/sources/fixtures/box/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
+    image: ${ELASTICSEARCH_DRA_DOCKER_IMAGE}
     container_name: elasticsearch
     environment:
       - cluster.name=docker-cluster
@@ -24,57 +24,8 @@ services:
     networks:
       - esnet
 
-  kibana:
-    image: docker.elastic.co/kibana/kibana:${VERSION}
-    ports:
-      - 5601:5601
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    depends_on:
-        - "elasticsearch"
-    profiles:
-      - "enterprise-search"
-    environment:
-      ELASTICSEARCH_URL: http://host.docker.internal:9200
-      ELASTICSEARCH_HOSTS: http://host.docker.internal:9200
-      ENTERPRISESEARCH_HOST: http://host.docker.internal:3002
-      ELASTICSEARCH_USERNAME: kibana_system
-      ELASTICSEARCH_PASSWORD: changeme
-    networks:
-      - esnet
-
-  enterprise_search:
-    image: docker.elastic.co/enterprise-search/enterprise-search:${VERSION}
-    profiles:
-      - "enterprise-search"
-    depends_on:
-      - "elasticsearch"
-    environment:
-      - ENT_SEARCH_DEFAULT_PASSWORD=changeme
-      - elasticsearch.username=elastic
-      - elasticsearch.password=changeme
-      - elasticsearch.host=http://host.docker.internal:9200
-      - allow_es_settings_modification=true
-      - kibana.host=http://host.docker.internal:5601
-      - kibana.external_url=http://localhost:5601
-      - secret_management.encryption_keys=["4a2cd3f81d39bf28738c10db0ca782095ffac07279561809eecc722e0c20eb09"]
-      - JAVA_OPTS=-Xms2g -Xmx2g
-      - email.account.enabled=true
-      - email.account.smtp.auth=plain
-      - email.account.smtp.starttls.enable=false
-      - email.account.smtp.host=host.docker.internal
-      - email.account.smtp.port=1025
-      - [email protected]
-      - DEBUG=true
-    ports:
-      - 3002:3002
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    networks:
-      - esnet
-
   box:
-    build: 
+    build:
       context: ../../../../
       dockerfile: ${DOCKERFILE_FTEST_PATH}
     command: .venv/bin/python tests/sources/fixtures/box/fixture.py