feat(specs): add estimate path and responses [skip-bc] (#4057)

Co-authored-by: Pierre Millot <[email protected]>

specs/abtesting/common/schemas/ABTest.yml

@@ -154,10 +154,13 @@ MinimumDetectableEffect:
       description: |
         Smallest difference in an observable metric between variants.
         For example, to detect a 10% difference between variants, set this value to 0.1.
-    effect:
-      $ref: '#/Effect'
+    metric:
+      $ref: '#/EffectMetric'
+  required:
+    - size
+    - metric
 
-Effect:
+EffectMetric:
   type: string
   description: Metric for which you want to detect the smallest relative difference.
   enum:

specs/abtesting/common/schemas/EstimateABTestResponse.yml

@@ -0,0 +1,18 @@
+EstimateABTestResponse:
+  type: object
+  properties:
+    durationDays:
+      type: integer
+      format: int64
+      description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic.
+      example: 21
+    controlSampleSize:
+      type: integer
+      format: int64
+      description: Number of tracked searches needed to be able to detect the configured effect for the control variant.
+      example: 23415
+    experimentSampleSize:
+      type: integer
+      format: int64
+      description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant.
+      example: 23415

specs/abtesting/paths/estimate.yml

@@ -0,0 +1,62 @@
+post:
+  tags:
+    - abtest
+  operationId: estimateABTest
+  x-acl:
+    - analytics
+  summary: Estimate the sample size and duration of an A/B test
+  description: Given the traffic percentage and the expected effect size, this endpoint estimates the sample size and duration of an A/B test based on historical traffic.
+  requestBody:
+    required: true
+    content:
+      application/json:
+        schema:
+          title: estimateABTestRequest
+          type: object
+          additionalProperties: false
+          properties:
+            configuration:
+              title: estimateConfiguration
+              type: object
+              description: A/B test configuration for estimating the sample size and duration using minimum detectable effect.
+              properties:
+                outliers:
+                  $ref: '../common/schemas/ABTest.yml#/Outliers'
+                emptySearch:
+                  $ref: '../common/schemas/ABTest.yml#/EmptySearch'
+                minimumDetectableEffect:
+                  $ref: '../common/schemas/ABTest.yml#/MinimumDetectableEffect'
+              required:
+                - minimumDetectableEffect
+            variants:
+              type: array
+              description: A/B test variants.
+              minItems: 2
+              maxItems: 2
+              items:
+                $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant'
+          required:
+            - configuration
+            - variants
+  responses:
+    '200':
+      description: OK
+      headers:
+        x-ratelimit-limit:
+          $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit'
+        x-ratelimit-remaining:
+          $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining'
+        x-ratelimit-reset:
+          $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset'
+      content:
+        application/json:
+          schema:
+            $ref: '../common/schemas/EstimateABTestResponse.yml#/EstimateABTestResponse'
+    '400':
+      $ref: '../../common/responses/BadRequest.yml'
+    '402':
+      $ref: '../../common/responses/FeatureNotEnabled.yml'
+    '403':
+      $ref: '../../common/responses/MethodNotAllowed.yml'
+    '404':
+      $ref: '../../common/responses/IndexNotFound.yml'

specs/abtesting/spec.yml

@@ -79,7 +79,7 @@ tags:
     description: |
       Manage A/B tests.
 
-      A/B tests are configurations of two indices, usually your production index and an index with different settings that you want to test.
+      A/B tests are configurations one or more indices, usually your production index and an index with different settings that you want to test.
 x-tagGroups:
   - name: General
     tags:
@@ -99,6 +99,8 @@ paths:
     $ref: 'paths/stopABTest.yml'
   /2/abtests/schedule:
     $ref: 'paths/scheduleABTest.yml'
+  /2/abtests/estimate:
+    $ref: 'paths/estimate.yml'
 
   # ###############
   # ### Helpers ###

tests/CTS/requests/abtesting/estimateABTest.json

@@ -0,0 +1,51 @@
+[
+  {
+    "testName": "estimate AB Test sample size",
+    "parameters": {
+      "configuration": {
+        "emptySearch": {
+          "exclude": true
+        },
+        "minimumDetectableEffect": {
+          "size": 0.03,
+          "metric": "conversionRate"
+        }
+      },
+      "variants": [
+        {
+          "index": "AB_TEST_1",
+          "trafficPercentage": 50
+        },
+        {
+          "index": "AB_TEST_2",
+          "trafficPercentage": 50
+        }
+      ]
+    },
+    "request": {
+      "path": "/2/abtests/estimate",
+      "method": "POST",
+      "body": {
+        "configuration": {
+          "emptySearch": {
+            "exclude": true
+          },
+          "minimumDetectableEffect": {
+            "size": 0.03,
+            "metric": "conversionRate"
+          }
+        },
+        "variants": [
+          {
+            "index": "AB_TEST_1",
+            "trafficPercentage": 50
+          },
+          {
+            "index": "AB_TEST_2",
+            "trafficPercentage": 50
+          }
+        ]
+      }
+    }
+  }
+]