[Draft] Play around with index_mode in Fleet

The draft PR has several purpose

* Better understand how the setup of data stream settings works on my end
* Leave comments to further discuss the expected end behaviour
* Learn more about the Fleet code base on my end

This PR is not expected to be merged but to have conversations.

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx

@@ -57,6 +57,8 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
     experimentalDataFeatures ?? [],
     registryDataStream
   );
+
+  // Additional dependency here on TSDB feature set
   const isSyntheticSourceEnabledByDefault =
     registryDataStream.elasticsearch?.source_mode === 'synthetic';
 
@@ -65,6 +67,7 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
       typeof syntheticSourceExperimentalValue !== 'undefined'
         ? syntheticSourceExperimentalValue
         : isSyntheticSourceEnabledByDefault,
+    // I initially was looking for this but couldn't find it as I was looking for time_series. Should we align on naming?
     tsdb:
       getExperimentalFeatureValue('tsdb', experimentalDataFeatures ?? [], registryDataStream) ??
       false,
@@ -93,6 +96,11 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
       });
     }
 
+    // This sets the features on change, will it update data stream itself?
+    // Is this were rollover should happen?
+    // Ideally there would be a "data management" part in Fleet (or outside) where on a change
+    // the policy API could just make a call with the requested change and would get back if it was applied or
+    // not, not having to think about templates, data streams etc.
     setNewExperimentalDataFeatures(newExperimentalDataStreamFeatures);
   };
 

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx

@@ -134,6 +134,10 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
         (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset
       );
 
+      // It seems this logic does check for the data stream setting. What if template and data stream are not in sync (no rollover yet)
+      // Is there a way in Elasticsearch to say: For data stream X, what template will apply? The reason I'm asking
+      // is that there is a chance that users made some modification in the settings or that for a namespace an additional
+      // template was set in place. So if we only look at the data stream, the info might be wrong / outdated.
       if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') {
         if (!packagePolicy.package) return;
         if (!packagePolicy.package?.experimental_data_stream_features)
@@ -143,6 +147,7 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
         const match = packagePolicy.package!.experimental_data_stream_features.find(
           (feat) => feat.data_stream === dsName
         );
+        // If the user updates the TSDB setting, will this make its way to the template? Could not find where.
         if (match) {
           if (!match.features.tsdb) {
             match.features.tsdb = true;

x-pack/plugins/fleet/server/services/epm/archive/parse.ts

@@ -538,6 +538,7 @@ export function parseDataStreamElasticsearchEntry(
     parsedElasticsearchEntry.privileges = expandedElasticsearch.privileges;
   }
 
+  // we parse the settings from the data stream, what abou tthe values from the index template?
   if (expandedElasticsearch?.source_mode) {
     parsedElasticsearchEntry.source_mode = expandedElasticsearch.source_mode;
   }

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -270,6 +270,10 @@ export function buildComponentTemplates(params: {
 
   const indexTemplateSettings = registryElasticsearch?.['index_template.settings'] ?? {};
 
+  // TODO: Is this the place that index_mode should be added as it belongs into template.settings.index.mode
+  // On package upgrade, what happens if the previous package version had index_mode: time_series and now it is
+  // switched back? This would work for the template but not when the data stream is rolled over
+  // Ideally TSDB can be disabled again but is not possible today -> validation on the package side needed?
   const templateSettings = merge(defaultSettings, indexTemplateSettings);
 
   const indexTemplateMappings = registryElasticsearch?.['index_template.mappings'] ?? {};
@@ -281,6 +285,7 @@ export function buildComponentTemplates(params: {
     (dynampingTemplate) => Object.keys(dynampingTemplate)[0]
   );
 
+  // In case of index_mode: time_series this should be set to true by default except it is overwritten
   const sourceModeSynthetic =
     params.experimentalDataStreamFeature?.features.synthetic_source !== false &&
     (params.experimentalDataStreamFeature?.features.synthetic_source === true ||

x-pack/test/fleet_api_integration/apis/epm/get.ts

@@ -182,6 +182,9 @@ export default function (providerContext: FtrProviderContext) {
       const dataStream = packageInfo?.data_streams?.find(
         ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
       );
+      // These are tests for source_mode set in the package. Couldn't find any integration tests around the policy part?
+      // Is this all covered by unit tests? I'm especially curious how an upgrade around this feature is tested?
+      // And how do you test it manually with uploading a package when you run from source -> run your own registry?
       expect(dataStream?.elasticsearch?.source_mode).equal('default');
     });
     it('returns package info from the registry if ?full=false', async function () {

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/manifest.yml

@@ -13,4 +13,5 @@ streams:
       show_user: true
 
 elasticsearch:
-  source_mode: "synthetic"
+  # Good to see these packages for testing. Is there a flow to test upgrades?
+  source_mode: "synthetic"