[index patterns] Fleep app - Keep saved object field list until field…

… caps provides fields (elastic#85370)

# Conflicts:
#	src/plugins/data/public/public.api.md

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-public.indexpattern.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-public.indexpattern.allownoindex.md)
+
+## IndexPattern.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+readonly allowNoIndex: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.getassavedobjectbody.md

@@ -19,6 +19,7 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
 ```
 <b>Returns:</b>
@@ -33,5 +34,6 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     }`
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md

@@ -20,6 +20,7 @@ export declare class IndexPattern implements IIndexPattern
 
 |  Property | Modifiers | Type | Description |
 |  --- | --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-public.indexpattern.allownoindex.md) |  | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [deleteFieldFormat](./kibana-plugin-plugins-data-public.indexpattern.deletefieldformat.md) |  | <code>(fieldName: string) =&gt; void</code> |  |
 |  [fieldAttrs](./kibana-plugin-plugins-data-public.indexpattern.fieldattrs.md) |  | <code>FieldAttrs</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpattern.fieldformatmap.md) |  | <code>Record&lt;string, any&gt;</code> |  |
@@ -51,16 +52,15 @@ export declare class IndexPattern implements IIndexPattern
 |  [getComputedFields()](./kibana-plugin-plugins-data-public.indexpattern.getcomputedfields.md) |  |  |
 |  [getFieldByName(name)](./kibana-plugin-plugins-data-public.indexpattern.getfieldbyname.md) |  |  |
 |  [getFormatterForField(field)](./kibana-plugin-plugins-data-public.indexpattern.getformatterforfield.md) |  | Provide a field, get its formatter |
-|  [getIndex()](./kibana-plugin-plugins-data-public.indexpattern.getindex.md) |  |  |
 |  [getFormatterForFieldNoDefault(fieldname)](./kibana-plugin-plugins-data-public.indexpattern.getformatterforfieldnodefault.md) |  | Get formatter for a given field name. Return undefined if none exists |
+|  [getIndex()](./kibana-plugin-plugins-data-public.indexpattern.getindex.md) |  |  |
 |  [getNonScriptedFields()](./kibana-plugin-plugins-data-public.indexpattern.getnonscriptedfields.md) |  |  |
 |  [getScriptedFields()](./kibana-plugin-plugins-data-public.indexpattern.getscriptedfields.md) |  |  |
 |  [getSourceFiltering()](./kibana-plugin-plugins-data-public.indexpattern.getsourcefiltering.md) |  | Get the source filtering configuration for that index. |
 |  [getTimeField()](./kibana-plugin-plugins-data-public.indexpattern.gettimefield.md) |  |  |
 |  [isTimeBased()](./kibana-plugin-plugins-data-public.indexpattern.istimebased.md) |  |  |
 |  [isTimeNanosBased()](./kibana-plugin-plugins-data-public.indexpattern.istimenanosbased.md) |  |  |
 |  [isUnsupportedTimePattern()](./kibana-plugin-plugins-data-public.indexpattern.isunsupportedtimepattern.md) |  |  |
-|  [popularizeField(fieldName, unit)](./kibana-plugin-plugins-data-public.indexpattern.popularizefield.md) |  |  |
 |  [removeScriptedField(fieldName)](./kibana-plugin-plugins-data-public.indexpattern.removescriptedfield.md) |  | Remove scripted field from field list |
 |  [setFieldAttrs(fieldName, attrName, value)](./kibana-plugin-plugins-data-public.indexpattern.setfieldattrs.md) |  |  |
 |  [setFieldCount(fieldName, count)](./kibana-plugin-plugins-data-public.indexpattern.setfieldcount.md) |  |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternAttributes](./kibana-plugin-plugins-data-public.indexpatternattributes.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md)
+
+## IndexPatternAttributes.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+allowNoIndex?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternattributes.md

@@ -14,6 +14,7 @@ export interface IndexPatternAttributes
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternattributes.allownoindex.md) | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [fieldAttrs](./kibana-plugin-plugins-data-public.indexpatternattributes.fieldattrs.md) | <code>string</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-public.indexpatternattributes.fieldformatmap.md) | <code>string</code> |  |
 |  [fields](./kibana-plugin-plugins-data-public.indexpatternattributes.fields.md) | <code>string</code> |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternSpec](./kibana-plugin-plugins-data-public.indexpatternspec.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md)
+
+## IndexPatternSpec.allowNoIndex property
+
+<b>Signature:</b>
+
+```typescript
+allowNoIndex?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternspec.md

@@ -14,6 +14,7 @@ export interface IndexPatternSpec
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-public.indexpatternspec.allownoindex.md) | <code>boolean</code> |  |
 |  [fieldAttrs](./kibana-plugin-plugins-data-public.indexpatternspec.fieldattrs.md) | <code>FieldAttrs</code> |  |
 |  [fieldFormats](./kibana-plugin-plugins-data-public.indexpatternspec.fieldformats.md) | <code>Record&lt;string, SerializedFieldFormat&gt;</code> |  |
 |  [fields](./kibana-plugin-plugins-data-public.indexpatternspec.fields.md) | <code>IndexPatternFieldMap</code> |  |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IndexPattern](./kibana-plugin-plugins-data-server.indexpattern.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-server.indexpattern.allownoindex.md)
+
+## IndexPattern.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+readonly allowNoIndex: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.getassavedobjectbody.md

@@ -19,6 +19,7 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
 ```
 <b>Returns:</b>
@@ -33,5 +34,6 @@ getAsSavedObjectBody(): {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     }`
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpattern.md

@@ -20,6 +20,7 @@ export declare class IndexPattern implements IIndexPattern
 
 |  Property | Modifiers | Type | Description |
 |  --- | --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-server.indexpattern.allownoindex.md) |  | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [deleteFieldFormat](./kibana-plugin-plugins-data-server.indexpattern.deletefieldformat.md) |  | <code>(fieldName: string) =&gt; void</code> |  |
 |  [fieldAttrs](./kibana-plugin-plugins-data-server.indexpattern.fieldattrs.md) |  | <code>FieldAttrs</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-server.indexpattern.fieldformatmap.md) |  | <code>Record&lt;string, any&gt;</code> |  |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IndexPatternAttributes](./kibana-plugin-plugins-data-server.indexpatternattributes.md) &gt; [allowNoIndex](./kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md)
+
+## IndexPatternAttributes.allowNoIndex property
+
+prevents errors when index pattern exists before indices
+
+<b>Signature:</b>
+
+```typescript
+allowNoIndex?: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternattributes.md

@@ -14,6 +14,7 @@ export interface IndexPatternAttributes
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [allowNoIndex](./kibana-plugin-plugins-data-server.indexpatternattributes.allownoindex.md) | <code>boolean</code> | prevents errors when index pattern exists before indices |
 |  [fieldAttrs](./kibana-plugin-plugins-data-server.indexpatternattributes.fieldattrs.md) | <code>string</code> |  |
 |  [fieldFormatMap](./kibana-plugin-plugins-data-server.indexpatternattributes.fieldformatmap.md) | <code>string</code> |  |
 |  [fields](./kibana-plugin-plugins-data-server.indexpatternattributes.fields.md) | <code>string</code> |  |

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -74,6 +74,10 @@ export class IndexPattern implements IIndexPattern {
   private fieldFormats: FieldFormatsStartCommon;
   // make private once manual field refresh is removed
   public fieldAttrs: FieldAttrs;
+  /**
+   * prevents errors when index pattern exists before indices
+   */
+  public readonly allowNoIndex: boolean = false;
 
   constructor({
     spec = {},
@@ -110,6 +114,7 @@ export class IndexPattern implements IIndexPattern {
     this.typeMeta = spec.typeMeta;
     this.fieldAttrs = spec.fieldAttrs || {};
     this.intervalName = spec.intervalName;
+    this.allowNoIndex = spec.allowNoIndex || false;
   }
 
   /**
@@ -204,6 +209,7 @@ export class IndexPattern implements IIndexPattern {
       fieldFormats: this.fieldFormatMap,
       fieldAttrs: this.fieldAttrs,
       intervalName: this.intervalName,
+      allowNoIndex: this.allowNoIndex,
     };
   }
 
@@ -332,6 +338,7 @@ export class IndexPattern implements IIndexPattern {
       fieldFormatMap,
       type: this.type,
       typeMeta: this.typeMeta ? JSON.stringify(this.typeMeta) : undefined,
+      allowNoIndex: this.allowNoIndex ? this.allowNoIndex : undefined,
     };
   }
 

src/plugins/data/common/index_patterns/index_patterns/index_patterns.test.ts

@@ -115,6 +115,21 @@ describe('IndexPatterns', () => {
     SOClientGetDelay = 0;
   });
 
+  test('allowNoIndex flag preserves existing fields when index is missing', async () => {
+    const id = '2';
+    setDocsourcePayload(id, {
+      id: 'foo',
+      version: 'foo',
+      attributes: {
+        title: 'something',
+        allowNoIndex: true,
+        fields: '[{"name":"field"}]',
+      },
+    });
+
+    expect((await indexPatterns.get(id)).fields.length).toBe(1);
+  });
+
   test('savedObjectCache pre-fetches only title', async () => {
     expect(await indexPatterns.getIds()).toEqual(['id']);
     expect(savedObjectsClient.find).toHaveBeenCalledWith({

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -227,6 +227,7 @@ export class IndexPatternsService {
       metaFields,
       type: options.type,
       rollupIndex: options.rollupIndex,
+      allowNoIndex: options.allowNoIndex,
     });
   };
 
@@ -286,10 +287,21 @@ export class IndexPatternsService {
     options: GetFieldsOptions,
     fieldAttrs: FieldAttrs = {}
   ) => {
-    const scriptedFields = Object.values(fields).filter((field) => field.scripted);
+    const fieldsAsArr = Object.values(fields);
+    const scriptedFields = fieldsAsArr.filter((field) => field.scripted);
     try {
+      let updatedFieldList: FieldSpec[];
       const newFields = (await this.getFieldsForWildcard(options)) as FieldSpec[];
-      return this.fieldArrayToMap([...newFields, ...scriptedFields], fieldAttrs);
+
+      // If allowNoIndex, only update field list if field caps finds fields. To support
+      // beats creating index pattern and dashboard before docs
+      if (!options.allowNoIndex || (newFields && newFields.length > 5)) {
+        updatedFieldList = [...newFields, ...scriptedFields];
+      } else {
+        updatedFieldList = fieldsAsArr;
+      }
+
+      return this.fieldArrayToMap(updatedFieldList, fieldAttrs);
     } catch (err) {
       if (err instanceof IndexPatternMissingIndices) {
         this.onNotification({ title: (err as any).message, color: 'danger', iconType: 'alert' });
@@ -339,6 +351,7 @@ export class IndexPatternsService {
         typeMeta,
         type,
         fieldAttrs,
+        allowNoIndex,
       },
     } = savedObject;
 
@@ -360,6 +373,7 @@ export class IndexPatternsService {
       type,
       fieldFormats: parsedFieldFormatMap,
       fieldAttrs: parsedFieldAttrs,
+      allowNoIndex,
     };
   };
 
@@ -389,6 +403,7 @@ export class IndexPatternsService {
           metaFields: await this.config.get(UI_SETTINGS.META_FIELDS),
           type,
           rollupIndex: typeMeta?.params?.rollup_index,
+          allowNoIndex: spec.allowNoIndex,
         },
         spec.fieldAttrs
       );

src/plugins/data/common/index_patterns/types.ts

@@ -50,6 +50,10 @@ export interface IndexPatternAttributes {
   sourceFilters?: string;
   fieldFormatMap?: string;
   fieldAttrs?: string;
+  /**
+   * prevents errors when index pattern exists before indices
+   */
+  allowNoIndex?: boolean;
 }
 
 export interface FieldAttrs {
@@ -112,6 +116,7 @@ export interface GetFieldsOptions {
   lookBack?: boolean;
   metaFields?: string[];
   rollupIndex?: string;
+  allowNoIndex?: boolean;
 }
 
 export interface GetFieldsOptionsTimePattern {
@@ -204,6 +209,7 @@ export interface IndexPatternSpec {
   type?: string;
   fieldFormats?: Record<string, SerializedFieldFormat>;
   fieldAttrs?: FieldAttrs;
+  allowNoIndex?: boolean;
 }
 
 export interface SourceFilter {

src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts

@@ -64,12 +64,13 @@ export class IndexPatternsApiClient implements IIndexPatternsApiClient {
     }).then((resp: any) => resp.fields);
   }
 
-  getFieldsForWildcard({ pattern, metaFields, type, rollupIndex }: GetFieldsOptions) {
+  getFieldsForWildcard({ pattern, metaFields, type, rollupIndex, allowNoIndex }: GetFieldsOptions) {
     return this._request(this._getUrl(['_fields_for_wildcard']), {
       pattern,
       meta_fields: metaFields,
       type,
       rollup_index: rollupIndex,
-    }).then((resp: any) => resp.fields);
+      allow_no_index: allowNoIndex,
+    }).then((resp: any) => resp.fields || []);
   }
 }

src/plugins/data/public/public.api.md

@@ -1253,6 +1253,7 @@ export class IndexPattern implements IIndexPattern {
     // Warning: (ae-forgotten-export) The symbol "IndexPatternDeps" needs to be exported by the entry point index.d.ts
     constructor({ spec, fieldFormats, shortDotsEnable, metaFields, }: IndexPatternDeps);
     addScriptedField(name: string, script: string, fieldType?: string): Promise<void>;
+    readonly allowNoIndex: boolean;
     // (undocumented)
     readonly deleteFieldFormat: (fieldName: string) => void;
     // Warning: (ae-forgotten-export) The symbol "FieldAttrs" needs to be exported by the entry point index.d.ts
@@ -1293,6 +1294,7 @@ export class IndexPattern implements IIndexPattern {
         fieldFormatMap: string | undefined;
         type: string | undefined;
         typeMeta: string | undefined;
+        allowNoIndex: true | undefined;
     };
     // (undocumented)
     getComputedFields(): {
@@ -1389,6 +1391,7 @@ export type IndexPatternAggRestrictions = Record<string, {
 //
 // @public (undocumented)
 export interface IndexPatternAttributes {
+    allowNoIndex?: boolean;
     // (undocumented)
     fieldAttrs?: string;
     // (undocumented)
@@ -1519,6 +1522,8 @@ export type IndexPatternSelectProps = Required<Omit<EuiComboBoxProps<any>, 'isLo
 //
 // @public (undocumented)
 export interface IndexPatternSpec {
+    // (undocumented)
+    allowNoIndex?: boolean;
     // (undocumented)
     fieldAttrs?: FieldAttrs;
     // (undocumented)
@@ -2567,7 +2572,7 @@ export const UI_SETTINGS: {
 // src/plugins/data/common/es_query/filters/phrase_filter.ts:33:3 - (ae-forgotten-export) The symbol "PhraseFilterMeta" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/es_query/filters/phrases_filter.ts:31:3 - (ae-forgotten-export) The symbol "PhrasesFilterMeta" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts
-// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:128:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
+// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:133:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/search/aggs/types.ts:145:51 - (ae-forgotten-export) The symbol "AggTypesRegistryStart" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/search/search_source/search_source.ts:197:7 - (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/field_formats/field_formats_service.ts:67:3 - (ae-forgotten-export) The symbol "FormatFactory" needs to be exported by the entry point index.d.ts

src/plugins/data/server/index_patterns/index_patterns_api_client.ts

@@ -30,8 +30,14 @@ export class IndexPatternsApiServer implements IIndexPatternsApiClient {
   constructor(elasticsearchClient: ElasticsearchClient) {
     this.esClient = elasticsearchClient;
   }
-  async getFieldsForWildcard({ pattern, metaFields, type, rollupIndex }: GetFieldsOptions) {
-    const indexPatterns = new IndexPatternsFetcher(this.esClient);
+  async getFieldsForWildcard({
+    pattern,
+    metaFields,
+    type,
+    rollupIndex,
+    allowNoIndex,
+  }: GetFieldsOptions) {
+    const indexPatterns = new IndexPatternsFetcher(this.esClient, allowNoIndex);
     return await indexPatterns.getFieldsForWildcard({
       pattern,
       metaFields,