From 1282341020ecef5dbc7f6c47bef24ad26f9c2e99 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Tue, 28 Apr 2020 16:00:22 -0600 Subject: [PATCH] [SIEM][Detections] Adds large list support using REST endpoints ## Summary * Adds large list support using REST endpoints. Status: --- * Currently ready to be merged behind the feature flag of it being disabled with ongoing work happening after it is merged. * REST Endpoints shouldn't have large refactoring at this point * Team meeting occurred where the pieces were discussed in person. What is left? --- - [ ] Add other data types. At the moment `ip` and `keyword` are the two types of lists. See: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html - [x] Unit tests - [x] Lots of misc TODO's in the code base still - [ ] Import loads everything into memory first when it should attempt streaming - [ ] Add end to end backend tests - [x] Add transform and io-ts validation for returns Testing --- Ensure you set this in your ENV before starting Kibana: ```ts export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true ``` Download or create a large list file such as this one filled with IP's: https://cinsscore.com/list/ci-badguys.txt Go to your REST endpoint folder of scripts: ```ts cd kibana/x-pack/plugins/lists/server/scripts ``` Do a hard reset: ```ts ./hard_reset ``` Then import it as either a data type of `ip`: ```ts ./import_list_items_by_filename.sh ip ~/Downloads/ci-badguys-smaller.txt ``` Or as a `keyword` ```ts ./import_list_items_by_filename.sh keyword ~/Downloads/ci-badguys-smaller.txt ``` Then you can export it through: ```ts ./export_list_items.sh ci-badgusy-smaller.txt ``` For all the other endpoints and testing of the CRUD operations you have access to: ```ts delete_all_lists.sh delete_list.sh delete_list_index.sh delete_list_item.sh delete_list_item_by_id.sh delete_list_item_by_value.sh export_list_items.sh export_list_items_to_file.sh get_list.sh get_list_item_by_id.sh get_list_item_by_value.sh import_list_items.sh import_list_items_by_filename.sh lists_index_exists.sh patch_list.sh patch_list_item.sh post_list.sh post_list_index.sh post_list_item.sh ``` ### Checklist Delete any items that are not applicable to this PR. - [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process) --- .eslintrc.js | 95 ++++ .github/CODEOWNERS | 2 +- x-pack/plugins/lists/common/constants.ts | 12 + .../lists/common/schemas/common/index.ts | 7 + .../lists/common/schemas/common/schemas.ts | 62 +++ .../elastic_query/create_es_bulk_type.ts | 17 + .../common/schemas/elastic_query/index.ts | 10 + .../index_es_list_item_schema.ts | 37 ++ .../elastic_query/index_es_list_schema.ts | 37 ++ .../update_es_list_item_schema.ts | 24 + .../elastic_query/update_es_list_schema.ts | 29 ++ .../common/schemas/elastic_response/index.ts | 8 + .../search_es_list_item_schema.ts | 37 ++ .../elastic_response/search_es_list_schema.ts | 37 ++ x-pack/plugins/lists/common/schemas/index.ts | 11 + .../request/create_list_item_schema.ts | 22 + .../request/create_list_schema.test.ts | 31 ++ .../schemas/request/create_list_schema.ts | 23 + .../request/delete_list_item_schema.ts | 21 + .../schemas/request/delete_list_schema.ts | 19 + .../request/export_list_item_query_schema.ts | 20 + .../request/import_list_item_query_schema.ts | 17 + .../request/import_list_item_schema.ts | 32 ++ .../lists/common/schemas/request/index.ts | 19 + .../common/schemas/request/mocks/utils.ts | 15 + .../schemas/request/patch_list_item_schema.ts | 21 + .../schemas/request/patch_list_schema.ts | 22 + .../schemas/request/read_list_item_schema.ts | 17 + .../schemas/request/read_list_schema.ts | 19 + .../request/update_list_item_schema.ts | 21 + .../schemas/request/update_list_schema.ts | 22 + .../schemas/response/acknowledge_schema.ts | 11 + .../lists/common/schemas/response/index.ts | 10 + .../response/list_item_index_exist_schema.ts | 14 + .../schemas/response/list_item_schema.ts | 42 ++ .../common/schemas/response/list_schema.ts | 39 ++ .../common/schemas/types/non_empty_string.ts | 27 + .../plugins/lists/common/siem_common_deps.ts | 8 + x-pack/plugins/lists/kibana.json | 10 + x-pack/plugins/lists/server/config.ts | 15 + x-pack/plugins/lists/server/create_config.ts | 21 + .../lists/server/error_with_status_code.ts | 16 + x-pack/plugins/lists/server/index.ts | 14 + x-pack/plugins/lists/server/plugin.ts | 86 ++++ .../server/routes/create_list_index_route.ts | 82 +++ .../server/routes/create_list_item_route.ts | 74 +++ .../lists/server/routes/create_list_route.ts | 69 +++ .../server/routes/delete_list_index_route.ts | 97 ++++ .../server/routes/delete_list_item_route.ts | 89 ++++ .../lists/server/routes/delete_list_route.ts | 59 +++ .../server/routes/export_list_item_route.ts | 63 +++ .../server/routes/import_list_item_route.ts | 105 ++++ x-pack/plugins/lists/server/routes/index.ts | 21 + .../lists/server/routes/init_routes.ts | 49 ++ .../server/routes/patch_list_item_route.ts | 63 +++ .../lists/server/routes/patch_list_route.ts | 59 +++ .../server/routes/read_list_index_route.ts | 67 +++ .../server/routes/read_list_item_route.ts | 93 ++++ .../lists/server/routes/read_list_route.ts | 59 +++ .../server/routes/update_list_item_route.ts | 63 +++ .../lists/server/routes/update_list_route.ts | 59 +++ .../server/routes/utils/get_list_client.ts | 19 + .../lists/server/routes/utils/index.ts | 6 + .../server/scripts/check_env_variables.sh | 41 ++ .../lists/server/scripts/delete_all_lists.sh | 38 ++ .../lists/server/scripts/delete_list.sh | 16 + .../lists/server/scripts/delete_list_index.sh | 16 + .../server/scripts/delete_list_item_by_id.sh | 16 + .../scripts/delete_list_item_by_value.sh | 16 + .../lists/server/scripts/export_list_items.sh | 21 + .../scripts/export_list_items_to_file.sh | 26 + .../plugins/lists/server/scripts/get_list.sh | 15 + .../server/scripts/get_list_item_by_id.sh | 15 + .../server/scripts/get_list_item_by_value.sh | 15 + .../lists/server/scripts/hard_reset.sh | 14 + .../lists/server/scripts/import_list_items.sh | 22 + .../scripts/import_list_items_by_filename.sh | 24 + .../server/scripts/lists/files/hosts.txt | 2 + .../lists/server/scripts/lists/files/ips.txt | 9 + .../scripts/lists/new/list_everything.json | 13 + .../server/scripts/lists/new/list_ip.json | 6 + .../scripts/lists/new/list_ip_item.json | 5 + .../lists/new/list_ip_item_everything.json | 12 + .../scripts/lists/new/list_ip_no_id.json | 5 + .../scripts/lists/new/list_keyword.json | 6 + .../scripts/lists/new/list_keyword_item.json | 4 + .../scripts/lists/patches/list_ip_item.json | 4 + .../lists/patches/simplest_updated_name.json | 4 + .../scripts/lists/updates/list_ip_item.json | 4 + .../scripts/lists/updates/simple_update.json | 5 + .../server/scripts/lists_index_exists.sh | 16 + .../lists/server/scripts/patch_list.sh | 30 ++ .../lists/server/scripts/patch_list_item.sh | 30 ++ .../plugins/lists/server/scripts/post_list.sh | 30 ++ .../lists/server/scripts/post_list_index.sh | 17 + .../lists/server/scripts/post_list_item.sh | 30 ++ .../lists/server/scripts/update_list.sh | 30 ++ .../lists/server/scripts/update_list_item.sh | 30 ++ .../services/items/buffer_lines.test.ts | 88 ++++ .../server/services/items/buffer_lines.ts | 50 ++ .../services/items/create_list_item.test.ts | 54 ++ .../server/services/items/create_list_item.ts | 73 +++ .../items/create_list_items_bulk.test.ts | 49 ++ .../services/items/create_list_items_bulk.ts | 70 +++ .../services/items/delete_list_item.test.ts | 51 ++ .../server/services/items/delete_list_item.ts | 33 ++ .../items/delete_list_item_by_value.test.ts | 57 +++ .../items/delete_list_item_by_value.ts | 52 ++ .../services/items/get_list_item.test.ts | 36 ++ .../server/services/items/get_list_item.ts | 46 ++ .../items/get_list_item_by_value.test.ts | 33 ++ .../services/items/get_list_item_by_value.ts | 33 ++ .../items/get_list_item_by_values.test.ts | 60 +++ .../services/items/get_list_item_by_values.ts | 44 ++ .../items/get_list_item_index.test.ts | 38 ++ .../services/items/get_list_item_index.ts | 22 + .../items/get_list_item_template.test.ts | 30 ++ .../services/items/get_list_item_template.ts | 23 + .../lists/server/services/items/index.ts | 19 + .../services/items/list_item_mappings.json | 33 ++ .../services/items/list_item_policy.json | 14 + .../services/items/update_list_item.test.ts | 41 ++ .../server/services/items/update_list_item.ts | 71 +++ .../write_lines_to_bulk_list_items.test.ts | 160 ++++++ .../items/write_lines_to_bulk_list_items.ts | 101 ++++ .../items/write_list_items_to_stream.test.ts | 290 +++++++++++ .../items/write_list_items_to_stream.ts | 160 ++++++ .../lists/server/services/lists/client.ts | 465 ++++++++++++++++++ .../server/services/lists/client_types.ts | 116 +++++ .../server/services/lists/create_list.test.ts | 54 ++ .../server/services/lists/create_list.ts | 67 +++ .../server/services/lists/delete_list.test.ts | 76 +++ .../server/services/lists/delete_list.ts | 46 ++ .../server/services/lists/get_list.test.ts | 41 ++ .../lists/server/services/lists/get_list.ts | 42 ++ .../services/lists/get_list_index.test.ts | 38 ++ .../server/services/lists/get_list_index.ts | 19 + .../services/lists/get_list_template.test.ts | 30 ++ .../services/lists/get_list_template.ts | 20 + .../lists/server/services/lists/index.ts | 12 + .../server/services/lists/list_mappings.json | 33 ++ .../server/services/lists/list_policy.json | 14 + .../server/services/lists/update_list.test.ts | 41 ++ .../server/services/lists/update_list.ts | 72 +++ .../get_create_list_item_bulk_options_mock.ts | 32 ++ .../get_create_list_item_options_mock.ts | 32 ++ .../mocks/get_create_list_options_mock.ts | 33 ++ .../services/mocks/get_data_client_mock.ts | 34 ++ ..._delete_list_item_by_value_options_mock.ts | 18 + .../get_delete_list_item_options_mock.ts | 16 + .../mocks/get_delete_list_options_mock.ts | 17 + ...mport_list_items_to_stream_options_mock.ts | 20 + .../mocks/get_index_es_list_item_mock.ts | 20 + .../services/mocks/get_index_es_list_mock.ts | 29 ++ .../get_list_item_by_value_options_mock.ts | 18 + .../get_list_item_by_values_options_mock.ts | 18 + .../mocks/get_list_item_response_mock.ts | 22 + .../services/mocks/get_list_response_mock.ts | 22 + .../mocks/get_search_es_list_item_mock.ts | 21 + .../services/mocks/get_search_es_list_mock.ts | 21 + .../mocks/get_search_list_item_mock.ts | 33 ++ .../services/mocks/get_search_list_mock.ts | 33 ++ .../server/services/mocks/get_shard_mock.ts | 14 + .../get_update_list_item_options_mock.ts | 26 + .../mocks/get_update_list_options_mock.ts | 28 ++ .../get_write_buffer_to_items_options_mock.ts | 19 + ...write_list_items_to_stream_options_mock.ts | 49 ++ .../lists/server/services/mocks/index.ts | 29 ++ .../mocks/lists_services_mock_constants.ts | 30 ++ .../server/services/mocks/test_readable.ts | 11 + .../utils/derive_type_from_es_type.test.ts | 38 ++ .../utils/derive_type_from_es_type.ts | 27 + .../utils/get_query_filter_from_type_value.ts | 33 ++ .../lists/server/services/utils/get_space.ts | 17 + .../lists/server/services/utils/get_user.ts | 23 + .../lists/server/services/utils/index.ts | 12 + .../utils/transform_elastic_to_list_item.ts | 74 +++ .../transform_list_item_to_elastic_query.ts | 30 ++ .../plugins/lists/server/siem_server_deps.ts | 21 + x-pack/plugins/lists/server/types.ts | 29 ++ .../detection_engine/scripts/hard_reset.sh | 6 + 181 files changed, 6833 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/lists/common/constants.ts create mode 100644 x-pack/plugins/lists/common/schemas/common/index.ts create mode 100644 x-pack/plugins/lists/common/schemas/common/schemas.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_query/index.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_response/index.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/index.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/create_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/index.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/mocks/utils.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/read_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/update_list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/request/update_list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/response/acknowledge_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/response/index.ts create mode 100644 x-pack/plugins/lists/common/schemas/response/list_item_index_exist_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/response/list_item_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/response/list_schema.ts create mode 100644 x-pack/plugins/lists/common/schemas/types/non_empty_string.ts create mode 100644 x-pack/plugins/lists/common/siem_common_deps.ts create mode 100644 x-pack/plugins/lists/kibana.json create mode 100644 x-pack/plugins/lists/server/config.ts create mode 100644 x-pack/plugins/lists/server/create_config.ts create mode 100644 x-pack/plugins/lists/server/error_with_status_code.ts create mode 100644 x-pack/plugins/lists/server/index.ts create mode 100644 x-pack/plugins/lists/server/plugin.ts create mode 100644 x-pack/plugins/lists/server/routes/create_list_index_route.ts create mode 100644 x-pack/plugins/lists/server/routes/create_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/create_list_route.ts create mode 100644 x-pack/plugins/lists/server/routes/delete_list_index_route.ts create mode 100644 x-pack/plugins/lists/server/routes/delete_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/delete_list_route.ts create mode 100644 x-pack/plugins/lists/server/routes/export_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/import_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/index.ts create mode 100644 x-pack/plugins/lists/server/routes/init_routes.ts create mode 100644 x-pack/plugins/lists/server/routes/patch_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/patch_list_route.ts create mode 100644 x-pack/plugins/lists/server/routes/read_list_index_route.ts create mode 100644 x-pack/plugins/lists/server/routes/read_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/read_list_route.ts create mode 100644 x-pack/plugins/lists/server/routes/update_list_item_route.ts create mode 100644 x-pack/plugins/lists/server/routes/update_list_route.ts create mode 100644 x-pack/plugins/lists/server/routes/utils/get_list_client.ts create mode 100644 x-pack/plugins/lists/server/routes/utils/index.ts create mode 100755 x-pack/plugins/lists/server/scripts/check_env_variables.sh create mode 100755 x-pack/plugins/lists/server/scripts/delete_all_lists.sh create mode 100755 x-pack/plugins/lists/server/scripts/delete_list.sh create mode 100755 x-pack/plugins/lists/server/scripts/delete_list_index.sh create mode 100755 x-pack/plugins/lists/server/scripts/delete_list_item_by_id.sh create mode 100755 x-pack/plugins/lists/server/scripts/delete_list_item_by_value.sh create mode 100755 x-pack/plugins/lists/server/scripts/export_list_items.sh create mode 100755 x-pack/plugins/lists/server/scripts/export_list_items_to_file.sh create mode 100755 x-pack/plugins/lists/server/scripts/get_list.sh create mode 100755 x-pack/plugins/lists/server/scripts/get_list_item_by_id.sh create mode 100755 x-pack/plugins/lists/server/scripts/get_list_item_by_value.sh create mode 100755 x-pack/plugins/lists/server/scripts/hard_reset.sh create mode 100755 x-pack/plugins/lists/server/scripts/import_list_items.sh create mode 100755 x-pack/plugins/lists/server/scripts/import_list_items_by_filename.sh create mode 100644 x-pack/plugins/lists/server/scripts/lists/files/hosts.txt create mode 100644 x-pack/plugins/lists/server/scripts/lists/files/ips.txt create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_everything.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_ip.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json create mode 100644 x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json create mode 100755 x-pack/plugins/lists/server/scripts/lists_index_exists.sh create mode 100755 x-pack/plugins/lists/server/scripts/patch_list.sh create mode 100755 x-pack/plugins/lists/server/scripts/patch_list_item.sh create mode 100755 x-pack/plugins/lists/server/scripts/post_list.sh create mode 100755 x-pack/plugins/lists/server/scripts/post_list_index.sh create mode 100755 x-pack/plugins/lists/server/scripts/post_list_item.sh create mode 100755 x-pack/plugins/lists/server/scripts/update_list.sh create mode 100755 x-pack/plugins/lists/server/scripts/update_list_item.sh create mode 100644 x-pack/plugins/lists/server/services/items/buffer_lines.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/buffer_lines.ts create mode 100644 x-pack/plugins/lists/server/services/items/create_list_item.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/create_list_item.ts create mode 100644 x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts create mode 100644 x-pack/plugins/lists/server/services/items/delete_list_item.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/delete_list_item.ts create mode 100644 x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/delete_list_item_by_value.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_by_value.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_by_value.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_index.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_index.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/get_list_item_template.ts create mode 100644 x-pack/plugins/lists/server/services/items/index.ts create mode 100644 x-pack/plugins/lists/server/services/items/list_item_mappings.json create mode 100644 x-pack/plugins/lists/server/services/items/list_item_policy.json create mode 100644 x-pack/plugins/lists/server/services/items/update_list_item.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/update_list_item.ts create mode 100644 x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts create mode 100644 x-pack/plugins/lists/server/services/items/write_list_items_to_stream.test.ts create mode 100644 x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts create mode 100644 x-pack/plugins/lists/server/services/lists/client.ts create mode 100644 x-pack/plugins/lists/server/services/lists/client_types.ts create mode 100644 x-pack/plugins/lists/server/services/lists/create_list.test.ts create mode 100644 x-pack/plugins/lists/server/services/lists/create_list.ts create mode 100644 x-pack/plugins/lists/server/services/lists/delete_list.test.ts create mode 100644 x-pack/plugins/lists/server/services/lists/delete_list.ts create mode 100644 x-pack/plugins/lists/server/services/lists/get_list.test.ts create mode 100644 x-pack/plugins/lists/server/services/lists/get_list.ts create mode 100644 x-pack/plugins/lists/server/services/lists/get_list_index.test.ts create mode 100644 x-pack/plugins/lists/server/services/lists/get_list_index.ts create mode 100644 x-pack/plugins/lists/server/services/lists/get_list_template.test.ts create mode 100644 x-pack/plugins/lists/server/services/lists/get_list_template.ts create mode 100644 x-pack/plugins/lists/server/services/lists/index.ts create mode 100644 x-pack/plugins/lists/server/services/lists/list_mappings.json create mode 100644 x-pack/plugins/lists/server/services/lists/list_policy.json create mode 100644 x-pack/plugins/lists/server/services/lists/update_list.test.ts create mode 100644 x-pack/plugins/lists/server/services/lists/update_list.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_create_list_item_bulk_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_create_list_item_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_create_list_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_data_client_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_delete_list_item_by_value_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_delete_list_item_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_delete_list_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_import_list_items_to_stream_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_index_es_list_item_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_index_es_list_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_list_item_by_value_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_list_item_by_values_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_list_item_response_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_list_response_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_search_es_list_item_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_search_es_list_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_search_list_item_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_search_list_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_shard_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_update_list_item_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_update_list_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_write_buffer_to_items_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/get_write_list_items_to_stream_options_mock.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/index.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/lists_services_mock_constants.ts create mode 100644 x-pack/plugins/lists/server/services/mocks/test_readable.ts create mode 100644 x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts create mode 100644 x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts create mode 100644 x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts create mode 100644 x-pack/plugins/lists/server/services/utils/get_space.ts create mode 100644 x-pack/plugins/lists/server/services/utils/get_user.ts create mode 100644 x-pack/plugins/lists/server/services/utils/index.ts create mode 100644 x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts create mode 100644 x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts create mode 100644 x-pack/plugins/lists/server/siem_server_deps.ts create mode 100644 x-pack/plugins/lists/server/types.ts diff --git a/.eslintrc.js b/.eslintrc.js index e45a2a96f29d7..c9b41ec711b7f 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -742,6 +742,101 @@ module.exports = { }, }, + /** + * Lists overrides + */ + { + // typescript and javascript for front and back end + files: ['x-pack/plugins/lists/**/*.{js,ts,tsx}'], + plugins: ['eslint-plugin-node'], + env: { + mocha: true, + jest: true, + }, + rules: { + 'accessor-pairs': 'error', + 'array-callback-return': 'error', + 'no-array-constructor': 'error', + complexity: 'error', + 'consistent-return': 'error', + 'func-style': ['error', 'expression'], + 'import/order': [ + 'error', + { + groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'], + 'newlines-between': 'always', + }, + ], + 'sort-imports': [ + 'error', + { + ignoreDeclarationSort: true, + }, + ], + 'node/no-deprecated-api': 'error', + 'no-bitwise': 'error', + 'no-continue': 'error', + 'no-dupe-keys': 'error', + 'no-duplicate-case': 'error', + 'no-duplicate-imports': 'error', + 'no-empty-character-class': 'error', + 'no-empty-pattern': 'error', + 'no-ex-assign': 'error', + 'no-extend-native': 'error', + 'no-extra-bind': 'error', + 'no-extra-boolean-cast': 'error', + 'no-extra-label': 'error', + 'no-func-assign': 'error', + 'no-implicit-globals': 'error', + 'no-implied-eval': 'error', + 'no-invalid-regexp': 'error', + 'no-inner-declarations': 'error', + 'no-lone-blocks': 'error', + 'no-multi-assign': 'error', + 'no-misleading-character-class': 'error', + 'no-new-symbol': 'error', + 'no-obj-calls': 'error', + 'no-param-reassign': ['error', { props: true }], + 'no-process-exit': 'error', + 'no-prototype-builtins': 'error', + 'no-return-await': 'error', + 'no-self-compare': 'error', + 'no-shadow-restricted-names': 'error', + 'no-sparse-arrays': 'error', + 'no-this-before-super': 'error', + 'no-undef': 'error', + 'no-unreachable': 'error', + 'no-unsafe-finally': 'error', + 'no-useless-call': 'error', + 'no-useless-catch': 'error', + 'no-useless-concat': 'error', + 'no-useless-computed-key': 'error', + 'no-useless-escape': 'error', + 'no-useless-rename': 'error', + 'no-useless-return': 'error', + 'no-void': 'error', + 'one-var-declaration-per-line': 'error', + 'prefer-object-spread': 'error', + 'prefer-promise-reject-errors': 'error', + 'prefer-rest-params': 'error', + 'prefer-spread': 'error', + 'prefer-template': 'error', + 'require-atomic-updates': 'error', + 'symbol-description': 'error', + 'vars-on-top': 'error', + '@typescript-eslint/explicit-member-accessibility': 'error', + '@typescript-eslint/no-this-alias': 'error', + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/no-useless-constructor': 'error', + '@typescript-eslint/unified-signatures': 'error', + '@typescript-eslint/explicit-function-return-type': 'error', + '@typescript-eslint/no-non-null-assertion': 'error', + '@typescript-eslint/no-unused-vars': 'error', + 'no-template-curly-in-string': 'error', + 'sort-keys': 'error', + 'prefer-destructuring': 'error', + }, + }, /** * Alerting Services overrides */ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4a16952f82014..7d1122682be7b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -224,7 +224,7 @@ /x-pack/test/detection_engine_api_integration @elastic/siem /x-pack/test/api_integration/apis/siem @elastic/siem /x-pack/plugins/case @elastic/siem +/x-pack/plugins/lists @elastic/siem # Security Intelligence And Analytics -/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics /x-pack/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics diff --git a/x-pack/plugins/lists/common/constants.ts b/x-pack/plugins/lists/common/constants.ts new file mode 100644 index 0000000000000..dbe31fed66413 --- /dev/null +++ b/x-pack/plugins/lists/common/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * Lists routes + */ +export const LIST_URL = `/api/lists`; +export const LIST_INDEX = `${LIST_URL}/index`; +export const LIST_ITEM_URL = `${LIST_URL}/items`; diff --git a/x-pack/plugins/lists/common/schemas/common/index.ts b/x-pack/plugins/lists/common/schemas/common/index.ts new file mode 100644 index 0000000000000..a05e97ded38ee --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/common/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './schemas'; diff --git a/x-pack/plugins/lists/common/schemas/common/schemas.ts b/x-pack/plugins/lists/common/schemas/common/schemas.ts new file mode 100644 index 0000000000000..edc037ed7a0b1 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/common/schemas.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { NonEmptyString } from '../types/non_empty_string'; + +export const name = t.string; +export type Name = t.TypeOf; +export const nameOrUndefined = t.union([name, t.undefined]); +export type NameOrUndefined = t.TypeOf; + +export const description = t.string; +export type Description = t.TypeOf; +export const descriptionOrUndefined = t.union([description, t.undefined]); +export type DescriptionOrUndefined = t.TypeOf; + +export const list_id = NonEmptyString; +export const list_idOrUndefined = t.union([list_id, t.undefined]); +export type List_idOrUndefined = t.TypeOf; + +export const item = t.string; +export const created_at = t.string; // TODO: Make this into an ISO Date string check +export const updated_at = t.string; // TODO: Make this into an ISO Date string check +export const updated_by = t.string; +export const created_by = t.string; +export const file = t.object; + +export const id = NonEmptyString; +export type Id = t.TypeOf; +export const idOrUndefined = t.union([id, t.undefined]); +export type IdOrUndefined = t.TypeOf; + +export const ip = t.string; +export const ipOrUndefined = t.union([ip, t.undefined]); + +export const keyword = t.string; +export const keywordOrUndefined = t.union([keyword, t.undefined]); + +export const value = t.string; +export const valueOrUndefined = t.union([value, t.undefined]); + +export const tie_breaker_id = t.string; // TODO: Use UUID for this instead of a string for validation +export const _index = t.string; + +export const type = t.keyof({ ip: null, keyword: null }); // TODO: Add the other data types here + +export const typeOrUndefined = t.union([type, t.undefined]); +export type Type = t.TypeOf; + +export const meta = t.object; +export type Meta = t.TypeOf; +export const metaOrUndefined = t.union([meta, t.undefined]); +export type MetaOrUndefined = t.TypeOf; + +export const esDataTypeUnion = t.union([t.type({ ip }), t.type({ keyword })]); +export type EsDataTypeUnion = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts b/x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts new file mode 100644 index 0000000000000..4a825382c06e4 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_query/create_es_bulk_type.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { _index } from '../common/schemas'; + +export const createEsBulkTypeSchema = t.exact( + t.type({ + create: t.exact(t.type({ _index })), + }) +); + +export type CreateEsBulkTypeSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index.ts new file mode 100644 index 0000000000000..d70dd09849fa6 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export * from './update_es_list_schema'; +export * from './index_es_list_schema'; +export * from './update_es_list_item_schema'; +export * from './index_es_list_item_schema'; +export * from './create_es_bulk_type'; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts new file mode 100644 index 0000000000000..596498b64b771 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { + created_at, + created_by, + esDataTypeUnion, + list_id, + metaOrUndefined, + tie_breaker_id, + updated_at, + updated_by, +} from '../common/schemas'; + +export const indexEsListItemSchema = t.intersection([ + t.exact( + t.type({ + created_at, + created_by, + list_id, + meta: metaOrUndefined, + tie_breaker_id, + updated_at, + updated_by, + }) + ), + esDataTypeUnion, +]); + +export type IndexEsListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts new file mode 100644 index 0000000000000..e0924392628a9 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { + created_at, + created_by, + description, + metaOrUndefined, + name, + tie_breaker_id, + type, + updated_at, + updated_by, +} from '../common/schemas'; + +export const indexEsListSchema = t.exact( + t.type({ + created_at, + created_by, + description, + meta: metaOrUndefined, + name, + tie_breaker_id, + type, + updated_at, + updated_by, + }) +); + +export type IndexEsListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts new file mode 100644 index 0000000000000..e4cf46bc39429 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_item_schema.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { esDataTypeUnion, metaOrUndefined, updated_at, updated_by } from '../common/schemas'; + +export const updateEsListItemSchema = t.intersection([ + t.exact( + t.type({ + meta: metaOrUndefined, + updated_at, + updated_by, + }) + ), + esDataTypeUnion, +]); + +export type UpdateEsListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts new file mode 100644 index 0000000000000..8f23f3744e563 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { + descriptionOrUndefined, + metaOrUndefined, + nameOrUndefined, + updated_at, + updated_by, +} from '../common/schemas'; + +export const updateEsListSchema = t.exact( + t.type({ + description: descriptionOrUndefined, + meta: metaOrUndefined, + name: nameOrUndefined, + updated_at, + updated_by, + }) +); + +export type UpdateEsListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/index.ts b/x-pack/plugins/lists/common/schemas/elastic_response/index.ts new file mode 100644 index 0000000000000..6fbc6ef293064 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_response/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './search_es_list_item_schema'; +export * from './search_es_list_schema'; diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts new file mode 100644 index 0000000000000..902d3e6a9896e --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { + created_at, + created_by, + ipOrUndefined, + keywordOrUndefined, + list_id, + metaOrUndefined, + tie_breaker_id, + updated_at, + updated_by, +} from '../common/schemas'; + +export const searchEsListItemSchema = t.exact( + t.type({ + created_at, + created_by, + ip: ipOrUndefined, + keyword: keywordOrUndefined, + list_id, + meta: metaOrUndefined, + tie_breaker_id, + updated_at, + updated_by, + }) +); + +export type SearchEsListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts new file mode 100644 index 0000000000000..00a7c6f321d38 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { + created_at, + created_by, + description, + metaOrUndefined, + name, + tie_breaker_id, + type, + updated_at, + updated_by, +} from '../common/schemas'; + +export const searchEsListSchema = t.exact( + t.type({ + created_at, + created_by, + description, + meta: metaOrUndefined, + name, + tie_breaker_id, + type, + updated_at, + updated_by, + }) +); + +export type SearchEsListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/index.ts b/x-pack/plugins/lists/common/schemas/index.ts new file mode 100644 index 0000000000000..6a60a6df55691 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './common'; +export * from './request'; +export * from './response'; +export * from './elastic_query'; +export * from './elastic_response'; diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts new file mode 100644 index 0000000000000..8168e5a9838f2 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { idOrUndefined, list_id, metaOrUndefined, value } from '../common/schemas'; + +export const createListItemSchema = t.exact( + t.type({ + id: idOrUndefined, + list_id, + meta: metaOrUndefined, + value, + }) +); + +export type CreateListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts new file mode 100644 index 0000000000000..ba791a55d17eb --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { left } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; + +import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps'; + +import { getListRequest } from './mocks/utils'; +import { createListSchema } from './create_list_schema'; + +describe('create_list_schema', () => { + // TODO: Finish the tests for this + test('it should validate a typical lists request', () => { + const payload = getListRequest(); + const decoded = createListSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual({ + description: 'Description of a list item', + id: 'some-list-id', + name: 'Name of a list item', + type: 'ip', + }); + }); +}); diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts new file mode 100644 index 0000000000000..353a4ecdafa0c --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { description, idOrUndefined, metaOrUndefined, name, type } from '../common/schemas'; + +export const createListSchema = t.exact( + t.type({ + description, + id: idOrUndefined, + meta: metaOrUndefined, + name, + type, + }) +); + +export type CreateListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts new file mode 100644 index 0000000000000..f4c1fb5c43eb0 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/delete_list_item_schema.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { idOrUndefined, list_idOrUndefined, valueOrUndefined } from '../common/schemas'; + +export const deleteListItemSchema = t.exact( + t.type({ + id: idOrUndefined, + list_id: list_idOrUndefined, + value: valueOrUndefined, + }) +); + +export type DeleteListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts new file mode 100644 index 0000000000000..fd6aa5b85f81a --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { id } from '../common/schemas'; + +export const deleteListSchema = t.exact( + t.type({ + id, + }) +); + +export type DeleteListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts b/x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts new file mode 100644 index 0000000000000..14b201bf8089d --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/export_list_item_query_schema.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { list_id } from '../common/schemas'; + +export const exportListItemQuerySchema = t.exact( + t.type({ + list_id, + // TODO: Add file_name here with a default value + }) +); + +export type ExportListItemQuerySchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts new file mode 100644 index 0000000000000..b8467d141bdd8 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { list_idOrUndefined, typeOrUndefined } from '../common/schemas'; + +export const importListItemQuerySchema = t.exact( + t.type({ list_id: list_idOrUndefined, type: typeOrUndefined }) +); + +export type ImportListItemQuerySchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts new file mode 100644 index 0000000000000..0cf01db8617f0 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import { Readable } from 'stream'; + +import * as t from 'io-ts'; + +import { file } from '../common/schemas'; + +export const importListItemSchema = t.exact( + t.type({ + file, + }) +); + +export interface HapiReadableStream extends Readable { + hapi: { + filename: string; + }; +} + +/** + * Special interface since we are streaming in a file through a reader + */ +export interface ImportListItemSchema { + file: HapiReadableStream; +} diff --git a/x-pack/plugins/lists/common/schemas/request/index.ts b/x-pack/plugins/lists/common/schemas/request/index.ts new file mode 100644 index 0000000000000..d332ab1eb1bab --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './create_list_item_schema'; +export * from './create_list_schema'; +export * from './delete_list_item_schema'; +export * from './delete_list_schema'; +export * from './export_list_item_query_schema'; +export * from './import_list_item_schema'; +export * from './patch_list_item_schema'; +export * from './patch_list_schema'; +export * from './read_list_item_schema'; +export * from './read_list_schema'; +export * from './import_list_item_query_schema'; +export * from './update_list_schema'; +export * from './update_list_item_schema'; diff --git a/x-pack/plugins/lists/common/schemas/request/mocks/utils.ts b/x-pack/plugins/lists/common/schemas/request/mocks/utils.ts new file mode 100644 index 0000000000000..e5d189db8490b --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/mocks/utils.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateListSchema } from '../create_list_schema'; + +export const getListRequest = (): CreateListSchema => ({ + description: 'Description of a list item', + id: 'some-list-id', + meta: undefined, + name: 'Name of a list item', + type: 'ip', +}); diff --git a/x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts new file mode 100644 index 0000000000000..3e8198a5109b3 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/patch_list_item_schema.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { id, metaOrUndefined, valueOrUndefined } from '../common/schemas'; + +export const patchListItemSchema = t.exact( + t.type({ + id, + meta: metaOrUndefined, + value: valueOrUndefined, + }) +); + +export type PatchListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts new file mode 100644 index 0000000000000..efcb81fc8be2a --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { descriptionOrUndefined, id, metaOrUndefined, nameOrUndefined } from '../common/schemas'; + +export const patchListSchema = t.exact( + t.type({ + description: descriptionOrUndefined, + id, + meta: metaOrUndefined, + name: nameOrUndefined, + }) +); + +export type PatchListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts new file mode 100644 index 0000000000000..9ea14a2a21ed8 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/read_list_item_schema.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { idOrUndefined, list_idOrUndefined, valueOrUndefined } from '../common/schemas'; + +export const readListItemSchema = t.exact( + t.type({ id: idOrUndefined, list_id: list_idOrUndefined, value: valueOrUndefined }) +); + +export type ReadListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/read_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/read_list_schema.ts new file mode 100644 index 0000000000000..8803346709c31 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/read_list_schema.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { id } from '../common/schemas'; + +export const readListSchema = t.exact( + t.type({ + id, + }) +); + +export type ReadListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/update_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/update_list_item_schema.ts new file mode 100644 index 0000000000000..e1f88bae66e0f --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/update_list_item_schema.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { id, metaOrUndefined, value } from '../common/schemas'; + +export const updateListItemSchema = t.exact( + t.type({ + id, + meta: metaOrUndefined, + value, + }) +); + +export type UpdateListItemSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/request/update_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/update_list_schema.ts new file mode 100644 index 0000000000000..d51ed60c41b56 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/request/update_list_schema.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { description, id, metaOrUndefined, name } from '../common/schemas'; + +export const updateListSchema = t.exact( + t.type({ + description, + id, + meta: metaOrUndefined, + name, + }) +); + +export type UpdateListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/response/acknowledge_schema.ts b/x-pack/plugins/lists/common/schemas/response/acknowledge_schema.ts new file mode 100644 index 0000000000000..55aaf587ac06b --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/acknowledge_schema.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const acknowledgeSchema = t.type({ acknowledged: t.boolean }); + +export type AcknowledgeSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/response/index.ts b/x-pack/plugins/lists/common/schemas/response/index.ts new file mode 100644 index 0000000000000..3f11adf58d8d4 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './list_item_schema'; +export * from './list_schema'; +export * from './acknowledge_schema'; +export * from './list_item_index_exist_schema'; diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_index_exist_schema.ts b/x-pack/plugins/lists/common/schemas/response/list_item_index_exist_schema.ts new file mode 100644 index 0000000000000..bf2bf21d2c216 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/list_item_index_exist_schema.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const listItemIndexExistSchema = t.type({ + list_index: t.boolean, + list_item_index: t.boolean, +}); + +export type ListItemIndexExistSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts new file mode 100644 index 0000000000000..6c2f2ed9a7095 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +/* eslint-disable @typescript-eslint/camelcase */ + +import { + created_at, + created_by, + id, + list_id, + metaOrUndefined, + tie_breaker_id, + type, + updated_at, + updated_by, + value, +} from '../common/schemas'; + +export const listItemSchema = t.exact( + t.type({ + created_at, + created_by, + id, + list_id, + meta: metaOrUndefined, + tie_breaker_id, + type, + updated_at, + updated_by, + value, + }) +); + +export type ListItemSchema = t.TypeOf; + +export const listItemArraySchema = t.array(listItemSchema); +export type ListItemArraySchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.ts new file mode 100644 index 0000000000000..cad449766ceb4 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/camelcase */ + +import * as t from 'io-ts'; + +import { + created_at, + created_by, + description, + id, + metaOrUndefined, + name, + tie_breaker_id, + type, + updated_at, + updated_by, +} from '../common/schemas'; + +export const listSchema = t.exact( + t.type({ + created_at, + created_by, + description, + id, + meta: metaOrUndefined, + name, + tie_breaker_id, + type, + updated_at, + updated_by, + }) +); + +export type ListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/types/non_empty_string.ts b/x-pack/plugins/lists/common/schemas/types/non_empty_string.ts new file mode 100644 index 0000000000000..d1e2094bbcad3 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/types/non_empty_string.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; + +export type NonEmptyStringC = t.Type; + +/** + * Types the NonEmptyString as: + * - A string that is not empty + */ +export const NonEmptyString: NonEmptyStringC = new t.Type( + 'NonEmptyString', + t.string.is, + (input, context): Either => { + if (typeof input === 'string' && input.trim() !== '') { + return t.success(input); + } else { + return t.failure(input, context); + } + }, + t.identity +); diff --git a/x-pack/plugins/lists/common/siem_common_deps.ts b/x-pack/plugins/lists/common/siem_common_deps.ts new file mode 100644 index 0000000000000..5e74753a6f0bd --- /dev/null +++ b/x-pack/plugins/lists/common/siem_common_deps.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { getPaths, foldLeftRight } from '../../siem/server/utils/build_validation/__mocks__/utils'; +export { exactCheck } from '../../siem/server/utils/build_validation/exact_check'; diff --git a/x-pack/plugins/lists/kibana.json b/x-pack/plugins/lists/kibana.json new file mode 100644 index 0000000000000..b7aaac6d3fc76 --- /dev/null +++ b/x-pack/plugins/lists/kibana.json @@ -0,0 +1,10 @@ +{ + "configPath": ["xpack", "lists"], + "id": "lists", + "kibanaVersion": "kibana", + "requiredPlugins": [], + "optionalPlugins": ["spaces", "security"], + "server": true, + "ui": false, + "version": "8.0.0" +} diff --git a/x-pack/plugins/lists/server/config.ts b/x-pack/plugins/lists/server/config.ts new file mode 100644 index 0000000000000..3e7995b2ce8d0 --- /dev/null +++ b/x-pack/plugins/lists/server/config.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TypeOf, schema } from '@kbn/config-schema'; + +export const ConfigSchema = schema.object({ + enabled: schema.boolean({ defaultValue: false }), + listIndex: schema.string({ defaultValue: '.lists' }), + listItemIndex: schema.string({ defaultValue: '.items' }), +}); + +export type ConfigType = TypeOf; diff --git a/x-pack/plugins/lists/server/create_config.ts b/x-pack/plugins/lists/server/create_config.ts new file mode 100644 index 0000000000000..3158fabda935f --- /dev/null +++ b/x-pack/plugins/lists/server/create_config.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { map } from 'rxjs/operators'; +import { PluginInitializerContext } from 'kibana/server'; +import { Observable } from 'rxjs'; + +import { ConfigType } from './config'; + +export const createConfig$ = ( + context: PluginInitializerContext +): Observable> => { + return context.config.create().pipe(map(config => config)); +}; diff --git a/x-pack/plugins/lists/server/error_with_status_code.ts b/x-pack/plugins/lists/server/error_with_status_code.ts new file mode 100644 index 0000000000000..f9bbbc4abad27 --- /dev/null +++ b/x-pack/plugins/lists/server/error_with_status_code.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export class ErrorWithStatusCode extends Error { + private readonly statusCode: number; + + constructor(message: string, statusCode: number) { + super(message); + this.statusCode = statusCode; + } + + public getStatusCode = (): number => this.statusCode; +} diff --git a/x-pack/plugins/lists/server/index.ts b/x-pack/plugins/lists/server/index.ts new file mode 100644 index 0000000000000..c1e577aa60195 --- /dev/null +++ b/x-pack/plugins/lists/server/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { PluginInitializerContext } from '../../../../src/core/server'; + +import { ConfigSchema } from './config'; +import { ListPlugin } from './plugin'; + +export const config = { schema: ConfigSchema }; +export const plugin = (initializerContext: PluginInitializerContext): ListPlugin => + new ListPlugin(initializerContext); diff --git a/x-pack/plugins/lists/server/plugin.ts b/x-pack/plugins/lists/server/plugin.ts new file mode 100644 index 0000000000000..4473d68d3c646 --- /dev/null +++ b/x-pack/plugins/lists/server/plugin.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { first } from 'rxjs/operators'; +import { ElasticsearchServiceSetup, Logger, PluginInitializerContext } from 'kibana/server'; +import { CoreSetup } from 'src/core/server'; + +import { SecurityPluginSetup } from '../../security/server'; +import { SpacesServiceSetup } from '../../spaces/server'; + +import { ConfigType } from './config'; +import { initRoutes } from './routes/init_routes'; +import { ListClient } from './services/lists/client'; +import { ContextProvider, ContextProviderReturn, PluginsSetup } from './types'; +import { createConfig$ } from './create_config'; + +export class ListPlugin { + private readonly logger: Logger; + private spaces: SpacesServiceSetup | undefined | null; + private config: ConfigType | undefined | null; + private elasticsearch: ElasticsearchServiceSetup | undefined | null; + private security: SecurityPluginSetup | undefined | null; + + constructor(private readonly initializerContext: PluginInitializerContext) { + this.logger = this.initializerContext.logger.get(); + } + + public async setup(core: CoreSetup, plugins: PluginsSetup): Promise { + const config = await createConfig$(this.initializerContext) + .pipe(first()) + .toPromise(); + + this.logger.error( + 'You have activated the lists values feature flag which is NOT currently supported for Elastic Security! You should turn this feature flag off immediately by un-setting "xpack.lists.enabled: true" in kibana.yml and restarting Kibana' + ); + this.spaces = plugins.spaces?.spacesService; + this.config = config; + this.elasticsearch = core.elasticsearch; + this.security = plugins.security; + + core.http.registerRouteHandlerContext('lists', this.createRouteHandlerContext()); + const router = core.http.createRouter(); + initRoutes(router); + } + + public start(): void { + this.logger.debug('Starting plugin'); + } + + public stop(): void { + this.logger.debug('Stopping plugin'); + } + + private createRouteHandlerContext = (): ContextProvider => { + return async (context, request): ContextProviderReturn => { + const { spaces, config, security, elasticsearch } = this; + const { + core: { + elasticsearch: { dataClient }, + }, + } = context; + if (config == null) { + throw new TypeError('Configuration is required for this plugin to operate'); + } else if (elasticsearch == null) { + throw new TypeError('Elastic Search is required for this plugin to operate'); + } else if (security == null) { + // TODO: This might be null, test authentication being turned off. + throw new TypeError('Security plugin is required for this plugin to operate'); + } else { + return { + getListClient: (): ListClient => + new ListClient({ + config, + dataClient, + request, + security, + spaces, + }), + }; + } + }; + }; +} diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/create_list_index_route.ts new file mode 100644 index 0000000000000..1c893fb757c5d --- /dev/null +++ b/x-pack/plugins/lists/server/routes/create_list_index_route.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { buildSiemResponse, transformError, validate } from '../siem_server_deps'; +import { LIST_INDEX } from '../../common/constants'; +import { acknowledgeSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const createListIndexRoute = (router: IRouter): void => { + router.post( + { + options: { + tags: ['access:lists'], + }, + path: LIST_INDEX, + validate: false, + }, + async (context, _, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const lists = getListClient(context); + const listIndexExists = await lists.getListIndexExists(); + const listItemIndexExists = await lists.getListItemIndexExists(); + + if (listIndexExists && listItemIndexExists) { + return siemResponse.error({ + body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`, + statusCode: 409, + }); + } else { + const policyExists = await lists.getListPolicyExists(); + const policyListItemExists = await lists.getListItemPolicyExists(); + + if (!policyExists) { + await lists.setListPolicy(); + } + if (!policyListItemExists) { + await lists.setListItemPolicy(); + } + + const templateExists = await lists.getListTemplateExists(); + const templateListItemsExists = await lists.getListItemTemplateExists(); + + if (!templateExists) { + await lists.setListTemplate(); + } + + if (!templateListItemsExists) { + await lists.setListItemTemplate(); + } + + if (!listIndexExists) { + await lists.createListBootStrapIndex(); + } + if (!listItemIndexExists) { + await lists.createListItemBootStrapIndex(); + } + + const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/create_list_item_route.ts b/x-pack/plugins/lists/server/routes/create_list_item_route.ts new file mode 100644 index 0000000000000..68622e98cbc52 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/create_list_item_route.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { createListItemSchema, listItemSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const createListItemRoute = (router: IRouter): void => { + router.post( + { + options: { + tags: ['access:lists'], + }, + path: LIST_ITEM_URL, + validate: { + body: buildRouteValidation(createListItemSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { id, list_id: listId, value, meta } = request.body; + const lists = getListClient(context); + const list = await lists.getList({ id: listId }); + if (list == null) { + return siemResponse.error({ + body: `list id: "${listId}" does not exist`, + statusCode: 404, + }); + } else { + const listItem = await lists.getListItemByValue({ listId, type: list.type, value }); + if (listItem.length !== 0) { + return siemResponse.error({ + body: `list_id: "${listId}" already contains the given value: ${value}`, + statusCode: 409, + }); + } else { + const createdListItem = await lists.createListItem({ + id, + listId, + meta, + type: list.type, + value, + }); + const [validated, errors] = validate(createdListItem, listItemSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/create_list_route.ts b/x-pack/plugins/lists/server/routes/create_list_route.ts new file mode 100644 index 0000000000000..0f3c404c53cfd --- /dev/null +++ b/x-pack/plugins/lists/server/routes/create_list_route.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { createListSchema, listSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const createListRoute = (router: IRouter): void => { + router.post( + { + options: { + tags: ['access:lists'], + }, + path: LIST_URL, + validate: { + body: buildRouteValidation(createListSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { name, description, id, type, meta } = request.body; + const lists = getListClient(context); + const listExists = await lists.getListIndexExists(); + if (!listExists) { + return siemResponse.error({ + body: `To create a list, the index must exist first. Index "${lists.getListIndex()}" does not exist`, + statusCode: 400, + }); + } else { + if (id != null) { + const list = await lists.getList({ id }); + if (list != null) { + return siemResponse.error({ + body: `list id: "${id}" already exists`, + statusCode: 409, + }); + } + } + const list = await lists.createList({ description, id, meta, name, type }); + const [validated, errors] = validate(list, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts new file mode 100644 index 0000000000000..424c3f45aac40 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_INDEX } from '../../common/constants'; +import { buildSiemResponse, transformError, validate } from '../siem_server_deps'; +import { acknowledgeSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +/** + * Deletes all of the indexes, template, ilm policies, and aliases. You can check + * this by looking at each of these settings from ES after a deletion: + * + * GET /_template/.lists-default + * GET /.lists-default-000001/ + * GET /_ilm/policy/.lists-default + * GET /_alias/.lists-default + * + * GET /_template/.items-default + * GET /.items-default-000001/ + * GET /_ilm/policy/.items-default + * GET /_alias/.items-default + * + * And ensuring they're all gone + */ +export const deleteListIndexRoute = (router: IRouter): void => { + router.delete( + { + options: { + tags: ['access:lists'], + }, + path: LIST_INDEX, + validate: false, + }, + async (context, _, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const lists = getListClient(context); + const listIndexExists = await lists.getListIndexExists(); + const listItemIndexExists = await lists.getListItemIndexExists(); + + if (!listIndexExists && !listItemIndexExists) { + return siemResponse.error({ + body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`, + statusCode: 404, + }); + } else { + if (listIndexExists) { + await lists.deleteListIndex(); + } + if (listItemIndexExists) { + await lists.deleteListItemIndex(); + } + + const listsPolicyExists = await lists.getListPolicyExists(); + const listItemPolicyExists = await lists.getListItemPolicyExists(); + + if (listsPolicyExists) { + await lists.deleteListPolicy(); + } + if (listItemPolicyExists) { + await lists.deleteListItemPolicy(); + } + + const listsTemplateExists = await lists.getListTemplateExists(); + const listItemTemplateExists = await lists.getListItemTemplateExists(); + + if (listsTemplateExists) { + await lists.deleteListTemplate(); + } + if (listItemTemplateExists) { + await lists.deleteListItemTemplate(); + } + + const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts b/x-pack/plugins/lists/server/routes/delete_list_item_route.ts new file mode 100644 index 0000000000000..51b4eb9f02cc2 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/delete_list_item_route.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { deleteListItemSchema, listItemArraySchema, listItemSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const deleteListItemRoute = (router: IRouter): void => { + router.delete( + { + options: { + tags: ['access:lists'], + }, + path: LIST_ITEM_URL, + validate: { + query: buildRouteValidation(deleteListItemSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { id, list_id: listId, value } = request.query; + const lists = getListClient(context); + if (id != null) { + const deleted = await lists.deleteListItem({ id }); + if (deleted == null) { + return siemResponse.error({ + body: `list item with id: "${id}" item not found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(deleted, listItemSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } else if (listId != null && value != null) { + const list = await lists.getList({ id: listId }); + if (list == null) { + return siemResponse.error({ + body: `list_id: "${listId}" does not exist`, + statusCode: 404, + }); + } else { + const deleted = await lists.deleteListItemByValue({ listId, type: list.type, value }); + if (deleted == null || deleted.length === 0) { + return siemResponse.error({ + body: `list_id: "${listId}" with ${value} was not found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(deleted, listItemArraySchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } + } else { + return siemResponse.error({ + body: `Either "list_id" or "id" needs to be defined in the request`, + statusCode: 400, + }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/delete_list_route.ts new file mode 100644 index 0000000000000..e89355b7689c5 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/delete_list_route.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { deleteListSchema, listSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const deleteListRoute = (router: IRouter): void => { + router.delete( + { + options: { + tags: ['access:lists'], + }, + path: LIST_URL, + validate: { + query: buildRouteValidation(deleteListSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const lists = getListClient(context); + const { id } = request.query; + const deleted = await lists.deleteList({ id }); + if (deleted == null) { + return siemResponse.error({ + body: `list id: "${id}" was not found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(deleted, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/export_list_item_route.ts b/x-pack/plugins/lists/server/routes/export_list_item_route.ts new file mode 100644 index 0000000000000..32b99bfc512bf --- /dev/null +++ b/x-pack/plugins/lists/server/routes/export_list_item_route.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Stream } from 'stream'; + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; +import { exportListItemQuerySchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const exportListItemRoute = (router: IRouter): void => { + router.post( + { + options: { + tags: ['access:lists'], + }, + path: `${LIST_ITEM_URL}/_export`, + validate: { + query: buildRouteValidation(exportListItemQuerySchema), + // TODO: Do we want to add a body here like export_rules_route and allow a size limit? + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { list_id: listId } = request.query; + const lists = getListClient(context); + const list = await lists.getList({ id: listId }); + if (list == null) { + return siemResponse.error({ + body: `list_id: ${listId} does not exist`, + statusCode: 400, + }); + } else { + // TODO: Allow the API to override the name of the file to export + const fileName = list.name; + + const stream = new Stream.PassThrough(); + lists.exportListItemsToStream({ listId, stream, stringToAppend: '\n' }); + return response.ok({ + body: stream, + headers: { + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Content-Type': 'text/plain', + }, + }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/import_list_item_route.ts new file mode 100644 index 0000000000000..a3b6a520a4ecf --- /dev/null +++ b/x-pack/plugins/lists/server/routes/import_list_item_route.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { + ImportListItemSchema, + importListItemQuerySchema, + importListItemSchema, + listSchema, +} from '../../common/schemas'; + +import { getListClient } from '.'; + +export const importListItemRoute = (router: IRouter): void => { + router.post( + { + options: { + body: { + output: 'stream', + }, + tags: ['access:lists'], + }, + path: `${LIST_ITEM_URL}/_import`, + validate: { + body: buildRouteValidation( + importListItemSchema + ), + query: buildRouteValidation(importListItemQuerySchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { list_id: listId, type } = request.query; + const lists = getListClient(context); + if (listId != null) { + const list = await lists.getList({ id: listId }); + if (list == null) { + return siemResponse.error({ + body: `list id: "${listId}" does not exist`, + statusCode: 409, + }); + } + await lists.importListItemsToStream({ + listId, + meta: undefined, + stream: request.body.file, + type: list.type, + }); + + const [validated, errors] = validate(list, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } else if (type != null) { + const { filename } = request.body.file.hapi; + // TODO: Should we prevent the same file from being uploaded multiple times? + const list = await lists.createListIfItDoesNotExist({ + description: `File uploaded from file system of ${filename}`, + id: filename, + meta: undefined, + name: filename, + type, + }); + await lists.importListItemsToStream({ + listId: list.id, + meta: undefined, + stream: request.body.file, + type: list.type, + }); + const [validated, errors] = validate(list, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } else { + return siemResponse.error({ + body: 'Either type or list_id need to be defined in the query', + statusCode: 400, + }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts new file mode 100644 index 0000000000000..4951cddc56939 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './create_list_index_route'; +export * from './create_list_item_route'; +export * from './create_list_route'; +export * from './delete_list_index_route'; +export * from './delete_list_item_route'; +export * from './delete_list_route'; +export * from './export_list_item_route'; +export * from './import_list_item_route'; +export * from './init_routes'; +export * from './patch_list_item_route'; +export * from './patch_list_route'; +export * from './read_list_index_route'; +export * from './read_list_item_route'; +export * from './read_list_route'; +export * from './utils'; diff --git a/x-pack/plugins/lists/server/routes/init_routes.ts b/x-pack/plugins/lists/server/routes/init_routes.ts new file mode 100644 index 0000000000000..924dd086ee708 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/init_routes.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { updateListRoute } from './update_list_route'; +import { updateListItemRoute } from './update_list_item_route'; + +import { + createListIndexRoute, + createListItemRoute, + createListRoute, + deleteListIndexRoute, + deleteListItemRoute, + deleteListRoute, + exportListItemRoute, + importListItemRoute, + patchListItemRoute, + patchListRoute, + readListIndexRoute, + readListItemRoute, + readListRoute, +} from '.'; + +export const initRoutes = (router: IRouter): void => { + // lists + createListRoute(router); + readListRoute(router); + updateListRoute(router); + deleteListRoute(router); + patchListRoute(router); + + // lists items + createListItemRoute(router); + readListItemRoute(router); + updateListItemRoute(router); + deleteListItemRoute(router); + patchListItemRoute(router); + exportListItemRoute(router); + importListItemRoute(router); + + // indexes of lists + createListIndexRoute(router); + readListIndexRoute(router); + deleteListIndexRoute(router); +}; diff --git a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts new file mode 100644 index 0000000000000..e18fd0618b133 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { listItemSchema, patchListItemSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const patchListItemRoute = (router: IRouter): void => { + router.patch( + { + options: { + tags: ['access:lists'], + }, + path: LIST_ITEM_URL, + validate: { + body: buildRouteValidation(patchListItemSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { value, id, meta } = request.body; + const lists = getListClient(context); + const listItem = await lists.updateListItem({ + id, + meta, + value, + }); + if (listItem == null) { + return siemResponse.error({ + body: `list item id: "${id}" not found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(listItem, listItemSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/patch_list_route.ts b/x-pack/plugins/lists/server/routes/patch_list_route.ts new file mode 100644 index 0000000000000..9d3fa4db8ccd0 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/patch_list_route.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { listSchema, patchListSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const patchListRoute = (router: IRouter): void => { + router.patch( + { + options: { + tags: ['access:lists'], + }, + path: LIST_URL, + validate: { + body: buildRouteValidation(patchListSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { name, description, id, meta } = request.body; + const lists = getListClient(context); + const list = await lists.updateList({ description, id, meta, name }); + if (list == null) { + return siemResponse.error({ + body: `list id: "${id}" found found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(list, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/read_list_index_route.ts new file mode 100644 index 0000000000000..248fc72666d70 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/read_list_index_route.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_INDEX } from '../../common/constants'; +import { buildSiemResponse, transformError, validate } from '../siem_server_deps'; +import { listItemIndexExistSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const readListIndexRoute = (router: IRouter): void => { + router.get( + { + options: { + tags: ['access:lists'], + }, + path: LIST_INDEX, + validate: false, + }, + async (context, _, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const lists = getListClient(context); + const listIndexExists = await lists.getListIndexExists(); + const listItemIndexExists = await lists.getListItemIndexExists(); + + if (listIndexExists || listItemIndexExists) { + const [validated, errors] = validate( + { list_index: listIndexExists, lists_item_index: listItemIndexExists }, + listItemIndexExistSchema + ); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } else if (!listIndexExists && listItemIndexExists) { + return siemResponse.error({ + body: `index ${lists.getListIndex()} does not exist`, + statusCode: 404, + }); + } else if (!listItemIndexExists && listIndexExists) { + return siemResponse.error({ + body: `index ${lists.getListItemIndex()} does not exist`, + statusCode: 404, + }); + } else { + return siemResponse.error({ + body: `index ${lists.getListIndex()} and index ${lists.getListItemIndex()} does not exist`, + statusCode: 404, + }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/read_list_item_route.ts b/x-pack/plugins/lists/server/routes/read_list_item_route.ts new file mode 100644 index 0000000000000..0a60cba786f04 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/read_list_item_route.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { listItemArraySchema, listItemSchema, readListItemSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const readListItemRoute = (router: IRouter): void => { + router.get( + { + options: { + tags: ['access:lists'], + }, + path: LIST_ITEM_URL, + validate: { + query: buildRouteValidation(readListItemSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { id, list_id: listId, value } = request.query; + const lists = getListClient(context); + if (id != null) { + const listItem = await lists.getListItem({ id }); + if (listItem == null) { + return siemResponse.error({ + body: `list item id: "${id}" does not exist`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(listItem, listItemSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } else if (listId != null && value != null) { + const list = await lists.getList({ id: listId }); + if (list == null) { + return siemResponse.error({ + body: `list id: "${listId}" does not exist`, + statusCode: 404, + }); + } else { + const listItem = await lists.getListItemByValue({ + listId, + type: list.type, + value, + }); + if (listItem.length === 0) { + return siemResponse.error({ + body: `list_id: "${listId}" item of ${value} does not exist`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(listItem, listItemArraySchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } + } else { + return siemResponse.error({ + body: `Either "list_id" or "id" needs to be defined in the request`, + statusCode: 400, + }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/read_list_route.ts b/x-pack/plugins/lists/server/routes/read_list_route.ts new file mode 100644 index 0000000000000..c30eadfca0b65 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/read_list_route.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { listSchema, readListSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const readListRoute = (router: IRouter): void => { + router.get( + { + options: { + tags: ['access:lists'], + }, + path: LIST_URL, + validate: { + query: buildRouteValidation(readListSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { id } = request.query; + const lists = getListClient(context); + const list = await lists.getList({ id }); + if (list == null) { + return siemResponse.error({ + body: `list id: "${id}" does not exist`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(list, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/update_list_item_route.ts b/x-pack/plugins/lists/server/routes/update_list_item_route.ts new file mode 100644 index 0000000000000..494d57b93b8e4 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/update_list_item_route.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_ITEM_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { listItemSchema, updateListItemSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const updateListItemRoute = (router: IRouter): void => { + router.put( + { + options: { + tags: ['access:lists'], + }, + path: LIST_ITEM_URL, + validate: { + body: buildRouteValidation(updateListItemSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { value, id, meta } = request.body; + const lists = getListClient(context); + const listItem = await lists.updateListItem({ + id, + meta, + value, + }); + if (listItem == null) { + return siemResponse.error({ + body: `list item id: "${id}" not found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(listItem, listItemSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/update_list_route.ts b/x-pack/plugins/lists/server/routes/update_list_route.ts new file mode 100644 index 0000000000000..6ace61e46a780 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/update_list_route.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'kibana/server'; + +import { LIST_URL } from '../../common/constants'; +import { + buildRouteValidation, + buildSiemResponse, + transformError, + validate, +} from '../siem_server_deps'; +import { listSchema, updateListSchema } from '../../common/schemas'; + +import { getListClient } from '.'; + +export const updateListRoute = (router: IRouter): void => { + router.put( + { + options: { + tags: ['access:lists'], + }, + path: LIST_URL, + validate: { + body: buildRouteValidation(updateListSchema), + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + try { + const { name, description, id, meta } = request.body; + const lists = getListClient(context); + const list = await lists.updateList({ description, id, meta, name }); + if (list == null) { + return siemResponse.error({ + body: `list id: "${id}" found found`, + statusCode: 404, + }); + } else { + const [validated, errors] = validate(list, listSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/utils/get_list_client.ts b/x-pack/plugins/lists/server/routes/utils/get_list_client.ts new file mode 100644 index 0000000000000..a16163ec0fa3a --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/get_list_client.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandlerContext } from 'kibana/server'; + +import { ListClient } from '../../services/lists/client'; +import { ErrorWithStatusCode } from '../../error_with_status_code'; + +export const getListClient = (context: RequestHandlerContext): ListClient => { + const lists = context.lists?.getListClient(); + if (lists == null) { + throw new ErrorWithStatusCode('Lists is not found as a plugin', 404); + } else { + return lists; + } +}; diff --git a/x-pack/plugins/lists/server/routes/utils/index.ts b/x-pack/plugins/lists/server/routes/utils/index.ts new file mode 100644 index 0000000000000..a601bdfc003c5 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export * from './get_list_client'; diff --git a/x-pack/plugins/lists/server/scripts/check_env_variables.sh b/x-pack/plugins/lists/server/scripts/check_env_variables.sh new file mode 100755 index 0000000000000..fb3bbbe0fad18 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/check_env_variables.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Add this to the start of any scripts to detect if env variables are set + +set -e + +if [ -z "${ELASTICSEARCH_USERNAME}" ]; then + echo "Set ELASTICSEARCH_USERNAME in your environment" + exit 1 +fi + +if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then + echo "Set ELASTICSEARCH_PASSWORD in your environment" + exit 1 +fi + +if [ -z "${ELASTICSEARCH_URL}" ]; then + echo "Set ELASTICSEARCH_URL in your environment" + exit 1 +fi + +if [ -z "${KIBANA_URL}" ]; then + echo "Set KIBANA_URL in your environment" + exit 1 +fi + +if [ -z "${TASK_MANAGER_INDEX}" ]; then + echo "Set TASK_MANAGER_INDEX in your environment" + exit 1 +fi + +if [ -z "${KIBANA_INDEX}" ]; then + echo "Set KIBANA_INDEX in your environment" + exit 1 +fi diff --git a/x-pack/plugins/lists/server/scripts/delete_all_lists.sh b/x-pack/plugins/lists/server/scripts/delete_all_lists.sh new file mode 100755 index 0000000000000..5b65bb14414c7 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/delete_all_lists.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_all_lists.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html + + +# Delete all the main lists that have children items +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \ + --data '{ + "query": { + "exists": { "field": "siem_list" } + } + }' \ + | jq . + +# Delete all the list children items as well +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \ + --data '{ + "query": { + "exists": { "field": "siem_list_item" } + } + }' \ + | jq . diff --git a/x-pack/plugins/lists/server/scripts/delete_list.sh b/x-pack/plugins/lists/server/scripts/delete_list.sh new file mode 100755 index 0000000000000..9934ce61c7107 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/delete_list.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_list_by_list_id.sh ${list_id} +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE ${KIBANA_URL}${SPACE_URL}/api/lists?id="$1" | jq . diff --git a/x-pack/plugins/lists/server/scripts/delete_list_index.sh b/x-pack/plugins/lists/server/scripts/delete_list_index.sh new file mode 100755 index 0000000000000..85f06ffbd6670 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/delete_list_index.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_signal_index.sh +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE ${KIBANA_URL}${SPACE_URL}/api/lists/index | jq . diff --git a/x-pack/plugins/lists/server/scripts/delete_list_item_by_id.sh b/x-pack/plugins/lists/server/scripts/delete_list_item_by_id.sh new file mode 100755 index 0000000000000..ab14d8c8a80ed --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/delete_list_item_by_id.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_list_item_by_id.sh?id={id} +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE ${KIBANA_URL}${SPACE_URL}/api/lists/items?id=$1 | jq . diff --git a/x-pack/plugins/lists/server/scripts/delete_list_item_by_value.sh b/x-pack/plugins/lists/server/scripts/delete_list_item_by_value.sh new file mode 100755 index 0000000000000..6d3213ccb8793 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/delete_list_item_by_value.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_list_item_by_value.sh?list_id=${some_id}&value=${some_ip} +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE "${KIBANA_URL}${SPACE_URL}/api/lists/items?list_id=$1&value=$2" | jq . diff --git a/x-pack/plugins/lists/server/scripts/export_list_items.sh b/x-pack/plugins/lists/server/scripts/export_list_items.sh new file mode 100755 index 0000000000000..ba355854c77cc --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/export_list_items.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a defaults if no argument is specified +LIST_ID=${1:-ips.txt} + +# Example to export +# ./export_list_items.sh > /tmp/ips.txt + +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_export?list_id=${LIST_ID}" diff --git a/x-pack/plugins/lists/server/scripts/export_list_items_to_file.sh b/x-pack/plugins/lists/server/scripts/export_list_items_to_file.sh new file mode 100755 index 0000000000000..5efad01e9a68e --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/export_list_items_to_file.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a defaults if no argument is specified +FOLDER=${1:-/tmp} + +# Example to export +# ./export_list_items_to_file.sh + +# Change current working directory as exports cause Kibana to restart +pushd ${FOLDER} > /dev/null + +curl -s -k -OJ \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_export?list_id=list-ip" + +popd > /dev/null diff --git a/x-pack/plugins/lists/server/scripts/get_list.sh b/x-pack/plugins/lists/server/scripts/get_list.sh new file mode 100755 index 0000000000000..7f0e4e3062266 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/get_list.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./get_list.sh {list_id} +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}${SPACE_URL}/api/lists?id="$1" | jq . diff --git a/x-pack/plugins/lists/server/scripts/get_list_item_by_id.sh b/x-pack/plugins/lists/server/scripts/get_list_item_by_id.sh new file mode 100755 index 0000000000000..31d26e195815f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/get_list_item_by_id.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./get_list_item_by_id ${id} +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items?id=$1" | jq . diff --git a/x-pack/plugins/lists/server/scripts/get_list_item_by_value.sh b/x-pack/plugins/lists/server/scripts/get_list_item_by_value.sh new file mode 100755 index 0000000000000..24ca27b0c949d --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/get_list_item_by_value.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./get_list_item_by_value.sh ${list_id} ${value} +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items?list_id=$1&value=$2" | jq . diff --git a/x-pack/plugins/lists/server/scripts/hard_reset.sh b/x-pack/plugins/lists/server/scripts/hard_reset.sh new file mode 100755 index 0000000000000..861928866369b --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/hard_reset.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# re-create the list and list item indexes +./delete_list_index.sh +./post_list_index.sh diff --git a/x-pack/plugins/lists/server/scripts/import_list_items.sh b/x-pack/plugins/lists/server/scripts/import_list_items.sh new file mode 100755 index 0000000000000..a39409cd08267 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/import_list_items.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a defaults if no argument is specified +LIST_ID=${1:-list-ip} +FILE=${2:-./lists/files/ips.txt} + +# ./import_list_items.sh list-ip ./lists/files/ips.txt +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_import?list_id=${LIST_ID}" \ + --form file=@${FILE} \ + | jq .; diff --git a/x-pack/plugins/lists/server/scripts/import_list_items_by_filename.sh b/x-pack/plugins/lists/server/scripts/import_list_items_by_filename.sh new file mode 100755 index 0000000000000..4ec55cb4c5f7b --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/import_list_items_by_filename.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a defaults if no argument is specified +TYPE=${1:-ip} +FILE=${2:-./lists/files/ips.txt} + +# Example to import ips from ./lists/files/ips.txt +# ./import_list_items_by_filename.sh ip ./lists/files/ips.txt + +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST "${KIBANA_URL}${SPACE_URL}/api/lists/items/_import?type=${TYPE}" \ + --form file=@${FILE} \ + | jq .; diff --git a/x-pack/plugins/lists/server/scripts/lists/files/hosts.txt b/x-pack/plugins/lists/server/scripts/lists/files/hosts.txt new file mode 100644 index 0000000000000..aee32e3a4bd92 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/hosts.txt @@ -0,0 +1,2 @@ +kibana +rock01 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/ips.txt b/x-pack/plugins/lists/server/scripts/lists/files/ips.txt new file mode 100644 index 0000000000000..cf8ebcacae5a1 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/ips.txt @@ -0,0 +1,9 @@ +127.0.0.1 +127.0.0.2 +127.0.0.3 +127.0.0.4 +127.0.0.5 +127.0.0.6 +127.0.0.7 +127.0.0.8 +127.0.0.9 diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_everything.json b/x-pack/plugins/lists/server/scripts/lists/new/list_everything.json new file mode 100644 index 0000000000000..196b3b149ab82 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_everything.json @@ -0,0 +1,13 @@ +{ + "id": "list-ip-everything", + "name": "Simple list with an ip", + "description": "This list describes bad internet ip", + "type": "ip", + "meta": { + "level_1_meta": { + "level_2_meta": { + "level_3_key": "some_value_ui" + } + } + } +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip.json b/x-pack/plugins/lists/server/scripts/lists/new/list_ip.json new file mode 100644 index 0000000000000..3e12ef1754f07 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_ip.json @@ -0,0 +1,6 @@ +{ + "id": "list-ip", + "name": "Simple list with an ip", + "description": "This list describes bad internet ip", + "type": "ip" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json b/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json new file mode 100644 index 0000000000000..1516fa5057e50 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json @@ -0,0 +1,5 @@ +{ + "id": "hand_inserted_item_id", + "list_id": "list-ip", + "value": "127.0.0.1" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json b/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json new file mode 100644 index 0000000000000..9730c1b7523f1 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json @@ -0,0 +1,12 @@ +{ + "id": "hand_inserted_item_id_everything", + "list_id": "list-ip", + "value": "127.0.0.2", + "meta": { + "level_1_meta": { + "level_2_meta": { + "level_3_key": "some_value_ui" + } + } + } +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json b/x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json new file mode 100644 index 0000000000000..4a95a62b67c3e --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json @@ -0,0 +1,5 @@ +{ + "name": "Simple list with an ip", + "description": "This list describes bad internet ip", + "type": "ip" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json b/x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json new file mode 100644 index 0000000000000..e8f5fa7e38a06 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json @@ -0,0 +1,6 @@ +{ + "id": "list-keyword", + "name": "Simple list with a keyword", + "description": "This list describes bad host names", + "type": "keyword" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json b/x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json new file mode 100644 index 0000000000000..b736e7b96ad98 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json @@ -0,0 +1,4 @@ +{ + "list_id": "list-keyword", + "value": "kibana" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json b/x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json new file mode 100644 index 0000000000000..00c3496e71b35 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json @@ -0,0 +1,4 @@ +{ + "id": "hand_inserted_item_id", + "value": "255.255.255.255" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json b/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json new file mode 100644 index 0000000000000..1a57ab8b6a3b9 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json @@ -0,0 +1,4 @@ +{ + "id": "list-ip", + "name": "Changed the name here to something else" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json b/x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json new file mode 100644 index 0000000000000..00c3496e71b35 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json @@ -0,0 +1,4 @@ +{ + "id": "hand_inserted_item_id", + "value": "255.255.255.255" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json b/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json new file mode 100644 index 0000000000000..936a070ede52c --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json @@ -0,0 +1,5 @@ +{ + "id": "list-ip", + "name": "Changed the name here to something else", + "description": "Some other description here for you" +} diff --git a/x-pack/plugins/lists/server/scripts/lists_index_exists.sh b/x-pack/plugins/lists/server/scripts/lists_index_exists.sh new file mode 100755 index 0000000000000..7dfbd5b1bada5 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists_index_exists.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./lists_index_exists.sh +curl -s -k -f \ + -H 'Content-Type: application/json' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + ${KIBANA_URL}${SPACE_URL}/api/lists/index | jq . diff --git a/x-pack/plugins/lists/server/scripts/patch_list.sh b/x-pack/plugins/lists/server/scripts/patch_list.sh new file mode 100755 index 0000000000000..3a517a52dbd21 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/patch_list.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +LISTS=(${@:-./lists/patches/simplest_updated_name.json}) + +# Example: ./patch_list.sh +# Example: ./patch_list.sh ./lists/patches/simplest_updated_name.json +for LIST in "${LISTS[@]}" +do { + [ -e "$LIST" ] || continue + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PATCH ${KIBANA_URL}${SPACE_URL}/api/lists \ + -d @${LIST} \ + | jq .; +} & +done + +wait diff --git a/x-pack/plugins/lists/server/scripts/patch_list_item.sh b/x-pack/plugins/lists/server/scripts/patch_list_item.sh new file mode 100755 index 0000000000000..406b03dc6499c --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/patch_list_item.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +LISTS=(${@:-./lists/patches/list_ip_item.json}) + +# Example: ./patch_list.sh +# Example: ./patch_list.sh ./lists/patches/list_ip_item.json +for LIST in "${LISTS[@]}" +do { + [ -e "$LIST" ] || continue + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PATCH ${KIBANA_URL}${SPACE_URL}/api/lists/items \ + -d @${LIST} \ + | jq .; +} & +done + +wait diff --git a/x-pack/plugins/lists/server/scripts/post_list.sh b/x-pack/plugins/lists/server/scripts/post_list.sh new file mode 100755 index 0000000000000..6aaffee0bc4b2 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/post_list.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +LISTS=(${@:-./lists/new/list_ip.json}) + +# Example: ./post_list.sh +# Example: ./post_list.sh ./lists/new/list_ip.json +for LIST in "${LISTS[@]}" +do { + [ -e "$LIST" ] || continue + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/lists \ + -d @${LIST} \ + | jq .; +} & +done + +wait diff --git a/x-pack/plugins/lists/server/scripts/post_list_index.sh b/x-pack/plugins/lists/server/scripts/post_list_index.sh new file mode 100755 index 0000000000000..b7c372d3947e3 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/post_list_index.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./post_signal_index.sh +curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/lists/index | jq . diff --git a/x-pack/plugins/lists/server/scripts/post_list_item.sh b/x-pack/plugins/lists/server/scripts/post_list_item.sh new file mode 100755 index 0000000000000..b55a60420674f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/post_list_item.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +LISTS=(${@:-./lists/new/list_ip_item.json}) + +# Example: ./post_list.sh +# Example: ./post_list.sh ./lists/new/list_ip_item.json +for LIST in "${LISTS[@]}" +do { + [ -e "$LIST" ] || continue + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/lists/items \ + -d @${LIST} \ + | jq .; +} & +done + +wait diff --git a/x-pack/plugins/lists/server/scripts/update_list.sh b/x-pack/plugins/lists/server/scripts/update_list.sh new file mode 100755 index 0000000000000..4d93544d568a8 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/update_list.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +LISTS=(${@:-./lists/updates/simple_update.json}) + +# Example: ./update_list.sh +# Example: ./update_list.sh ./lists/updates/simple_update.json +for LIST in "${LISTS[@]}" +do { + [ -e "$LIST" ] || continue + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PUT ${KIBANA_URL}${SPACE_URL}/api/lists \ + -d @${LIST} \ + | jq .; +} & +done + +wait diff --git a/x-pack/plugins/lists/server/scripts/update_list_item.sh b/x-pack/plugins/lists/server/scripts/update_list_item.sh new file mode 100755 index 0000000000000..e3153bfd25b19 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/update_list_item.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default if no argument is specified +LISTS=(${@:-./lists/updates/list_ip_item.json}) + +# Example: ./patch_list.sh +# Example: ./patch_list.sh ./lists/updates/list_ip_item.json +for LIST in "${LISTS[@]}" +do { + [ -e "$LIST" ] || continue + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PATCH ${KIBANA_URL}${SPACE_URL}/api/lists/items \ + -d @${LIST} \ + | jq .; +} & +done + +wait diff --git a/x-pack/plugins/lists/server/services/items/buffer_lines.test.ts b/x-pack/plugins/lists/server/services/items/buffer_lines.test.ts new file mode 100644 index 0000000000000..946e1c240be31 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/buffer_lines.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TestReadable } from '../mocks/test_readable'; + +import { BufferLines } from './buffer_lines'; + +describe('buffer_lines', () => { + test('it can read a single line', done => { + const input = new TestReadable(); + input.push('line one\n'); + input.push(null); + const bufferedLine = new BufferLines({ input }); + let linesToTest: string[] = []; + bufferedLine.on('lines', (lines: string[]) => { + linesToTest = [...linesToTest, ...lines]; + }); + bufferedLine.on('close', () => { + expect(linesToTest).toEqual(['line one']); + done(); + }); + }); + + test('it can read two lines', done => { + const input = new TestReadable(); + input.push('line one\n'); + input.push('line two\n'); + input.push(null); + const bufferedLine = new BufferLines({ input }); + let linesToTest: string[] = []; + bufferedLine.on('lines', (lines: string[]) => { + linesToTest = [...linesToTest, ...lines]; + }); + bufferedLine.on('close', () => { + expect(linesToTest).toEqual(['line one', 'line two']); + done(); + }); + }); + + test('two identical lines are collapsed into just one line without duplicates', done => { + const input = new TestReadable(); + input.push('line one\n'); + input.push('line one\n'); + input.push(null); + const bufferedLine = new BufferLines({ input }); + let linesToTest: string[] = []; + bufferedLine.on('lines', (lines: string[]) => { + linesToTest = [...linesToTest, ...lines]; + }); + bufferedLine.on('close', () => { + expect(linesToTest).toEqual(['line one']); + done(); + }); + }); + + test('it can close out without writing any lines', done => { + const input = new TestReadable(); + input.push(null); + const bufferedLine = new BufferLines({ input }); + let linesToTest: string[] = []; + bufferedLine.on('lines', (lines: string[]) => { + linesToTest = [...linesToTest, ...lines]; + }); + bufferedLine.on('close', () => { + expect(linesToTest).toEqual([]); + done(); + }); + }); + + test('it can read 200 lines', done => { + const input = new TestReadable(); + const bufferedLine = new BufferLines({ input }); + let linesToTest: string[] = []; + const size200: string[] = new Array(200).fill(null).map((_, index) => `${index}\n`); + size200.forEach(element => input.push(element)); + input.push(null); + bufferedLine.on('lines', (lines: string[]) => { + linesToTest = [...linesToTest, ...lines]; + }); + bufferedLine.on('close', () => { + expect(linesToTest.length).toEqual(200); + done(); + }); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/buffer_lines.ts b/x-pack/plugins/lists/server/services/items/buffer_lines.ts new file mode 100644 index 0000000000000..fd8fe7077fd58 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/buffer_lines.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import readLine from 'readline'; +import { Readable } from 'stream'; + +const BUFFER_SIZE = 100; + +export class BufferLines extends Readable { + private set = new Set(); + constructor({ input }: { input: NodeJS.ReadableStream }) { + super({ encoding: 'utf-8' }); + const readline = readLine.createInterface({ + input, + }); + + readline.on('line', line => { + this.push(line); + }); + + readline.on('close', () => { + this.push(null); + }); + } + + public _read(): void { + // No operation but this is required to be implemented + } + + public push(line: string | null): boolean { + if (line == null) { + this.emit('lines', Array.from(this.set)); + this.set.clear(); + this.emit('close'); + return true; + } else { + this.set.add(line); + if (this.set.size > BUFFER_SIZE) { + this.emit('lines', Array.from(this.set)); + this.set.clear(); + return true; + } else { + return true; + } + } + } +} diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts new file mode 100644 index 0000000000000..b2bca241c468c --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + LIST_ITEM_ID, + LIST_ITEM_INDEX, + getCreateListItemOptionsMock, + getIndexESListItemMock, + getListItemResponseMock, +} from '../mocks'; + +import { createListItem } from './create_list_item'; + +describe('crete_list_item', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list item as expected with the id changed out for the elastic id', async () => { + const options = getCreateListItemOptionsMock(); + const listItem = await createListItem(options); + const expected = getListItemResponseMock(); + expected.id = 'elastic-id-123'; + expect(listItem).toEqual(expected); + }); + + test('It calls "callAsCurrentUser" with body, index, and listIndex', async () => { + const options = getCreateListItemOptionsMock(); + await createListItem(options); + const body = getIndexESListItemMock(); + const expected = { + body, + id: LIST_ITEM_ID, + index: LIST_ITEM_INDEX, + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('index', expected); + }); + + test('It returns an auto-generated id if id is sent in undefined', async () => { + const options = getCreateListItemOptionsMock(); + options.id = undefined; + const list = await createListItem(options); + const expected = getListItemResponseMock(); + expected.id = 'elastic-id-123'; + expect(list).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.ts b/x-pack/plugins/lists/server/services/items/create_list_item.ts new file mode 100644 index 0000000000000..da1e192bf2412 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/create_list_item.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { CreateDocumentResponse } from 'elasticsearch'; + +import { + IdOrUndefined, + IndexEsListItemSchema, + ListItemSchema, + MetaOrUndefined, + Type, +} from '../../../common/schemas'; +import { DataClient } from '../../types'; +import { transformListItemToElasticQuery } from '../utils'; + +export interface CreateListItemOptions { + id: IdOrUndefined; + listId: string; + type: Type; + value: string; + dataClient: DataClient; + listItemIndex: string; + user: string; + meta: MetaOrUndefined; + dateNow?: string; + tieBreaker?: string; +} + +export const createListItem = async ({ + id, + listId, + type, + value, + dataClient, + listItemIndex, + user, + meta, + dateNow, + tieBreaker, +}: CreateListItemOptions): Promise => { + const createdAt = dateNow ?? new Date().toISOString(); + const tieBreakerId = tieBreaker ?? uuid.v4(); + const baseBody = { + created_at: createdAt, + created_by: user, + list_id: listId, + meta, + tie_breaker_id: tieBreakerId, + updated_at: createdAt, + updated_by: user, + }; + const body: IndexEsListItemSchema = { + ...baseBody, + ...transformListItemToElasticQuery({ type, value }), + }; + + const response: CreateDocumentResponse = await dataClient.callAsCurrentUser('index', { + body, + id, + index: listItemIndex, + }); + + return { + id: response._id, + type, + value, + ...baseBody, + }; +}; diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts new file mode 100644 index 0000000000000..9263b975b20e7 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexEsListItemSchema } from '../../../common/schemas'; +import { + LIST_ITEM_INDEX, + TIE_BREAKERS, + VALUE_2, + getCreateListItemBulkOptionsMock, + getIndexESListItemMock, +} from '../mocks'; + +import { createListItemsBulk } from './create_list_items_bulk'; + +describe('crete_list_item_bulk', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('It calls "callAsCurrentUser" with body, index, and the bulk items', async () => { + const options = getCreateListItemBulkOptionsMock(); + await createListItemsBulk(options); + const firstRecord: IndexEsListItemSchema = getIndexESListItemMock(); + const secondRecord: IndexEsListItemSchema = getIndexESListItemMock(VALUE_2); + [firstRecord.tie_breaker_id, secondRecord.tie_breaker_id] = TIE_BREAKERS; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('bulk', { + body: [ + { create: { _index: LIST_ITEM_INDEX } }, + firstRecord, + { create: { _index: LIST_ITEM_INDEX } }, + secondRecord, + ], + index: LIST_ITEM_INDEX, + }); + }); + + test('It should not call the dataClient when the values are empty', async () => { + const options = getCreateListItemBulkOptionsMock(); + options.value = []; + expect(options.dataClient.callAsCurrentUser).not.toBeCalled(); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts new file mode 100644 index 0000000000000..7100a5f8eaabc --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; + +import { transformListItemToElasticQuery } from '../utils'; +import { DataClient } from '../../types'; +import { + CreateEsBulkTypeSchema, + IndexEsListItemSchema, + MetaOrUndefined, + Type, +} from '../../../common/schemas'; + +export interface CreateListItemsBulkOptions { + listId: string; + type: Type; + value: string[]; + dataClient: DataClient; + listItemIndex: string; + user: string; + meta: MetaOrUndefined; + dateNow?: string; + tieBreaker?: string[]; +} + +export const createListItemsBulk = async ({ + listId, + type, + value, + dataClient, + listItemIndex, + user, + meta, + dateNow, + tieBreaker, +}: CreateListItemsBulkOptions): Promise => { + // It causes errors if you try to add items to bulk that do not exist within ES + if (!value.length) { + return; + } + const body = value.reduce>( + (accum, singleValue, index) => { + const createdAt = dateNow ?? new Date().toISOString(); + const tieBreakerId = + tieBreaker != null && tieBreaker[index] != null ? tieBreaker[index] : uuid.v4(); + const elasticBody: IndexEsListItemSchema = { + created_at: createdAt, + created_by: user, + list_id: listId, + meta, + tie_breaker_id: tieBreakerId, + updated_at: createdAt, + updated_by: user, + ...transformListItemToElasticQuery({ type, value: singleValue }), + }; + const createBody: CreateEsBulkTypeSchema = { create: { _index: listItemIndex } }; + return [...accum, createBody, elasticBody]; + }, + [] + ); + + await dataClient.callAsCurrentUser('bulk', { + body, + index: listItemIndex, + }); +}; diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts b/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts new file mode 100644 index 0000000000000..795c579462b69 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LIST_ITEM_ID, LIST_ITEM_INDEX, getListItemResponseMock } from '../mocks'; +import { getDeleteListItemOptionsMock } from '../mocks/get_delete_list_item_options_mock'; + +import { getListItem } from './get_list_item'; +import { deleteListItem } from './delete_list_item'; + +jest.mock('./get_list_item', () => ({ + getListItem: jest.fn(), +})); + +describe('delete_list_item', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Delete returns a null if "getListItem" returns a null', async () => { + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteListItemOptionsMock(); + const deletedListItem = await deleteListItem(options); + expect(deletedListItem).toEqual(null); + }); + + test('Delete returns the same list item if a list item is returned from "getListItem"', async () => { + const listItem = getListItemResponseMock(); + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(listItem); + const options = getDeleteListItemOptionsMock(); + const deletedListItem = await deleteListItem(options); + expect(deletedListItem).toEqual(listItem); + }); + test('Delete calls "delete" if a list item is returned from "getListItem"', async () => { + const listItem = getListItemResponseMock(); + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(listItem); + const options = getDeleteListItemOptionsMock(); + await deleteListItem(options); + const deleteQuery = { + id: LIST_ITEM_ID, + index: LIST_ITEM_INDEX, + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('delete', deleteQuery); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item.ts b/x-pack/plugins/lists/server/services/items/delete_list_item.ts new file mode 100644 index 0000000000000..ffce2d3b2af81 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/delete_list_item.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Id, ListItemSchema } from '../../../common/schemas'; +import { DataClient } from '../../types'; + +import { getListItem } from '.'; + +export interface DeleteListItemOptions { + id: Id; + dataClient: DataClient; + listItemIndex: string; +} + +export const deleteListItem = async ({ + id, + dataClient, + listItemIndex, +}: DeleteListItemOptions): Promise => { + const listItem = await getListItem({ dataClient, id, listItemIndex }); + if (listItem == null) { + return null; + } else { + await dataClient.callAsCurrentUser('delete', { + id, + index: listItemIndex, + }); + } + return listItem; +}; diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts b/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts new file mode 100644 index 0000000000000..dee890445f9a3 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getDeleteListItemByValueOptionsMock, getListItemResponseMock } from '../mocks'; + +import { getListItemByValues } from './get_list_item_by_values'; +import { deleteListItemByValue } from './delete_list_item_by_value'; + +jest.mock('./get_list_item_by_values', () => ({ + getListItemByValues: jest.fn(), +})); + +describe('delete_list_item_by_value', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Delete returns a an empty array if the list items are also empty', async () => { + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); + const options = getDeleteListItemByValueOptionsMock(); + const deletedListItem = await deleteListItemByValue(options); + expect(deletedListItem).toEqual([]); + }); + + test('Delete returns the list item if a list item is returned from "getListByValues"', async () => { + const listItems = [getListItemResponseMock()]; + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce(listItems); + const options = getDeleteListItemByValueOptionsMock(); + const deletedListItem = await deleteListItemByValue(options); + expect(deletedListItem).toEqual(listItems); + }); + + test('Delete calls "deleteByQuery" if a list item is returned from "getListByValues"', async () => { + const listItems = [getListItemResponseMock()]; + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce(listItems); + const options = getDeleteListItemByValueOptionsMock(); + await deleteListItemByValue(options); + const deleteByQuery = { + body: { + query: { + bool: { + filter: [{ term: { list_id: 'some-list-id' } }, { terms: { ip: ['127.0.0.1'] } }], + }, + }, + }, + index: '.items', + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('deleteByQuery', deleteByQuery); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.ts b/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.ts new file mode 100644 index 0000000000000..f2f5ec3078e62 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ListItemArraySchema, Type } from '../../../common/schemas'; +import { getQueryFilterFromTypeValue } from '../utils'; +import { DataClient } from '../../types'; + +import { getListItemByValues } from './get_list_item_by_values'; + +export interface DeleteListItemByValueOptions { + listId: string; + type: Type; + value: string; + dataClient: DataClient; + listItemIndex: string; +} + +export const deleteListItemByValue = async ({ + listId, + value, + type, + dataClient, + listItemIndex, +}: DeleteListItemByValueOptions): Promise => { + const listItems = await getListItemByValues({ + dataClient, + listId, + listItemIndex, + type, + value: [value], + }); + const values = listItems.map(listItem => listItem.value); + const filter = getQueryFilterFromTypeValue({ + listId, + type, + value: values, + }); + await dataClient.callAsCurrentUser('deleteByQuery', { + body: { + query: { + bool: { + filter, + }, + }, + }, + index: listItemIndex, + }); + return listItems; +}; diff --git a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts new file mode 100644 index 0000000000000..937993f1d8f71 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LIST_ID, LIST_INDEX, getDataClientMock, getListItemResponseMock } from '../mocks'; +import { getSearchListItemMock } from '../mocks/get_search_list_item_mock'; + +import { getListItem } from './get_list_item'; + +describe('get_list_item', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list item as expected if the list item is found', async () => { + const data = getSearchListItemMock(); + const dataClient = getDataClientMock(data); + const list = await getListItem({ dataClient, id: LIST_ID, listItemIndex: LIST_INDEX }); + const expected = getListItemResponseMock(); + expect(list).toEqual(expected); + }); + + test('it returns null if the search is empty', async () => { + const data = getSearchListItemMock(); + data.hits.hits = []; + const dataClient = getDataClientMock(data); + const list = await getListItem({ dataClient, id: LIST_ID, listItemIndex: LIST_INDEX }); + expect(list).toEqual(null); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item.ts b/x-pack/plugins/lists/server/services/items/get_list_item.ts new file mode 100644 index 0000000000000..1c91b69801648 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { Id, ListItemSchema, SearchEsListItemSchema } from '../../../common/schemas'; +import { DataClient } from '../../types'; +import { deriveTypeFromItem, transformElasticToListItem } from '../utils'; + +interface GetListItemOptions { + id: Id; + dataClient: DataClient; + listItemIndex: string; +} + +export const getListItem = async ({ + id, + dataClient, + listItemIndex, +}: GetListItemOptions): Promise => { + const listItemES: SearchResponse = await dataClient.callAsCurrentUser( + 'search', + { + body: { + query: { + term: { + _id: id, + }, + }, + }, + ignoreUnavailable: true, + index: listItemIndex, + } + ); + + if (listItemES.hits.hits.length) { + const type = deriveTypeFromItem({ item: listItemES.hits.hits[0]._source }); + const listItems = transformElasticToListItem({ response: listItemES, type }); + return listItems[0]; + } else { + return null; + } +}; diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_value.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_value.test.ts new file mode 100644 index 0000000000000..d30b3c795550f --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_value.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getListItemByValueOptionsMocks, getListItemResponseMock } from '../mocks'; + +import { getListItemByValues } from './get_list_item_by_values'; +import { getListItemByValue } from './get_list_item_by_value'; + +jest.mock('./get_list_item_by_values', () => ({ + getListItemByValues: jest.fn(), +})); + +describe('get_list_by_value', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Calls get_list_item_by_values with its input', async () => { + const listItemMock = getListItemResponseMock(); + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([listItemMock]); + const options = getListItemByValueOptionsMocks(); + const listItem = await getListItemByValue(options); + const expected = getListItemResponseMock(); + expect(listItem).toEqual([expected]); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_value.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_value.ts new file mode 100644 index 0000000000000..a6efcbc0d3ffb --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_value.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ListItemArraySchema, Type } from '../../../common/schemas'; +import { DataClient } from '../../types'; + +import { getListItemByValues } from '.'; + +export interface GetListItemByValueOptions { + listId: string; + dataClient: DataClient; + listItemIndex: string; + type: Type; + value: string; +} + +export const getListItemByValue = async ({ + listId, + dataClient, + listItemIndex, + type, + value, +}: GetListItemByValueOptions): Promise => + getListItemByValues({ + dataClient, + listId, + listItemIndex, + type, + value: [value], + }); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts new file mode 100644 index 0000000000000..55b170487d95a --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE, VALUE_2, getDataClientMock } from '../mocks'; +import { getSearchListItemMock } from '../mocks/get_search_list_item_mock'; + +import { getListItemByValues } from './get_list_item_by_values'; + +describe('get_list_item_by_values', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Returns a an empty array if the ES query is also empty', async () => { + const data = getSearchListItemMock(); + data.hits.hits = []; + const dataClient = getDataClientMock(data); + const listItem = await getListItemByValues({ + dataClient, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [VALUE, VALUE_2], + }); + expect(listItem).toEqual([]); + }); + + test('Returns transformed list item if the data exists within ES', async () => { + const data = getSearchListItemMock(); + const dataClient = getDataClientMock(data); + const listItem = await getListItemByValues({ + dataClient, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [VALUE, VALUE_2], + }); + expect(listItem).toEqual([ + { + created_at: '2020-04-20T15:25:31.830Z', + created_by: 'some user', + id: 'some-list-item-id', + list_id: 'some-list-id', + meta: {}, + tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e', + type: 'ip', + updated_at: '2020-04-20T15:25:31.830Z', + updated_by: 'some user', + value: '127.0.0.1', + }, + ]); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts new file mode 100644 index 0000000000000..1e5c0b4a6655c --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { ListItemArraySchema, SearchEsListItemSchema, Type } from '../../../common/schemas'; +import { DataClient } from '../../types'; +import { getQueryFilterFromTypeValue, transformElasticToListItem } from '../utils'; + +export interface GetListItemByValuesOptions { + listId: string; + dataClient: DataClient; + listItemIndex: string; + type: Type; + value: string[]; +} + +export const getListItemByValues = async ({ + listId, + dataClient, + listItemIndex, + type, + value, +}: GetListItemByValuesOptions): Promise => { + const response: SearchResponse = await dataClient.callAsCurrentUser( + 'search', + { + body: { + query: { + bool: { + filter: getQueryFilterFromTypeValue({ listId, type, value }), + }, + }, + }, + ignoreUnavailable: true, + index: listItemIndex, + size: value.length, // This has a limit on the number which is 10k + } + ); + return transformElasticToListItem({ response, type }); +}; diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_index.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_index.test.ts new file mode 100644 index 0000000000000..0ea8320e966bd --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_index.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { httpServerMock } from 'src/core/server/mocks'; +import { KibanaRequest } from 'src/core/server'; + +import { getSpace } from '../utils'; + +import { getListItemIndex } from './get_list_item_index'; + +jest.mock('../utils', () => ({ + getSpace: jest.fn(), +})); + +describe('get_list_item_index', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Returns the list item index when there is a space', async () => { + ((getSpace as unknown) as jest.Mock).mockReturnValueOnce('test-space'); + const rawRequest = httpServerMock.createRawRequest({}); + const request = KibanaRequest.from(rawRequest); + const listIndex = getListItemIndex({ + listsItemsIndexName: 'lists-items-index', + request, + spaces: undefined, + }); + expect(listIndex).toEqual('lists-items-index-test-space'); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_index.ts b/x-pack/plugins/lists/server/services/items/get_list_item_index.ts new file mode 100644 index 0000000000000..c9f1bfd4d44e4 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KibanaRequest } from 'kibana/server'; + +import { SpacesServiceSetup } from '../../../../spaces/server'; +import { getSpace } from '../utils'; + +interface GetListItemIndexOptions { + spaces: SpacesServiceSetup | undefined | null; + request: KibanaRequest; + listsItemsIndexName: string; +} + +export const getListItemIndex = ({ + spaces, + request, + listsItemsIndexName, +}: GetListItemIndexOptions): string => `${listsItemsIndexName}-${getSpace({ request, spaces })}`; diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts new file mode 100644 index 0000000000000..9c85fa6ff0256 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getListItemTemplate } from './get_list_item_template'; + +jest.mock('./list_item_mappings.json', () => ({ + listMappings: {}, +})); + +describe('get_list_item_template', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list template with the string filled in', async () => { + const template = getListItemTemplate('some_index'); + expect(template).toEqual({ + index_patterns: ['some_index-*'], + mappings: { listMappings: {} }, + settings: { index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } } }, + }); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts new file mode 100644 index 0000000000000..95f4a09b40648 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import listsItemsMappings from './list_item_mappings.json'; + +export const getListItemTemplate = (index: string): Record => { + const template = { + index_patterns: [`${index}-*`], + mappings: listsItemsMappings, + settings: { + index: { + lifecycle: { + name: index, + rollover_alias: index, + }, + }, + }, + }; + return template; +}; diff --git a/x-pack/plugins/lists/server/services/items/index.ts b/x-pack/plugins/lists/server/services/items/index.ts new file mode 100644 index 0000000000000..ee1d83fabca31 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './buffer_lines'; +export * from './create_list_item'; +export * from './create_list_items_bulk'; +export * from './delete_list_item_by_value'; +export * from './get_list_item_by_value'; +export * from './get_list_item'; +export * from './get_list_item_by_values'; +export * from './update_list_item'; +export * from './write_lines_to_bulk_list_items'; +export * from './write_list_items_to_stream'; +export * from './get_list_item_template'; +export * from './delete_list_item'; +export * from './get_list_item_index'; diff --git a/x-pack/plugins/lists/server/services/items/list_item_mappings.json b/x-pack/plugins/lists/server/services/items/list_item_mappings.json new file mode 100644 index 0000000000000..ca69c26df52b5 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/list_item_mappings.json @@ -0,0 +1,33 @@ +{ + "dynamic": "strict", + "properties": { + "tie_breaker_id": { + "type": "keyword" + }, + "list_id": { + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "keyword": { + "type": "keyword" + }, + "meta": { + "enabled": "false", + "type": "object" + }, + "created_at": { + "type": "date" + }, + "updated_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + } + } +} diff --git a/x-pack/plugins/lists/server/services/items/list_item_policy.json b/x-pack/plugins/lists/server/services/items/list_item_policy.json new file mode 100644 index 0000000000000..a4c84f73e7896 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/list_item_policy.json @@ -0,0 +1,14 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50gb" + } + } + } + } + } +} diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts new file mode 100644 index 0000000000000..4ef4110bc0742 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getListItemResponseMock, getUpdateListItemOptionsMock } from '../mocks'; + +import { updateListItem } from './update_list_item'; +import { getListItem } from './get_list_item'; + +jest.mock('./get_list_item', () => ({ + getListItem: jest.fn(), +})); + +describe('update_list_item', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list item as expected with the id changed out for the elastic id when there is a list item to update', async () => { + const list = getListItemResponseMock(); + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(list); + const options = getUpdateListItemOptionsMock(); + const updatedList = await updateListItem(options); + const expected = getListItemResponseMock(); + expected.id = 'elastic-id-123'; + expect(updatedList).toEqual(expected); + }); + + test('it returns null when there is not a list item to update', async () => { + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(null); + const options = getUpdateListItemOptionsMock(); + const updatedList = await updateListItem(options); + expect(updatedList).toEqual(null); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts new file mode 100644 index 0000000000000..ce4f8125d77af --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateDocumentResponse } from 'elasticsearch'; + +import { + Id, + ListItemSchema, + MetaOrUndefined, + UpdateEsListItemSchema, +} from '../../../common/schemas'; +import { transformListItemToElasticQuery } from '../utils'; +import { DataClient } from '../../types'; + +import { getListItem } from './get_list_item'; + +export interface UpdateListItemOptions { + id: Id; + value: string | null | undefined; + dataClient: DataClient; + listItemIndex: string; + user: string; + meta: MetaOrUndefined; + dateNow?: string; +} + +export const updateListItem = async ({ + id, + value, + dataClient, + listItemIndex, + user, + meta, + dateNow, +}: UpdateListItemOptions): Promise => { + const updatedAt = dateNow ?? new Date().toISOString(); + const listItem = await getListItem({ dataClient, id, listItemIndex }); + if (listItem == null) { + return null; + } else { + const doc: UpdateEsListItemSchema = { + meta, + updated_at: updatedAt, + updated_by: user, + ...transformListItemToElasticQuery({ type: listItem.type, value: value ?? listItem.value }), + }; + + const response: CreateDocumentResponse = await dataClient.callAsCurrentUser('update', { + body: { + doc, + }, + id: listItem.id, + index: listItemIndex, + }); + return { + created_at: listItem.created_at, + created_by: listItem.created_by, + id: response._id, + list_id: listItem.list_id, + meta: meta ?? listItem.meta, + tie_breaker_id: listItem.tie_breaker_id, + type: listItem.type, + updated_at: updatedAt, + updated_by: listItem.updated_by, + value: value ?? listItem.value, + }; + } +}; diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts new file mode 100644 index 0000000000000..f064543f1ec93 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + getImportListItemsToStreamOptionsMock, + getListItemResponseMock, + getWriteBufferToItemsOptionsMock, +} from '../mocks'; + +import { + LinesResult, + importListItemsToStream, + writeBufferToItems, +} from './write_lines_to_bulk_list_items'; + +import { getListItemByValues } from '.'; + +jest.mock('./get_list_item_by_values', () => ({ + getListItemByValues: jest.fn(), +})); + +describe('write_lines_to_bulk_list_items', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('importListItemsToStream', () => { + test('It imports a set of items to a write buffer by calling "getListItemByValues" with an empty buffer', async () => { + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); + const options = getImportListItemsToStreamOptionsMock(); + const promise = importListItemsToStream(options); + options.stream.push(null); + await promise; + expect(getListItemByValues).toBeCalledWith(expect.objectContaining({ value: [] })); + }); + + test('It imports a set of items to a write buffer by calling "getListItemByValues" with a single value given', async () => { + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); + const options = getImportListItemsToStreamOptionsMock(); + const promise = importListItemsToStream(options); + options.stream.push('127.0.0.1\n'); + options.stream.push(null); + await promise; + expect(getListItemByValues).toBeCalledWith(expect.objectContaining({ value: ['127.0.0.1'] })); + }); + + test('It imports a set of items to a write buffer by calling "getListItemByValues" with two values given', async () => { + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); + const options = getImportListItemsToStreamOptionsMock(); + const promise = importListItemsToStream(options); + options.stream.push('127.0.0.1\n'); + options.stream.push('127.0.0.2\n'); + options.stream.push(null); + await promise; + expect(getListItemByValues).toBeCalledWith( + expect.objectContaining({ value: ['127.0.0.1', '127.0.0.2'] }) + ); + }); + }); + + describe('writeBufferToItems', () => { + test('It returns no duplicates and no lines processed when given empty items', async () => { + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); + const options = getWriteBufferToItemsOptionsMock(); + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 0, + linesProcessed: 0, + }; + expect(linesResult).toEqual(expected); + }); + + test('It returns no lines processed when given items but no buffer', async () => { + const data = getListItemResponseMock(); + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); + const options = getWriteBufferToItemsOptionsMock(); + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 0, + linesProcessed: 0, + }; + expect(linesResult).toEqual(expected); + }); + + test('It returns 1 lines processed when given a buffer item that is not a duplicate', async () => { + const data = getListItemResponseMock(); + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); + const options = getWriteBufferToItemsOptionsMock(); + options.buffer = ['255.255.255.255']; + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 0, + linesProcessed: 1, + }; + expect(linesResult).toEqual(expected); + }); + + test('It filters a duplicate value out and reports it as a duplicate', async () => { + const data = getListItemResponseMock(); + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); + const options = getWriteBufferToItemsOptionsMock(); + options.buffer = [data.value]; + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 1, + linesProcessed: 0, + }; + expect(linesResult).toEqual(expected); + }); + + test('It filters a duplicate value out and reports it as a duplicate and processing a second value as not a duplicate', async () => { + const data = getListItemResponseMock(); + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); + const options = getWriteBufferToItemsOptionsMock(); + options.buffer = ['255.255.255.255', data.value]; + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 1, + linesProcessed: 1, + }; + expect(linesResult).toEqual(expected); + }); + + test('It filters a duplicate value out and reports it as a duplicate and processing two other values', async () => { + const data = getListItemResponseMock(); + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); + const options = getWriteBufferToItemsOptionsMock(); + options.buffer = ['255.255.255.255', '192.168.0.1', data.value]; + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 1, + linesProcessed: 2, + }; + expect(linesResult).toEqual(expected); + }); + + test('It filters two duplicate values out and reports processes a single value', async () => { + const dataItem1 = getListItemResponseMock(); + dataItem1.value = '127.0.0.1'; + const dataItem2 = getListItemResponseMock(); + dataItem2.value = '127.0.0.2'; + ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([dataItem1, dataItem2]); + const options = getWriteBufferToItemsOptionsMock(); + options.buffer = [dataItem1.value, dataItem2.value, '192.168.0.0.1']; + const linesResult = await writeBufferToItems(options); + const expected: LinesResult = { + duplicatesFound: 2, + linesProcessed: 1, + }; + expect(linesResult).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts new file mode 100644 index 0000000000000..1fe1023e28ab9 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Readable } from 'stream'; + +import { MetaOrUndefined, Type } from '../../../common/schemas'; +import { DataClient } from '../../types'; + +import { BufferLines } from './buffer_lines'; +import { getListItemByValues } from './get_list_item_by_values'; +import { createListItemsBulk } from './create_list_items_bulk'; + +export interface ImportListItemsToStreamOptions { + listId: string; + stream: Readable; + dataClient: DataClient; + listItemIndex: string; + type: Type; + user: string; + meta: MetaOrUndefined; +} + +export const importListItemsToStream = ({ + listId, + stream, + dataClient, + listItemIndex, + type, + user, + meta, +}: ImportListItemsToStreamOptions): Promise => { + return new Promise(resolve => { + const readBuffer = new BufferLines({ input: stream }); + readBuffer.on('lines', async (lines: string[]) => { + await writeBufferToItems({ + buffer: lines, + dataClient, + listId, + listItemIndex, + meta, + type, + user, + }); + }); + + readBuffer.on('close', () => { + resolve(); + }); + }); +}; + +export interface WriteBufferToItemsOptions { + listId: string; + dataClient: DataClient; + listItemIndex: string; + buffer: string[]; + type: Type; + user: string; + meta: MetaOrUndefined; +} + +export interface LinesResult { + linesProcessed: number; + duplicatesFound: number; +} + +export const writeBufferToItems = async ({ + listId, + dataClient, + listItemIndex, + buffer, + type, + user, + meta, +}: WriteBufferToItemsOptions): Promise => { + const items = await getListItemByValues({ + dataClient, + listId, + listItemIndex, + type, + value: buffer, + }); + const duplicatesRemoved = buffer.filter( + bufferedValue => !items.some(item => item.value === bufferedValue) + ); + const linesProcessed = duplicatesRemoved.length; + const duplicatesFound = buffer.length - duplicatesRemoved.length; + await createListItemsBulk({ + dataClient, + listId, + listItemIndex, + meta, + type, + user, + value: duplicatesRemoved, + }); + return { duplicatesFound, linesProcessed }; +}; diff --git a/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.test.ts b/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.test.ts new file mode 100644 index 0000000000000..63e9aeb61bad0 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.test.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + LIST_ID, + LIST_ITEM_INDEX, + getDataClientMock, + getExportListItemsToStreamOptionsMock, + getResponseOptionsMock, + getWriteNextResponseOptions, + getWriteResponseHitsToStreamOptionsMock, +} from '../mocks'; +import { getSearchListItemMock } from '../mocks/get_search_list_item_mock'; + +import { + exportListItemsToStream, + getResponse, + getSearchAfterFromResponse, + writeNextResponse, + writeResponseHitsToStream, +} from '.'; + +describe('write_list_items_to_stream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('exportListItemsToStream', () => { + test('It exports empty list items to the stream as an empty array', done => { + const options = getExportListItemsToStreamOptionsMock(); + const firstResponse = getSearchListItemMock(); + firstResponse.hits.hits = []; + options.dataClient = getDataClientMock(firstResponse); + exportListItemsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.on('finish', () => { + expect(chunks).toEqual([]); + done(); + }); + }); + + test('It exports single list item to the stream', done => { + const options = getExportListItemsToStreamOptionsMock(); + exportListItemsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.on('finish', () => { + expect(chunks).toEqual(['127.0.0.1']); + done(); + }); + }); + + test('It exports two list items to the stream', done => { + const options = getExportListItemsToStreamOptionsMock(); + const firstResponse = getSearchListItemMock(); + const secondResponse = getSearchListItemMock(); + firstResponse.hits.hits = [...firstResponse.hits.hits, ...secondResponse.hits.hits]; + options.dataClient = getDataClientMock(firstResponse); + exportListItemsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.on('finish', () => { + expect(chunks).toEqual(['127.0.0.1', '127.0.0.1']); + done(); + }); + }); + + test('It exports two list items to the stream with two separate calls', done => { + const options = getExportListItemsToStreamOptionsMock(); + + const firstResponse = getSearchListItemMock(); + firstResponse.hits.hits[0].sort = ['some-sort-value']; + const secondResponse = getSearchListItemMock(); + secondResponse.hits.hits[0]._source.ip = '255.255.255.255'; + + const jestCalls = jest.fn().mockResolvedValueOnce(firstResponse); + jestCalls.mockResolvedValueOnce(secondResponse); + + const dataClient = getDataClientMock(firstResponse); + dataClient.callAsCurrentUser = jestCalls; + options.dataClient = dataClient; + + exportListItemsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.on('finish', () => { + expect(chunks).toEqual(['127.0.0.1', '255.255.255.255']); + done(); + }); + }); + }); + + describe('writeNextResponse', () => { + test('It returns an empty searchAfter response when there is no sort defined', async () => { + const options = getWriteNextResponseOptions(); + const searchAfter = await writeNextResponse(options); + expect(searchAfter).toEqual(undefined); + }); + + test('It returns a searchAfter response when there is a sort defined', async () => { + const listItem = getSearchListItemMock(); + listItem.hits.hits[0].sort = ['sort-value-1']; + const options = getWriteNextResponseOptions(); + options.dataClient = getDataClientMock(listItem); + const searchAfter = await writeNextResponse(options); + expect(searchAfter).toEqual(['sort-value-1']); + }); + + test('It returns a searchAfter response of undefined when the response is empty', async () => { + const listItem = getSearchListItemMock(); + listItem.hits.hits = []; + const options = getWriteNextResponseOptions(); + options.dataClient = getDataClientMock(listItem); + const searchAfter = await writeNextResponse(options); + expect(searchAfter).toEqual(undefined); + }); + }); + + describe('getSearchAfterFromResponse', () => { + test('It returns undefined if the hits array is empty', () => { + const response = getSearchListItemMock(); + response.hits.hits = []; + const searchAfter = getSearchAfterFromResponse({ response }); + expect(searchAfter).toEqual(undefined); + }); + + test('It returns undefined if the hits array does not have a sort', () => { + const response = getSearchListItemMock(); + response.hits.hits[0].sort = undefined; + const searchAfter = getSearchAfterFromResponse({ response }); + expect(searchAfter).toEqual(undefined); + }); + + test('It returns a sort of a single array if that single item exists', () => { + const response = getSearchListItemMock(); + response.hits.hits[0].sort = ['sort-value-1', 'sort-value-2']; + const searchAfter = getSearchAfterFromResponse({ response }); + expect(searchAfter).toEqual(['sort-value-1', 'sort-value-2']); + }); + + test('It returns a sort of the last array element of size 2', () => { + const response = getSearchListItemMock(); + const response2 = getSearchListItemMock(); + response2.hits.hits[0].sort = ['sort-value']; + response.hits.hits = [...response.hits.hits, ...response2.hits.hits]; + const searchAfter = getSearchAfterFromResponse({ response }); + expect(searchAfter).toEqual(['sort-value']); + }); + }); + + describe('getResponse', () => { + test('It returns a simple response with the default size of 100', async () => { + const options = getResponseOptionsMock(); + options.searchAfter = ['string 1', 'string 2']; + await getResponse(options); + const expected = { + body: { + query: { term: { list_id: LIST_ID } }, + search_after: ['string 1', 'string 2'], + sort: [{ tie_breaker_id: 'asc' }], + }, + ignoreUnavailable: true, + index: LIST_ITEM_INDEX, + size: 100, + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('search', expected); + }); + + test('It returns a simple response with expected values and size changed', async () => { + const options = getResponseOptionsMock(); + options.searchAfter = ['string 1', 'string 2']; + options.size = 33; + await getResponse(options); + const expected = { + body: { + query: { term: { list_id: LIST_ID } }, + search_after: ['string 1', 'string 2'], + sort: [{ tie_breaker_id: 'asc' }], + }, + ignoreUnavailable: true, + index: LIST_ITEM_INDEX, + size: 33, + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('search', expected); + }); + }); + + describe('writeResponseHitsToStream', () => { + test('it will push into the stream the mock response', done => { + const options = getWriteResponseHitsToStreamOptionsMock(); + writeResponseHitsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.end(() => { + expect(chunks).toEqual(['127.0.0.1']); + done(); + }); + }); + + test('it will push into the stream an empty mock response', done => { + const options = getWriteResponseHitsToStreamOptionsMock(); + options.response.hits.hits = []; + writeResponseHitsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.on('finish', () => { + expect(chunks).toEqual([]); + done(); + }); + options.stream.end(); + }); + + test('it will push into the stream 2 mock responses', done => { + const options = getWriteResponseHitsToStreamOptionsMock(); + const secondResponse = getSearchListItemMock(); + options.response.hits.hits = [...options.response.hits.hits, ...secondResponse.hits.hits]; + writeResponseHitsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.end(() => { + expect(chunks).toEqual(['127.0.0.1', '127.0.0.1']); + done(); + }); + }); + + test('it will push an additional string given to it such as a new line character', done => { + const options = getWriteResponseHitsToStreamOptionsMock(); + const secondResponse = getSearchListItemMock(); + options.response.hits.hits = [...options.response.hits.hits, ...secondResponse.hits.hits]; + options.stringToAppend = '\n'; + writeResponseHitsToStream(options); + + let chunks: string[] = []; + options.stream.on('data', (chunk: Buffer) => { + chunks = [...chunks, chunk.toString()]; + }); + + options.stream.end(() => { + expect(chunks).toEqual(['127.0.0.1\n', '127.0.0.1\n']); + done(); + }); + }); + + test('it will throw an exception with a status code if the hit_source is not a data type we expect', () => { + const options = getWriteResponseHitsToStreamOptionsMock(); + options.response.hits.hits[0]._source.ip = undefined; + options.response.hits.hits[0]._source.keyword = undefined; + const expected = `Encountered an error where hit._source was an unexpected type: ${JSON.stringify( + options.response.hits.hits[0]._source + )}`; + expect(() => writeResponseHitsToStream(options)).toThrow(expected); + }); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts b/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts new file mode 100644 index 0000000000000..0e0ae7b924e17 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { PassThrough } from 'stream'; + +import { SearchResponse } from 'elasticsearch'; + +import { SearchEsListItemSchema } from '../../../common/schemas'; +import { DataClient } from '../../types'; +import { ErrorWithStatusCode } from '../../error_with_status_code'; + +/** + * How many results to page through from the network at a time + * using search_after + */ +export const SIZE = 100; + +export interface ExportListItemsToStreamOptions { + listId: string; + dataClient: DataClient; + listItemIndex: string; + stream: PassThrough; + stringToAppend: string | null | undefined; +} + +export const exportListItemsToStream = ({ + listId, + dataClient, + stream, + listItemIndex, + stringToAppend, +}: ExportListItemsToStreamOptions): void => { + // Use a timeout to start the reading process on the next tick. + // and prevent the async await from bubbling up to the caller + setTimeout(async () => { + let searchAfter = await writeNextResponse({ + dataClient, + listId, + listItemIndex, + searchAfter: undefined, + stream, + stringToAppend, + }); + while (searchAfter != null) { + searchAfter = await writeNextResponse({ + dataClient, + listId, + listItemIndex, + searchAfter, + stream, + stringToAppend, + }); + } + stream.end(); + }); +}; + +export interface WriteNextResponseOptions { + listId: string; + dataClient: DataClient; + listItemIndex: string; + stream: PassThrough; + searchAfter: string[] | undefined; + stringToAppend: string | null | undefined; +} + +export const writeNextResponse = async ({ + listId, + dataClient, + stream, + listItemIndex, + searchAfter, + stringToAppend, +}: WriteNextResponseOptions): Promise => { + const response = await getResponse({ + dataClient, + listId, + listItemIndex, + searchAfter, + }); + + if (response.hits.hits.length) { + writeResponseHitsToStream({ response, stream, stringToAppend }); + return getSearchAfterFromResponse({ response }); + } else { + return undefined; + } +}; + +export const getSearchAfterFromResponse = ({ + response, +}: { + response: SearchResponse; +}): string[] | undefined => + response.hits.hits.length > 0 + ? response.hits.hits[response.hits.hits.length - 1].sort + : undefined; + +export interface GetResponseOptions { + dataClient: DataClient; + listId: string; + searchAfter: undefined | string[]; + listItemIndex: string; + size?: number; +} + +export const getResponse = async ({ + dataClient, + searchAfter, + listId, + listItemIndex, + size = SIZE, +}: GetResponseOptions): Promise> => { + return dataClient.callAsCurrentUser('search', { + body: { + query: { + term: { + list_id: listId, + }, + }, + search_after: searchAfter, + sort: [{ tie_breaker_id: 'asc' }], + }, + ignoreUnavailable: true, + index: listItemIndex, + size, + }); +}; + +export interface WriteResponseHitsToStreamOptions { + response: SearchResponse; + stream: PassThrough; + stringToAppend: string | null | undefined; +} + +export const writeResponseHitsToStream = ({ + response, + stream, + stringToAppend, +}: WriteResponseHitsToStreamOptions): void => { + const stringToAppendOrEmpty = stringToAppend ?? ''; + + response.hits.hits.forEach(hit => { + if (hit._source.ip != null) { + stream.push(`${hit._source.ip}${stringToAppendOrEmpty}`); + } else if (hit._source.keyword != null) { + stream.push(`${hit._source.keyword}${stringToAppendOrEmpty}`); + } else { + throw new ErrorWithStatusCode( + `Encountered an error where hit._source was an unexpected type: ${JSON.stringify( + hit._source + )}`, + 400 + ); + } + }); +}; diff --git a/x-pack/plugins/lists/server/services/lists/client.ts b/x-pack/plugins/lists/server/services/lists/client.ts new file mode 100644 index 0000000000000..32578fc739f26 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/client.ts @@ -0,0 +1,465 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KibanaRequest, ScopedClusterClient } from 'src/core/server'; + +import { SecurityPluginSetup } from '../../../../security/server'; +import { SpacesServiceSetup } from '../../../../spaces/server'; +import { ListItemArraySchema, ListItemSchema, ListSchema } from '../../../common/schemas'; +import { ConfigType } from '../../config'; +import { + createList, + deleteList, + getList, + getListIndex, + getListTemplate, + updateList, +} from '../../services/lists'; +import { + createListItem, + deleteListItem, + deleteListItemByValue, + exportListItemsToStream, + getListItem, + getListItemByValue, + getListItemByValues, + getListItemIndex, + getListItemTemplate, + importListItemsToStream, + updateListItem, +} from '../../services/items'; +import { getUser } from '../../services/utils'; +import { + createBootstrapIndex, + deleteAllIndex, + deletePolicy, + deleteTemplate, + getIndexExists, + getPolicyExists, + getTemplateExists, + setPolicy, + setTemplate, +} from '../../siem_server_deps'; +import listsItemsPolicy from '../items/list_item_policy.json'; + +import listPolicy from './list_policy.json'; +import { + ConstructorOptions, + CreateListIfItDoesNotExistOptions, + CreateListItemOptions, + CreateListOptions, + DeleteListItemByValueOptions, + DeleteListItemOptions, + DeleteListOptions, + ExportListItemsToStreamOptions, + GetListItemByValueOptions, + GetListItemOptions, + GetListItemsByValueOptions, + GetListOptions, + ImportListItemsToStreamOptions, + UpdateListItemOptions, + UpdateListOptions, +} from './client_types'; + +// TODO: Consider an interface and a factory +export class ListClient { + private readonly spaces: SpacesServiceSetup | undefined | null; + private readonly config: ConfigType; + private readonly dataClient: Pick< + ScopedClusterClient, + 'callAsCurrentUser' | 'callAsInternalUser' + >; + private readonly request: KibanaRequest; + private readonly security: SecurityPluginSetup; + + constructor({ request, spaces, config, dataClient, security }: ConstructorOptions) { + this.request = request; + this.spaces = spaces; + this.config = config; + this.dataClient = dataClient; + this.security = security; + } + + public getListIndex = (): string => { + const { + spaces, + request, + config: { listIndex: listsIndexName }, + } = this; + return getListIndex({ listsIndexName, request, spaces }); + }; + + public getListItemIndex = (): string => { + const { + spaces, + request, + config: { listItemIndex: listsItemsIndexName }, + } = this; + return getListItemIndex({ listsItemsIndexName, request, spaces }); + }; + + public getList = async ({ id }: GetListOptions): Promise => { + const { dataClient } = this; + const listIndex = this.getListIndex(); + return getList({ dataClient, id, listIndex }); + }; + + public createList = async ({ + id, + name, + description, + type, + meta, + }: CreateListOptions): Promise => { + const { dataClient, security, request } = this; + const listIndex = this.getListIndex(); + const user = getUser({ request, security }); + return createList({ dataClient, description, id, listIndex, meta, name, type, user }); + }; + + public createListIfItDoesNotExist = async ({ + id, + name, + description, + type, + meta, + }: CreateListIfItDoesNotExistOptions): Promise => { + const list = await this.getList({ id }); + if (list == null) { + return this.createList({ description, id, meta, name, type }); + } else { + return list; + } + }; + + public getListIndexExists = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return getIndexExists(callAsCurrentUser, listIndex); + }; + + public getListItemIndexExists = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return getIndexExists(callAsCurrentUser, listItemIndex); + }; + + public createListBootStrapIndex = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return createBootstrapIndex(callAsCurrentUser, listIndex); + }; + + public createListItemBootStrapIndex = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return createBootstrapIndex(callAsCurrentUser, listItemIndex); + }; + + public getListPolicyExists = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return getPolicyExists(callAsCurrentUser, listIndex); + }; + + public getListItemPolicyExists = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listsItemIndex = this.getListItemIndex(); + return getPolicyExists(callAsCurrentUser, listsItemIndex); + }; + + public getListTemplateExists = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return getTemplateExists(callAsCurrentUser, listIndex); + }; + + public getListItemTemplateExists = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return getTemplateExists(callAsCurrentUser, listItemIndex); + }; + + public getListTemplate = (): Record => { + const listIndex = this.getListIndex(); + return getListTemplate(listIndex); + }; + + public getListItemTemplate = (): Record => { + const listItemIndex = this.getListItemIndex(); + return getListItemTemplate(listItemIndex); + }; + + public setListTemplate = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const template = this.getListTemplate(); + const listIndex = this.getListIndex(); + return setTemplate(callAsCurrentUser, listIndex, template); + }; + + public setListItemTemplate = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const template = this.getListItemTemplate(); + const listItemIndex = this.getListItemIndex(); + return setTemplate(callAsCurrentUser, listItemIndex, template); + }; + + public setListPolicy = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return setPolicy(callAsCurrentUser, listIndex, listPolicy); + }; + + public setListItemPolicy = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return setPolicy(callAsCurrentUser, listItemIndex, listsItemsPolicy); + }; + + public deleteListIndex = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return deleteAllIndex(callAsCurrentUser, `${listIndex}-*`); + }; + + public deleteListItemIndex = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return deleteAllIndex(callAsCurrentUser, `${listItemIndex}-*`); + }; + + public deleteListPolicy = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return deletePolicy(callAsCurrentUser, listIndex); + }; + + public deleteListItemPolicy = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return deletePolicy(callAsCurrentUser, listItemIndex); + }; + + public deleteListTemplate = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listIndex = this.getListIndex(); + return deleteTemplate(callAsCurrentUser, listIndex); + }; + + public deleteListItemTemplate = async (): Promise => { + const { + dataClient: { callAsCurrentUser }, + } = this; + const listItemIndex = this.getListItemIndex(); + return deleteTemplate(callAsCurrentUser, listItemIndex); + }; + + public deleteListItem = async ({ id }: DeleteListItemOptions): Promise => { + const { dataClient } = this; + const listItemIndex = this.getListItemIndex(); + return deleteListItem({ dataClient, id, listItemIndex }); + }; + + public deleteListItemByValue = async ({ + listId, + value, + type, + }: DeleteListItemByValueOptions): Promise => { + const { dataClient } = this; + const listItemIndex = this.getListItemIndex(); + return deleteListItemByValue({ + dataClient, + listId, + listItemIndex, + type, + value, + }); + }; + + public deleteList = async ({ id }: DeleteListOptions): Promise => { + const { dataClient } = this; + const listIndex = this.getListIndex(); + const listItemIndex = this.getListItemIndex(); + return deleteList({ + dataClient, + id, + listIndex, + listItemIndex, + }); + }; + + public exportListItemsToStream = ({ + stringToAppend, + listId, + stream, + }: ExportListItemsToStreamOptions): void => { + const { dataClient } = this; + const listItemIndex = this.getListItemIndex(); + exportListItemsToStream({ + dataClient, + listId, + listItemIndex, + stream, + stringToAppend, + }); + }; + + public importListItemsToStream = async ({ + type, + listId, + stream, + meta, + }: ImportListItemsToStreamOptions): Promise => { + const { dataClient, security, request } = this; + const listItemIndex = this.getListItemIndex(); + const user = getUser({ request, security }); + return importListItemsToStream({ + dataClient, + listId, + listItemIndex, + meta, + stream, + type, + user, + }); + }; + + public getListItemByValue = async ({ + listId, + value, + type, + }: GetListItemByValueOptions): Promise => { + const { dataClient } = this; + const listItemIndex = this.getListItemIndex(); + return getListItemByValue({ + dataClient, + listId, + listItemIndex, + type, + value, + }); + }; + + public createListItem = async ({ + id, + listId, + value, + type, + meta, + }: CreateListItemOptions): Promise => { + const { dataClient, security, request } = this; + const listItemIndex = this.getListItemIndex(); + const user = getUser({ request, security }); + return createListItem({ + dataClient, + id, + listId, + listItemIndex, + meta, + type, + user, + value, + }); + }; + + public updateListItem = async ({ + id, + value, + meta, + }: UpdateListItemOptions): Promise => { + const { dataClient, security, request } = this; + const user = getUser({ request, security }); + const listItemIndex = this.getListItemIndex(); + return updateListItem({ + dataClient, + id, + listItemIndex, + meta, + user, + value, + }); + }; + + public updateList = async ({ + id, + name, + description, + meta, + }: UpdateListOptions): Promise => { + const { dataClient, security, request } = this; + const user = getUser({ request, security }); + const listIndex = this.getListIndex(); + return updateList({ + dataClient, + description, + id, + listIndex, + meta, + name, + user, + }); + }; + + public getListItem = async ({ id }: GetListItemOptions): Promise => { + const { dataClient } = this; + const listItemIndex = this.getListItemIndex(); + return getListItem({ + dataClient, + id, + listItemIndex, + }); + }; + + public getListItemByValues = async ({ + type, + listId, + value, + }: GetListItemsByValueOptions): Promise => { + const { dataClient } = this; + const listItemIndex = this.getListItemIndex(); + return getListItemByValues({ + dataClient, + listId, + listItemIndex, + type, + value, + }); + }; +} diff --git a/x-pack/plugins/lists/server/services/lists/client_types.ts b/x-pack/plugins/lists/server/services/lists/client_types.ts new file mode 100644 index 0000000000000..c3b6a484d8787 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/client_types.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { PassThrough, Readable } from 'stream'; + +import { KibanaRequest } from 'kibana/server'; + +import { SecurityPluginSetup } from '../../../../security/server'; +import { SpacesServiceSetup } from '../../../../spaces/server'; +import { + Description, + DescriptionOrUndefined, + Id, + IdOrUndefined, + MetaOrUndefined, + Name, + NameOrUndefined, + Type, +} from '../../../common/schemas'; +import { ConfigType } from '../../config'; +import { DataClient } from '../../types'; + +export interface ConstructorOptions { + config: ConfigType; + dataClient: DataClient; + request: KibanaRequest; + spaces: SpacesServiceSetup | undefined | null; + security: SecurityPluginSetup; +} + +export interface GetListOptions { + id: Id; +} + +export interface DeleteListOptions { + id: Id; +} + +export interface DeleteListItemOptions { + id: Id; +} + +export interface CreateListOptions { + id: IdOrUndefined; + name: Name; + description: Description; + type: Type; + meta: MetaOrUndefined; +} + +export interface CreateListIfItDoesNotExistOptions { + id: Id; + name: Name; + description: Description; + type: Type; + meta: MetaOrUndefined; +} + +export interface DeleteListItemByValueOptions { + listId: string; + value: string; + type: Type; +} + +export interface GetListItemByValueOptions { + listId: string; + value: string; + type: Type; +} + +export interface ExportListItemsToStreamOptions { + stringToAppend: string | null | undefined; + listId: string; + stream: PassThrough; +} + +export interface ImportListItemsToStreamOptions { + listId: string; + type: Type; + stream: Readable; + meta: MetaOrUndefined; +} + +export interface CreateListItemOptions { + id: IdOrUndefined; + listId: string; + type: Type; + value: string; + meta: MetaOrUndefined; +} + +export interface UpdateListItemOptions { + id: Id; + value: string | null | undefined; + meta: MetaOrUndefined; +} + +export interface UpdateListOptions { + id: Id; + name: NameOrUndefined; + description: DescriptionOrUndefined; + meta: MetaOrUndefined; +} + +export interface GetListItemOptions { + id: Id; +} + +export interface GetListItemsByValueOptions { + type: Type; + listId: string; + value: string[]; +} diff --git a/x-pack/plugins/lists/server/services/lists/create_list.test.ts b/x-pack/plugins/lists/server/services/lists/create_list.test.ts new file mode 100644 index 0000000000000..d6ba435155c60 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/create_list.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + LIST_ID, + LIST_INDEX, + getCreateListOptionsMock, + getIndexESListMock, + getListResponseMock, +} from '../mocks'; + +import { createList } from './create_list'; + +describe('crete_list', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list as expected with the id changed out for the elastic id', async () => { + const options = getCreateListOptionsMock(); + const list = await createList(options); + const expected = getListResponseMock(); + expected.id = 'elastic-id-123'; + expect(list).toEqual(expected); + }); + + test('It calls "callAsCurrentUser" with body, index, and listIndex', async () => { + const options = getCreateListOptionsMock(); + await createList(options); + const body = getIndexESListMock(); + const expected = { + body, + id: LIST_ID, + index: LIST_INDEX, + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('index', expected); + }); + + test('It returns an auto-generated id if id is sent in undefined', async () => { + const options = getCreateListOptionsMock(); + options.id = undefined; + const list = await createList(options); + const expected = getListResponseMock(); + expected.id = 'elastic-id-123'; + expect(list).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/lists/server/services/lists/create_list.ts b/x-pack/plugins/lists/server/services/lists/create_list.ts new file mode 100644 index 0000000000000..dcf87b3ad1ef1 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/create_list.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { CreateDocumentResponse } from 'elasticsearch'; + +import { DataClient } from '../../types'; +import { + Description, + IdOrUndefined, + IndexEsListSchema, + ListSchema, + MetaOrUndefined, + Name, + Type, +} from '../../../common/schemas'; + +export interface CreateListOptions { + id: IdOrUndefined; + type: Type; + name: Name; + description: Description; + dataClient: DataClient; + listIndex: string; + user: string; + meta: MetaOrUndefined; + dateNow?: string; + tieBreaker?: string; +} + +export const createList = async ({ + id, + name, + type, + description, + dataClient, + listIndex, + user, + meta, + dateNow, + tieBreaker, +}: CreateListOptions): Promise => { + const createdAt = dateNow ?? new Date().toISOString(); + const body: IndexEsListSchema = { + created_at: createdAt, + created_by: user, + description, + meta, + name, + tie_breaker_id: tieBreaker ?? uuid.v4(), + type, + updated_at: createdAt, + updated_by: user, + }; + const response: CreateDocumentResponse = await dataClient.callAsCurrentUser('index', { + body, + id, + index: listIndex, + }); + return { + id: response._id, + ...body, + }; +}; diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts new file mode 100644 index 0000000000000..f32273e3e7f76 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + LIST_ID, + LIST_INDEX, + LIST_ITEM_INDEX, + getDeleteListOptionsMock, + getListResponseMock, +} from '../mocks'; + +import { getList } from './get_list'; +import { deleteList } from './delete_list'; + +jest.mock('./get_list', () => ({ + getList: jest.fn(), +})); + +describe('delete_list', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Delete returns a null if the list is also null', async () => { + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteListOptionsMock(); + const deletedList = await deleteList(options); + expect(deletedList).toEqual(null); + }); + + test('Delete returns the list if a list is returned from getList', async () => { + const list = getListResponseMock(); + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + const deletedList = await deleteList(options); + expect(deletedList).toEqual(list); + }); + + test('Delete calls "deleteByQuery" and "delete" if a list is returned from getList', async () => { + const list = getListResponseMock(); + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + await deleteList(options); + const deleteByQuery = { + body: { query: { term: { list_id: LIST_ID } } }, + index: LIST_ITEM_INDEX, + }; + expect(options.dataClient.callAsCurrentUser).toBeCalledWith('deleteByQuery', deleteByQuery); + }); + + test('Delete calls "delete" second if a list is returned from getList', async () => { + const list = getListResponseMock(); + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + await deleteList(options); + const deleteQuery = { + id: LIST_ID, + index: LIST_INDEX, + }; + expect(options.dataClient.callAsCurrentUser).toHaveBeenNthCalledWith(2, 'delete', deleteQuery); + }); + + test('Delete does not call data client if the list returns null', async () => { + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(null); + const options = getDeleteListOptionsMock(); + await deleteList(options); + expect(options.dataClient.callAsCurrentUser).not.toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.ts b/x-pack/plugins/lists/server/services/lists/delete_list.ts new file mode 100644 index 0000000000000..653a8da74a105 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/delete_list.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Id, ListSchema } from '../../../common/schemas'; +import { DataClient } from '../../types'; + +import { getList } from './get_list'; + +export interface DeleteListOptions { + id: Id; + dataClient: DataClient; + listIndex: string; + listItemIndex: string; +} + +export const deleteList = async ({ + id, + dataClient, + listIndex, + listItemIndex, +}: DeleteListOptions): Promise => { + const list = await getList({ dataClient, id, listIndex }); + if (list == null) { + return null; + } else { + await dataClient.callAsCurrentUser('deleteByQuery', { + body: { + query: { + term: { + list_id: id, + }, + }, + }, + index: listItemIndex, + }); + + await dataClient.callAsCurrentUser('delete', { + id, + index: listIndex, + }); + return list; + } +}; diff --git a/x-pack/plugins/lists/server/services/lists/get_list.test.ts b/x-pack/plugins/lists/server/services/lists/get_list.test.ts new file mode 100644 index 0000000000000..1f9a33c191764 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/get_list.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + LIST_ID, + LIST_INDEX, + getDataClientMock, + getListResponseMock, + getSearchListMock, +} from '../mocks'; + +import { getList } from './get_list'; + +describe('get_list', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list as expected if the list is found', async () => { + const data = getSearchListMock(); + const dataClient = getDataClientMock(data); + const list = await getList({ dataClient, id: LIST_ID, listIndex: LIST_INDEX }); + const expected = getListResponseMock(); + expect(list).toEqual(expected); + }); + + test('it returns null if the search is empty', async () => { + const data = getSearchListMock(); + data.hits.hits = []; + const dataClient = getDataClientMock(data); + const list = await getList({ dataClient, id: LIST_ID, listIndex: LIST_INDEX }); + expect(list).toEqual(null); + }); +}); diff --git a/x-pack/plugins/lists/server/services/lists/get_list.ts b/x-pack/plugins/lists/server/services/lists/get_list.ts new file mode 100644 index 0000000000000..216703f08f069 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/get_list.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { Id, ListSchema, SearchEsListSchema } from '../../../common/schemas'; +import { DataClient } from '../../types'; + +interface GetListOptions { + id: Id; + dataClient: DataClient; + listIndex: string; +} + +export const getList = async ({ + id, + dataClient, + listIndex, +}: GetListOptions): Promise => { + const result: SearchResponse = await dataClient.callAsCurrentUser('search', { + body: { + query: { + term: { + _id: id, + }, + }, + }, + ignoreUnavailable: true, + index: listIndex, + }); + if (result.hits.hits.length) { + return { + id: result.hits.hits[0]._id, + ...result.hits.hits[0]._source, + }; + } else { + return null; + } +}; diff --git a/x-pack/plugins/lists/server/services/lists/get_list_index.test.ts b/x-pack/plugins/lists/server/services/lists/get_list_index.test.ts new file mode 100644 index 0000000000000..22a738a340b25 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/get_list_index.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { httpServerMock } from 'src/core/server/mocks'; +import { KibanaRequest } from 'src/core/server'; + +import { getSpace } from '../utils'; + +import { getListIndex } from './get_list_index'; + +jest.mock('../utils', () => ({ + getSpace: jest.fn(), +})); + +describe('get_list_index', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Returns the list index when there is a space', async () => { + ((getSpace as unknown) as jest.Mock).mockReturnValueOnce('test-space'); + const rawRequest = httpServerMock.createRawRequest({}); + const request = KibanaRequest.from(rawRequest); + const listIndex = getListIndex({ + listsIndexName: 'lists-index', + request, + spaces: undefined, + }); + expect(listIndex).toEqual('lists-index-test-space'); + }); +}); diff --git a/x-pack/plugins/lists/server/services/lists/get_list_index.ts b/x-pack/plugins/lists/server/services/lists/get_list_index.ts new file mode 100644 index 0000000000000..70b85fc97ebfa --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/get_list_index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KibanaRequest } from 'kibana/server'; + +import { SpacesServiceSetup } from '../../../../spaces/server'; +import { getSpace } from '../utils'; + +interface GetListIndexOptions { + spaces: SpacesServiceSetup | undefined | null; + request: KibanaRequest; + listsIndexName: string; +} + +export const getListIndex = ({ spaces, request, listsIndexName }: GetListIndexOptions): string => + `${listsIndexName}-${getSpace({ request, spaces })}`; diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts new file mode 100644 index 0000000000000..e25eaaafd855e --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getListTemplate } from './get_list_template'; + +jest.mock('./list_mappings.json', () => ({ + listMappings: {}, +})); + +describe('get_list_template', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list template with the string filled in', async () => { + const template = getListTemplate('some_index'); + expect(template).toEqual({ + index_patterns: ['some_index-*'], + mappings: { listMappings: {} }, + settings: { index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } } }, + }); + }); +}); diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.ts new file mode 100644 index 0000000000000..9d93a051f2d10 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import listMappings from './list_mappings.json'; + +export const getListTemplate = (index: string): Record => ({ + index_patterns: [`${index}-*`], + mappings: listMappings, + settings: { + index: { + lifecycle: { + name: index, + rollover_alias: index, + }, + }, + }, +}); diff --git a/x-pack/plugins/lists/server/services/lists/index.ts b/x-pack/plugins/lists/server/services/lists/index.ts new file mode 100644 index 0000000000000..f704ef0b05b82 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './create_list'; +export * from './delete_list'; +export * from './get_list'; +export * from './get_list_template'; +export * from './update_list'; +export * from './get_list_index'; diff --git a/x-pack/plugins/lists/server/services/lists/list_mappings.json b/x-pack/plugins/lists/server/services/lists/list_mappings.json new file mode 100644 index 0000000000000..1136a53da787d --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/list_mappings.json @@ -0,0 +1,33 @@ +{ + "dynamic": "strict", + "properties": { + "name": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "meta": { + "enabled": "false", + "type": "object" + }, + "created_at": { + "type": "date" + }, + "updated_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + } + } +} diff --git a/x-pack/plugins/lists/server/services/lists/list_policy.json b/x-pack/plugins/lists/server/services/lists/list_policy.json new file mode 100644 index 0000000000000..a4c84f73e7896 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/list_policy.json @@ -0,0 +1,14 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50gb" + } + } + } + } + } +} diff --git a/x-pack/plugins/lists/server/services/lists/update_list.test.ts b/x-pack/plugins/lists/server/services/lists/update_list.test.ts new file mode 100644 index 0000000000000..09bf0ee69c981 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/update_list.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getListResponseMock, getUpdateListOptionsMock } from '../mocks'; + +import { updateList } from './update_list'; +import { getList } from './get_list'; + +jest.mock('./get_list', () => ({ + getList: jest.fn(), +})); + +describe('update_list', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns a list as expected with the id changed out for the elastic id when there is a list to update', async () => { + const list = getListResponseMock(); + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(list); + const options = getUpdateListOptionsMock(); + const updatedList = await updateList(options); + const expected = getListResponseMock(); + expected.id = 'elastic-id-123'; + expect(updatedList).toEqual(expected); + }); + + test('it returns null when there is not a list to update', async () => { + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(null); + const options = getUpdateListOptionsMock(); + const updatedList = await updateList(options); + expect(updatedList).toEqual(null); + }); +}); diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts new file mode 100644 index 0000000000000..55f110e9a8291 --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateDocumentResponse } from 'elasticsearch'; + +import { + DescriptionOrUndefined, + Id, + ListSchema, + MetaOrUndefined, + NameOrUndefined, + UpdateEsListSchema, +} from '../../../common/schemas'; +import { DataClient } from '../../types'; + +import { getList } from '.'; + +export interface UpdateListOptions { + id: Id; + dataClient: DataClient; + listIndex: string; + user: string; + name: NameOrUndefined; + description: DescriptionOrUndefined; + meta: MetaOrUndefined; + dateNow?: string; +} + +export const updateList = async ({ + id, + name, + description, + dataClient, + listIndex, + user, + meta, + dateNow, +}: UpdateListOptions): Promise => { + const updatedAt = dateNow ?? new Date().toISOString(); + const list = await getList({ dataClient, id, listIndex }); + if (list == null) { + return null; + } else { + const doc: UpdateEsListSchema = { + description, + meta, + name, + updated_at: updatedAt, + updated_by: user, + }; + const response: CreateDocumentResponse = await dataClient.callAsCurrentUser('update', { + body: { doc }, + id, + index: listIndex, + }); + return { + created_at: list.created_at, + created_by: list.created_by, + description: description ?? list.description, + id: response._id, + meta, + name: name ?? list.name, + tie_breaker_id: list.tie_breaker_id, + type: list.type, + updated_at: updatedAt, + updated_by: user, + }; + } +}; diff --git a/x-pack/plugins/lists/server/services/mocks/get_create_list_item_bulk_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_create_list_item_bulk_options_mock.ts new file mode 100644 index 0000000000000..0f4d92cabaa7a --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_create_list_item_bulk_options_mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateListItemsBulkOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { + DATE_NOW, + LIST_ID, + LIST_ITEM_INDEX, + META, + TIE_BREAKERS, + TYPE, + USER, + VALUE, + VALUE_2, +} from './lists_services_mock_constants'; + +export const getCreateListItemBulkOptionsMock = (): CreateListItemsBulkOptions => ({ + dataClient: getDataClientMock(), + dateNow: DATE_NOW, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + meta: META, + tieBreaker: TIE_BREAKERS, + type: TYPE, + user: USER, + value: [VALUE, VALUE_2], +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_create_list_item_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_create_list_item_options_mock.ts new file mode 100644 index 0000000000000..960db293f1124 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_create_list_item_options_mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateListItemOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { + DATE_NOW, + LIST_ID, + LIST_ITEM_ID, + LIST_ITEM_INDEX, + META, + TIE_BREAKER, + TYPE, + USER, +} from './lists_services_mock_constants'; + +export const getCreateListItemOptionsMock = (): CreateListItemOptions => ({ + dataClient: getDataClientMock(), + dateNow: DATE_NOW, + id: LIST_ITEM_ID, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + meta: META, + tieBreaker: TIE_BREAKER, + type: TYPE, + user: USER, + value: '127.0.0.1', +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_create_list_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_create_list_options_mock.ts new file mode 100644 index 0000000000000..1a005a76547f5 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_create_list_options_mock.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateListOptions } from '../lists'; + +import { getDataClientMock } from './get_data_client_mock'; +import { + DATE_NOW, + DESCRIPTION, + LIST_ID, + LIST_INDEX, + META, + NAME, + TIE_BREAKER, + TYPE, + USER, +} from './lists_services_mock_constants'; + +export const getCreateListOptionsMock = (): CreateListOptions => ({ + dataClient: getDataClientMock(), + dateNow: DATE_NOW, + description: DESCRIPTION, + id: LIST_ID, + listIndex: LIST_INDEX, + meta: META, + name: NAME, + tieBreaker: TIE_BREAKER, + type: TYPE, + user: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_data_client_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_data_client_mock.ts new file mode 100644 index 0000000000000..6e4cc40efeed7 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_data_client_mock.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateDocumentResponse } from 'elasticsearch'; + +import { LIST_INDEX } from './lists_services_mock_constants'; +import { getShardMock } from './get_shard_mock'; + +interface DataClientReturn { + callAsCurrentUser: () => Promise; + callAsInternalUser: () => Promise; +} + +export const getEmptyCreateDocumentResponseMock = (): CreateDocumentResponse => ({ + _id: 'elastic-id-123', + _index: LIST_INDEX, + _shards: getShardMock(), + _type: '', + _version: 1, + created: true, + result: '', +}); + +export const getDataClientMock = ( + callAsCurrentUserData: unknown = getEmptyCreateDocumentResponseMock() +): DataClientReturn => ({ + callAsCurrentUser: jest.fn().mockResolvedValue(callAsCurrentUserData), + callAsInternalUser: (): Promise => { + throw new Error('This function should not be calling "callAsInternalUser"'); + }, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_delete_list_item_by_value_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_delete_list_item_by_value_options_mock.ts new file mode 100644 index 0000000000000..58fd319589ea3 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_delete_list_item_by_value_options_mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DeleteListItemByValueOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE } from './lists_services_mock_constants'; + +export const getDeleteListItemByValueOptionsMock = (): DeleteListItemByValueOptions => ({ + dataClient: getDataClientMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: VALUE, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_delete_list_item_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_delete_list_item_options_mock.ts new file mode 100644 index 0000000000000..1e7167547a6de --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_delete_list_item_options_mock.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DeleteListItemOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ITEM_ID, LIST_ITEM_INDEX } from './lists_services_mock_constants'; + +export const getDeleteListItemOptionsMock = (): DeleteListItemOptions => ({ + dataClient: getDataClientMock(), + id: LIST_ITEM_ID, + listItemIndex: LIST_ITEM_INDEX, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_delete_list_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_delete_list_options_mock.ts new file mode 100644 index 0000000000000..9d70dae969362 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_delete_list_options_mock.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DeleteListOptions } from '../lists'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_INDEX, LIST_ITEM_INDEX } from './lists_services_mock_constants'; + +export const getDeleteListOptionsMock = (): DeleteListOptions => ({ + dataClient: getDataClientMock(), + id: LIST_ID, + listIndex: LIST_INDEX, + listItemIndex: LIST_ITEM_INDEX, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_import_list_items_to_stream_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_import_list_items_to_stream_options_mock.ts new file mode 100644 index 0000000000000..4cc6d85cd947a --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_import_list_items_to_stream_options_mock.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { ImportListItemsToStreamOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_ITEM_INDEX, META, TYPE, USER } from './lists_services_mock_constants'; +import { TestReadable } from './test_readable'; + +export const getImportListItemsToStreamOptionsMock = (): ImportListItemsToStreamOptions => ({ + dataClient: getDataClientMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + meta: META, + stream: new TestReadable(), + type: TYPE, + user: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_index_es_list_item_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_index_es_list_item_mock.ts new file mode 100644 index 0000000000000..574e4afcb36f0 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_index_es_list_item_mock.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexEsListItemSchema } from '../../../common/schemas'; + +import { DATE_NOW, LIST_ID, META, TIE_BREAKER, USER, VALUE } from './lists_services_mock_constants'; + +export const getIndexESListItemMock = (ip = VALUE): IndexEsListItemSchema => ({ + created_at: DATE_NOW, + created_by: USER, + ip, + list_id: LIST_ID, + meta: META, + tie_breaker_id: TIE_BREAKER, + updated_at: DATE_NOW, + updated_by: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_index_es_list_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_index_es_list_mock.ts new file mode 100644 index 0000000000000..4e4d8d9c572e4 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_index_es_list_mock.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexEsListSchema } from '../../../common/schemas'; + +import { + DATE_NOW, + DESCRIPTION, + META, + NAME, + TIE_BREAKER, + TYPE, + USER, +} from './lists_services_mock_constants'; + +export const getIndexESListMock = (): IndexEsListSchema => ({ + created_at: DATE_NOW, + created_by: USER, + description: DESCRIPTION, + meta: META, + name: NAME, + tie_breaker_id: TIE_BREAKER, + type: TYPE, + updated_at: DATE_NOW, + updated_by: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_list_item_by_value_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_list_item_by_value_options_mock.ts new file mode 100644 index 0000000000000..ab1bde48e7ebf --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_list_item_by_value_options_mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GetListItemByValueOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE } from './lists_services_mock_constants'; + +export const getListItemByValueOptionsMocks = (): GetListItemByValueOptions => ({ + dataClient: getDataClientMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: VALUE, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_list_item_by_values_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_list_item_by_values_options_mock.ts new file mode 100644 index 0000000000000..c15d417d10289 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_list_item_by_values_options_mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GetListItemByValuesOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE, VALUE_2 } from './lists_services_mock_constants'; + +export const getListItemByValuesOptionsMocks = (): GetListItemByValuesOptions => ({ + dataClient: getDataClientMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [VALUE, VALUE_2], +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_list_item_response_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_list_item_response_mock.ts new file mode 100644 index 0000000000000..1a30282ddaeba --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_list_item_response_mock.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ListItemSchema } from '../../../common/schemas'; + +import { DATE_NOW, LIST_ID, LIST_ITEM_ID, USER, VALUE } from './lists_services_mock_constants'; + +export const getListItemResponseMock = (): ListItemSchema => ({ + created_at: DATE_NOW, + created_by: USER, + id: LIST_ITEM_ID, + list_id: LIST_ID, + meta: {}, + tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e', + type: 'ip', + updated_at: DATE_NOW, + updated_by: USER, + value: VALUE, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_list_response_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_list_response_mock.ts new file mode 100644 index 0000000000000..ea068d774c4ed --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_list_response_mock.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ListSchema } from '../../../common/schemas'; + +import { DATE_NOW, DESCRIPTION, LIST_ID, NAME, USER } from './lists_services_mock_constants'; + +export const getListResponseMock = (): ListSchema => ({ + created_at: DATE_NOW, + created_by: USER, + description: DESCRIPTION, + id: LIST_ID, + meta: {}, + name: NAME, + tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e', + type: 'ip', + updated_at: DATE_NOW, + updated_by: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_search_es_list_item_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_search_es_list_item_mock.ts new file mode 100644 index 0000000000000..5e9fd8995c0eb --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_search_es_list_item_mock.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchEsListItemSchema } from '../../../common/schemas'; + +import { DATE_NOW, LIST_ID, USER, VALUE } from './lists_services_mock_constants'; + +export const getSearchEsListItemMock = (): SearchEsListItemSchema => ({ + created_at: DATE_NOW, + created_by: USER, + ip: VALUE, + keyword: undefined, + list_id: LIST_ID, + meta: {}, + tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e', + updated_at: DATE_NOW, + updated_by: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_search_es_list_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_search_es_list_mock.ts new file mode 100644 index 0000000000000..6a565437617ba --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_search_es_list_mock.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchEsListSchema } from '../../../common/schemas'; + +import { DATE_NOW, DESCRIPTION, NAME, USER } from './lists_services_mock_constants'; + +export const getSearchEsListMock = (): SearchEsListSchema => ({ + created_at: DATE_NOW, + created_by: USER, + description: DESCRIPTION, + meta: {}, + name: NAME, + tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e', + type: 'ip', + updated_at: DATE_NOW, + updated_by: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_search_list_item_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_search_list_item_mock.ts new file mode 100644 index 0000000000000..9f877c8168cca --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_search_list_item_mock.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { SearchEsListItemSchema } from '../../../common/schemas'; + +import { getShardMock } from './get_shard_mock'; +import { LIST_INDEX, LIST_ITEM_ID } from './lists_services_mock_constants'; +import { getSearchEsListItemMock } from './get_search_es_list_item_mock'; + +export const getSearchListItemMock = (): SearchResponse => ({ + _scroll_id: '123', + _shards: getShardMock(), + hits: { + hits: [ + { + _id: LIST_ITEM_ID, + _index: LIST_INDEX, + _score: 0, + _source: getSearchEsListItemMock(), + _type: '', + }, + ], + max_score: 0, + total: 1, + }, + timed_out: false, + took: 10, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_search_list_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_search_list_mock.ts new file mode 100644 index 0000000000000..9728139eab42a --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_search_list_mock.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { SearchEsListSchema } from '../../../common/schemas'; + +import { getShardMock } from './get_shard_mock'; +import { LIST_ID, LIST_INDEX } from './lists_services_mock_constants'; +import { getSearchEsListMock } from './get_search_es_list_mock'; + +export const getSearchListMock = (): SearchResponse => ({ + _scroll_id: '123', + _shards: getShardMock(), + hits: { + hits: [ + { + _id: LIST_ID, + _index: LIST_INDEX, + _score: 0, + _source: getSearchEsListMock(), + _type: '', + }, + ], + max_score: 0, + total: 1, + }, + timed_out: false, + took: 10, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_shard_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_shard_mock.ts new file mode 100644 index 0000000000000..4cc6577d5e531 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_shard_mock.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ShardsResponse } from 'elasticsearch'; + +export const getShardMock = (): ShardsResponse => ({ + failed: 0, + skipped: 0, + successful: 0, + total: 0, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_update_list_item_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_update_list_item_options_mock.ts new file mode 100644 index 0000000000000..b60d6f5113e06 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_update_list_item_options_mock.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { UpdateListItemOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { + DATE_NOW, + LIST_ITEM_ID, + LIST_ITEM_INDEX, + META, + USER, + VALUE, +} from './lists_services_mock_constants'; + +export const getUpdateListItemOptionsMock = (): UpdateListItemOptions => ({ + dataClient: getDataClientMock(), + dateNow: DATE_NOW, + id: LIST_ITEM_ID, + listItemIndex: LIST_ITEM_INDEX, + meta: META, + user: USER, + value: VALUE, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_update_list_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_update_list_options_mock.ts new file mode 100644 index 0000000000000..e56ebc24bdae1 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_update_list_options_mock.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { UpdateListOptions } from '../lists'; + +import { getDataClientMock } from './get_data_client_mock'; +import { + DATE_NOW, + DESCRIPTION, + LIST_ID, + LIST_INDEX, + META, + NAME, + USER, +} from './lists_services_mock_constants'; + +export const getUpdateListOptionsMock = (): UpdateListOptions => ({ + dataClient: getDataClientMock(), + dateNow: DATE_NOW, + description: DESCRIPTION, + id: LIST_ID, + listIndex: LIST_INDEX, + meta: META, + name: NAME, + user: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_write_buffer_to_items_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_write_buffer_to_items_options_mock.ts new file mode 100644 index 0000000000000..9a77453b65d6a --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_write_buffer_to_items_options_mock.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { WriteBufferToItemsOptions } from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_ITEM_INDEX, META, TYPE, USER } from './lists_services_mock_constants'; + +export const getWriteBufferToItemsOptionsMock = (): WriteBufferToItemsOptions => ({ + buffer: [], + dataClient: getDataClientMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + meta: META, + type: TYPE, + user: USER, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/get_write_list_items_to_stream_options_mock.ts b/x-pack/plugins/lists/server/services/mocks/get_write_list_items_to_stream_options_mock.ts new file mode 100644 index 0000000000000..96724c2a88045 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/get_write_list_items_to_stream_options_mock.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Stream } from 'stream'; + +import { + ExportListItemsToStreamOptions, + GetResponseOptions, + WriteNextResponseOptions, + WriteResponseHitsToStreamOptions, +} from '../items'; + +import { getDataClientMock } from './get_data_client_mock'; +import { LIST_ID, LIST_ITEM_INDEX } from './lists_services_mock_constants'; +import { getSearchListItemMock } from './get_search_list_item_mock'; + +export const getExportListItemsToStreamOptionsMock = (): ExportListItemsToStreamOptions => ({ + dataClient: getDataClientMock(getSearchListItemMock()), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + stream: new Stream.PassThrough(), + stringToAppend: undefined, +}); + +export const getWriteNextResponseOptions = (): WriteNextResponseOptions => ({ + dataClient: getDataClientMock(getSearchListItemMock()), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + searchAfter: [], + stream: new Stream.PassThrough(), + stringToAppend: undefined, +}); + +export const getResponseOptionsMock = (): GetResponseOptions => ({ + dataClient: getDataClientMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + searchAfter: [], + size: 100, +}); + +export const getWriteResponseHitsToStreamOptionsMock = (): WriteResponseHitsToStreamOptions => ({ + response: getSearchListItemMock(), + stream: new Stream.PassThrough(), + stringToAppend: undefined, +}); diff --git a/x-pack/plugins/lists/server/services/mocks/index.ts b/x-pack/plugins/lists/server/services/mocks/index.ts new file mode 100644 index 0000000000000..516264149fac7 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './get_data_client_mock'; +export * from './get_delete_list_options_mock'; +export * from './get_create_list_options_mock'; +export * from './get_list_response_mock'; +export * from './get_search_list_mock'; +export * from './get_shard_mock'; +export * from './lists_services_mock_constants'; +export * from './get_update_list_options_mock'; +export * from './get_create_list_item_options_mock'; +export * from './get_list_item_response_mock'; +export * from './get_index_es_list_mock'; +export * from './get_index_es_list_item_mock'; +export * from './get_create_list_item_bulk_options_mock'; +export * from './get_delete_list_item_by_value_options_mock'; +export * from './get_delete_list_item_options_mock'; +export * from './get_list_item_by_values_options_mock'; +export * from './get_search_es_list_mock'; +export * from './get_search_es_list_item_mock'; +export * from './get_list_item_by_value_options_mock'; +export * from './get_update_list_item_options_mock'; +export * from './get_write_buffer_to_items_options_mock'; +export * from './get_import_list_items_to_stream_options_mock'; +export * from './get_write_list_items_to_stream_options_mock'; diff --git a/x-pack/plugins/lists/server/services/mocks/lists_services_mock_constants.ts b/x-pack/plugins/lists/server/services/mocks/lists_services_mock_constants.ts new file mode 100644 index 0000000000000..d174211f348ea --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/lists_services_mock_constants.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +export const DATE_NOW = '2020-04-20T15:25:31.830Z'; +export const USER = 'some user'; +export const LIST_INDEX = '.lists'; +export const LIST_ITEM_INDEX = '.items'; +export const NAME = 'some name'; +export const DESCRIPTION = 'some description'; +export const LIST_ID = 'some-list-id'; +export const LIST_ITEM_ID = 'some-list-item-id'; +export const TIE_BREAKER = '6a76b69d-80df-4ab2-8c3e-85f466b06a0e'; +export const TIE_BREAKERS = [ + '21530991-4051-46ec-bc35-2afa09a1b0b5', + '3c662054-ae37-4aa9-9936-3e8e2ea26775', + '60e49a20-3a23-48b6-8bf9-ed5e3b70f7a0', + '38814080-a40f-4358-992a-3b875f9b7dec', + '29fa61be-aaaf-411c-a78a-7059e3f723f1', + '9c19c959-cb9d-4cd2-99e4-1ea2baf0ef0e', + 'd409308c-f94b-4b3a-8234-bbd7a80c9140', + '87824c99-cd83-45c4-8aa6-4ad95dfea62c', + '7b940c17-9355-479f-b882-f3e575718f79', + '5983ad0c-4ef4-4fa0-8308-80ab9ecc4f74', +]; +export const META = {}; +export const TYPE = 'ip'; +export const VALUE = '127.0.0.1'; +export const VALUE_2 = '255.255.255'; diff --git a/x-pack/plugins/lists/server/services/mocks/test_readable.ts b/x-pack/plugins/lists/server/services/mocks/test_readable.ts new file mode 100644 index 0000000000000..52ad6de484005 --- /dev/null +++ b/x-pack/plugins/lists/server/services/mocks/test_readable.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Readable } from 'stream'; + +export class TestReadable extends Readable { + public _read(): void {} +} diff --git a/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts b/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts new file mode 100644 index 0000000000000..6e5dca7d54e5b --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getSearchEsListItemMock } from '../mocks'; +import { Type } from '../../../common/schemas'; + +import { deriveTypeFromItem } from './derive_type_from_es_type'; + +describe('derive_type_from_es_type', () => { + test('it returns the item ip if it exists', () => { + const item = getSearchEsListItemMock(); + const derivedType = deriveTypeFromItem({ item }); + const expected: Type = 'ip'; + expect(derivedType).toEqual(expected); + }); + + test('it returns the item keyword if it exists', () => { + const item = getSearchEsListItemMock(); + item.ip = undefined; + item.keyword = 'some keyword'; + const derivedType = deriveTypeFromItem({ item }); + const expected: Type = 'keyword'; + expect(derivedType).toEqual(expected); + }); + + test('it throws an error with status code if neither one exists', () => { + const item = getSearchEsListItemMock(); + item.ip = undefined; + item.keyword = undefined; + const expected = `Was expecting a valid type from the Elastic Search List Item such as ip or keyword but did not found one here ${JSON.stringify( + item + )}`; + expect(() => deriveTypeFromItem({ item })).toThrowError(expected); + }); +}); diff --git a/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts b/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts new file mode 100644 index 0000000000000..7a65e74bf4947 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchEsListItemSchema, Type } from '../../../common/schemas'; +import { ErrorWithStatusCode } from '../../error_with_status_code'; + +interface DeriveTypeFromItemOptions { + item: SearchEsListItemSchema; +} + +export const deriveTypeFromItem = ({ item }: DeriveTypeFromItemOptions): Type => { + if (item.ip != null) { + return 'ip'; + } else if (item.keyword != null) { + return 'keyword'; + } else { + throw new ErrorWithStatusCode( + `Was expecting a valid type from the Elastic Search List Item such as ip or keyword but did not found one here ${JSON.stringify( + item + )}`, + 400 + ); + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts new file mode 100644 index 0000000000000..3f50efe0c6c56 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Type } from '../../../common/schemas'; + +export type QueryFilterType = Array< + { term: { list_id: string } } | { terms: { ip: string[] } } | { terms: { keyword: string[] } } +>; + +export const getQueryFilterFromTypeValue = ({ + type, + value, + listId, +}: { + type: Type; + value: string[]; + listId: string; + // We disable the consistent return since we want to use typescript for exhaustive type checks + // eslint-disable-next-line consistent-return +}): QueryFilterType => { + const filter: QueryFilterType = [{ term: { list_id: listId } }]; + switch (type) { + case 'ip': { + return [...filter, ...[{ terms: { ip: value } }]]; + } + case 'keyword': { + return [...filter, ...[{ terms: { keyword: value } }]]; + } + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/get_space.ts b/x-pack/plugins/lists/server/services/utils/get_space.ts new file mode 100644 index 0000000000000..e23f963b2c40d --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/get_space.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KibanaRequest } from 'kibana/server'; + +import { SpacesServiceSetup } from '../../../../spaces/server'; + +export const getSpace = ({ + spaces, + request, +}: { + spaces: SpacesServiceSetup | undefined | null; + request: KibanaRequest; +}): string => spaces?.getSpaceId(request) ?? 'default'; diff --git a/x-pack/plugins/lists/server/services/utils/get_user.ts b/x-pack/plugins/lists/server/services/utils/get_user.ts new file mode 100644 index 0000000000000..1ddad047da722 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/get_user.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KibanaRequest } from 'kibana/server'; + +import { SecurityPluginSetup } from '../../../../security/server'; + +interface GetUserOptions { + security: SecurityPluginSetup; + request: KibanaRequest; +} + +export const getUser = ({ security, request }: GetUserOptions): string => { + const authenticatedUser = security.authc.getCurrentUser(request); + if (authenticatedUser != null) { + return authenticatedUser.username; + } else { + return 'elastic'; + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/index.ts b/x-pack/plugins/lists/server/services/utils/index.ts new file mode 100644 index 0000000000000..f256b6cb8f2d5 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './get_query_filter_from_type_value'; +export * from './transform_elastic_to_list_item'; +export * from './transform_list_item_to_elastic_query'; +export * from './get_user'; +export * from './derive_type_from_es_type'; +export * from './get_space'; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts new file mode 100644 index 0000000000000..9cf673081d320 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { ListItemArraySchema, SearchEsListItemSchema, Type } from '../../../common/schemas'; +import { ErrorWithStatusCode } from '../../error_with_status_code'; + +export const transformElasticToListItem = ({ + response, + type, +}: { + response: SearchResponse; + type: Type; +}): ListItemArraySchema => { + return response.hits.hits.map(hit => { + const { + _id, + _source: { + created_at, + updated_at, + updated_by, + created_by, + list_id, + tie_breaker_id, + ip, + keyword, + meta, + }, + } = hit; + + const baseTypes = { + created_at, + created_by, + id: _id, + list_id, + meta, + tie_breaker_id, + type, + updated_at, + updated_by, + }; + + switch (type) { + case 'ip': { + if (ip == null) { + throw new ErrorWithStatusCode('Was expecting ip to not be null/undefined', 400); + } + return { + ...baseTypes, + value: ip, + }; + } + case 'keyword': { + if (keyword == null) { + throw new ErrorWithStatusCode('Was expecting keyword to not be null/undefined', 400); + } + return { + ...baseTypes, + value: keyword, + }; + } + } + // TypeScript is not happy unless I have this line here + return assertUnreachable(); + }); +}; + +export const assertUnreachable = (): never => { + throw new Error('Unknown type in elastic_to_list_items'); +}; diff --git a/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts b/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts new file mode 100644 index 0000000000000..e68851dc3582b --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EsDataTypeUnion, Type } from '../../../common/schemas'; + +export const transformListItemToElasticQuery = ({ + type, + value, +}: { + type: Type; + value: string; + // We disable the consistent return since we want to use typescript for exhaustive type checks + // eslint-disable-next-line consistent-return +}): EsDataTypeUnion => { + switch (type) { + case 'ip': { + return { + ip: value, + }; + } + case 'keyword': { + return { + keyword: value, + }; + } + } +}; diff --git a/x-pack/plugins/lists/server/siem_server_deps.ts b/x-pack/plugins/lists/server/siem_server_deps.ts new file mode 100644 index 0000000000000..e78debc8e4349 --- /dev/null +++ b/x-pack/plugins/lists/server/siem_server_deps.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { + transformError, + buildSiemResponse, +} from '../../siem/server/lib/detection_engine/routes/utils'; +export { deleteTemplate } from '../../siem/server/lib/detection_engine/index/delete_template'; +export { deletePolicy } from '../../siem/server/lib/detection_engine/index/delete_policy'; +export { deleteAllIndex } from '../../siem/server/lib/detection_engine/index/delete_all_index'; +export { setPolicy } from '../../siem/server/lib/detection_engine/index/set_policy'; +export { setTemplate } from '../../siem/server/lib/detection_engine/index/set_template'; +export { getTemplateExists } from '../../siem/server/lib/detection_engine/index/get_template_exists'; +export { getPolicyExists } from '../../siem/server/lib/detection_engine/index/get_policy_exists'; +export { createBootstrapIndex } from '../../siem/server/lib/detection_engine/index/create_bootstrap_index'; +export { getIndexExists } from '../../siem/server/lib/detection_engine/index/get_index_exists'; +export { buildRouteValidation } from '../../siem/server/utils/build_validation/route_validation'; +export { validate } from '../../siem/server/lib/detection_engine/routes/rules/validate'; diff --git a/x-pack/plugins/lists/server/types.ts b/x-pack/plugins/lists/server/types.ts new file mode 100644 index 0000000000000..7d509c4e27167 --- /dev/null +++ b/x-pack/plugins/lists/server/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IContextProvider, RequestHandler, ScopedClusterClient } from 'kibana/server'; + +import { SecurityPluginSetup } from '../../security/server'; +import { SpacesPluginSetup } from '../../spaces/server'; + +import { ListClient } from './services/lists/client'; + +export type DataClient = Pick; +export type ContextProvider = IContextProvider, 'lists'>; + +export interface PluginsSetup { + security: SecurityPluginSetup; + spaces: SpacesPluginSetup | undefined | null; +} + +export type ContextProviderReturn = Promise<{ getListClient: () => ListClient }>; +declare module 'src/core/server' { + interface RequestHandlerContext { + lists?: { + getListClient: () => ListClient; + }; + } +} diff --git a/x-pack/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh b/x-pack/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh index a56d788d69c16..22b602f935187 100755 --- a/x-pack/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh +++ b/x-pack/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh @@ -9,9 +9,15 @@ set -e ./check_env_variables.sh +# Clean up and remove all actions and alerts from SIEM +# within saved objects ./delete_all_actions.sh ./delete_all_alerts.sh ./delete_all_alert_tasks.sh + +# delete all the statuses from the signal index ./delete_all_statuses.sh + +# re-create the signal index ./delete_signal_index.sh ./post_signal_index.sh