Configuring Elasticsearch Index for Time Series Data

Here's how to configure an Elasticsearch index for time series data. In this video, Nama Illo, Education Architect at Elastic, explains what goes into setting up an Ingest Pipeline and ILM Policy and then configuring an Index Template to use them. Specifically, this video covers: Index fundamentals Time series data, index summary Ingest pipeline overview Processors Index lifecycle (IL) policy exploration rollover, rollover alias, phases Index template configuration Logistics, index patterns, naming scheme, data streams, component templates, settings, mapping, aliases Brief demo #Elasticsearch #index #TimeSeriesData #ElasticsearchTutorial #TechCommunity Code Snippet: Component Templates PUT _component_template/demo__settings_ilm_policy { "template": { "settings": { "index": { "lifecycle": { "name": "demo_rollover_policy" } } } } } PUT _component_template/demo_mappings_component { "template": { "mappings": { "_routing": { "required": false }, "numeric_detection": false, "dynamic_date_formats": [ "strict_date_optional_time", "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z" ], "dynamic": true, "_source": { "excludes": [], "includes": [], "enabled": true }, "dynamic_templates": [], "date_detection": true, "properties": { "hello": { "type": "keyword" } } } } } PUT _component_template/demo_pipeline_timestamp { "template": { "settings": { "index": { "default_pipeline": "demo_pipeline" } } } } PUT _component_template/demo_rollover_alias_logs { "template": { "settings": { "index": { "lifecycle": { "rollover_alias": "my_logs" } } } } } PUT _component_template/demo_rollover_alias_metric { "template": { "settings": { "index": { "lifecycle": { "rollover_alias": "my_metrics" } } } } } PUT _component_template/demo_routing_template { "template": { "settings": { "index": { "routing": { "allocation": { "include": { "_tier_preference": "data_hot" } } } } } } } PUT _component_template/demo_shards_template { "template": { "settings": { "index": { "number_of_shards": "2", "number_of_replicas": "0" } } } } Pipeline to set timestamp PUT _ingest/pipeline/demo_pipeline { "processors": [ { "set": { "field": "hello", "value": "world" } }, { "date": { "field": "_ingest.timestamp", "formats": [ "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "ISO8601", "UNIX", "UNIX_MS" ] } } ] } IL Policy to illustrate rollover PUT _ilm/policy/demo_rollover_policy { "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "rollover": { "max_age": "10s", "max_docs": 5 } } }, "warm": { "min_age": "40s", "actions": { "forcemerge": { "max_num_segments": 1 }, "readonly": {}, "shrink": { "number_of_shards": 1 }, "migrate": { "enabled": false }, "allocate": { "number_of_replicas": 3 } } }, "delete": { "min_age": "4m", "actions": { "delete": { "delete_searchable_snapshot": true } } } } } } Index template used for logs: PUT _index_template/demo_logs_template { "priority": 501, "index_patterns": [ "demo_logs*" ], "composed_of": [ "demo__settings_ilm_policy", "demo_mappings_component", "demo_pipeline_timestamp", "demo_rollover_alias_logs", "demo_routing_template", "demo_shards_template" ] } ##Template for metrics using data stream PUT _index_template/demo_metrics_template { "priority": 500, "index_patterns": [ "demo_metrics*" ], "data_stream": { "hidden": false, "allow_custom_routing": false }, "composed_of": [ "demo__settings_ilm_policy", "demo_mappings_component", "demo_pipeline_timestamp", "demo_rollover_alias_metric", "demo_routing_template", "demo_shards_template" ] } GET _cat/indices #we create two different templates each using an ingest processor to process data and IL policy to manage rollover conditions. Each illustrating different naming patterns and how to communicate using aliases. #I want to create an index to auto rollover and can be communicated using the name - "my_logs" PUT demo_logs-000001 { "aliases": { "my_logs": { "is_write_index" : true } } } GET my_logs GET my_logs/_ilm/explain POST my_logs/_doc/ { "field_1" : "value 1", "field_2" : "value 2" } GET my_logs/_search GET _cat/indices GET my_logs #I want to create a second index to auto rollover and can be communicated using the name - "my_metrics" ##this won't work and the 0001 is not needed anymore PUT demo_metrics-000001 { "aliases": { "my_metrics": { "is_write_index" : true } } } #you just send the data without a bootstrap. it just has to match the index pattern defined POST demo_metrics/_doc { "field_1" : "value 1", "field_2" : "value 2" } #notice the name we are using to index docs now GET _data_stream/_stats GET demo_metrics GET demo_metrics/_search #Will we be able to search using the name "my_metrics"? GET my_metrics/_search POST demo_metrics/_alias/my_metrics { "is_write_index": true } POST my_metrics/_doc/ { "field_1" : "value 1", "field_2" : "value 2" } GET my_metrics/_search GET _cat/indices GET my_metrics GET my_metrics/_ilm/explain