Elasticsearch API
Basics
  1. # Syntax
  2. # GET _API/parameter
  3.  
  4. # Cluster health
  5. GET _cluster/health
  6.  
  7. # Node info
  8. GET _nodes/stats
  9.  
  10. # Get index info
  11. # GET index_name
  12. GET c_house_price
  13.  
  14. # List indices
  15. GET /_cat/indices
Create, Read, Update, and Delete (CRUD)
  • Create index
    1. # PUT index_name
    2. PUT c_house_price
    3.  
    4. # Create index with mapping
    5. PUT ecommerce_data
    6. {
    7. "mappings": {
    8. "properties": {
    9. "Country": {
    10. "type": "keyword"
    11. },
    12. "CustomerID": {
    13. "type": "long"
    14. },
    15. "Description": {
    16. "type": "text"
    17. },
    18. "InvoiceDate": {
    19. "type": "date",
    20. "format": "M/d/yyyy H:m"
    21. },
    22. "InvoiceNo": {
    23. "type": "keyword"
    24. },
    25. "Quantity": {
    26. "type": "long"
    27. },
    28. "StockCode": {
    29. "type": "keyword"
    30. },
    31. "UnitPrice": {
    32. "type": "double"
    33. }
    34. }
    35. }
    36. }
  • Insert document
    1. # POST index_name/_doc
    2. # PUT does not work
    3. POST c_house_price/_doc
    4. {
    5. "first_name": "Lin",
    6. "last_name": "Chen"
    7. }
    8.  
    9. # Insert a document with a specific id
    10. # PUT index_name/_doc/id
    11. # POST index_name/_doc/id
    12. # if id already exists, the existing document is overwritten by the new document
    13. PUT c_house_price/_doc/1
    14. {
    15. "first_name": "John",
    16. "last_name": "Starburst"
    17. }
    18.  
    19. # if id already exist, return 409 error
    20. # PUT index_name/_create/id
    21. # POST index_name/_create/id
    22. PUT c_house_price/_create/3
    23. {
    24. "first_name": "Lin",
    25. "candy": "Jolly Ranchers"
    26. }
    27.  
    28. # reindex to create an index from an original index
    29. # run the task asynchronously to avoid timeout
    30. POST _reindex?wait_for_completion=false
    31. {
    32. "source": {
    33. "index": "e_commerce"
    34. },
    35. "dest": {
    36. "index": "ecommerce_data"
    37. }
    38. }
    39.  
    40. # use task id to check the progress
    41. GET _tasks/task_id
  • Read a document
    1. # GET index_name/_doc/id
    2. GET c_house_price/_doc/1
  • Update a document
    1. # POST index_name/_update/id
    2. POST c_house_price/_update/1
    3. {
    4. "doc":{
    5. "last_name": "Chen"
    6. }
    7. }
  • Delete a document
    1. # DELETE index_name/_doc/id
    2. DELETE c_house_price/_doc/1
    3.  
    4. # Delete an index
    5. # DELETE index_name
    6. DELETE c_house_price
    7.  
    8. # Delete by query
    9. POST ecommerce_data/_delete_by_query
    10. {
    11. "query": {
    12. "range": {
    13. "UnitPrice": {
    14. "lte": 0
    15. }
    16. }
    17. }
    18. }
    Bulk
  • Insert
    1. POST _bulk
    2. {"index": {"_index": "baseline_1", "_id":1}}
    3. {"Absolute vorticity": 0.0001307,"Apparent temperature": 232.5}
    4. {"index": {"_index": "baseline_1", "_id":2}}
    5. {"Absolute vorticity": 0.0001307,"Apparent temperature": 233.6}
  • Update
    1. POST _bulk
    2. {"update": {"_index": "baseline_1", "_id": 1}}
    3. {"doc": {"Apparent temperature": 242.5}}
    4. {"update": {"_index": "baseline_1", "_id":2}}
    5. {"doc": {"Apparent temperature": 243.6}}
  • Delete
    1. POST _bulk
    2. {"delete": {"_index": "baseline_1", "_id": 1}}
    3. {"delete": {"_index": "baseline_1", "_id":2}}
    Query
  • Query Types
    1. # Search
    2. # GET index_name/_search
    3. GET news_headlines/_search # return 10,000 hits
    4. # GET index_pattern/_search
    5. GET c*/_search
    6.  
    7. # return exact total number of hits
    8. GET news_headlines/_search
    9. {
    10. "track_total_hits": true
    11. }
  • match all query
    1. POST news_headlines/_search
    2. {
    3. "query":{
    4. "match_all":{}
    5. }
    6. }
  • Search between two date
    1. GET enter_name_of_the_index_here/_search
    2. {
    3. "query": {
    4. "Specify the type of query here": {
    5. "Enter name of the field here": {
    6. "gte": "Enter lowest value of the range here",
    7. "lte": "Enter highest value of the range here"
    8. }
    9. }
    10. }
    11. }
    1. GET news_headlines/_search
    2. {
    3. "query":{
    4. "range": {
    5. "date": {
    6. "gte": "2017-05-28T00:00:00.000-04:00",
    7. "lt": "2017-12-26T00:00:00.000-05:00"
    8. }
    9. }
    10. }
    11. }
  • Match Query
    1. GET Enter_name_of_index_here/_search
    2. {
    3. "query": {
    4. "match": {
    5. "Specify the field you want to search": {
    6. "query": "Enter search terms"
    7. }
    8. }
    9. }
    10. }
    1. # all, if any one of the term match, return as a hit
    2. GET news_headlines/_search
    3. {
    4. "query": {
    5. "match": {
    6. "headline": {
    7. "query": "Khloe Kardashian Kendall Jenner" # search keywords in headline
    8. }
    9. }
    10. }
    11. }
    12.  
    13. # and, all terms need to match
    14. GET news_headlines/_search
    15. {
    16. "query": {
    17. "match": {
    18. "headline": {
    19. "query": "Khloe Kardashian Kendall Jenner",
    20. "operator": "and"
    21. }
    22. }
    23. }
    24.  
    25. # specify the minimum number of terms a document should have to be included
    26. GET news_headlines/_search
    27. {
    28. "query": {
    29. "match": {
    30. "headline": {
    31. "query": "Khloe Kardashian Kendall Jenner",
    32. "minimum_should_match": 3
    33. }
    34. }
    35. }
    36. }
  • Match_phase Query
    1. GET Enter_name_of_index_here/_search
    2. {
    3. "query": {
    4. "match_phrase": {
    5. "Specify the field you want to search": {
    6. "query": "Enter search terms"
    7. }
    8. }
    9. }
    10. }
    1. GET news_headlines/_search
    2. {
    3. "query": {
    4. "match_phrase": {
    5. "headline": {
    6. "query": "Shape of You"
    7. }
    8. }
    9. }
    10. }
  • Query Multiple Fields
    1. GET Enter_the_name_of_the_index_here/_search
    2. {
    3. "query": {
    4. "multi_match": {
    5. "query": "Enter search terms here",
    6. "fields": [
    7. "List the field you want to search over",
    8. "List the field you want to search over",
    9. "List the field you want to search over"
    10. ]
    11. }
    12. }
    13. }
    1. GET news_headlines/_search
    2. {
    3. "query": {
    4. "multi_match": {
    5. "query": "Michelle Obama",
    6. "fields": [
    7. "headline",
    8. "short_description",
    9. "authors"
    10. ]
    11. }
    12. }
    13. }
    1. # designate one field to carry more weight than the others
    2. # by ^number
    3. GET Enter_the_name_of_the_index_here/_search
    4. {
    5. "query": {
    6. "multi_match": {
    7. "query": "Enter search terms",
    8. "fields": [
    9. "List field you want to boost^2",
    10. "List field you want to search over",
    11. "List field you want to search over^3"
    12. ]
    13. }
    14. }
    15. }
    1. GET news_headlines/_search
    2. {
    3. "query": {
    4. "multi_match": {
    5. "query": "Michelle Obama",
    6. "fields": [
    7. "headline^2",
    8. "short_description",
    9. "authors"
    10. ]
    11. }
    12. }
    13. }
    1. # improve precision with phrase type match
    2. GET Enter_the_name_of_the_index_here/_search
    3. {
    4. "query": {
    5. "multi_match": {
    6. "query": "Enter search phrase",
    7. "fields": [
    8. "List field you want to boost^2",
    9. "List field you want to search over",
    10. "List field you want to search over"
    11. ],
    12. "type": "phrase"
    13. }
    14. }
    15. }
    1. GET news_headlines/_search
    2. {
    3. "query": {
    4. "multi_match": {
    5. "query": "party planning",
    6. "fields": [
    7. "headline^2",
    8. "short_description"
    9. ],
    10. "type": "phrase"
    11. }
    12. }
    13. }
  • Combine multiple queries
    1. GET index_name/_search
    2. {
    3. "query":{
    4. "bool":{
    5. "must":[{}], # items must appear in matching documents, AND
    6. "must_not":[{}], # NOT
    7. "should":[{}], # at least one of items appear in matching documents, OR
    8. "filter":[{}] # query filter
    9. }
    10. }
    11. }
    12.  
    13. # must
    14. GET news_headlines/_search
    15. {
    16. "query": {
    17. "bool": {
    18. "must": [
    19. {
    20. "match_phrase": {
    21. "headline": "Michelle Obama"
    22. }
    23. },
    24. {
    25. "match": {
    26. "category": "POLITICS"
    27. }
    28. }
    29. ]
    30. }
    31. }
    32. }
    33.  
    34. # must_not
    35. GET news_headlines/_search
    36. {
    37. "query": {
    38. "bool": {
    39. "must": {
    40. "match_phrase": {
    41. "headline": "Michelle Obama"
    42. }
    43. },
    44. "must_not":[
    45. {
    46. "match": {
    47. "category": "WEDDINGS"
    48. }
    49. }
    50. ]
    51. }
    52. }
    53. }
    54.  
    55. # should
    56. GET news_headlines/_search
    57. {
    58. "query": {
    59. "bool": {
    60. "must": [
    61. {
    62. "match_phrase": {
    63. "headline": "Michelle Obama"
    64. }
    65. }
    66. ],
    67. "should":[
    68. {
    69. "match_phrase": {
    70. "category": "BLACK VOICES"
    71. }
    72. }
    73. ]
    74. }
    75. }
    76. }
    77.  
    78. # filter
    79. GET news_headlines/_search
    80. {
    81. "query": {
    82. "bool": {
    83. "must": [
    84. {
    85. "match_phrase": {
    86. "headline": "Michelle Obama"
    87. }
    88. }
    89. ],
    90. "filter":{
    91. "range":{
    92. "date": {
    93. "gte": "2014-03-25",
    94. "lte": "2016-03-25"
    95. }
    96. }
    97. }
    98. }
    99. }
    100. }
    101.  
    102. GET news_headlines/_search
    103. {
    104. "query": {
    105. "bool": {
    106. "should": [
    107. {
    108. "match_phrase": {
    109. "category": "ENTERTAINMENT"
    110. }
    111. }
    112. ],
    113. "filter": [
    114. {
    115. "exists": {
    116. "field": "headline"
    117. }
    118. }
    119. ]
    120. }
    121. }
    122. }
  • Geo queries
    1. POST /geo_example/_doc?refresh
    2. {
    3. "name": "Chapter One, London, UK",
    4. "location": {
    5. "type": "point",
    6. "coordinates": [11.660544, 57.800286]
    7. }
    8. }
    Aggregation
  • Metric Aggregations
  • Bucket Aggregations
    1. GET Enter_name_of_the_index_here/_search
    2. {
    3. "aggs": {
    4. "Name your aggregation here": {
    5. "Specify aggregation type here": { # terms, stats, geodistance
    6. "field": "Name the field you want to aggregate here",
    7. "size": "State how many buckets you want returned here"
    8. }
    9. }
    10. }
    11. }
    1. GET news_headlines/_search
    2. {
    3. "aggs": {
    4. "by_category": { # aggregation name
    5. "terms": { # aggregation type
    6. "field": "category", # filed name
    7. "size": 100 # number of buckets returns
    8. }
    9. }
    10. }
    11. }
  • Metric Aggregations
    1. GET Enter_name_of_the_index_here/_search
    2. {
    3. "aggs": {
    4. "Name your aggregations here": {
    5. "sum": {
    6. "field": "Name the field you want to aggregate on here"
    7. }
    8. }
    9. }
    10. }
    1. # sum
    2. GET ecommerce_data/_search
    3. {
    4. "size": 0, # prevents Elasticsearch from fetching the top 10 hits
    5. "aggs": {
    6. "sum_unit_price": {
    7. "sum": {
    8. "field": "UnitPrice"
    9. }
    10. }
    11. }
    12. }
    1. # stats, list count, min, max, avg, sum
    2. GET ecommerce_data/_search
    3. {
    4. "size": 0,
    5. "aggs": {
    6. "all_stats_unit_price": {
    7. "stats": {
    8. "field": "UnitPrice"
    9. }
    10. }
    11. }
    12. }
    1. # Cardinality Aggregation
    2. GET ecommerce_data/_search
    3. {
    4. "size":0,
    5. "aggs": {
    6. "sum_unit_price": {
    7. "cardinality": {
    8. "field": "CustomerID"
    9. }
    10. }
    11. }
    12. }
    1. # Limiting the scope of an aggregation
    2. GET ecommerce_data/_search
    3. {
    4. "size": 0,
    5. "query": {
    6. "match": {
    7. "Country": "Germany"
    8. }
    9. },
    10. "aggs": {
    11. "germany_average_unit_price": {
    12. "avg": {
    13. "field": "UnitPrice"
    14. }
    15. }
    16. }
    17. }
  • Bucket Aggregations
    1. # Date Histogram Aggregation
    2. GET ecommerce_data/_search
    3. {
    4. "size": 0,
    5. "aggs": {
    6. "Name your aggregations here": {
    7. "date_histogram": {
    8. "field":"Name the field you want to aggregate on here",
    9. "fixed_interval": "Specify the interval here"
    10. }
    11. }
    12. }
    13. }
    14.  
    15. GET ecommerce_data/_search
    16. {
    17. "size": 0,
    18. "aggs": {
    19. "transactions_by_8_hrs": {
    20. "date_histogram": {
    21. "field": "InvoiceDate",
    22. "fixed_interval": "8h"
    23. }
    24. }
    25. }
    26. }
    27.  
    28. GET ecommerce_data/_search
    29. {
    30. "size": 0,
    31. "aggs": {
    32. "Name your aggregations here": {
    33. "date_histogram": {
    34. "field":"Name the field you want to aggregate on here",
    35. "calendar_interval": "Specify the interval here"
    36. }
    37. }
    38. }
    39. }
    40.  
    41. GET ecommerce_data/_search
    42. {
    43. "size": 0,
    44. "aggs": {
    45. "transactions_by_month": {
    46. "date_histogram": {
    47. "field": "InvoiceDate",
    48. "calendar_interval": "1M"
    49. "order": {
    50. "_key": "desc"
    51. }
    52. }
    53. }
    54. }
    55. }
    1. # Histogram Aggregation
    2. GET ecommerce_data/_search
    3. {
    4. "size": 0,
    5. "aggs": {
    6. "Name your aggregations here": {
    7. "histogram": {
    8. "field":"Name the field you want to aggregate on here",
    9. "interval": Specify the interval here
    10. }
    11. }
    12. }
    13. }
    14.  
    15. GET ecommerce_data/_search
    16. {
    17. "size": 0,
    18. "aggs": {
    19. "transactions_per_price_interval": {
    20. "histogram": {
    21. "field": "UnitPrice",
    22. "interval": 10
    23. }
    24. }
    25. }
    26. }
    1. # Range Aggregation
    2. GET Enter_name_of_the_index_here/_search
    3. {
    4. "size": 0,
    5. "aggs": {
    6. "Name your aggregations here": {
    7. "range": {
    8. "field": "Name the field you want to aggregate on here",
    9. "ranges": [
    10. {
    11. "to": x
    12. },
    13. {
    14. "from": x,
    15. "to": y
    16. },
    17. {
    18. "from": z
    19. }
    20. ]
    21. }
    22. }
    23. }
    24. }
    25.  
    26. GET ecommerce_data/_search
    27. {
    28. "size": 0,
    29. "aggs": {
    30. "transactions_per_custom_price_ranges": {
    31. "range": {
    32. "field": "UnitPrice",
    33. "ranges": [
    34. {
    35. "to": 50
    36. },
    37. {
    38. "from": 50,
    39. "to": 200
    40. },
    41. {
    42. "from": 200
    43. }
    44. ]
    45. }
    46. }
    47. }
    48. }
    1. # Terms Aggregation
    2. GET Enter_name_of_the_index_here/_search
    3. {
    4. "aggs": {
    5. "Name your aggregations here": {
    6. "terms": {
    7. "field": "Name the field you want to aggregate on here",
    8. "size": State how many top results you want returned here
    9. }
    10. }
    11. }
    12. }
    13.  
    14. GET ecommerce_data/_search
    15. {
    16. "size": 0,
    17. "aggs": {
    18. "top_5_customers": {
    19. "terms": {
    20. "field": "CustomerID",
    21. "size": 5
    22. }
    23. }
    24. }
    25. }
  • Combined Aggregations
    1. GET ecommerce_data/_search
    2. {
    3. "size": 0,
    4. "aggs": {
    5. "transactions_per_day": {
    6. "date_histogram": {
    7. "field": "InvoiceDate",
    8. "calendar_interval": "day"
    9. },
    10. "aggs": {
    11. "daily_revenue": {
    12. "sum": {
    13. "script": {
    14. "source": "doc['UnitPrice'].value * doc['Quantity'].value"
    15. }
    16. }
    17. }
    18. }
    19. }
    20. }
    21. }
    22.  
    23. GET ecommerce_data/_search
    24. {
    25. "size": 0,
    26. "aggs": {
    27. "transactions_per_day": {
    28. "date_histogram": {
    29. "field": "InvoiceDate",
    30. "calendar_interval": "day"
    31. },
    32. "aggs": {
    33. "daily_revenue": {
    34. "sum": {
    35. "script": {
    36. "source": "doc['UnitPrice'].value * doc['Quantity'].value"
    37. }
    38. }
    39. },
    40. "number_of_unique_customers_per_day": {
    41. "cardinality": {
    42. "field": "CustomerID"
    43. }
    44. }
    45. }
    46. }
    47. }
    48. }
    Query and Aggregation
  • Pull documents with query, then analyze the query data and create summary with aggregations
    1. GET Enter_name_of_the_index_here/_search
    2. {
    3. "query": {
    4. "Enter match or match_phrase here": { "Enter the name of the field": "Enter the value you are looking for" }
    5. },
    6. "aggs": {
    7. "Name your aggregation here": {
    8. "Specify aggregation type here": {
    9. "field": "Name the field you want to aggregate here",
    10. "size": "State how many buckets you want returned here"
    11. }
    12. }
    13. }
    14. }
    1. GET news_headlines/_search
    2. {
    3. "query": {
    4. "match": {
    5. "category": "ENTERTAINMENT"
    6. }
    7. },
    8. "aggs": {
    9. "popular_in_entertainment": { # aggregation name
    10. "significant_text": { # aggregation type
    11. "field": "headline"
    12. }
    13. }
    14. }
    15. }
    Mapping
  • defines how a document and its fields are indexed and stored
  • help optimize the performance of Elasticsearch and save disk space
  • Rules
  • String types
  • View Mapping
    1. GET Enter_name_of_the_index_here/_mapping
    2.  
    3. GET temp_index/_mapping
  • Create an index with Mapping
    1. PUT produce_index
    2. {
    3. "mappings": {
    4. "properties": {
    5. "botanical_name": {
    6. "enabled": false # disabled to save disk space
    7. },
    8. "country_of_origin": { # text and keyword
    9. "type": "text",
    10. "fields": {
    11. "keyword": {
    12. "type": "keyword"
    13. }
    14. }
    15. },
    16. "date_purchased": { # date
    17. "type": "date"
    18. },
    19. "description": { # text
    20. "type": "text"
    21. },
    22. "name": { # text
    23. "type": "text"
    24. },
    25. "produce_type": { # keyword
    26. "type": "keyword"
    27. },
    28. "quantity": {
    29. "type": "long"
    30. },
    31. "unit_price": {
    32. "type": "float"
    33. },
    34. "vendor_details": {
    35. "enabled": false
    36. }
    37. }
    38. }
    39. }
  • Update Mapping
    1. # step 1, create a new index
    2. PUT produce_v2
    3. {
    4. "mappings": {
    5. "properties": {
    6. "botanical_name": {
    7. "type": "text"
    8. },
    9. "country_of_origin": {
    10. "type": "text",
    11. "fields": {
    12. "keyword": {
    13. "type": "keyword",
    14. "ignore_above": 256
    15. }
    16. }
    17. },
    18. "date_purchased": {
    19. "type": "date"
    20. },
    21. "description": {
    22. "type": "text"
    23. },
    24. "name": {
    25. "type": "text"
    26. },
    27. "organic": {
    28. "type": "boolean"
    29. },
    30. "produce_type": {
    31. "type": "keyword"
    32. },
    33. "quantity": {
    34. "type": "long"
    35. },
    36. "unit_price": {
    37. "type": "float"
    38. },
    39. "vendor_details": {
    40. "type": "object",
    41. "enabled": false
    42. }
    43. }
    44. }
    45. }
    1. # step 2: Reindex the data from the original index(produce_index) to the one you just created(produce_v2)
    2. # reindex copys existing data from a source index to a destination index
    3. POST _reindex
    4. {
    5. "source": {
    6. "index": "produce_index"
    7. },
    8. "dest": {
    9. "index": "produce_v2"
    10. }
    11. }
    Runtime Field
  • Enable to create and query fields that are evaluated only at query time
  • Runtime fields are not physically saved in the index
    1. # step 1. create a runtime field and add it to the mapping of the existing index
    2. PUT Enter-name-of-index/_mapping
    3. {
    4. "runtime": {
    5. "Name-your-runtime-field-here": {
    6. "type": "Specify-field-type-here",
    7. "script": {
    8. "source": "Specify the formula you want executed"
    9. }
    10. }
    11. }
    12. }
    1. PUT produce_v2/_mapping
    2. {
    3. "runtime": {
    4. "total": {
    5. "type": "double",
    6. "script": {
    7. "source": "emit(doc['unit_price'].value* doc['quantity'].value)"
    8. }
    9. }
    10. }
    11. }
    1. # step 2. use runtime fields
    2. GET Enter_name_of_the_index_here/_search
    3. {
    4. "size": 0,
    5. "aggs": {
    6. "Name your aggregations here": {
    7. "Specify the aggregation type here": {
    8. "field": "Name the field you want to aggregate on here"
    9. }
    10. }
    11. }
    12. }
    1. GET produce_v2/_search
    2. {
    3. "size": 0,
    4. "aggs": {
    5. "total_expense": {
    6. "sum": {
    7. "field": "total"
    8. }
    9. }
    10. }
    11. }
    Troubleshooting
  • 5XX errors
  • 4XX errors
  • Reference
  • Field data types
  • Tutorialspoint
  • REST APIs
  • Beginner's Crash Course to Elastic Stack at Yourtube
  • Cheatsheet
  • Beginner's Crash Course to Elastic Stack Series at Github
  • Getting Started with Kibana (tutorial) at Youtube