esrally自定义tracks实践
生活随笔
收集整理的這篇文章主要介紹了
esrally自定义tracks实践
小編覺得挺不錯的,現在分享給大家,幫大家做個參考.
一、使用官方的log類型，通過改造challenge來實現parallel的功能，實現對同一個index進行讀寫混合的測試需求
{
"name": "append-no-conflicts",
"description": "Indexes the whole document corpus using Elasticsearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Rally will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only. After that a couple of queries are run.",
"default": true,
"schedule": [
{
"operation": "delete-index"
},
{
"operation": {
"operation-type": "create-index",
"settings": {{index_settings | default({}) | tojson}}
}
},
{
"name": "check-cluster-health",
"operation": {
"operation-type": "cluster-health",
"index": "logs-*",
"request-params": {
"wait_for_status": "{{cluster_health | default('green')}}",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
},
{
"parallel": {
"completed-by": "index-append", #表示當index-append完成時結束并發
"tasks": [
{
"operation": "index-append",
"warmup-time-period": 240,
"clients": {{bulk_indexing_clients | default(8)}}
},
{
"operation": "default",
"clients": 1,
"warmup-iterations": 500,
"iterations": 100,
"target-throughput": 8
},
{
"operation": "term",
"clients": 1,
"warmup-iterations": 500,
"iterations": 100,
"target-throughput": 50
},
{
"operation": "range",
"clients": 1,
"warmup-iterations": 100,
"iterations": 100,
"target-throughput": 1
},
{
"operation": "hourly_agg",
"clients": 1,
"warmup-iterations": 100,
"iterations": 100,
"target-throughput": 0.2
},
{
"operation": "scroll",
"clients": 1,
"warmup-iterations": 100,
"iterations": 200,
"#COMMENT": "Throughput is considered per request. So we issue one scroll request per second which will retrieve 25 pages",
"target-throughput": 1
}
]
}
}
]
},
二、根據實際業務日志來生成data和track使性能測試結果更加貼合實際業務
1、從已有集群中的數據自動生成log和track
esrally版本要求2.0以上
參考：https://esrally.readthedocs.io/en/2.1.0/adding_tracks.html?highlight=custom
esrally create-track --track=acme --target-hosts=127.0.0.1:9200 --indices="products,companies" --output-path=~/tracks
最終生成效果：
airkafka_pm02_2021-02-01-documents-1k.json：xxx-1k表示用于test_mode模式，數據量較小，用于測試
airkafka_pm02_2021-02-01.json：是對應log的index
airkafka_pm02_2021-02-01-documents.json.offset：log日志偏移量
{
"version": 2,
"description": "Tracker-generated track for nsh",
"indices": [
{
"name": "airkafka_pm02_2021-02-12",
"body": "airkafka_pm02_2021-02-12.json", #可以改成track.json所在目錄下的其他自定義的index
"types": ["doc"] #由于es7以上版本不支持type字段，因此生成的track.json沒有該字段。如果是對es7以下版本進行測試，需要增加該字段
},
{
"name": "airkafka_pm02_2021-02-01",
"body": "airkafka_pm02_2021-02-12.json",
"types": ["doc"]
}
],
"corpora": [
{
"name": "nsh",
"target-type": "doc", #該字段對應的是bulk插入的_type字段，必須要指定，不然會報type missing的錯誤
"documents": [
{
"target-index": "airkafka_pm02_2021-02-12",
"source-file": "airkafka_pm02_2021-02-12-documents.json.bz2",
"document-count": 14960567,
"compressed-bytes": 814346714,
"uncompressed-bytes": 12138377222
},
{
"target-index": "airkafka_pm02_2021-02-01",
"source-file": "airkafka_pm02_2021-02-01-documents.json.bz2",
"document-count": 24000503, #需要跟實際的documents文件里的數量一致
"compressed-bytes": 1296215463,
"uncompressed-bytes": 19551041674
}
]
}
],
"operations": [ #自動生成的track.json里不會區分operatin和challenge，可以自己拆分定義，按照這個模板來就行
{
"name": "index-append",
"operation-type": "bulk",
"bulk-size": {{bulk_size | default(5000)}},
"ingest-percentage": 100,
"corpora": "nsh" #要改成上面corpora的name
},
{
"name": "default", #name可以改成其他自定義的
"operation-type": "search", #operation-type只支持search
"index": "airkafka_pm02_2021-*",
"body": {
"query": {
"match_all": {}
}
}
},
{
"name": "term",
"operation-type": "search",
"index": "airkafka_pm02_2021-*", #index也可以自定義
"body": { #body里的query語句可以根據業務需求自定義
"query": {
"term": {
"log_id.raw": {
"value": "gm_client_app_profile_log"
}
}
}
}
},
{
"name": "range",
"operation-type": "search",
"index": "airkafka_pm02_2021-*",
"body": {
"query": {
"range": {
"deveice_level": {
"gte": 0,
"lt": 3
}
}
}
}
},
{
"name": "hourly_agg",
"operation-type": "search",
"index": "airkafka_pm02_2021-*",
"body": {
"size": 0,
"aggs": {
"by_hour": {
"date_histogram": {
"field": "@timestamp",
"interval": "hour"
}
}
}
}
},
{
"name": "scroll",
"operation-type": "search",
"index": "airkafka_pm02_2021-*",
"pages": 25,
"results-per-page": 1000,
"body": {
"query": {
"match_all": {}
}
}
}
],
"challenges": [ #可以自定義多個不同的challenge，然后命令行里指定需要運行的challenge
{
"name": "append-no-conflicts",
"description": "Indexes the whole document corpus using Elasticsearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Rally will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only. After that a couple of queries are run.",
"default": true,
"schedule": [
{
"operation": "delete-index"
},
{
"operation": {
"operation-type": "create-index",
"settings": {}
}
},
{
"name": "check-cluster-health",
"operation": {
"operation-type": "cluster-health",
"index": "airkafka_pm02_2021-*",
"request-params": {
"wait_for_status": "green",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
},
{
"parallel": {
"completed-by": "index-append",
"tasks": [
{
"operation": "index-append",
"warmup-time-period": 240,
"clients": {{bulk_indexing_clients | default(8)}}
},
{
"operation": "default",
"clients": 1,
"warmup-iterations": 500,
"iterations": 100,
"target-throughput": 8 #限定最大的tps，類似于jmeter里的目標加壓。此時service time和letency的大小不一致，service time小于letency，真正具有參考意義的是service time
},
{
"operation": "term",
"clients": 1,
"warmup-iterations": 500,
"iterations": 100,
"target-throughput": 50
},
{
"operation": "range",
"clients": 1,
"warmup-iterations": 100,
"iterations": 100,
"target-throughput": 1
},
{
"operation": "hourly_agg",
"clients": 1,
"warmup-iterations": 100,
"iterations": 100,
"target-throughput": 0.2
},
{
"operation": "scroll",
"clients": 1,
"warmup-iterations": 100,
"iterations": 200,
"#COMMENT": "Throughput is considered per request. So we issue one scroll request per second which will retrieve 25 pages",
"target-throughput": 1
}
]
}
}
]
},
{
"name": "append-no-conflicts-index-only",
"description": "Indexes the whole document corpus using Elasticsearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Rally will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.",
"schedule": [
{
"operation": "delete-index"
},
{
"operation": {
"operation-type": "create-index",
"settings": {}
}
},
{
"name": "check-cluster-health",
"operation": {
"operation-type": "cluster-health",
"index": "airkafka_pm02_2021-*",
"request-params": {
"wait_for_status": "green",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
},
{
"operation": "index-append",
"warmup-time-period": 240,
"clients": 8
},
{
"name": "refresh-after-index",
"operation": "refresh"
},
{
"operation": {
"operation-type": "force-merge",
"request-timeout": 7200
}
},
{
"name": "refresh-after-force-merge",
"operation": "refresh"
},
{
"name": "wait-until-merges-finish",
"operation": {
"operation-type": "index-stats",
"index": "_all",
"condition": {
"path": "_all.total.merges.current",
"expected-value": 0
},
"retry-until-success": true,
"include-in-reporting": false
}
}
]
}
]
}
"target-throughput": 50 #不指定則表示esrally盡最大可能發送消息，即測最大的性能，指定則是按照指定的tps發送。注意，如果指定，service time和letency是不一樣的，letency要大于service time，實際的es性能需要看service time
在自定義track的時候出現的一些錯誤以及解決辦法
1、https://discuss.elastic.co/t/esrally-got-the-benchmark-ended-already-during-warmup-when-running-custom-track/186076/3
2、--on-error=abort 打開該開關，esrally將會在第一次出錯時就停止，同時記錄錯誤日志，建議調試tracks打開
3、在調試track的時候，把bulk_size設置小一點，這樣error時同樣的日志會比較少，方便查看完整日志
4、service time和letency的區別
總結
以上是生活随笔為你收集整理的esrally自定义tracks实践的全部內容，希望文章能夠幫你解決所遇到的問題。
- 上一篇： linux6 epel yum源,Cen
- 下一篇： 基于Three.js、Go语言和Mong