1
2
3
4
5

# 获取HTTP请求量的增长率
rate(http_requests_total[5m])

# 统计前十
topk(10, http_requests_total)

1
2
3
4
5
6

# 获取一段时间内的变化情况
dalta(cpu_temp_celsius{host="zeus"}[2h])

# 简单线性回归，预测未来数据
predict_linear(node_filesystem_free{job="node"}[2h], 4 * 3600) < 0

1
2
3

tar xvfz prometheus-*.tar.gz

cd prometheus-*

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: 'codelab-monitor'

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s

    static_configs:
      - targets: ['localhost:9090']

1
2

cd prometheus-2.3.2.linux-amd64
./prometheus --config.file="prometheus.yml"

1
2
3
4
5
6
7
8

#表达式
prometheus_target_interval_length_seconds

#表达式
prometheus_target_interval_length_seconds{quantile="0.99"}

#计算返回的时间序列数
count(prometheus_target_interval_length_seconds)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

mkdir ./sample && cd sample

git clone https://github.com/prometheus/client_golang.git
cd client_golang/examples/random
go get -d
go build

# Start 3 example targets in separate terminals:
./random -listen-address=:9091
./random -listen-address=:9092
./random -listen-address=:9093

#访问
http://localhost:9091/metrices
http://localhost:9092/metrices
http://localhost:9093/metrices

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

scrape_configs:
  - job_name:       'example-random'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s

    static_configs:
      - targets: ['localhost:8080', 'localhost:8081']
        labels:
          group: 'production'

      - targets: ['localhost:8082']
        labels:
          group: 'canary'

1
2
3
4
5
6

#job_service:rpc_durations_seconds_count:avg_rate5m
groups:
- name: example
  rules:
  - record: job_service:rpc_durations_seconds_count:avg_rate5m
    expr: avg(rate(rpc_durations_seconds_count[5m])) by (job, service)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  evaluation_interval: 15s # Evaluate rules every 15 seconds.

  # Attach these extra labels to all timeseries collected by this Prometheus instance.
  external_labels:
    monitor: 'codelab-monitor'

rule_files:
  - 'prometheus.rules.yml'

scrape_configs:
  - job_name: 'prometheus'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s

    static_configs:
      - targets: ['localhost:9090']

  - job_name:       'example-random'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s

    static_configs:
      - targets: ['localhost:8091', 'localhost:8092']
        labels:
          group: 'production'

      - targets: ['localhost:9093']
        labels:
          group: 'canary'

1
2
3
4
5

#bind-mount
docker run -p 9090:9090 -v /tmp/prometheus.yml:/etc/prometheus.yml  prom/prometheus

#volume
docker run -p 9090:9090 -v /promethe-data  prom/prometheus  --config.file=/prometheus-data/prometheus.yml

1
2
3

FROM prom/prometheus
ADD prometheus.yml /etc/prometheus/
xxx

1

docker build -t my-prometheus .

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

global:
  # How frequently to scrape targets by default.
  [ scrape_interval: <duration> | default = 1m ]

  # How long until a scrape request times out.
  [ scrape_timeout: <duration> | default = 10s ]

  # How frequently to evaluate rules.
  [ evaluation_interval: <duration> | default = 1m ]

  # The labels to add to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    [ <labelname>: <labelvalue> ... ]

# Rule files specifies a list of globs. Rules and alerts are read from
# all matching files.
rule_files:
  [ - <filepath_glob> ... ]

# A list of scrape configurations.
scrape_configs:
  [ - <scrape_config> ... ]

# Alerting specifies settings related to the Alertmanager.
alerting:
  alert_relabel_configs:
    [ - <relabel_config> ... ]
  alertmanagers:
    [ - <alertmanager_config> ... ]

# Settings related to the remote write feature.
remote_write:
  [ - <remote_write> ... ]

# Settings related to the remote read feature.
remote_read:
  [ - <remote_read> ... ]

1
2
3

go get github.com/prometheus/prometheus/cmd/promtool

promtool check rules /path/to/example.rules.yml

1
2

groups:
  [ - <rule_group> ]

1
2
3
4
5

groups:
  - name: example
    relues:
    - record: job:http_inprogress_requests:sum
      expr: sum(http_inprogress_requests) by (job)

1
2
3
4
5
6
7
8

# The name of the group. Must be unique within a file.
name: <string>

# How often rules in the group are evaluated.
[ interval: <duration> | default = global.evaluation_interval ]

rules:
  [ - <rule> ... ]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# The name of the time series to output to. Must be a valid metric name.
record: <string>

# The PromQL expression to evaluate. Every evaluation cycle this is
# evaluated at the current time, and the result recorded as a new set of
# time series with the metric name as given by 'record'.
expr: <string>

# Labels to add or overwrite before storing the result.
labels:
  [ <labelname>: <labelvalue>]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# The name of the alert. Must be a valid metric name.
alert: <string>

# The PromQL expression to evaluate. Every evaluation cycle this is
# evaluated at the current time, and all resultant time series become
# pending/firing alerts.
expr: <string>

# Alerts are considered firing once they have been returned for this long.
# Alerts which have not yet fired for long enough are considered pending.
[ for: <duration> | default = 0s ]

# Labels to add or overwrite for each alert.
labels:
  [ <labelname>: <tmp_string>]

# Annotations to add to each alert.
annotations:
  [ <labelname>: <tmpl_string> ]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

groups:
- name: example
  rules:
  - alerts: HighRequestLatency
    expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5
    for: 10m
    labels:
      severity: page
    annotations:
      summary: High request latency

1
2
3
4
5

# To insert a firing element's label values:
{{ $labels.<labelname> }}

# To insert the numeric expression value of the firing element:
{{ $value }}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

groups:
- name: example
  rules:

  # Alert for any instance that is unreachable for >5 minutes.
  - alert: InstanceDown
    expr: up == 0
    for: 5m
    labels:
      severity: page
    annotations:
      summary: "Instance {{ $labels.instance }} down"
      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

  # Alert for any instance that has a median request latency >1s.
  - alert: APIHighRequestLatency
    expr: api_http_request_latencies_second{quantile="0.5"} > 1
    for: 10m
    annotations:
      summary: "High request latency on {{ $labels.instance }}"
      description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }})s"

1
2
3
4
5
6
7
8

alert: InstanceDown
expr: up == 0
for: 5m
labels:
  severity: page
annotations:
  summary: "Instance {{$labels.instance}} down"
  description: "{{$labels.instance}} of job {{$labels.job}} has been down for more than 5 minutes."

1
2
3

{{ range query "up" }}
  {{ .Labels.instance }} {{ .Value }}
{{ end }}

1
2
3

{{ with query "some_metric{instance='someinstance'}" }}
  {{ . | first | value | humanize }
{{ end }}

1
2
3

{{ with printf "node_memory_MemTotal{job='node', instance='%s'}" .Params.instance | query}}
  {{ . | first | value | humanize1024 }}B
{{ end }}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

<table>
{{ range printf "node_network_receive_bytes{job='node',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device"}}
<tr><th colspan=2>{{ .Labels.device }}</th></tr>
<tr>
  <td>Received</td>
  <td>{{ with printf "rate(node_network_receive_bytes{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device | query }}{{ . | first | value | humanize }}B/s{{end}}</td>
</tr>
<tr>
  <td>Transmitted</td>
  <td>{{ with printf "rate(node_network_transmit_bytes{job='node',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device | query }}{{ . | first | value | humanize }}B/s{{end}}</td>
  </tr>{{ end }}
</table>

1
2
3
4
5
6
7

{{ /* Define the template */ }}
{{define "myTemplate"}}
  do something
{{ end }}

{{/* Use the template */}}
{{template "myTemplate"}}

1
2
3
4
5

{{define "myMultiArgTemplate"}}
  First argument: {{.arg0}}
  Second argument: {{.arg1}}
{{ end }}
{{ template "myMultiArgTemplate" (args 1 2)}}

1
2
3
4

type sample struct {
    Labels map[string]string
    Value float64
}

1
2
3

"this is a string"
'these are unescaped: \n \\ \t'
`these are not unescaped: \n ' " \t`"'`

1

-2.43

1

http_requests_total

1

http_requests_total{job="prometheus", group="canary"}

1

http_requests_total{environment=~"staging|testing|development", method!="GET"}

1
2
3
4
5
6

{job=~".*"} # Bad!

{job=~".+"}              # Good!
{job=~".*",method="get"} # Good!

{__name__=~"job:.*"}

1

http_requests_total{job="prometheus"}[5m]

1
2
3
4
5
6
7

http_requests_total offset 5m

sum(http_requests_total{method="GET"} offset 5m) // GOOD.

sum(http_requests_total{method="GET"}) offset 5m // INVALID.

rate(http_requests_total[5m] offset 1w)

1
2

# resolution可选。Default is the global evaluation interval.
Syntax: <instant_query> '[' <range> ':' [<resolution>] ']' [ offset <duration> ]

1
2

<vector expr> <bin-op> ignoring(<label list>) <vector expr>
<vector expr> <bin-op> on(<label list>) <vector expr>

1
2
3
4
5
6
7
8
9

method_code:http_errors:rate5m{method="get", code="500"}  24
method_code:http_errors:rate5m{method="get", code="404"}  30
method_code:http_errors:rate5m{method="put", code="501"}  3
method_code:http_errors:rate5m{method="post", code="500"} 6
method_code:http_errors:rate5m{method="post", code="404"} 21

method:http_requests:rate5m{method="get"}  600
method:http_requests:rate5m{method="del"}  34
method:http_requests:rate5m{method="post"} 120

1

method_code:http_errors:rate5m{code="500"} / ignoring(code) method:http_requests:rate5m

1
2
3
4

<vector expr> <bin-op> ignoring(<label list>) group_left(<label list>) <vector expr>
<vector expr> <bin-op> ignoring(<label list>) group_right(<label list>) <vector expr>
<vector expr> <bin-op> on(<label list>) group_left(<label list>) <vector expr>
<vector expr> <bin-op> on(<label list>) group_right(<label list>) <vector expr>

1

method_code:http_errors:rate5m / ignoring(code) group_left method:http_request:rate5m

1
2
3
4

{method="get", code="500"}  0.04            //  24 / 600
{method="get", code="404"}  0.05            //  30 / 600
{method="post", code="500"} 0.05            //   6 / 120
{method="post", code="404"} 0.175           //  21 / 120

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

<aggr-op> [without|by (<label list>)] ([parameter,] <vector expression>)

<aggr-op>([parameter,] <vector expression>) [without|by (<label list>)]

sum without (instance) (http_requests_total)

sum by (application, group) (http_requests_total)

count_values("version", build_version)

topk(5, http_requests_total)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

abs()
abs(v instant-vector) 返回输入向量的所有样本的绝对值

absent()
absent(v instant-vector)，判断是否存在

absent(nonexistent{job="myjob"})
# => {job="myjob"}

absent(nonexistent{job="myjob",instance=~".*"})
# => {job="myjob"}

absent(sum(nonexistent{job="myjob"}))
# => {}

ceil()
ceil(v instant-vector) 将v中所有元素的样本值向上四舍五入到最接近的整数

floor()
floor(v instant-vector) 与ceil()相反，将v中所有元素的样本值向下四舍五入到最接近的整数

changes()
changes(v range-vector) 输入一个区间向量，返回这个区间向量内每个样本数据值变化的次数（瞬时向量）。如果样本数据值没有发生变化，则返回结果为1

clamp_max()
clamp_max(v instant-vector, max scalar) 输入一个瞬时向量和最大值，样本数据值若大于max，则改为max，否则不变

clamp_min()
clamp_min(v instant-vector, min scalar) 输入一个瞬时向量和最小值，样本数据值若小于min，则改为min，否则不变

day_of_month()
day_of_month(v=vector(time()) instant-vector) 值范围为1-31

day_of_week()
day_of_week(v=vector(time()) instant-vector) 值范围为0-6

days_in_month()
days_in_month(v=vector(time()) instant-vector) 月份的天数，值范围为28-31

minute()
minute(v=vector(time()) instant-vector) 函数返回给定UTC时间当前小时的第多少分钟，范围为0-59

month()
month(v=vector(time()) instant-vector) 函数返回给定UTC时间当前属于第几个月，范围为1-12

year()
year(v=vector(time()) instant-vector) 返回被给定 UTC 时间的当前年份

hour()
hour(v=vector(time()) instant-vector) 值范围为0-23

delta()
delta(v range-vector) 它计算一个区间向量v的第一个元素和最后一个元素之间的差值，返回一个瞬时向量
delta(cpu_temp_celsius{host="zeus"}[2h]) # 现在和两小时前的CPU温度差

idelta()
idelta(v range-vector) 计算最后两个样本之间的差

deriv()
deriv(v range-vector) 使用简单的线性回归计算区间向量v中各个时间序列的导数

exp()
exp(v instant-vector) 输入一个瞬时向量，返回各个样本值的e的指数值

histogram_quantile()
histogram_quantile(φ float, b instant-vector)
histogram_quantile(0.9, rate(http_request_duration_seconds_bucket[10m])) # 计算过去10分钟内请求持续在90%
histogram_quantile(0.9, sum(rate(http_request_duration_seconds_bucket[10m])) by (job, le)) # 聚合

holt_winters()
holt_winters(v range-vector, sf scalar, tf scalar) 基于区间向量v，生成时间序列数据平滑值

increase()
increase(v range-vector) 获取区间向量中的第一个和最后一个样本并返回其增长量
increase(http_requests_total{job="api-server"}[5m]) # 区间向量中每个时间序列过去5分钟内HTTP请求数的增长数

label_join()

label_replace()

ln()
ln(v instant-vector) 计算瞬时向量v中所有样本数据的自然对数

log2()
log2(v instant-vector) 函数计算瞬时向量v中所有样本数据的二进制对数

log10()
log10(v instant-vector) 计算瞬时向量v中所有样本数据的十进制对数

predict_linear()
predict_linear(v range-vector, t scalar) 函数可以预测时间序列v在t秒后的值
predict_linear(node_filesystem_free{job="node"}[2h], 4 * 3600) < 0 # 基于2小时的样本数据，来预测主机可用磁盘空间的是否在4个小时候被占满

rate()
rate(v range-vector) 直接计算区间向量 v 在时间窗口内平均增长速率
rate(http_requests_total[5m]) 区间向量中每个时间序列过去5分钟内HTTP请求数的每秒增长率

irate()
irate(v range-vector) 用于计算区间向量的增长率，但是其反应出的是瞬时增长率。通过区间向量中最后两个两本数据来计算区间向量的增长速率。
irate(http_requests_total{job="api-server"}[5m]) # 区间向量中每个时间序列过去 5 分钟内最后两个样本数据的 HTTP 请求数的增长率

resets()
resets(v range-vector) 的参数是一个区间向量。对于每个时间序列，它都返回一个计数器重置的次数。两个连续样本之间的值的减少被认为是一次计数器重置。

round()
round(v instant-vector, to_nearest=1 scalar) 与ceil和floor函数类似，返回向量中所有样本值的最接近的整数

scalar()
scalar(v instant-vector) 函数的参数是一个单元素的瞬时向量,它返回其唯一的时间序列的值作为一个标量

sort()
sort(v instant-vector) 函数对向量按元素的值进行升序排序

sort_desc()
降序排列

sqrt()
sqrt(v instant-vector) 计算向量v 所有元素的平方根

vector()
vector(s scalar)

<aggregation>_over_time()
avg_over_time(range-vector) : 区间向量内每个度量指标的平均值。
min_over_time(range-vector) : 区间向量内每个度量指标的最小值。
max_over_time(range-vector) : 区间向量内每个度量指标的最大值。
sum_over_time(range-vector) : 区间向量内每个度量指标的求和。
count_over_time(range-vector) : 区间向量内每个度量指标的样本数据个数。
quantile_over_time(scalar, range-vector) : 区间向量内每个度量指标的样本数据值分位数，φ-quantile (0 ≤ φ ≤ 1)。
stddev_over_time(range-vector) : 区间向量内每个度量指标的总体标准差。
stdvar_over_time(range-vector) : 区间向量内每个度量指标的总体标准方差。

1

http_requests_total

1

http_requests_total{job="apiserver", handler="/api/comments"}

1

http_requests_total{job="apiserver", handler="/api/comments"}[5m]

1

http_requests_total{job=~".*server"}

1

http_requests_total{status!~"4.."}

1

rate(http_requests_total[5m])[30m:1m]

1

max_over_time(deriv(rate(distance_covered_total[5s])[30s:5s])[10m:])

1

rate(http_requests_total[5m])

1
2
3

sum by (job) (
  rate(http_requests_total[5m])
)

1
2
3
4
5
6
7

(instance_memory_limit_bytes - instance_memory_usage_bytes) / 1024 / 1024


# 同样的表达式，只不过通过应用相加
sum by (app, proc) (
  instance_memory_limit_bytes - instance_memory_usage_bytes
) / 1024 / 1024

1

topk(3, sum by (app, proc) (rate(instance_cpu_time_ns[5m])))

1

count(instance_cpu_time_ns) by (app)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

{
  "status": "success" | "error",
  "data": <data>,

  // Only set if status is "error". The data field may still hold
  // additional data.
  "errorType": "<string>",
  "error": "<string>",

  // Only if there were warnings while executing the request.
  // There will still be data in the data field.
  "warnings": ["<string>"]
}

1
2

GET /api/v1/query
POST /api/v1/query

1
2
3
4
5
6
7
8

# 表达式
query=<string>

# 时间戳，可选。默认使用当前系统时间
time=<rfc3339 | unix_timestamp>

# 超时，可选。默认使用全局的-query.timeout参数
timeout=<duration>

1
2
3
4

{
  "resultType": "matrix" | "vector" | "scalar" | "string",
  "result": <value>
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

$ curl 'http://localhost:9090/api/v1/query?query=up&time=2015-07-01T20:10:51.781Z'
{
   "status" : "success",
   "data" : {
      "resultType" : "vector",
      "result" : [
         {
            "metric" : {
               "__name__" : "up",
               "job" : "prometheus",
               "instance" : "localhost:9090"
            },
            "value": [ 1435781451.781, "1" ]
         },
         {
            "metric" : {
               "__name__" : "up",
               "job" : "node",
               "instance" : "localhost:9100"
            },
            "value" : [ 1435781451.781, "0" ]
         }
      ]
   }
}

1
2

GET /api/v1/query_range
POST /api/v1/query_range

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 表达式
query=<string>

# 时间戳
start=<rfc3339 | unix_timestamp>
end=<rfc3339 | unix_timestamp>

# 查询时间步长，时间区间内每step秒执行一次
step=<duration | float>

# 超时，可选
timeout=<duration>

1
2
3
4

{
  "resultType": "matrix",
  "result": <value>
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

$ curl 'http://localhost:9090/api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s'
{
   "status" : "success",
   "data" : {
      "resultType" : "matrix",
      "result" : [
         {
            "metric" : {
               "__name__" : "up",
               "job" : "prometheus",
               "instance" : "localhost:9090"
            },
            "values" : [
               [ 1435781430.781, "1" ],
               [ 1435781445.781, "1" ],
               [ 1435781460.781, "1" ]
            ]
         },
         {
            "metric" : {
               "__name__" : "up",
               "job" : "node",
               "instance" : "localhost:9091"
            },
            "values" : [
               [ 1435781430.781, "0" ],
               [ 1435781445.781, "0" ],
               [ 1435781460.781, "1" ]
            ]
         }
      ]
   }
}

1
2

GET /api/v1/series
POST /api/v1/series

1
2
3
4
5
6

# 标签选择器是 series_selector。必须至少提供一个match[]参数
match[]=<series_selector>

# 时间戳
start=<rfc3339 | unix_timestamp>
end=<rfc3339 | unix_timestamp>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

$ curl -g 'http://localhost:9090/api/v1/series?' --data-urlencode='match[]=up' --data-urlencode='match[]=process_start_time_seconds{job="prometheus"}'
{
   "status" : "success",
   "data" : [
      {
         "__name__" : "up",
         "job" : "prometheus",
         "instance" : "localhost:9090"
      },
      {
         "__name__" : "up",
         "job" : "node",
         "instance" : "localhost:9091"
      },
      {
         "__name__" : "process_start_time_seconds",
         "job" : "prometheus",
         "instance" : "localhost:9090"
      }
   ]
}

1
2

GET /api/v1/labels
POST /api/v1/labels

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

$ curl 'localhost:9090/api/v1/labels'
{
    "status": "success",
    "data": [
        "__name__",
        "call",
        "code",
        "config",
        "dialer_name",
        "endpoint",
        "event",
        "goversion",
        "handler",
        "instance",
        "interval",
        "job",
        "le",
        "listener_name",
        "name",
        "quantile",
        "reason",
        "role",
        "scrape_job",
        "slice",
        "version"
    ]
}

1

GET /api/v1/label/<label_name>/values

1
2
3
4
5
6
7
8

$ curl http://localhost:9090/api/v1/label/job/values
{
   "status" : "success",
   "data" : [
      "node",
      "prometheus"
   ]
}

1
2
3
4
5
6
7

[
  {
    "metric": { "<label_name>": "<label_value>", ... },
    "values": [ [ <unix_time>, "<sample_value>" ], ... ]
  },
  ...
]

1
2
3
4
5
6
7

[
  {
    "metric": { "<label_name>": "<label_value>", ... },
    "value": [ <unix_time>, "<sample_value>" ]
  },
  ...
]

1

[ <unix_time>, "<scalar_value>" ]

1

[ <unix_time>, "<string_value>" ]

1

GET /api/v1/targets

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

curl http://localhost:9090/api/v1/targets
{
  "status": "success",
  "data": {
    "activeTargets": [
      {
        "discoveredLabels": {
          "__address__": "127.0.0.1:9090",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "job": "prometheus"
        },
        "labels": {
          "instance": "127.0.0.1:9090",
          "job": "prometheus"
        },
        "scrapeUrl": "http://127.0.0.1:9090/metrics",
        "lastError": "",
        "lastScrape": "2017-01-17T15:07:44.723715405+01:00",
        "health": "up"
      }
    ],
    "droppedTargets": [
      {
        "discoveredLabels": {
          "__address__": "127.0.0.1:9100",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "job": "node"
        },
      }
    ]
  }
}

1

GET /api/v1/rules

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

curl http://localhost:9090/api/v1/rules

{
    "data": {
        "groups": [
            {
                "rules": [
                    {
                        "alerts": [
                            {
                                "activeAt": "2018-07-04T20:27:12.60602144+02:00",
                                "annotations": {
                                    "summary": "High request latency"
                                },
                                "labels": {
                                    "alertname": "HighRequestLatency",
                                    "severity": "page"
                                },
                                "state": "firing",
                                "value": "1e+00"
                            }
                        ],
                        "annotations": {
                            "summary": "High request latency"
                        },
                        "duration": 600,
                        "health": "ok",
                        "labels": {
                            "severity": "page"
                        },
                        "name": "HighRequestLatency",
                        "query": "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5",
                        "type": "alerting"
                    },
                    {
                        "health": "ok",
                        "name": "job:http_inprogress_requests:sum",
                        "query": "sum(http_inprogress_requests) by (job)",
                        "type": "recording"
                    }
                ],
                "file": "/rules.yaml",
                "interval": 60,
                "name": "example"
            }
        ]
    },
    "status": "success"
}

1

GET /api/v1/alerts

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

curl http://localhost:9090/api/v1/alerts

{
    "data": {
        "alerts": [
            {
                "activeAt": "2018-07-04T20:27:12.60602144+02:00",
                "annotations": {},
                "labels": {
                    "alertname": "my-alert"
                },
                "state": "firing",
                "value": "1e+00"
            }
        ]
    },
    "status": "success"
}

1

GET /api/v1/targets/metadata

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

curl -G http://localhost:9091/api/v1/targets/metadata \
    --data-urlencode 'metric=go_goroutines' \
    --data-urlencode 'match_target={job="prometheus"}' \
    --data-urlencode 'limit=2'
{
  "status": "success",
  "data": [
    {
      "target": {
        "instance": "127.0.0.1:9090",
        "job": "prometheus"
      },
      "type": "gauge",
      "help": "Number of goroutines that currently exist.",
      "unit": ""
    },
    {
      "target": {
        "instance": "127.0.0.1:9091",
        "job": "prometheus"
      },
      "type": "gauge",
      "help": "Number of goroutines that currently exist.",
      "unit": ""
    }
  ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

curl -G http://localhost:9091/api/v1/targets/metadata \
    --data-urlencode 'match_target={instance="127.0.0.1:9090"}'
{
  "status": "success",
  "data": [
    // ...
    {
      "target": {
        "instance": "127.0.0.1:9090",
        "job": "prometheus"
      },
      "metric": "prometheus_treecache_zookeeper_failures_total",
      "type": "counter",
      "help": "The total number of ZooKeeper failures.",
      "unit": ""
    },
    {
      "target": {
        "instance": "127.0.0.1:9090",
        "job": "prometheus"
      },
      "metric": "prometheus_tsdb_reloads_total",
      "type": "counter",
      "help": "Number of times the database reloaded block data from disk.",
      "unit": ""
    },
    // ...
  ]
}

1

GET /api/v1/alertmanagers

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

curl http://localhost:9090/api/v1/alertmanagers
{
  "status": "success",
  "data": {
    "activeAlertmanagers": [
      {
        "url": "http://127.0.0.1:9090/api/v1/alerts"
      }
    ],
    "droppedAlertmanagers": [
      {
        "url": "http://127.0.0.1:9093/api/v1/alerts"
      }
    ]
  }
}

1

GET /api/v1/status/config

1
2
3
4
5
6
7

curl http://localhost:9090/api/v1/status/config
{
  "status": "success",
  "data": {
    "yaml": "<content of the loaded config file in YAML>",
  }
}

1

GET /api/v1/status/flags

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

curl http://localhost:9090/api/v1/status/flags
{
  "status": "success",
  "data": {
    "alertmanager.notification-queue-capacity": "10000",
    "alertmanager.timeout": "10s",
    "log.level": "info",
    "query.lookback-delta": "5m",
    "query.max-concurrency": "20",
    ...
  }
}

1

GET /api/v1/status/runtimeinfo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

curl http://localhost:9090/api/v1/status/runtimeinfo
{
  "status": "success",
  "data": {
    "startTime": "2019-11-02T17:23:59.301361365+01:00",
    "CWD": "/",
    "reloadConfigSuccess": true,
    "lastConfigTime": "2019-11-02T17:23:59+01:00",
    "chunkCount": 873,
    "timeSeriesCount": 873,
    "corruptionCount": 0,
    "goroutineCount": 48,
    "GOMAXPROCS": 4,
    "GOGC": "",
    "GODEBUG": "",
    "storageRetention": "15d"
  }
}

1

GET /api/v1/status/buildinfo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

curl http://localhost:9090/api/v1/status/buildinfo
{
  "status": "success",
  "data": {
    "version": "2.13.1",
    "revision": "cb7cbad5f9a2823a622aaa668833ca04f50a0ea7",
    "branch": "master",
    "buildUser": "julius@desktop",
    "buildDate": "20191102-16:19:59",
    "goVersion": "go1.13.1"
  }
}

1
2

POST /api/v1/admin/tsdb/snapshot
PUT /api/v1/admin/tsdb/snapshot

1
2
3
4
5
6
7

curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot
{
  "status": "success",
  "data": {
    "name": "20171210T211224Z-2be650b6d019eb54"
  }
}

1
2

POST /api/v1/admin/tsdb/delete_series
PUT /api/v1/admin/tsdb/delete_series

1

curl -X POST -g 'http://localhost:9090/api/v1/admin/tsdb/delete_series?match[]=up&match[]=process_start_time_seconds{job="prometheus"}'

1
2

POST /api/v1/admin/tsdb/clean_tombstones
PUT /api/v1/admin/tsdb/clean_tombstones

1

curl -XPOST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

./data
├── 01BKGV7JBM69T2G1BGBGM6KB12
│   └── meta.json
├── 01BKGTZQ1SYQJTR4PB43C8PD98
│   ├── chunks
│   │   └── 000001
│   ├── tombstones
│   ├── index
│   └── meta.json
├── 01BKGTZQ1HHWHV8FBJXW1Y3W0K
│   └── meta.json
├── 01BKGV7JC0RY8A6MACW02A2PJD
│   ├── chunks
│   │   └── 000001
│   ├── tombstones
│   ├── index
│   └── meta.json
└── wal
    ├── 00000002
    └── checkpoint.000001

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 数据存储路径，默认data/
--storage.tsdb.path

# 样本数据在存储中保存的时间。超过该时间限制的数据就会被删除。默认15d
-storage.tsdb.retention.time

# 每个块的最大字节数（不包括 wal 文件）。如果超过限制，最早的样本数据会被优先删除。支持的单位有 KB, MB, GB, PB。默认0，即为不限制
--storage.tsdb.retention.size

# 压缩wal
--storage.tsdb.wal-compression

1

needed_disk_space = retention_time_seconds * ingested_samples_per_second * bytes_per_sample

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

scrape_configs:
  - job_name: 'federate'
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{__name__=~".+"}'
    static_configs:
      - targets:
        - 'source-prometheus-1:9090'
    file_sd_configs:
      - files:
        - 'discovery/federate/*.yml'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# discovery/federate/federate.yml

- labels:
    env: test-02
  targets:
    - source-prometheus-2:9090

- labels:
    env: test-03
  targets:
    - source-prometheus-3:9090

1
2

#sudo yum install <rpm package url>
sudo yum install https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

[grafana]
name=grafana
baseurl=https://packagecloud.io/grafana/stable/el/7/$basearch
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packagecloud.io/gpg.key https://grafanarel.s3.amazonaws.com/RPM-GPG-KEY-grafana
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt


sudo yum install -y grafana


#启动
systemctl start grafana-server


#命令行工具
grafana-cli

1
2
3
4
5
6
7

# Download and unpack Grafana from binary tar (adjust version as appropriate).
curl -L -O https://grafanarel.s3.amazonaws.com/builds/grafana-2.5.0.linux-x64.tar.gz
tar zxf grafana-2.5.0.linux-x64.tar.gz

# Start Grafana.
cd grafana-2.5.0/
./bin/grafana-server web

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

#基础栗子
docker run -d -p 3000:3000 grafana/grafana


#配置化
docker run \
  -d \
  -p 3000:3000 \
  --name=grafana \
  -e "GF_SERVER_ROOT_URL=http://grafana.server.name" \
  -e "GF_SECURITY_ADMIN_PASSWORD=secret" \
  grafana/grafana:version


#默认环境变量值
GF_PATHS_CONFIG	/etc/grafana/grafana.ini
GF_PATHS_DATA	/var/lib/grafana
GF_PATHS_HOME	/usr/share/grafana
GF_PATHS_LOGS	/var/log/grafana
GF_PATHS_PLUGINS	/var/lib/grafana/plugins
GF_PATHS_PROVISIONING	/etc/grafana/provisioning

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# my global config
global:
  scrape_interval:     1m # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 1m # Evaluate rules every 15 seconds. The default is every 1 minute.
  scrape_timeout: 1m # scrape_timeout is set to the global default (10s).
  # for thanos/fedaration
  external_labels:
    region: ali # hw|tx
    replica: prom-01

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets: ["localhost:9093"]

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - "rules/*.yml"  # 通用告警规则模板
  - "rules/nodes/*.yml"  # 各主机告警规则
  - "rules/services/*.yml"  # 各主机告警规则

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: node-process-exporter
    file_sd_configs:
      - files:
        - "discovery/nodes/*.yml"
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

  # fedefation
  - job_name: 'federate'
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{__name__=~".+"}'
    file_sd_configs:
      - files:
        - 'discovery/federate/*.yml'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# global config
global:
  resolve_timeout: 5m
  # smtp config
  # slack config
  # wechat config
  # http config

# templates config
templates: []

# route tree
route:
  receiver: 'webhook-prometheusalert'
  group_by:
    - hostgroup
    - hostname
    - alertname
    - instance
  group_wait: 30s
  group_interval: 1m
  # notification again
  repeat_interval: 10m
  # continue config
  # match: [ <labelname>: <labelvalue>, ... ]
  # match_re: [ <labelname>: <regex>, ... ]
  routes:
  - match:
      severity: warning
      repeat_interval: 12h
  - match_re:
      severity: notice|info
      repeat_interval: 24h

# receiver lists
receivers:
- name: 'webhook-prometheusalert'
  webhook_configs:
      - url: 'http://prometheusAlert:8080/prometheus/alert'

# list of inhibit rules
inhibit_rules:
  # aliyun
  - source_match:
      severity: 'critical'
    target_match_re:
      severity: 'warning'
    # source_match_re
    # target_match_re
    # Labels that must have an equal value in the source and target
    equal: ['instance','job','instanceId','kind']
  # nodes
  - source_match:
      severity: 'critical'
    target_match_re:
      severity: 'warning'
    equal:
    - instance
    - job
    - hostname
    - kind

1
2
3
4
5
6
7

- labels:
    hostgroup: test
    hostname: localhost
  targets:
    - localhost:9100 # node-exporter
    - localhost:9256 # process-exporter
    - localhost:8181 # easeprobe

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

groups:
- name: node-cpu
  rules:
  # cpu核数
  - record: instance:node_cpus:count
    expr: count without (cpu, mode) (node_cpu_seconds_total{mode="idle"})
  # 每个cpu使用率
  - record: instance_cpu:node_cpu_seconds_not_idle:rate2m
    expr: sum without (mode) (1 - rate(node_cpu_seconds_total{mode="idle"}[2m]))
  # 总cpu使用率
  - record: instance:node_cpu_utilization:ratio
    expr: avg without (cpu) (instance_cpu:node_cpu_seconds_not_idle:rate2m)

  - alert: cpu使用率大于85%
    expr: instance:node_cpu_utilization:ratio * 100 > 85
    for: 3m
    labels:
      severity: warning
      level: 2
      kind: CpuUsage
    annotations:
      summary: "cpu使用率大于85%"
      description: "{{ $labels.hostname }}的cpu使用率: {{ $value | humanize }}%"

  - alert: cpu负载大于Cores
    expr: node_load1 > instance:node_cpus:count
    for: 3m
    labels:
      severity: warning
      level: 2
      kind: CpuLoad
    annotations:
      summary: "cpu负载大于cpu核数"
      description: "{{ $labels.hostname }}的cpu负载: {{ $value }}"

  - alert: 主机上下文切换忙
    expr: (rate(node_context_switches_total[5m]) / instance:node_cpus:count) > 2000
    for: 5m
    labels:
      severity: warning
      level: 2
    annotations:
      summary: "主机上下文切换忙"
      description: "主机{{ $labels.hostname }}的上下文切换大于2000/s: {{ $value | humanize }}/s"

- name: node-memory
  rules:
  # 内存可用率
  - record: instance:node_memory_available:ratio
    expr: >
      (
        node_memory_MemAvailable_bytes or
        (
          node_memory_Buffers_bytes +
          node_memory_Cached_bytes +
          node_memory_MemFree_bytes +
          node_memory_Slab_bytes
        )
      ) /
      node_memory_MemTotal_bytes      
  # 内存使用率
  - record: instance:node_memory_utilization:ratio
    expr: 1 - instance:node_memory_available:ratio

  - alert: 主机内存面临压力
    expr: rate(node_vmstat_pgmajfault[1m]) > 1000
    for: 5m
    labels:
      severity: warning
      level: 2
    annotations:
      summary: "主机内存面临压力: {{ $labels.instance }}"
      description: "节点内存面临压力。High rate of major page faults: {{ $value }}"

  - alert: 内存使用率大于85%
    expr: instance:node_memory_utilization:ratio * 100 > 85
    for: 3m
    labels:
      severity: warning
      level: 2
      kind: MemoryUsage
    annotations:
      summary: "内存使用率超过85%"
      description: "{{ $labels.hostname }}的内存使用率: {{ $value | humanize }}%"

  - alert: 主机检测到oom kill
    expr: >
      increase(node_vmstat_oom_kill[1h]) > 2
      or
      increase(syslog_oom_kills_total[1h]) > 2      
    labels:
      severity: warning
      level: 2
    annotations:
      summary: "主机检测到oom kill"
      description: "{{ $labels.hostname }}检测到oom kill: {{ $value }}"


- name: node-filesystem
  rules:
  # 磁盘分区可用率
  - record: instance:node_filesystem_avail:ratio
    expr: node_filesystem_avail_bytes{device=~"(/dev/.+|tank/dataset)"} / node_filesystem_size_bytes{device=~"(/dev/.+|tank/dataset)"}
  # 磁盘分区使用率
  - record: instance:node_filesystem_utilization:ratio
    expr: 1 - instance:node_filesystem_avail:ratio
  # 分区inode可用率
  - record: instance:node_filesystem_files_avail:ratio
    expr: node_filesystem_files_free / node_filesystem_files
  # 分区inode使用率
  - record: instance:node_filesystem_files_utilization:ratio
    expr: 1 - instance:node_filesystem_files_avail:ratio

  - alert: 磁盘分区使用率大于85%
    expr: instance:node_filesystem_utilization:ratio{fstype=~"(ext.|xfs|zfs)"} * 100 > 85
    for: 10m
    labels:
      severity: warning
      level: 2
      kind: "{{ $labels.mountpoint }}"
    annotations:
      summary: "磁盘分区使用率大于85%"
      description: "{{ $labels.hostname }}的磁盘分区{{ $labels.mountpoint }}使用率为: {{ $value | humanize }}%"

  - alert: 分区inode使用率大于80%
    expr: instance:node_filesystem_files_utilization:ratio{fstype=~"(ext.|xfs)"} * 100 > 80
    for: 3m
    labels:
      severity: warning
      level: 2
    annotations:
      summary: "磁盘分区的inode使用率大于80%"
      description: "{{ $labels.hostname }}的磁盘分区{{ $labels.mountpoint }}的inode使用率: '{{ $value | humanize }}%'"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# https://github.com/ncabatoff/process-exporter
# /proc/<pid>/xxx
process_names:
  # java
  - name: "{{.Matches}}"
    cmdline:
    - '.+/bin/java -jar /opt/.+'
  # mongo
  - name: "{{.Matches}}"
    cmdline:
    - '.+/bin/mongo[ds] -f /opt/mongodb.+'

1
2
3
4
5

class YourClass {
  static final Counter requests = Counter.build()
      .name("requests_total")
      .help("Requests.").register();
}

1
2
3
4
5

# HELP node_cooling_device_max_state Maximum throttle state of the cooling device
# TYPE node_cooling_device_max_state gauge
node_cooling_device_max_state{name="0",type="Processor"} 7
node_cooling_device_max_state{name="1",type="Processor"} 7
node_cooling_device_max_state{name="2",type="Processor"} 7

1
2
3
4
5
6
7

node_exporter --collector.textfile.directory=/tmp/textfile

cd /tmp/textfile
cat test1.prom
# TYPE test1 gauge
# HELP test1 test
test1 1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

- record: instance_path:requests:rate5m
  expr: rate(requests_total{job="myjob"}[5m])

- record: path:requests:rate5m
  expr: sum without (instance)(instance_path:requests:rate5m{job="myjob"})

- record: instance_path:request_failures:rate5m
  expr: rate(request_failures_total{job="myjob"}[5m])

- record: instance_path:request_failures_per_requests:ratio_rate5m
  expr: |2
      instance_path:request_failures:rate5m{job="myjob"}
    /
      instance_path:requests:rate5m{job="myjob"}

- record: job:request_latency_seconds_count:avg_rate5m
  expr: avg without (instance, path)(instance:request_latency_seconds_count:rate5m{job="myjob"})