Skip to content

Commit 2e67128

Browse files
jcpunkjcpunk
committed
Add optional timestamp to metric output
Fixes #18
1 parent fdf2489 commit 2e67128

File tree

1 file changed

+38
-30
lines changed

1 file changed

+38
-30
lines changed

‎cvmfs-client-prometheus.sh‎

Lines changed: 38 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
HTTP_HEADER='FALSE'
44
USE_NON_STANDARD_MOUNTPOINTS='FALSE'
5+
EPOCHTIME=''
56

67
TMPFILE=$(mktemp)
78

@@ -42,10 +43,11 @@ declare -A CVMFS_EXTENDED_ATTRIBUTE_NAMES=(
4243

4344
#############################################################
4445
usage() {
45-
echo "Usage: $0 [-h|--help] [--http] [--non-standard-mountpoints]" >&2
46+
echo "Usage: $0 [-h|--help] [--http] [--non-standard-mountpoints] [--timestamp]" >&2
4647
echo '' >&2
4748
echo ' --http: add the HTTP protocol header to the output' >&2
4849
echo ' --non-standard-mountpoints: use cvmfs_config status instead of findmnt to discover repositories' >&2
50+
echo ' --timestamp: add a timestamp to each metric' >&2
4951
echo '' >&2
5052
echo 'NOTE: The user running this script must have read access' >&2
5153
echo ' to the CVMFS cache files!' >&2
@@ -60,11 +62,12 @@ generate_metric() {
6062
local help_text="$3"
6163
local metric_labels="$4"
6264
local metric_value="$5"
65+
local metric_timestamp="$6"
6366

6467
cat >>"${TMPFILE}" <<EOF
6568
# HELP $metric_name $help_text
6669
# TYPE $metric_name $metric_type
67-
${metric_name}{${metric_labels}} ${metric_value}
70+
${metric_name}{${metric_labels}} ${metric_value} ${metric_timestamp}
6871
EOF
6972
}
7073

@@ -134,7 +137,7 @@ get_cvmfs_repo_extended_attribute_gauge_metrics() {
134137
local result
135138
result=$(attr -g "${attribute}" "${repomountpoint}" | tail -n +2)
136139
local metric_name="${CVMFS_EXTENDED_ATTRIBUTE_NAMES[${attribute}]}"
137-
generate_metric "${metric_name}" 'gauge' "${CVMFS_EXTENDED_ATTRIBUTE_GAUGES[${attribute}]}" "repo=\"${fqrn}\"" "${result}"
140+
generate_metric "${metric_name}" 'gauge' "${CVMFS_EXTENDED_ATTRIBUTE_GAUGES[${attribute}]}" "repo=\"${fqrn}\"" "${result}" "${EPOCHTIME}"
138141
done
139142
}
140143

@@ -166,7 +169,7 @@ get_cvmfs_repo_proxy_metrics() {
166169
break
167170
fi
168171
done
169-
generate_metric "cvmfs_net_proxy" "gauge" "Shows all registered proxies for this repository." "repo=\"${fqrn}\",group=\"${my_proxy_group}\",url=\"${proxy}\"" 1
172+
generate_metric "cvmfs_net_proxy" "gauge" "Shows all registered proxies for this repository." "repo=\"${fqrn}\",group=\"${my_proxy_group}\",url=\"${proxy}\"" 1 "${EPOCHTIME}"
170173
done
171174
}
172175

@@ -188,44 +191,44 @@ get_cvmfs_repo_metrics() {
188191

189192
local cached_bytes
190193
cached_bytes=$(cvmfs_talk -i "${reponame}" cache size | tr -d ')(' | tr -s '[:space:]' | cut -d ' ' -f 6)
191-
generate_metric 'cvmfs_cache_cached_bytes' 'gauge' 'CVMFS currently cached bytes.' "repo=\"${fqrn}\"" "${cached_bytes}"
194+
generate_metric 'cvmfs_cache_cached_bytes' 'gauge' 'CVMFS currently cached bytes.' "repo=\"${fqrn}\"" "${cached_bytes}" "${EPOCHTIME}"
192195

193196
local pinned_bytes
194197
pinned_bytes=$(cvmfs_talk -i "${reponame}" cache size | tr -d ')(' | tr -s '[:space:]' | cut -d ' ' -f 10)
195-
generate_metric 'cvmfs_cache_pinned_bytes' 'gauge' 'CVMFS currently pinned bytes.' "repo=\"${fqrn}\"" "${pinned_bytes}"
198+
generate_metric 'cvmfs_cache_pinned_bytes' 'gauge' 'CVMFS currently pinned bytes.' "repo=\"${fqrn}\"" "${pinned_bytes}" "${EPOCHTIME}"
196199

197200
local total_cache_size_mb
198201
total_cache_size_mb=$(cvmfs_talk -i "${reponame}" parameters | grep CVMFS_QUOTA_LIMIT | tr '=' ' ' | tr -s '[:space:]' | cut -d ' ' -f 2)
199202
local total_cache_size
200203
total_cache_size=$((total_cache_size_mb * 1024 * 1024))
201-
generate_metric 'cvmfs_cache_total_size_bytes' 'gauge' 'CVMFS configured cache size via CVMFS_QUOTA_LIMIT.' "repo=\"${fqrn}\"" "${total_cache_size}"
204+
generate_metric 'cvmfs_cache_total_size_bytes' 'gauge' 'CVMFS configured cache size via CVMFS_QUOTA_LIMIT.' "repo=\"${fqrn}\"" "${total_cache_size}" "${EPOCHTIME}"
202205

203206
local cache_volume_max
204207
cache_volume_max=$(df -B1 "${cache_volume}" | tail -n 1 | tr -s '[:space:]' | cut -d ' ' -f 2)
205-
generate_metric 'cvmfs_cache_physical_size_bytes' 'gauge' 'CVMFS cache volume physical size.' "repo=\"${fqrn}\"" "${cache_volume_max}"
208+
generate_metric 'cvmfs_cache_physical_size_bytes' 'gauge' 'CVMFS cache volume physical size.' "repo=\"${fqrn}\"" "${cache_volume_max}" "${EPOCHTIME}"
206209

207210
local cache_volume_free
208211
cache_volume_free=$(df -B1 "${cache_volume}" | tail -n 1 | tr -s '[:space:]' | cut -d ' ' -f 4)
209-
generate_metric 'cvmfs_cache_physical_avail_bytes' 'gauge' 'CVMFS cache volume physical free space available.' "repo=\"${fqrn}\"" "${cache_volume_free}"
212+
generate_metric 'cvmfs_cache_physical_avail_bytes' 'gauge' 'CVMFS cache volume physical free space available.' "repo=\"${fqrn}\"" "${cache_volume_free}" "${EPOCHTIME}"
210213

211214
local cvmfs_mount_version
212215
cvmfs_mount_version=$(attr -g version "${repomountpoint}" | tail -n +2)
213216
local cvmfs_mount_revision
214217
cvmfs_mount_revision=$(attr -g revision "${repomountpoint}" | tail -n +2)
215-
generate_metric 'cvmfs_repo' 'gauge' 'Shows the version of CVMFS used by this repository.' "repo=\"${fqrn}\",mountpoint=\"${repomountpoint}\",version=\"${cvmfs_mount_version}\",revision=\"${cvmfs_mount_revision}\"" 1
218+
generate_metric 'cvmfs_repo' 'gauge' 'Shows the version of CVMFS used by this repository.' "repo=\"${fqrn}\",mountpoint=\"${repomountpoint}\",version=\"${cvmfs_mount_version}\",revision=\"${cvmfs_mount_revision}\"" 1 "${EPOCHTIME}"
216219

217220
# Generate numeric version and revision metrics
218221
local cvmfs_numeric_version
219222
cvmfs_numeric_version=$(convert_version_to_numeric "${cvmfs_mount_version}")
220-
generate_metric 'cvmfs_repo_version' 'gauge' 'CVMFS repository version as a numeric value for easier querying.' "repo=\"${fqrn}\"" "${cvmfs_numeric_version}"
223+
generate_metric 'cvmfs_repo_version' 'gauge' 'CVMFS repository version as a numeric value for easier querying.' "repo=\"${fqrn}\"" "${cvmfs_numeric_version}" "${EPOCHTIME}"
221224

222-
generate_metric 'cvmfs_repo_revision' 'gauge' 'CVMFS repository revision number.' "repo=\"${fqrn}\"" "${cvmfs_mount_revision}"
225+
generate_metric 'cvmfs_repo_revision' 'gauge' 'CVMFS repository revision number.' "repo=\"${fqrn}\"" "${cvmfs_mount_revision}" "${EPOCHTIME}"
223226

224227
local cvmfs_mount_rx_kb
225228
cvmfs_mount_rx_kb=$(attr -g rx "${repomountpoint}" | tail -n +2)
226229
local cvmfs_mount_rx
227230
cvmfs_mount_rx=$((cvmfs_mount_rx_kb * 1024))
228-
generate_metric 'cvmfs_net_rx_total' 'counter' 'Shows the overall amount of downloaded bytes since mounting.' "repo=\"${fqrn}\"" "${cvmfs_mount_rx}"
231+
generate_metric 'cvmfs_net_rx_total' 'counter' 'Shows the overall amount of downloaded bytes since mounting.' "repo=\"${fqrn}\"" "${cvmfs_mount_rx}" "${EPOCHTIME}"
229232

230233
local cvmfs_mount_uptime_minutes
231234
cvmfs_mount_uptime_minutes=$(attr -g uptime "${repomountpoint}" | tail -n +2)
@@ -237,8 +240,8 @@ get_cvmfs_repo_metrics() {
237240
rounded_now_to_minute=$((now - (now % 60)))
238241
cvmfs_mount_uptime=$((cvmfs_mount_uptime_minutes * 60))
239242
cvmfs_mount_epoch_time=$((rounded_now_to_minute - cvmfs_mount_uptime))
240-
generate_metric 'cvmfs_repo_uptime_seconds' 'counter' 'Shows the time since the repo was mounted.' "repo=\"${fqrn}\"" "${cvmfs_mount_uptime}"
241-
generate_metric 'cvmfs_repo_mount_epoch_timestamp' 'counter' 'Shows the epoch time the repo was mounted.' "repo=\"${fqrn}\"" "${cvmfs_mount_epoch_time}"
243+
generate_metric 'cvmfs_repo_uptime_seconds' 'counter' 'Shows the time since the repo was mounted.' "repo=\"${fqrn}\"" "${cvmfs_mount_uptime}" "${EPOCHTIME}"
244+
generate_metric 'cvmfs_repo_mount_epoch_timestamp' 'counter' 'Shows the epoch time the repo was mounted.' "repo=\"${fqrn}\"" "${cvmfs_mount_epoch_time}" "${EPOCHTIME}"
242245

243246
local cvmfs_repo_expires_min
244247
cvmfs_repo_expires_min=$(attr -g expires "${repomountpoint}" | tail -n +2)
@@ -248,27 +251,27 @@ get_cvmfs_repo_metrics() {
248251
else
249252
cvmfs_repo_expires=$((cvmfs_repo_expires_min * 60))
250253
fi
251-
generate_metric 'cvmfs_repo_expires_seconds' 'gauge' 'Shows the remaining life time of the mounted root file catalog in seconds. -1 if never expires.' "repo=\"${fqrn}\"" "${cvmfs_repo_expires}"
254+
generate_metric 'cvmfs_repo_expires_seconds' 'gauge' 'Shows the remaining life time of the mounted root file catalog in seconds. -1 if never expires.' "repo=\"${fqrn}\"" "${cvmfs_repo_expires}" "${EPOCHTIME}"
252255

253256
local cvmfs_mount_ndownload
254257
cvmfs_mount_ndownload=$(attr -g ndownload "${repomountpoint}" | tail -n +2)
255-
generate_metric 'cvmfs_net_ndownload_total' 'counter' 'Shows the overall number of downloaded files since mounting.' "repo=\"${fqrn}\"" "${cvmfs_mount_ndownload}"
258+
generate_metric 'cvmfs_net_ndownload_total' 'counter' 'Shows the overall number of downloaded files since mounting.' "repo=\"${fqrn}\"" "${cvmfs_mount_ndownload}" "${EPOCHTIME}"
256259

257260
local cvmfs_mount_nioerr
258261
cvmfs_mount_nioerr=$(attr -g nioerr "${repomountpoint}" | tail -n +2)
259-
generate_metric 'cvmfs_sys_nioerr_total' 'counter' 'Shows the total number of I/O errors encountered since mounting.' "repo=\"${fqrn}\"" "${cvmfs_mount_nioerr}"
262+
generate_metric 'cvmfs_sys_nioerr_total' 'counter' 'Shows the total number of I/O errors encountered since mounting.' "repo=\"${fqrn}\"" "${cvmfs_mount_nioerr}" "${EPOCHTIME}"
260263

261264
local cvmfs_mount_timeout
262265
cvmfs_mount_timeout=$(attr -g timeout "${repomountpoint}" | tail -n +2)
263-
generate_metric 'cvmfs_net_timeout' 'gauge' 'Shows the timeout for proxied connections in seconds.' "repo=\"${fqrn}\"" "${cvmfs_mount_timeout}"
266+
generate_metric 'cvmfs_net_timeout' 'gauge' 'Shows the timeout for proxied connections in seconds.' "repo=\"${fqrn}\"" "${cvmfs_mount_timeout}" "${EPOCHTIME}"
264267

265268
local cvmfs_mount_timeout_direct
266269
cvmfs_mount_timeout_direct=$(attr -g timeout_direct "${repomountpoint}" | tail -n +2)
267-
generate_metric 'cvmfs_net_timeout_direct' 'gauge' 'Shows the timeout for direct connections in seconds.' "repo=\"${fqrn}\"" "${cvmfs_mount_timeout_direct}"
270+
generate_metric 'cvmfs_net_timeout_direct' 'gauge' 'Shows the timeout for direct connections in seconds.' "repo=\"${fqrn}\"" "${cvmfs_mount_timeout_direct}" "${EPOCHTIME}"
268271

269272
local cvmfs_mount_timestamp_last_ioerr
270273
cvmfs_mount_timestamp_last_ioerr=$(attr -g timestamp_last_ioerr "${repomountpoint}" | tail -n +2)
271-
generate_metric 'cvmfs_sys_timestamp_last_ioerr' 'counter' 'Shows the timestamp of the last ioerror.' "repo=\"${fqrn}\"" "${cvmfs_mount_timestamp_last_ioerr}"
274+
generate_metric 'cvmfs_sys_timestamp_last_ioerr' 'counter' 'Shows the timestamp of the last ioerror.' "repo=\"${fqrn}\"" "${cvmfs_mount_timestamp_last_ioerr}" "${EPOCHTIME}"
272275

273276
local cvmfs_repo_pid_statline
274277
cvmfs_repo_pid_statline=$(</proc/"${repo_pid}"/stat)
@@ -282,8 +285,8 @@ get_cvmfs_repo_metrics() {
282285
local cvmfs_system_seconds
283286
cvmfs_user_seconds=$(printf "%.2f" "$(echo "scale=4; $cvmfs_utime / $CLOCK_TICK" | bc)")
284287
cvmfs_system_seconds=$(printf "%.2f" "$(echo "scale=4; $cvmfs_stime / $CLOCK_TICK" | bc)")
285-
generate_metric 'cvmfs_sys_cpu_user_total' 'counter' 'CPU time used in userspace by CVMFS mount in seconds.' "repo=\"${fqrn}\"" "${cvmfs_user_seconds}"
286-
generate_metric 'cvmfs_sys_cpu_system_total' 'counter' 'CPU time used in the kernel system calls by CVMFS mount in seconds.' "repo=\"${fqrn}\"" "${cvmfs_system_seconds}"
288+
generate_metric 'cvmfs_sys_cpu_user_total' 'counter' 'CPU time used in userspace by CVMFS mount in seconds.' "repo=\"${fqrn}\"" "${cvmfs_user_seconds}" "${EPOCHTIME}"
289+
generate_metric 'cvmfs_sys_cpu_system_total' 'counter' 'CPU time used in the kernel system calls by CVMFS mount in seconds.' "repo=\"${fqrn}\"" "${cvmfs_system_seconds}" "${EPOCHTIME}"
287290

288291
# Add memory usage metric
289292
if [[ -f "/proc/${repo_pid}/status" ]]; then
@@ -292,13 +295,13 @@ get_cvmfs_repo_metrics() {
292295
if [[ -n "${memory_usage_kb}" ]]; then
293296
local memory_usage_bytes
294297
memory_usage_bytes=$((memory_usage_kb * 1000))
295-
generate_metric 'cvmfs_sys_memory_usage_bytes' 'gauge' 'CVMFS process memory usage in bytes.' "repo=\"${fqrn}\"" "${memory_usage_bytes}"
298+
generate_metric 'cvmfs_sys_memory_usage_bytes' 'gauge' 'CVMFS process memory usage in bytes.' "repo=\"${fqrn}\"" "${memory_usage_bytes}" "${EPOCHTIME}"
296299
fi
297300
fi
298301

299302
local cvmfs_mount_active_proxy
300303
cvmfs_mount_active_proxy=$(attr -g proxy "${repomountpoint}" | tail -n +2)
301-
generate_metric 'cvmfs_net_active_proxy' 'gauge' 'Shows the active proxy in use for this mount.' "repo=\"${fqrn}\",proxy=\"${cvmfs_mount_active_proxy}\"" 1
304+
generate_metric 'cvmfs_net_active_proxy' 'gauge' 'Shows the active proxy in use for this mount.' "repo=\"${fqrn}\",proxy=\"${cvmfs_mount_active_proxy}\"" 1 "${EPOCHTIME}"
302305

303306
# Pull in xattr based metrics with simple labels
304307
get_cvmfs_repo_extended_attribute_gauge_metrics "${reponame}"
@@ -318,7 +321,7 @@ get_cvmfs_repo_metrics_new() {
318321
# Still need to get maxfd via xattr since it was removed from metrics prometheus
319322
local maxfd_value
320323
maxfd_value=$(attr -g maxfd "${repomountpoint}" | tail -n +2)
321-
generate_metric 'cvmfs_sys_maxfd' 'gauge' 'Shows the maximum number of file descriptors available to file system clients.' "repo=\"${reponame}\"" "${maxfd_value}"
324+
generate_metric 'cvmfs_sys_maxfd' 'gauge' 'Shows the maximum number of file descriptors available to file system clients.' "repo=\"${reponame}\"" "${maxfd_value}" "${EPOCHTIME}"
322325

323326
# Extract version and revision from the cvmfs_repo metric in TMPFILE and generate numeric metrics
324327
local cvmfs_repo_line
@@ -337,11 +340,11 @@ get_cvmfs_repo_metrics_new() {
337340
if [[ -n "${cvmfs_mount_version}" ]]; then
338341
local cvmfs_numeric_version
339342
cvmfs_numeric_version=$(convert_version_to_numeric "${cvmfs_mount_version}")
340-
generate_metric 'cvmfs_repo_version' 'gauge' 'CVMFS repository version as a numeric value for easier querying.' "repo=\"${reponame}\"" "${cvmfs_numeric_version}"
343+
generate_metric 'cvmfs_repo_version' 'gauge' 'CVMFS repository version as a numeric value for easier querying.' "repo=\"${reponame}\"" "${cvmfs_numeric_version}" "${EPOCHTIME}"
341344
fi
342345

343346
if [[ -n "${cvmfs_mount_revision}" ]]; then
344-
generate_metric 'cvmfs_repo_revision' 'gauge' 'CVMFS repository revision number.' "repo=\"${reponame}\"" "${cvmfs_mount_revision}"
347+
generate_metric 'cvmfs_repo_revision' 'gauge' 'CVMFS repository revision number.' "repo=\"${reponame}\"" "${cvmfs_mount_revision}" "${EPOCHTIME}"
345348
fi
346349
fi
347350

@@ -354,7 +357,7 @@ get_cvmfs_repo_metrics_new() {
354357
if [[ -n "${memory_usage_kb}" ]]; then
355358
local memory_usage_bytes
356359
memory_usage_bytes=$((memory_usage_kb * 1000))
357-
generate_metric 'cvmfs_sys_memory_usage_bytes' 'gauge' 'CVMFS process memory usage in bytes.' "repo=\"${reponame}\"" "${memory_usage_bytes}"
360+
generate_metric 'cvmfs_sys_memory_usage_bytes' 'gauge' 'CVMFS process memory usage in bytes.' "repo=\"${reponame}\"" "${memory_usage_bytes}" "${EPOCHTIME}"
358361
fi
359362
fi
360363
}
@@ -608,7 +611,7 @@ for cmd in attr bc cvmfs_config cvmfs_talk grep; do
608611
done
609612

610613
#############################################################
611-
args=$(getopt --options 'h' --longoptions 'help,http,non-standard-mountpoints' -- "$@")
614+
args=$(getopt --options 'h' --longoptions 'help,http,non-standard-mountpoints,timestamp' -- "$@")
612615
eval set -- "$args"
613616

614617
for arg in $@; do
@@ -628,6 +631,11 @@ for arg in $@; do
628631
USE_NON_STANDARD_MOUNTPOINTS='TRUE'
629632
shift
630633
;;
634+
--timestamp)
635+
# Add metric timestamp
636+
EPOCHTIME=$(date +%s)
637+
shift
638+
;;
631639
-h | --help)
632640
# get help
633641
shift

0 commit comments

Comments
 (0)