Add RocksDB Block Cache Usage Metrics

The metrics are called blaze_rocksdb_block_cache_usage_bytes and
blaze_rocksdb_block_cache_pinned_usage_bytes.

modules/metrics/src/blaze/metrics/core.clj

@@ -23,15 +23,31 @@
   (mapv datafy/datafy (.collect ^Collector collector)))
 
 
-(defn counter-metric [name help label-names samples]
+(defn counter-metric
+  "Creates a counter metric from `samples`.
+
+  The other arguments are:
+  * name: the name with must only contain word chars
+  * the help text
+  * a collection of label names
+  * a collection of samples"
+  [name help label-names samples]
   (let [m (CounterMetricFamily. ^String name ^String help ^List label-names)]
     (run!
       (fn [{:keys [label-values value]}] (.addMetric m label-values value))
       samples)
     m))
 
 
-(defn gauge-metric [name help label-names samples]
+(defn gauge-metric
+  "Creates a gauge metric from `samples`.
+
+  The other arguments are:
+  * name: the name with must only contain word chars
+  * the help text
+  * a collection of label names
+  * a collection of samples"
+  [name help label-names samples]
   (let [m (GaugeMetricFamily. ^String name ^String help ^List label-names)]
     (run!
       (fn [{:keys [label-values value]}] (.addMetric m label-values value))

modules/metrics/src/blaze/metrics/core_spec.clj

@@ -2,26 +2,23 @@
   (:require
     [blaze.metrics.core :as metrics]
     [blaze.metrics.spec]
-    [clojure.spec.alpha :as s]
-    [clojure.string :as str]))
+    [clojure.spec.alpha :as s]))
 
 
 (s/fdef metrics/collect
   :args (s/cat :collector :blaze.metrics/collector)
   :ret (s/coll-of :blaze.metrics/metric))
 
 
-(defn counter-name? [x]
-  (and (string? x) (str/ends-with? x "_total")))
-
-
 (s/fdef metrics/counter-metric
-  :args (s/cat :name counter-name? :help string? :label-names (s/coll-of string?)
+  :args (s/cat :name :blaze.metrics.counter/name :help string?
+               :label-names (s/coll-of string?)
                :samples (s/coll-of :blaze.metrics/sample)))
 
 
 (s/fdef metrics/gauge-metric
-  :args (s/cat :name string? :help string? :label-names (s/coll-of string?)
+  :args (s/cat :name :blaze.metrics.metric/name :help string?
+               :label-names (s/coll-of string?)
                :samples (s/coll-of :blaze.metrics/sample)))
 
 

modules/metrics/src/blaze/metrics/spec.clj

@@ -1,6 +1,7 @@
 (ns blaze.metrics.spec
   (:require
-    [clojure.spec.alpha :as s])
+    [clojure.spec.alpha :as s]
+    [clojure.string :as str])
   (:import
     [io.prometheus.client Collector CollectorRegistry]))
 
@@ -26,7 +27,11 @@
 
 
 (s/def :blaze.metrics.metric/name
-  string?)
+  (s/and string? #(re-matches #"\w+" %)))
+
+
+(s/def :blaze.metrics.counter/name
+  (s/and :blaze.metrics.metric/name #(str/ends-with? % "_total")))
 
 
 (s/def :blaze.metrics.metric/samples

modules/rocksdb/src/blaze/db/kv/rocksdb.clj

@@ -239,4 +239,10 @@
   (metrics/stats-collector stats))
 
 
+(defmethod ig/init-key ::block-cache-collector
+  [_ {:keys [block-cache]}]
+  (metrics/block-cache-collector block-cache))
+
+
 (derive ::stats-collector :blaze.metrics/collector)
+(derive ::block-cache-collector :blaze.metrics/collector)

modules/rocksdb/src/blaze/db/kv/rocksdb/metrics.clj

@@ -2,7 +2,7 @@
   (:require
     [blaze.metrics.core :as metrics])
   (:import
-    [org.rocksdb Statistics TickerType HistogramType]))
+    [org.rocksdb Cache Statistics TickerType HistogramType]))
 
 
 (set! *warn-on-reflection* true)
@@ -395,3 +395,19 @@
      (compaction-seconds-total stats)
      (compression-seconds-total stats)
      (decompression-seconds-total stats)]))
+
+
+(defn block-cache-collector [block-cache]
+  (metrics/collector
+    [(metrics/gauge-metric
+       "blaze_rocksdb_block_cache_usage_bytes"
+       "Returns the memory size for the entries in the RocksDB block cache."
+       []
+       [{:label-values []
+         :value (.getUsage ^Cache block-cache)}])
+     (metrics/gauge-metric
+       "blaze_rocksdb_block_cache_pinned_usage_bytes"
+       "Returns the memory size for the entries pinned in the RocksDB block cache."
+       []
+       [{:label-values []
+         :value (.getPinnedUsage ^Cache block-cache)}])]))

modules/rocksdb/test/blaze/db/kv/rocksdb/metrics_test.clj

@@ -1,13 +1,13 @@
 (ns blaze.db.kv.rocksdb.metrics-test
   (:require
-    [blaze.db.kv.rocksdb.metrics :refer [stats-collector]]
+    [blaze.db.kv.rocksdb.metrics :refer [stats-collector block-cache-collector]]
     [blaze.metrics.core :as metrics]
     [blaze.metrics.core-spec]
     [blaze.test-util :as tu]
     [clojure.spec.test.alpha :as st]
     [clojure.test :as test :refer [deftest is testing]])
   (:import
-    [org.rocksdb RocksDB Statistics]))
+    [org.rocksdb LRUCache RocksDB Statistics]))
 
 
 (set! *warn-on-reflection* true)
@@ -20,7 +20,7 @@
 (RocksDB/loadLibrary)
 
 
-(deftest collector-test
+(deftest stats-collector-test
   (let [collector (stats-collector [["foo" (Statistics.)]])
         metrics (metrics/collect collector)]
 
@@ -78,3 +78,19 @@
 
     (testing "every metric has the value 0.0"
       (is (every? (comp #{0.0} :value first :samples) metrics)))))
+
+
+(deftest block-cache-collector-test
+  (let [collector (block-cache-collector (LRUCache. 100))
+        metrics (metrics/collect collector)]
+
+    (testing "the metrics names are"
+      (is (= ["blaze_rocksdb_block_cache_usage_bytes"
+              "blaze_rocksdb_block_cache_pinned_usage_bytes"]
+             (mapv :name metrics))))
+
+    (testing "every metric is of type gauge"
+      (is (every? (comp #{:gauge} :type) metrics)))
+
+    (testing "every metric has the value 0.0"
+      (is (every? (comp #{0.0} :value first :samples) metrics)))))

modules/rocksdb/test/blaze/db/kv/rocksdb_test.clj

@@ -89,7 +89,9 @@
    ::rocksdb/block-cache {}
    ::rocksdb/env {}
    ::rocksdb/stats {}
-   ::rocksdb/stats-collector {}})
+   ::rocksdb/stats-collector {}
+   ::rocksdb/block-cache-collector
+   {:block-cache (ig/ref ::rocksdb/block-cache)}})
 
 
 (deftest valid-test

resources/blaze.edn

@@ -546,6 +546,9 @@
      ["transaction" #blaze/ref :blaze.db.transaction-kv-store/stats]
      ["resource" #blaze/ref :blaze.db.resource-kv-store/stats]]}
 
+   :blaze.db.kv.rocksdb/block-cache-collector
+   {:block-cache #blaze/ref :blaze.db.kv.rocksdb/block-cache}
+
    ;;
    ;; Local Page Store
    ;;
@@ -659,6 +662,9 @@
    {:stats
     [["index" #blaze/ref :blaze.db.index-kv-store/stats]]}
 
+   :blaze.db.kv.rocksdb/block-cache-collector
+   {:block-cache #blaze/ref :blaze.db.kv.rocksdb/block-cache}
+
    ;;
    ;; Kafka Transaction Log
    ;;