Add RocksDB Histogram Metrics

modules/rocksdb/src/blaze/db/kv/rocksdb.clj

@@ -10,11 +10,11 @@
   (:import
     [java.lang AutoCloseable]
     [java.nio ByteBuffer]
-    [java.util ArrayList EnumSet]
+    [java.util ArrayList]
     [org.rocksdb
      RocksDB RocksIterator WriteOptions WriteBatch Options ColumnFamilyHandle
      Statistics LRUCache CompactRangeOptions Snapshot ReadOptions
-     StatsLevel HistogramType Env Priority]))
+     StatsLevel Env Priority]))
 
 
 (set! *warn-on-reflection* true)
@@ -228,8 +228,8 @@
 (defmethod ig/init-key ::stats
   [_ _]
   (log/info "Init RocksDB statistics")
-  (doto (Statistics. (EnumSet/allOf HistogramType))
-    (.setStatsLevel StatsLevel/EXCEPT_DETAILED_TIMERS)))
+  (doto (Statistics.)
+    (.setStatsLevel StatsLevel/EXCEPT_TIME_FOR_MUTEX)))
 
 
 (defmethod ig/halt-key! ::stats

modules/rocksdb/src/blaze/db/kv/rocksdb/metrics.clj

@@ -2,7 +2,7 @@
   (:require
     [blaze.metrics.core :as metrics])
   (:import
-    [org.rocksdb Statistics TickerType]))
+    [org.rocksdb Statistics TickerType HistogramType]))
 
 
 (set! *warn-on-reflection* true)
@@ -15,10 +15,21 @@
        :value (value-f (.getTickerCount ^Statistics stats ticker-type))})))
 
 
+(defn- histogram-xf [histogram-type value-f]
+  (map
+    (fn [[name stats]]
+      {:label-values [name]
+       :value (value-f (.getSum (.getHistogramData ^Statistics stats histogram-type)))})))
+
+
 (defn- samples [ticker-type value-f stats]
   (into [] (sample-xf ticker-type value-f) stats))
 
 
+(defn- histogram-samples [histogram-type value-f stats]
+  (into [] (histogram-xf histogram-type value-f) stats))
+
+
 (defn- counter-metric
   ([name help ticker-type]
    (counter-metric name help ticker-type identity))
@@ -27,6 +38,12 @@
      (metrics/counter-metric name help ["name"] (samples ticker-type value-f stats)))))
 
 
+(defn- histogram-metric
+  ([name help histogram-type value-f]
+   (fn [stats]
+     (metrics/counter-metric name help ["name"] (histogram-samples histogram-type value-f stats)))))
+
+
 (def ^:private block-cache-data-miss-total
   (counter-metric
     "blaze_rocksdb_block_cache_data_miss_total"
@@ -301,6 +318,38 @@
     TickerType/WRITE_TIMEDOUT))
 
 
+(def ^:private flush-seconds-total
+  (histogram-metric
+    "blaze_rocksdb_flush_seconds_total"
+    "Returns the total number of seconds spent flushing memtables to disk."
+    HistogramType/FLUSH_TIME
+    #(/ (double %) 1e6)))
+
+
+(def ^:private compaction-seconds-total
+  (histogram-metric
+    "blaze_rocksdb_compaction_seconds_total"
+    "Returns the total number of seconds spent in compaction."
+    HistogramType/COMPACTION_TIME
+    #(/ (double %) 1e6)))
+
+
+(def ^:private compression-seconds-total
+  (histogram-metric
+    "blaze_rocksdb_compression_seconds_total"
+    "Returns the total number of seconds spent in compression."
+    HistogramType/COMPRESSION_TIMES_NANOS
+    #(/ (double %) 1e9)))
+
+
+(def ^:private decompression-seconds-total
+  (histogram-metric
+    "blaze_rocksdb_decompression_seconds_total"
+    "Returns the total number of seconds spent in decompression."
+    HistogramType/DECOMPRESSION_TIMES_NANOS
+    #(/ (double %) 1e9)))
+
+
 (defn stats-collector [stats]
   (metrics/collector
     [(block-cache-data-miss-total stats)
@@ -341,4 +390,8 @@
      (iterators-created-total stats)
      (wal-syncs-total stats)
      (wal-bytes-total stats)
-     (write-timeout-total stats)]))
+     (write-timeout-total stats)
+     (flush-seconds-total stats)
+     (compaction-seconds-total stats)
+     (compression-seconds-total stats)
+     (decompression-seconds-total stats)]))

modules/rocksdb/test/blaze/db/kv/rocksdb/metrics_test.clj

@@ -67,7 +67,11 @@
               "blaze_rocksdb_iterators_created"
               "blaze_rocksdb_wal_syncs"
               "blaze_rocksdb_wal_bytes"
-              "blaze_rocksdb_write_timeout"]
+              "blaze_rocksdb_write_timeout"
+              "blaze_rocksdb_flush_seconds"
+              "blaze_rocksdb_compaction_seconds"
+              "blaze_rocksdb_compression_seconds"
+              "blaze_rocksdb_decompression_seconds"]
              (mapv :name metrics))))
 
     (testing "every metric is of type counter"