Add KV Resource Store Byte Count Metric

modules/db-resource-store/src/blaze/db/resource_store/kv.clj

@@ -8,14 +8,24 @@
     [blaze.db.resource-store :as rs]
     [blaze.executors :as ex]
     [blaze.fhir.spec :as fhir-spec]
+    [blaze.module :refer [reg-collector]]
     [clojure.spec.alpha :as s]
     [cognitect.anomalies :as anom]
     [integrant.core :as ig]
+    [prometheus.alpha :as prom :refer [defhistogram]]
     [taoensso.timbre :as log])
   (:import
     [java.util.concurrent ExecutorService TimeUnit]))
 
 
+(defhistogram resource-bytes
+  "Stored resource sizes in bytes in key-value resource store."
+  {:namespace "blaze"
+   :subsystem "db"
+   :name "resource_store_kv_resource_bytes"}
+  (take 16 (iterate #(* 2 %) 32)))
+
+
 (defn- parse-msg [hash e]
   (format "Error while parsing resource content with hash `%s`: %s"
           (bs/hex hash) (ex-message e)))
@@ -42,7 +52,9 @@
 (def ^:private entry-freezer
   (map
     (fn [[k v]]
-      [(bs/to-byte-array k) (fhir-spec/unform-cbor v)])))
+      (let [content (fhir-spec/unform-cbor v)]
+        (prom/observe! resource-bytes (alength content))
+        [(bs/to-byte-array k) content]))))
 
 
 (defn- get-content [kv-store hash]
@@ -109,3 +121,7 @@
 
 
 (derive ::executor :blaze.metrics/thread-pool-executor)
+
+
+(reg-collector ::resource-bytes
+  resource-bytes)

resources/blaze.edn

@@ -299,6 +299,8 @@
    :blaze.db.resource-store.kv/executor
    {:num-threads #blaze/cfg ["DB_RESOURCE_STORE_KV_THREADS" int? 4]}
 
+   :blaze.db.resource-store.kv/resource-bytes {}
+
    ;;
    ;; In-Memory, Volatile Key-Value Store Resource Store Backend
    ;;
@@ -467,6 +469,8 @@
    :blaze.db.resource-store.kv/executor
    {:num-threads #blaze/cfg ["DB_RESOURCE_STORE_KV_THREADS" int? 4]}
 
+   :blaze.db.resource-store.kv/resource-bytes {}
+
    ;;
    ;; RocksDB Key-Value Store Resource Store Backend
    ;;
@@ -486,6 +490,9 @@
       :level0-file-num-compaction-trigger 2
       :min-write-buffer-number-to-merge 2
       :max-bytes-for-level-base-in-mb 256
+      ;; According to Synthea test data, a resource has an average size of
+      ;; 1300 bytes. So a block with a size of 16 kb will hold about 10
+      ;; resources in average.
       :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]
       :bloom-filter? true}}}