Add Admin UI

Closes: #1123

.clj-kondo/root/config.edn

@@ -69,6 +69,8 @@
     blaze.elm.compiler.external-data ed
     blaze.elm.expression expr
     blaze.executors ex
+    blaze.fhir.structure-definition-repo sdr
+    blaze.middleware.fhir.db db
     blaze.rest-api.header header
     blaze.scheduler sched
     blaze.test-util tu

.github/scripts/check-resource-totals.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+ACTUAL_TOTALS="$(curl -sH 'Accept: application/fhir+json' "$BASE/\$totals" | jq -r '.parameter[] | [.name, .valueUnsignedInt] | @csv')"
+EXPECTED_TOTALS="$(cat <<END
+"Condition",51599
+"DiagnosticReport",168095
+"Encounter",96869
+"Medication",1016
+"MedicationAdministration",1016
+"Observation",621619
+"Patient",1000
+"Procedure",158380
+END
+)"
+
+test "resource totals" "$ACTUAL_TOTALS" "$EXPECTED_TOTALS"

.github/scripts/check-total-number-of-resources.sh

@@ -3,4 +3,5 @@
 SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
 . "$SCRIPT_DIR/util.sh"
 
-test "total number of resources" "$(curl -sH 'Accept: application/fhir+json' http://localhost:8080/fhir | jq -r .total)" "$1"
+BASE="http://localhost:8080/fhir"
+test "total number of resources" "$(curl -sH 'Accept: application/fhir+json' "$BASE" | jq -r .total)" "$1"

.github/workflows/build.yml

@@ -333,6 +333,9 @@ jobs:
     - name: Check Total-Number of Resources are 92114
       run: .github/scripts/check-total-number-of-resources.sh 92114
 
+    - name: Check Resource Totals
+      run: .github/scripts/check-resource-totals.sh
+
     - name: Count Resources
       run: blazectl count-resources --server http://localhost:8080/fhir
 

deps.edn

@@ -25,6 +25,9 @@
   blaze.operation/patient-everything
   {:local/root "modules/operation-patient-everything"}
 
+  blaze.operation/totals
+  {:local/root "modules/operation-totals"}
+
   blaze/openid-auth
   {:local/root "modules/openid-auth"}
 

dev/blaze/dev/rocksdb.clj

@@ -37,4 +37,9 @@
 
   (rocksdb/tables (index-kv-store) :resource-as-of-index)
 
+  (rocksdb/column-family-meta-data (index-kv-store) :search-param-value-index)
+  (rocksdb/column-family-meta-data (index-kv-store) :compartment-search-param-value-index)
+  (rocksdb/column-family-meta-data (index-kv-store) :resource-as-of-index)
+  (rocksdb/column-family-meta-data (index-kv-store) :patient-last-change-index)
+
   )

modules/admin-api/.clj-kondo/config.edn

@@ -1,5 +1,6 @@
 {:config-paths
- ["../../../.clj-kondo/root"]
+ ["../../../.clj-kondo/root"
+  "../../anomaly/resources/clj-kondo.exports/blaze/anomaly"]
 
  :lint-as
  {blaze.admin-api-test/with-handler clojure.core/fn}}

modules/admin-api/deps.edn

@@ -12,15 +12,18 @@
   {:local/root "../rocksdb"}
 
   blaze/spec
-  {:local/root "../spec"}}
+  {:local/root "../spec"}
+
+  fi.metosin/reitit-openapi
+  {:mvn/version "0.7.0-alpha5"}}
 
  :aliases
  {:test
   {:extra-paths ["test"]
 
    :extra-deps
-   {blaze/module-test-util
-    {:local/root "../module-test-util"}}}
+   {blaze/db-stub
+    {:local/root "../db-stub"}}}
 
   :kaocha
   {:extra-deps
@@ -34,4 +37,5 @@
    {cloverage/cloverage
     {:mvn/version "1.2.4"}}
 
-   :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"]}}}
+   :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"
+               "-e" ".+spec"]}}}

modules/admin-api/src/blaze/admin_api.clj

@@ -1,45 +1,254 @@
 (ns blaze.admin-api
   (:require
+   [blaze.admin-api.spec]
+   [blaze.anomaly :refer [if-ok]]
    [blaze.async.comp :as ac]
    [blaze.db.kv.rocksdb :as rocksdb]
+   [blaze.elm.expression :as-alias expr]
    [blaze.spec]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
+   [reitit.openapi :as openapi]
    [reitit.ring]
    [reitit.ring.spec]
    [ring.util.response :as ring]
-   [taoensso.timbre :as log]))
+   [taoensso.timbre :as log])
+  (:import
+   [java.io File]
+   [java.nio.file Files]))
 
-(defn- rocksdb-column-families-handler [index-kv-store]
+(set! *warn-on-reflection* true)
+
+(defn- root-handler [{:keys [settings features]}]
   (fn [_]
-    (-> (ring/response
-         {:column-families (mapv name (rocksdb/column-families index-kv-store))})
+    (-> (ring/response {:settings settings :features features})
+        (ac/completed-future))))
+
+(def ^:private openapi-handler
+  (let [handler (openapi/create-openapi-handler)]
+    (fn [request]
+      (ac/completed-future (handler request)))))
+
+(defn- block-cache [long-property]
+  (if-ok [capacity (long-property "rocksdb.block-cache-capacity")
+          usage (long-property "rocksdb.block-cache-usage")]
+    {:capacity capacity
+     :usage usage}
+    (constantly nil)))
+
+(defn- database-stats [[db-name db]]
+  (let [long-property (partial rocksdb/long-property db)
+        agg-long-property (partial rocksdb/agg-long-property db)]
+    {:name db-name
+     :estimate-live-data-size (agg-long-property "rocksdb.estimate-live-data-size")
+     :usable-space (.getUsableSpace (Files/getFileStore (.toPath (File. ^String (rocksdb/path db)))))
+     :block-cache (block-cache long-property)
+     :compactions
+     {:pending (long-property "rocksdb.compaction-pending")
+      :running (long-property "rocksdb.num-running-compactions")}}))
+
+(defn- databases-handler [dbs]
+  (fn [_]
+    (-> (ring/response (mapv database-stats dbs))
+        (ac/completed-future))))
+
+(def ^:private db-stats-handler
+  (fn [{:keys [db] {db-name :db} :path-params}]
+    (-> (ring/response (database-stats [db-name db]))
         (ac/completed-future))))
 
-(defn- rocksdb-table-handler [index-kv-store]
-  (fn [{{:keys [column-family]} :path-params}]
-    (-> (ring/response {:tables (rocksdb/tables index-kv-store (keyword column-family))})
+(defn- column-family-data [db column-family]
+  (let [long-property (partial rocksdb/long-property db column-family)]
+    {:name (name column-family)
+     :estimate-num-keys (long-property "rocksdb.estimate-num-keys")
+     :estimate-live-data-size (long-property "rocksdb.estimate-live-data-size")
+     :live-sst-files-size (long-property "rocksdb.live-sst-files-size")
+     :size-all-mem-tables (long-property "rocksdb.size-all-mem-tables")}))
+
+(def ^:private column-families-handler
+  (fn [{:keys [db]}]
+    (-> (mapv (partial column-family-data db) (rocksdb/column-families db))
+        (ring/response)
+        (ac/completed-future))))
+
+(defn- column-family-not-found-msg [db-name column-family]
+  (format "The column family `%s` in database `%s` was not found." column-family db-name))
+
+(defn- column-family-not-found [db-name column-family]
+  (ring/not-found {:msg (column-family-not-found-msg db-name column-family)}))
+
+(def ^:private cf-metadata-handler
+  (fn [{:keys [db] {db-name :db :keys [column-family]} :path-params}]
+    (-> (if-ok [metadata (rocksdb/column-family-meta-data db (keyword column-family))]
+          (ring/response metadata)
+          (fn [_] (column-family-not-found db-name column-family)))
         (ac/completed-future))))
 
-(defn- router [{:keys [context-path index-kv-store] :or {context-path ""}}]
+(def ^:private cf-tables-handler
+  (fn [{:keys [db] {db-name :db :keys [column-family]} :path-params}]
+    (-> (if-ok [tables (rocksdb/tables db (keyword column-family))]
+          (ring/response tables)
+          (fn [_] (column-family-not-found db-name column-family)))
+        (ac/completed-future))))
+
+(defn- db-not-found [db-name]
+  (ring/not-found {:msg (format "The database `%s` was not found." db-name)}))
+
+(def ^:private wrap-db
+  {:name :wrap-db
+   :wrap (fn [handler dbs]
+           (fn [{{db-name :db} :path-params :as request}]
+             (if-let [db (dbs db-name)]
+               (handler (assoc request :db db))
+               (-> (db-not-found db-name)
+                   (ac/completed-future)))))})
+
+(defn- router
+  [{:keys [context-path dbs] :or {context-path ""} :as context}]
   (reitit.ring/router
-   ["/rocksdb"
-    {}
-    ["/index"
+   [""
+    {:openapi {:id :admin-api}
+     :middleware [openapi/openapi-feature]}
+    [""
+     {:get
+      {:handler (root-handler context)}}]
+    ["/openapi.json"
+     {:get
+      {:handler openapi-handler
+       :openapi
+       {:info
+        {:title "Blaze Admin API"
+         :description "The Blaze Admin API is used to monitor and control a Blaze server instance."
+         :version "0.22"}
+        :components
+        {:schemas
+         {"ColumnFamilyMetadata"
+          {:type "object"
+           :description "Metadata about a column family like total and level based number of files."
+           :properties
+           {:name {:type "string"}
+            :size {:type "number"}
+            :num-files {:type "number"}
+            :levels
+            {:type "array"
+             :items
+             {:type "object"
+              :description "Level metadata."
+              :properties
+              {:level {:type "number"}
+               :size {:type "number"}
+               :num-files {:type "number"}}}}}}
+          "BloomFilter"
+          {:type "object"}}
+         :parameters
+         {:db
+          {:name "db"
+           :in "path"
+           :description "The name of the database like index, transaction or resource."
+           :required true
+           :schema {:type "string"}}
+          :column-family
+          {:name "column-family"
+           :in "path"
+           :description "The name of the column family like default."
+           :required true
+           :schema {:type "string"}}}}}
+       :no-doc true}}]
+    ["/dbs"
      {}
-     ["/column-families"
-      {}
-      [""
-       {:get (rocksdb-column-families-handler index-kv-store)}]
-      ["/{column-family}"
+     [""
+      {:get
+       {:handler (databases-handler dbs)
+        :summary "Fetch the list of all database names."
+        :openapi
+        {:operation-id "getDatabases"
+         :responses
+         {200
+          {:description "List of database names."
+           :content
+           {"application/json"
+            {:schema
+             {:type "array"
+              :items {:type "string"}}}}}}}}}]
+     ["/{db}"
+      {:middleware [[wrap-db dbs]]}
+      ["/stats"
+       {:get
+        {:handler db-stats-handler
+         :summary "Fetch stats of a database."
+         :openapi
+         {:operation-id "getDatabaseStats"
+          :parameters
+          [{"$ref" "#/components/parameters/db"}]
+          :responses
+          {200
+           {:description "Database statistics."
+            :content
+            {"application/json"
+             {:schema
+              {:type "object"}}}}}}}}]
+      ["/column-families"
        {}
-       ["/tables"
-        {:get (rocksdb-table-handler index-kv-store)}]]]]]
+       [""
+        {:get
+         {:handler column-families-handler
+          :summary "Fetch the list of all column families of a database."
+          :openapi
+          {:operation-id "getDatabaseColumnFamilies"
+           :parameters
+           [{"$ref" "#/components/parameters/db"}]
+           :responses
+           {200
+            {:description "A list of column families."
+             :content
+             {"application/json"
+              {:schema
+               {:type "array"}}}}}}}}]
+       ["/{column-family}"
+        {}
+        ["/metadata"
+         {:get
+          {:handler cf-metadata-handler
+           :summary "Fetch the metadata of a column family of a database."
+           :openapi
+           {:operation-id "getColumnFamilyMetadata"
+            :parameters
+            [{"$ref" "#/components/parameters/db"}
+             {"$ref" "#/components/parameters/columnFamily"}]
+            :responses
+            {200
+             {:description "Column family metadata."
+              :content
+              {"application/json"
+               {:schema
+                {"$ref" "#/components/schemas/ColumnFamilyMetadata"}}}}}}}}]
+        ["/tables"
+         {:get
+          {:handler cf-tables-handler
+           :summary "Fetch the list of all tables of a column family of a database."
+           :openapi
+           {:operation-id "getColumnFamilyTables"
+            :parameters
+            [{"$ref" "#/components/parameters/db"}
+             {"$ref" "#/components/parameters/columnFamily"}]
+            :responses
+            {200
+             {:description "A list of column family tables."
+              :content
+              {"application/json"
+               {:schema
+                {:type "array"
+                 :items
+                 {:type "object"
+                  :properties
+                  {:data-size {:type "number"}
+                   :total-raw-key-size {:type "number"}}}}}}}}}}}]]]]]]
    {:path (str context-path "/__admin")
     :syntax :bracket}))
 
 (defmethod ig/pre-init-spec :blaze/admin-api [_]
-  (s/keys :req-un [:blaze/context-path :blaze.db/index-kv-store]))
+  (s/keys :req-un [:blaze/context-path ::dbs ::settings ::features]
+          :opt [::expr/cache]))
 
 (defmethod ig/init-key :blaze/admin-api
   [_ context]