Update dependency org.rocksdb/rocksdbjni to v8.8.1

Don't set compaction_readahead_size to zero by default because the new default of RocksDB is non-zero.
samply · Nov 26, 2023 · eb150b4 · eb150b4
1 parent 4d5f656
commit eb150b4
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 28 deletions.
diff --git a/docs/performance/import.md b/docs/performance/import.md
@@ -17,21 +17,22 @@ The following systems were used for performance evaluation:
 
 The following datasets were used:
 
-| Dataset | # Pat. ¹ | # Res. ² | # Obs. ³ |
-|---------|---------:|---------:|---------:|
-| 100k    |    100 k |    104 M |     59 M |
-| 100k-fh |    100 k |    317 M |    191 M |
-| 1M      |      1 M |   1044 M |    593 M |
+| Dataset | # Pat. ¹ | # Res. ² | # Obs. ³ | Disc Size |
+|---------|---------:|---------:|---------:|----------:|
+| 100k    |    100 k |    104 M |     59 M |   205 GiB |
+| 100k-fh |    100 k |    317 M |    191 M |   323 GiB |
+| 1M      |      1 M |   1044 M |    593 M |  1045 GiB |
 
 ¹ Number of Patients, ² Total Number of Resources, ³ Number of Observations
 
 ## Import
 
 ```sh
-blazectl --server http://localhost:8080/fhir upload -c8 data/synthea/data-100k-full-history/
+blazectl --server http://localhost:8080/fhir upload -c8 <dataset>
 ```
 
-| System | Dataset | Time (s) | Bytes In (GiB) | Resources/s |
-|--------|---------|---------:|---------------:|------------:|
-| LEA47  | 100k-fh |    39076 |          53.94 |        8102 |
-| LEA58  | 100k-fh |    28366 |          53.94 |       11161 |
+| System | Dataset | Time (h) | Resources/s |
+|--------|---------|---------:|------------:|
+| LEA47  | 100k    |    3,048 |        9469 |
+| LEA47  | 100k-fh |    9,797 |        8977 |
+| LEA58  | 100k-fh |    7,879 |       11161 |
diff --git a/modules/rocksdb/deps.edn b/modules/rocksdb/deps.edn
@@ -15,7 +15,7 @@
   {:local/root "../module-base"}
 
   org.rocksdb/rocksdbjni
-  {:mvn/version "8.6.7"}}
+  {:mvn/version "8.8.1"}}
 
  :aliases
  {:test

diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj
@@ -101,17 +101,17 @@
            max-background-jobs
            compaction-readahead-size]
     :or {wal-dir ""
-         max-background-jobs 2
-         compaction-readahead-size 0}}]
-  (doto (DBOptions.)
-    (.setStatistics ^Statistics stats)
-    (.setWalDir (str wal-dir))
-    (.setMaxBackgroundJobs (long max-background-jobs))
-    (.setCompactionReadaheadSize (long compaction-readahead-size))
-    (.setEnablePipelinedWrite true)
-    (.setCreateIfMissing true)
-    (.setCreateMissingColumnFamilies true)
-    (.setListeners ^List (list listener))))
+         max-background-jobs 2}}]
+  (cond-> (doto (DBOptions.)
+            (.setStatistics ^Statistics stats)
+            (.setWalDir (str wal-dir))
+            (.setMaxBackgroundJobs (long max-background-jobs))
+            (.setEnablePipelinedWrite true)
+            (.setCreateIfMissing true)
+            (.setCreateMissingColumnFamilies true)
+            (.setListeners ^List (list listener)))
+    (int? compaction-readahead-size)
+    (.setCompactionReadaheadSize (long compaction-readahead-size))))
 
 (defn write-options [{:keys [sync? disable-wal?]}]
   (cond-> (WriteOptions.)

diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb/spec.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb/spec.clj
@@ -39,7 +39,7 @@
   nat-int?)
 
 (s/def ::db-options/compaction-readahead-size
-  nat-int?)
+  (s/nilable nat-int?))
 
 (s/def ::rocksdb/db-options
   (s/keys :opt-un [::db-options/wal-dir

diff --git a/modules/rocksdb/test/blaze/db/kv/rocksdb/impl_test.clj b/modules/rocksdb/test/blaze/db/kv/rocksdb/impl_test.clj
@@ -146,13 +146,18 @@
     (given (datafy/datafy (impl/db-options (Statistics.) (event-listener) nil))
       :wal-dir := ""
       :max-background-jobs := 2
-      :compaction-readahead-size := 0
+      :compaction-readahead-size := 2097152
       :enable-pipelined-write := true
       :create-if-missing := true
       :create-missing-column-families := true)
 
+    (given (datafy/datafy (impl/db-options (Statistics.) (event-listener) {:compaction-readahead-size nil}))
+      :compaction-readahead-size := 2097152))
+
+  (testing "with non-defaults"
     (doseq [[key value] [[:wal-dir "wal"]
                          [:max-background-jobs 4]
+                         [:compaction-readahead-size 0]
                          [:compaction-readahead-size 10]]]
       (given (datafy/datafy (impl/db-options (Statistics.) (event-listener) {key value}))
         key := value))))

diff --git a/resources/blaze.edn b/resources/blaze.edn
@@ -374,7 +374,7 @@
      :max-background-jobs
      #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4]
      :compaction-readahead-size
-     #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]}
+     #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]}
     :column-families
     {:search-param-value-index
      {:write-buffer-size-in-mb 64
@@ -479,7 +479,7 @@
     :opts
     {:wal-dir #blaze/cfg ["TRANSACTION_DB_WAL_DIR" string? ""]
      :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4]
-     :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]}
+     :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]}
     :column-families
     {:default
      {:write-buffer-size-in-mb 8
@@ -513,7 +513,7 @@
     :opts
     {:wal-dir #blaze/cfg ["RESOURCE_DB_WAL_DIR" string? ""]
      :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4]
-     :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]}
+     :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]}
     :column-families
     {:default
      {:write-buffer-size-in-mb 64
@@ -577,7 +577,7 @@
     :opts
     {:wal-dir #blaze/cfg ["INDEX_DB_WAL_DIR" string? ""]
      :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4]
-     :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]}
+     :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]}
     :column-families
     {:search-param-value-index
      {:write-buffer-size-in-mb 64