diff --git a/docs/performance/import.md b/docs/performance/import.md index 31de63c42..a9665380e 100644 --- a/docs/performance/import.md +++ b/docs/performance/import.md @@ -17,21 +17,22 @@ The following systems were used for performance evaluation: The following datasets were used: -| Dataset | # Pat. ¹ | # Res. ² | # Obs. ³ | -|---------|---------:|---------:|---------:| -| 100k | 100 k | 104 M | 59 M | -| 100k-fh | 100 k | 317 M | 191 M | -| 1M | 1 M | 1044 M | 593 M | +| Dataset | # Pat. ¹ | # Res. ² | # Obs. ³ | Disc Size | +|---------|---------:|---------:|---------:|----------:| +| 100k | 100 k | 104 M | 59 M | 205 GiB | +| 100k-fh | 100 k | 317 M | 191 M | 323 GiB | +| 1M | 1 M | 1044 M | 593 M | 1045 GiB | ¹ Number of Patients, ² Total Number of Resources, ³ Number of Observations ## Import ```sh -blazectl --server http://localhost:8080/fhir upload -c8 data/synthea/data-100k-full-history/ +blazectl --server http://localhost:8080/fhir upload -c8 ``` -| System | Dataset | Time (s) | Bytes In (GiB) | Resources/s | -|--------|---------|---------:|---------------:|------------:| -| LEA47 | 100k-fh | 39076 | 53.94 | 8102 | -| LEA58 | 100k-fh | 28366 | 53.94 | 11161 | +| System | Dataset | Time (h) | Resources/s | +|--------|---------|---------:|------------:| +| LEA47 | 100k | 3,048 | 9469 | +| LEA47 | 100k-fh | 9,797 | 8977 | +| LEA58 | 100k-fh | 7,879 | 11161 | diff --git a/modules/rocksdb/deps.edn b/modules/rocksdb/deps.edn index d4686d742..b5398bed6 100644 --- a/modules/rocksdb/deps.edn +++ b/modules/rocksdb/deps.edn @@ -15,7 +15,7 @@ {:local/root "../module-base"} org.rocksdb/rocksdbjni - {:mvn/version "8.6.7"}} + {:mvn/version "8.8.1"}} :aliases {:test diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj index 8830edbf9..740156240 100644 --- a/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj +++ b/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj @@ -101,17 +101,17 @@ max-background-jobs compaction-readahead-size] :or {wal-dir "" - max-background-jobs 2 - compaction-readahead-size 0}}] - (doto (DBOptions.) - (.setStatistics ^Statistics stats) - (.setWalDir (str wal-dir)) - (.setMaxBackgroundJobs (long max-background-jobs)) - (.setCompactionReadaheadSize (long compaction-readahead-size)) - (.setEnablePipelinedWrite true) - (.setCreateIfMissing true) - (.setCreateMissingColumnFamilies true) - (.setListeners ^List (list listener)))) + max-background-jobs 2}}] + (cond-> (doto (DBOptions.) + (.setStatistics ^Statistics stats) + (.setWalDir (str wal-dir)) + (.setMaxBackgroundJobs (long max-background-jobs)) + (.setEnablePipelinedWrite true) + (.setCreateIfMissing true) + (.setCreateMissingColumnFamilies true) + (.setListeners ^List (list listener))) + (int? compaction-readahead-size) + (.setCompactionReadaheadSize (long compaction-readahead-size)))) (defn write-options [{:keys [sync? disable-wal?]}] (cond-> (WriteOptions.) diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb/spec.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb/spec.clj index 52453da53..c788c9bb0 100644 --- a/modules/rocksdb/src/blaze/db/kv/rocksdb/spec.clj +++ b/modules/rocksdb/src/blaze/db/kv/rocksdb/spec.clj @@ -39,7 +39,7 @@ nat-int?) (s/def ::db-options/compaction-readahead-size - nat-int?) + (s/nilable nat-int?)) (s/def ::rocksdb/db-options (s/keys :opt-un [::db-options/wal-dir diff --git a/modules/rocksdb/test/blaze/db/kv/rocksdb/impl_test.clj b/modules/rocksdb/test/blaze/db/kv/rocksdb/impl_test.clj index 7f13b60c2..000a51501 100644 --- a/modules/rocksdb/test/blaze/db/kv/rocksdb/impl_test.clj +++ b/modules/rocksdb/test/blaze/db/kv/rocksdb/impl_test.clj @@ -146,13 +146,18 @@ (given (datafy/datafy (impl/db-options (Statistics.) (event-listener) nil)) :wal-dir := "" :max-background-jobs := 2 - :compaction-readahead-size := 0 + :compaction-readahead-size := 2097152 :enable-pipelined-write := true :create-if-missing := true :create-missing-column-families := true) + (given (datafy/datafy (impl/db-options (Statistics.) (event-listener) {:compaction-readahead-size nil})) + :compaction-readahead-size := 2097152)) + + (testing "with non-defaults" (doseq [[key value] [[:wal-dir "wal"] [:max-background-jobs 4] + [:compaction-readahead-size 0] [:compaction-readahead-size 10]]] (given (datafy/datafy (impl/db-options (Statistics.) (event-listener) {key value})) key := value)))) diff --git a/resources/blaze.edn b/resources/blaze.edn index 112e895f9..e1548b14f 100644 --- a/resources/blaze.edn +++ b/resources/blaze.edn @@ -374,7 +374,7 @@ :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4] :compaction-readahead-size - #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]} + #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]} :column-families {:search-param-value-index {:write-buffer-size-in-mb 64 @@ -479,7 +479,7 @@ :opts {:wal-dir #blaze/cfg ["TRANSACTION_DB_WAL_DIR" string? ""] :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4] - :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]} + :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]} :column-families {:default {:write-buffer-size-in-mb 8 @@ -513,7 +513,7 @@ :opts {:wal-dir #blaze/cfg ["RESOURCE_DB_WAL_DIR" string? ""] :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4] - :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]} + :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]} :column-families {:default {:write-buffer-size-in-mb 64 @@ -577,7 +577,7 @@ :opts {:wal-dir #blaze/cfg ["INDEX_DB_WAL_DIR" string? ""] :max-background-jobs #blaze/cfg ["DB_MAX_BACKGROUND_JOBS" int? 4] - :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" int? 0]} + :compaction-readahead-size #blaze/cfg ["DB_COMPACTION_READAHEAD_SIZE" nat-int?]} :column-families {:search-param-value-index {:write-buffer-size-in-mb 64