From d0ed6764baaf4f16631bc2ea584a8fbd45417c06 Mon Sep 17 00:00:00 2001 From: Alexander Kiel Date: Thu, 21 Apr 2022 21:05:07 +0200 Subject: [PATCH] Use Bigger Files --- .../rocksdb/src/blaze/db/kv/rocksdb/impl.clj | 20 +++--- resources/blaze.edn | 64 +++++++++---------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj b/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj index 5e4fd0b89..0404ee4e6 100644 --- a/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj +++ b/modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj @@ -47,12 +47,12 @@ (.setLevelCompactionDynamicLevelBytes true) (.setCompressionType CompressionType/LZ4_COMPRESSION) (.setBottommostCompressionType CompressionType/ZSTD_COMPRESSION) - (.setWriteBufferSize (bit-shift-left ^long write-buffer-size-in-mb 20)) - (.setMaxWriteBufferNumber ^long max-write-buffer-number) - (.setMaxBytesForLevelBase (bit-shift-left ^long max-bytes-for-level-base-in-mb 20)) - (.setLevel0FileNumCompactionTrigger ^long level0-file-num-compaction-trigger) - (.setMinWriteBufferNumberToMerge ^long min-write-buffer-number-to-merge) - (.setTargetFileSizeBase (bit-shift-left ^long target-file-size-base-in-mb 20)) + (.setWriteBufferSize (bit-shift-left write-buffer-size-in-mb 20)) + (.setMaxWriteBufferNumber (long max-write-buffer-number)) + (.setMaxBytesForLevelBase (bit-shift-left max-bytes-for-level-base-in-mb 20)) + (.setLevel0FileNumCompactionTrigger (long level0-file-num-compaction-trigger)) + (.setMinWriteBufferNumberToMerge (long min-write-buffer-number-to-merge)) + (.setTargetFileSizeBase (bit-shift-left target-file-size-base-in-mb 20)) (.setTableFormatConfig (cond-> (doto (BlockBasedTableConfig.) @@ -62,7 +62,9 @@ (.setBlockSize block-size) (.setBlockCache block-cache)) bloom-filter? - (.setFilterPolicy (BloomFilter. 10 false))))) + (.setFilterPolicy (BloomFilter. 10 false)) + bloom-filter? + (.setWholeKeyFiltering true)))) memtable-whole-key-filtering? (.setMemtableWholeKeyFiltering true) optimize-filters-for-hits? @@ -82,8 +84,8 @@ compaction-readahead-size 0}}] (doto (DBOptions.) (.setStatistics ^Statistics stats) - (.setMaxBackgroundJobs ^long max-background-jobs) - (.setCompactionReadaheadSize ^long compaction-readahead-size) + (.setMaxBackgroundJobs (long max-background-jobs)) + (.setCompactionReadaheadSize (long compaction-readahead-size)) (.setEnablePipelinedWrite true) (.setCreateIfMissing true) (.setCreateMissingColumnFamilies true))) diff --git a/resources/blaze.edn b/resources/blaze.edn index 187e38126..3ab5b17b3 100644 --- a/resources/blaze.edn +++ b/resources/blaze.edn @@ -357,26 +357,26 @@ :column-families {:search-param-value-index {:write-buffer-size-in-mb 64 - :max-bytes-for-level-base-in-mb 256 - :target-file-size-base-in-mb 26 + :max-bytes-for-level-base-in-mb 512 + :target-file-size-base-in-mb 64 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :resource-value-index - {:write-buffer-size-in-mb 32 - :max-bytes-for-level-base-in-mb 128 - :target-file-size-base-in-mb 13 + {:write-buffer-size-in-mb 64 + :max-bytes-for-level-base-in-mb 512 + :target-file-size-base-in-mb 64 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :compartment-search-param-value-index {:write-buffer-size-in-mb 64 - :max-bytes-for-level-base-in-mb 256 - :target-file-size-base-in-mb 26 + :max-bytes-for-level-base-in-mb 512 + :target-file-size-base-in-mb 64 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :compartment-resource-type-index {:write-buffer-size-in-mb 4 :max-bytes-for-level-base-in-mb 16 - :target-file-size-base-in-mb 2 + :target-file-size-base-in-mb 4 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :active-search-params @@ -385,51 +385,51 @@ :tx-success-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] :reverse-comparator? true} :tx-error-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :t-by-instant-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] :reverse-comparator? true} :resource-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 - :target-file-size-base-in-mb 3 + :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :type-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 - :target-file-size-base-in-mb 3 + :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :system-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 - :target-file-size-base-in-mb 3 + :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :system-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}} [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats] {} @@ -550,26 +550,26 @@ :column-families {:search-param-value-index {:write-buffer-size-in-mb 64 - :max-bytes-for-level-base-in-mb 256 - :target-file-size-base-in-mb 26 + :max-bytes-for-level-base-in-mb 512 + :target-file-size-base-in-mb 64 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :resource-value-index - {:write-buffer-size-in-mb 32 - :max-bytes-for-level-base-in-mb 128 - :target-file-size-base-in-mb 13 + {:write-buffer-size-in-mb 64 + :max-bytes-for-level-base-in-mb 512 + :target-file-size-base-in-mb 64 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :compartment-search-param-value-index {:write-buffer-size-in-mb 64 - :max-bytes-for-level-base-in-mb 256 - :target-file-size-base-in-mb 26 + :max-bytes-for-level-base-in-mb 512 + :target-file-size-base-in-mb 64 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :compartment-resource-type-index {:write-buffer-size-in-mb 4 :max-bytes-for-level-base-in-mb 16 - :target-file-size-base-in-mb 2 + :target-file-size-base-in-mb 4 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :active-search-params @@ -578,51 +578,51 @@ :tx-success-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] :reverse-comparator? true} :tx-error-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :t-by-instant-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] :reverse-comparator? true} :resource-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 - :target-file-size-base-in-mb 3 + :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :type-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 - :target-file-size-base-in-mb 3 + :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :system-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 - :target-file-size-base-in-mb 3 + :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} :system-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 - :target-file-size-base-in-mb 1 + :target-file-size-base-in-mb 2 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}} :blaze.db.kv.rocksdb/block-cache