Skip to content

Commit

Permalink
Use Bigger Files
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexanderkiel
alexanderkiel committed Apr 24, 2022
1 parent aa69dc7 commit d0ed676
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 41 deletions.
20 changes: 11 additions & 9 deletions modules/rocksdb/src/blaze/db/kv/rocksdb/impl.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@
(.setLevelCompactionDynamicLevelBytes true)
(.setCompressionType CompressionType/LZ4_COMPRESSION)
(.setBottommostCompressionType CompressionType/ZSTD_COMPRESSION)
(.setWriteBufferSize (bit-shift-left ^long write-buffer-size-in-mb 20))
(.setMaxWriteBufferNumber ^long max-write-buffer-number)
(.setMaxBytesForLevelBase (bit-shift-left ^long max-bytes-for-level-base-in-mb 20))
(.setLevel0FileNumCompactionTrigger ^long level0-file-num-compaction-trigger)
(.setMinWriteBufferNumberToMerge ^long min-write-buffer-number-to-merge)
(.setTargetFileSizeBase (bit-shift-left ^long target-file-size-base-in-mb 20))
(.setWriteBufferSize (bit-shift-left write-buffer-size-in-mb 20))
(.setMaxWriteBufferNumber (long max-write-buffer-number))
(.setMaxBytesForLevelBase (bit-shift-left max-bytes-for-level-base-in-mb 20))
(.setLevel0FileNumCompactionTrigger (long level0-file-num-compaction-trigger))
(.setMinWriteBufferNumberToMerge (long min-write-buffer-number-to-merge))
(.setTargetFileSizeBase (bit-shift-left target-file-size-base-in-mb 20))
(.setTableFormatConfig
(cond->
(doto (BlockBasedTableConfig.)
Expand All @@ -62,7 +62,9 @@
(.setBlockSize block-size)
(.setBlockCache block-cache))
bloom-filter?
(.setFilterPolicy (BloomFilter. 10 false)))))
(.setFilterPolicy (BloomFilter. 10 false))
bloom-filter?
(.setWholeKeyFiltering true))))
memtable-whole-key-filtering?
(.setMemtableWholeKeyFiltering true)
optimize-filters-for-hits?
Expand All @@ -82,8 +84,8 @@
compaction-readahead-size 0}}]
(doto (DBOptions.)
(.setStatistics ^Statistics stats)
(.setMaxBackgroundJobs ^long max-background-jobs)
(.setCompactionReadaheadSize ^long compaction-readahead-size)
(.setMaxBackgroundJobs (long max-background-jobs))
(.setCompactionReadaheadSize (long compaction-readahead-size))
(.setEnablePipelinedWrite true)
(.setCreateIfMissing true)
(.setCreateMissingColumnFamilies true)))
Expand Down
64 changes: 32 additions & 32 deletions resources/blaze.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,26 +357,26 @@
:column-families
{:search-param-value-index
{:write-buffer-size-in-mb 64
:max-bytes-for-level-base-in-mb 256
:target-file-size-base-in-mb 26
:max-bytes-for-level-base-in-mb 512
:target-file-size-base-in-mb 64
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:resource-value-index
{:write-buffer-size-in-mb 32
:max-bytes-for-level-base-in-mb 128
:target-file-size-base-in-mb 13
{:write-buffer-size-in-mb 64
:max-bytes-for-level-base-in-mb 512
:target-file-size-base-in-mb 64
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:compartment-search-param-value-index
{:write-buffer-size-in-mb 64
:max-bytes-for-level-base-in-mb 256
:target-file-size-base-in-mb 26
:max-bytes-for-level-base-in-mb 512
:target-file-size-base-in-mb 64
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:compartment-resource-type-index
{:write-buffer-size-in-mb 4
:max-bytes-for-level-base-in-mb 16
:target-file-size-base-in-mb 2
:target-file-size-base-in-mb 4
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:active-search-params
Expand All @@ -385,51 +385,51 @@
:tx-success-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]
:reverse-comparator? true}

:tx-error-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:t-by-instant-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]
:reverse-comparator? true}

:resource-as-of-index
{:write-buffer-size-in-mb 8
:max-bytes-for-level-base-in-mb 32
:target-file-size-base-in-mb 3
:target-file-size-base-in-mb 8
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:type-as-of-index
{:write-buffer-size-in-mb 8
:max-bytes-for-level-base-in-mb 32
:target-file-size-base-in-mb 3
:target-file-size-base-in-mb 8
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:system-as-of-index
{:write-buffer-size-in-mb 8
:max-bytes-for-level-base-in-mb 32
:target-file-size-base-in-mb 3
:target-file-size-base-in-mb 8
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:type-stats-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:system-stats-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}}

[:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats] {}
Expand Down Expand Up @@ -550,26 +550,26 @@
:column-families
{:search-param-value-index
{:write-buffer-size-in-mb 64
:max-bytes-for-level-base-in-mb 256
:target-file-size-base-in-mb 26
:max-bytes-for-level-base-in-mb 512
:target-file-size-base-in-mb 64
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:resource-value-index
{:write-buffer-size-in-mb 32
:max-bytes-for-level-base-in-mb 128
:target-file-size-base-in-mb 13
{:write-buffer-size-in-mb 64
:max-bytes-for-level-base-in-mb 512
:target-file-size-base-in-mb 64
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:compartment-search-param-value-index
{:write-buffer-size-in-mb 64
:max-bytes-for-level-base-in-mb 256
:target-file-size-base-in-mb 26
:max-bytes-for-level-base-in-mb 512
:target-file-size-base-in-mb 64
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:compartment-resource-type-index
{:write-buffer-size-in-mb 4
:max-bytes-for-level-base-in-mb 16
:target-file-size-base-in-mb 2
:target-file-size-base-in-mb 4
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:active-search-params
Expand All @@ -578,51 +578,51 @@
:tx-success-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]
:reverse-comparator? true}

:tx-error-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:t-by-instant-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]
:reverse-comparator? true}

:resource-as-of-index
{:write-buffer-size-in-mb 8
:max-bytes-for-level-base-in-mb 32
:target-file-size-base-in-mb 3
:target-file-size-base-in-mb 8
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:type-as-of-index
{:write-buffer-size-in-mb 8
:max-bytes-for-level-base-in-mb 32
:target-file-size-base-in-mb 3
:target-file-size-base-in-mb 8
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:system-as-of-index
{:write-buffer-size-in-mb 8
:max-bytes-for-level-base-in-mb 32
:target-file-size-base-in-mb 3
:target-file-size-base-in-mb 8
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:type-stats-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}

:system-stats-index
{:write-buffer-size-in-mb 2
:max-bytes-for-level-base-in-mb 8
:target-file-size-base-in-mb 1
:target-file-size-base-in-mb 2
:block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]}}}

:blaze.db.kv.rocksdb/block-cache
Expand Down

0 comments on commit d0ed676

Please sign in to comment.