Skip to content

Commit cace074

Browse files
EdwardX29EdwardX29
committed
sstable: set minlz compression to default for v6+
set minlz to default compression for v6+, have a guard against using minlz for v5 and earlier.
1 parent 6018d0e commit cace074

26 files changed

+365
-360
lines changed

‎ingest_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ func TestIngestLoadRand(t *testing.T) {
191191
},
192192
path: paths[i],
193193
}
194-
expected[i].tableMetadata.Stats.CompressionType = block.SnappyCompression
194+
expected[i].tableMetadata.Stats.CompressionType = block.MinlzCompression
195195
expected[i].StatsMarkValid()
196196

197197
func() {

‎metamorphic/options.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -786,14 +786,16 @@ func RandomOptions(
786786
lopts.FilterPolicy = newTestingFilterPolicy(1 << rng.IntN(5))
787787
}
788788

789-
// We use either no compression, snappy compression or zstd compression.
790-
switch rng.IntN(3) {
789+
// We use either no compression, snappy compression, zstd compression, or minlz compression.
790+
switch rng.IntN(4) {
791791
case 0:
792792
lopts.Compression = func() block.Compression { return pebble.NoCompression }
793793
case 1:
794794
lopts.Compression = func() block.Compression { return pebble.ZstdCompression }
795-
default:
795+
case 2:
796796
lopts.Compression = func() block.Compression { return pebble.SnappyCompression }
797+
default:
798+
lopts.Compression = func() block.Compression { return pebble.MinlzCompression }
797799
}
798800
opts.Levels = []pebble.LevelOptions{lopts}
799801

‎options.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2160,7 +2160,7 @@ func (o *Options) MakeBlobWriterOptions(level int) blob.FileWriterOptions {
21602160

21612161
func resolveDefaultCompression(c Compression) Compression {
21622162
if c <= DefaultCompression || c >= block.NCompression {
2163-
c = SnappyCompression
2163+
c = MinlzCompression
21642164
}
21652165
return c
21662166
}

‎options_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func TestDefaultOptionsString(t *testing.T) {
127127
block_restart_interval=16
128128
block_size=4096
129129
block_size_threshold=90
130-
compression=Snappy
130+
compression=Minlz
131131
filter_policy=none
132132
filter_type=table
133133
index_block_size=4096

‎replay/testdata/replay

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tree
1111
614 000007.sst
1212
0 LOCK
1313
133 MANIFEST-000001
14-
1524 OPTIONS-000003
14+
1523 OPTIONS-000003
1515
0 marker.format-version.000001.013
1616
0 marker.manifest.000001.MANIFEST-000001
1717
simple/
@@ -21,7 +21,7 @@ tree
2121
25 000004.log
2222
586 000005.sst
2323
85 MANIFEST-000001
24-
1524 OPTIONS-000003
24+
1523 OPTIONS-000003
2525
0 marker.format-version.000001.013
2626
0 marker.manifest.000001.MANIFEST-000001
2727

@@ -79,7 +79,7 @@ cat build/OPTIONS-000003
7979
block_restart_interval=16
8080
block_size=4096
8181
block_size_threshold=90
82-
compression=Snappy
82+
compression=Minlz
8383
filter_policy=none
8484
filter_type=table
8585
index_block_size=4096

‎replay/testdata/replay_paced

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ tree
1414
0 LOCK
1515
133 MANIFEST-000001
1616
205 MANIFEST-000010
17-
1524 OPTIONS-000003
17+
1523 OPTIONS-000003
1818
0 marker.format-version.000001.013
1919
0 marker.manifest.000002.MANIFEST-000010
2020
high_read_amp/
@@ -26,7 +26,7 @@ tree
2626
39 000008.log
2727
560 000009.sst
2828
157 MANIFEST-000010
29-
1524 OPTIONS-000003
29+
1523 OPTIONS-000003
3030
0 marker.format-version.000001.013
3131
0 marker.manifest.000001.MANIFEST-000010
3232

‎sstable/options.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,9 @@ func (o WriterOptions) ensureDefaults() WriterOptions {
331331
o.Comparer = base.DefaultComparer
332332
}
333333
if o.Compression <= block.DefaultCompression || o.Compression >= block.NCompression {
334+
o.Compression = block.MinlzCompression
335+
}
336+
if o.Compression == block.MinlzCompression && o.TableFormat <= TableFormatPebblev5 {
334337
o.Compression = block.SnappyCompression
335338
}
336339
if o.IndexBlockSize <= 0 {

‎sstable/testdata/copy_span

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ d#0,SET: foobar
173173

174174
copy-span test3 test4 b.SET.10 cc.SET.0
175175
----
176-
copied 928 bytes
176+
copied 921 bytes
177177

178178
iter test4
179179
----
@@ -183,7 +183,7 @@ d#0,SET: foobar
183183

184184
copy-span test3 test5 a.SET.10 bb.SET.0
185185
----
186-
copied 936 bytes
186+
copied 918 bytes
187187

188188
iter test5
189189
----
@@ -213,7 +213,7 @@ d#0,SET: foobar
213213

214214
copy-span test32 test33 b.SET.10 cc.SET.0
215215
----
216-
copied 928 bytes
216+
copied 921 bytes
217217

218218
iter test33
219219
----
@@ -223,7 +223,7 @@ d#0,SET: foobar
223223

224224
copy-span test32 test34 a.SET.10 bb.SET.0
225225
----
226-
copied 936 bytes
226+
copied 918 bytes
227227

228228
iter test34
229229
----

‎sstable/testdata/rewriter_v6

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ sstable
4545
├── index offset: 363 length: 37
4646
├── top-index offset: 405 length: 52
4747
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
48-
├── properties offset: 536 length: 572
49-
├── meta-index offset: 1113 length: 88
50-
└── footer offset: 1206 length: 57
48+
├── properties offset: 536 length: 583
49+
├── meta-index offset: 1124 length: 88
50+
└── footer offset: 1217 length: 57
5151

5252
scan
5353
----
@@ -80,9 +80,9 @@ sstable
8080
├── index offset: 367 length: 37
8181
├── top-index offset: 409 length: 56
8282
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
83-
├── properties offset: 544 length: 572
84-
├── meta-index offset: 1121 length: 88
85-
└── footer offset: 1214 length: 57
83+
├── properties offset: 544 length: 583
84+
├── meta-index offset: 1132 length: 88
85+
└── footer offset: 1225 length: 57
8686

8787
scan
8888
----
@@ -115,9 +115,9 @@ sstable
115115
├── index offset: 367 length: 37
116116
├── top-index offset: 409 length: 56
117117
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
118-
├── properties offset: 544 length: 572
119-
├── meta-index offset: 1121 length: 88
120-
└── footer offset: 1214 length: 57
118+
├── properties offset: 544 length: 583
119+
├── meta-index offset: 1132 length: 88
120+
└── footer offset: 1225 length: 57
121121

122122
scan
123123
----
@@ -150,9 +150,9 @@ sstable
150150
├── index offset: 367 length: 37
151151
├── top-index offset: 409 length: 56
152152
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
153-
├── properties offset: 544 length: 572
154-
├── meta-index offset: 1121 length: 88
155-
└── footer offset: 1214 length: 57
153+
├── properties offset: 544 length: 583
154+
├── meta-index offset: 1132 length: 88
155+
└── footer offset: 1225 length: 57
156156

157157
scan
158158
----
@@ -186,9 +186,9 @@ sstable
186186
├── index offset: 363 length: 37
187187
├── top-index offset: 405 length: 52
188188
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
189-
├── properties offset: 536 length: 572
190-
├── meta-index offset: 1113 length: 88
191-
└── footer offset: 1206 length: 57
189+
├── properties offset: 536 length: 583
190+
├── meta-index offset: 1124 length: 88
191+
└── footer offset: 1217 length: 57
192192

193193
scan
194194
----

‎sstable/testdata/writer_blob_value_handles

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -111,37 +111,37 @@ sstable
111111
│ ├── 00000 block:0/154
112112
│ │
113113
│ └── trailer [compression=none checksum=0xda1c8436]
114-
├── properties offset: 200 length: 562
114+
├── properties offset: 200 length: 559
115115
│ ├── 00000 obsolete-key (13)
116116
│ ├── 00013 pebble.colblk.schema (65)
117117
│ ├── 00078 pebble.num.values.in.blob-files (32)
118118
│ ├── 00110 pebble.raw.point-tombstone.key.size (36)
119119
│ ├── 00146 rocksdb.block.based.table.index.type (40)
120120
│ ├── 00186 rocksdb.comparator (42)
121-
│ ├── 00228 rocksdb.compression (25)
122-
│ ├── 00253 rocksdb.compression_options (122)
123-
│ ├── 00375 rocksdb.data.size (19)
124-
│ ├── 00394 rocksdb.deleted.keys (21)
125-
│ ├── 00415 rocksdb.filter.size (20)
126-
│ ├── 00435 rocksdb.index.size (19)
127-
│ ├── 00454 rocksdb.merge.operands (23)
128-
│ ├── 00477 rocksdb.merge.operator (40)
129-
│ ├── 00517 rocksdb.num.data.blocks (24)
130-
│ ├── 00541 rocksdb.num.entries (20)
131-
│ ├── 00561 rocksdb.num.range-deletions (28)
132-
│ ├── 00589 rocksdb.property.collectors (41)
133-
│ ├── 00630 rocksdb.raw.key.size (21)
134-
│ ├── 00651 rocksdb.raw.value.size (24)
135-
│ └── trailer [compression=snappy checksum=0xf0c5768d]
136-
├── meta-index offset: 767 length: 46
137-
│ ├── 0000 rocksdb.properties block:200/562
121+
│ ├── 00228 rocksdb.compression (24)
122+
│ ├── 00252 rocksdb.compression_options (122)
123+
│ ├── 00374 rocksdb.data.size (19)
124+
│ ├── 00393 rocksdb.deleted.keys (21)
125+
│ ├── 00414 rocksdb.filter.size (20)
126+
│ ├── 00434 rocksdb.index.size (19)
127+
│ ├── 00453 rocksdb.merge.operands (23)
128+
│ ├── 00476 rocksdb.merge.operator (40)
129+
│ ├── 00516 rocksdb.num.data.blocks (24)
130+
│ ├── 00540 rocksdb.num.entries (20)
131+
│ ├── 00560 rocksdb.num.range-deletions (28)
132+
│ ├── 00588 rocksdb.property.collectors (41)
133+
│ ├── 00629 rocksdb.raw.key.size (21)
134+
│ ├── 00650 rocksdb.raw.value.size (24)
135+
│ └── trailer [compression=minlz checksum=0xefe5c665]
136+
├── meta-index offset: 764 length: 46
137+
│ ├── 0000 rocksdb.properties block:200/559
138138
│ │
139-
│ └── trailer [compression=none checksum=0xaa86f2de]
140-
└── footer offset: 818 length: 57
139+
│ └── trailer [compression=none checksum=0x3da86560]
140+
└── footer offset: 815 length: 57
141141
├── 000 checksum type: crc32c
142-
├── 001 meta: offset=767, length=46
142+
├── 001 meta: offset=764, length=46
143143
├── 004 index: offset=159, length=36
144-
├── 041 footer checksum: 0xe7d8ef2e
144+
├── 041 footer checksum: 0x35ccc1c0
145145
├── 045 version: 6
146146
└── 049 magic number: 0xf09faab3f09faab3
147147

0 commit comments

Comments
 (0)