Skip to content

internal/manifest: add heap maintaining virtual backing rewrite candidates #5282

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

internal/manifest: add heap maintaining virtual backing rewrite candidates #5282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
243571e
manifest/internal: adjust VirtualBackings map to track pointers
xinhaoz Sep 3, 2025
4005c9d
internal/manifest: create virtual_backings testdata dir
xinhaoz Sep 5, 2025
ca0b574
internal/manifest: add heap maintaining virtual backing rewrite candi…
xinhaoz Sep 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions internal/manifest/table_metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,11 @@ type TableBacking struct {
DiskFileNum base.DiskFileNum
Size uint64

// ReferencedBlobValueSizeTotal is the the sum of the length of uncompressed
// values in all blob files for which there exists a reference in the backing
// table.
ReferencedBlobValueSizeTotal uint64

// Reference count for the backing file, used to determine when a backing file
// is obsolete and can be removed.
//
Expand Down Expand Up @@ -474,9 +479,16 @@ func (m *TableMetadata) InitPhysicalBacking() {
if m.TableBacking != nil {
panic("backing already initialized")
}

var blobValueSizeTotal uint64
for i := range m.BlobReferences {
blobValueSizeTotal += m.BlobReferences[i].BackingValueSize
}

m.TableBacking = &TableBacking{
DiskFileNum: base.PhysicalTableDiskFileNum(m.TableNum),
Size: m.Size,
DiskFileNum: base.PhysicalTableDiskFileNum(m.TableNum),
Size: m.Size,
ReferencedBlobValueSizeTotal: blobValueSizeTotal,
}
}

Expand Down
2 changes: 1 addition & 1 deletion internal/manifest/table_metadata_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ func TestTableMetadataSize(t *testing.T) {
structSize, tableMetadataSize)
}

const tableBackingSize = 152
const tableBackingSize = 160
if structSize := unsafe.Sizeof(TableBacking{}); structSize != tableBackingSize {
t.Errorf("TableBacking struct size (%d bytes) is not expected size (%d bytes)",
structSize, tableBackingSize)
Expand Down
135 changes: 0 additions & 135 deletions internal/manifest/testdata/virtual_backings
View file
Open in desktop

This file was deleted.

131 changes: 131 additions & 0 deletions internal/manifest/testdata/virtual_backings/rewrite_candidates
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# Test rewrite candidates heap with blob value sizes.

# Add backings with values in blob files.
add n=1 size=100 blobValueSize=50
----
1 virtual backings, total size 100:
000001: size=100 refBlobValueSize=50 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
unused virtual backings: 000001

add n=2 size=200 blobValueSize=100
----
2 virtual backings, total size 300:
000001: size=100 refBlobValueSize=50 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000002: size=200 refBlobValueSize=100 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
unused virtual backings: 000001 000002

add n=3 size=150 blobValueSize=75
----
3 virtual backings, total size 450:
000001: size=100 refBlobValueSize=50 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000002: size=200 refBlobValueSize=100 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000003: size=150 refBlobValueSize=75 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
unused virtual backings: 000001 000002 000003

add n=4 size=300 blobValueSize=150
----
4 virtual backings, total size 750:
000001: size=100 refBlobValueSize=50 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000002: size=200 refBlobValueSize=100 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000003: size=150 refBlobValueSize=75 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000004: size=300 refBlobValueSize=150 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
unused virtual backings: 000001 000002 000003 000004

add n=5 size=250 blobValueSize=125
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000002: size=200 refBlobValueSize=100 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000003: size=150 refBlobValueSize=75 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000004: size=300 refBlobValueSize=150 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
unused virtual backings: 000001 000002 000003 000004 000005

# Add tables to create rewrite candidates with varying percentages.
add-table n=1 size=50 table=1
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000003: size=150 refBlobValueSize=75 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000004: size=300 refBlobValueSize=150 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
rewrite candidates heap: 000001(33.3%)
unused virtual backings: 000002 000003 000004 000005

add-table n=2 size=10 table=2
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=1 protectionCount=0 virtualizedSize=10 tables: [000002]
000003: size=150 refBlobValueSize=75 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000004: size=300 refBlobValueSize=150 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
rewrite candidates heap: 000002(3.3%) 000001(33.3%)
unused virtual backings: 000003 000004 000005

add-table n=3 size=90 table=3
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=1 protectionCount=0 virtualizedSize=10 tables: [000002]
000003: size=150 refBlobValueSize=75 useCount=1 protectionCount=0 virtualizedSize=90 tables: [000003]
000004: size=300 refBlobValueSize=150 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
rewrite candidates heap: 000002(3.3%) 000001(33.3%) 000003(40.0%)
unused virtual backings: 000004 000005

add-table n=4 size=45 table=4
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=1 protectionCount=0 virtualizedSize=10 tables: [000002]
000003: size=150 refBlobValueSize=75 useCount=1 protectionCount=0 virtualizedSize=90 tables: [000003]
000004: size=300 refBlobValueSize=150 useCount=1 protectionCount=0 virtualizedSize=45 tables: [000004]
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
rewrite candidates heap: 000002(3.3%) 000004(10.0%) 000003(40.0%) 000001(33.3%)
unused virtual backings: 000005

add-table n=5 size=100 table=5
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=1 protectionCount=0 virtualizedSize=10 tables: [000002]
000003: size=150 refBlobValueSize=75 useCount=1 protectionCount=0 virtualizedSize=90 tables: [000003]
000004: size=300 refBlobValueSize=150 useCount=1 protectionCount=0 virtualizedSize=45 tables: [000004]
000005: size=250 refBlobValueSize=125 useCount=1 protectionCount=0 virtualizedSize=100 tables: [000005]
rewrite candidates heap: 000002(3.3%) 000004(10.0%) 000003(40.0%) 000001(33.3%) 000005(26.7%)

add-table n=2 size=80 table=6
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=2 protectionCount=0 virtualizedSize=90 tables: [000002 000006]
000003: size=150 refBlobValueSize=75 useCount=1 protectionCount=0 virtualizedSize=90 tables: [000003]
000004: size=300 refBlobValueSize=150 useCount=1 protectionCount=0 virtualizedSize=45 tables: [000004]
000005: size=250 refBlobValueSize=125 useCount=1 protectionCount=0 virtualizedSize=100 tables: [000005]
rewrite candidates heap: 000004(10.0%) 000005(26.7%) 000003(40.0%) 000001(33.3%) 000002(30.0%)

# Remove some tables to demonstrate heap updates.
remove-table n=5 table=5
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=2 protectionCount=0 virtualizedSize=90 tables: [000002 000006]
000003: size=150 refBlobValueSize=75 useCount=1 protectionCount=0 virtualizedSize=90 tables: [000003]
000004: size=300 refBlobValueSize=150 useCount=1 protectionCount=0 virtualizedSize=45 tables: [000004]
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
rewrite candidates heap: 000004(10.0%) 000002(30.0%) 000003(40.0%) 000001(33.3%)
unused virtual backings: 000005

# Remove backing 4's table. Should remove from heap since virtualizedSize=0.
remove-table n=4 table=4
----
5 virtual backings, total size 1000:
000001: size=100 refBlobValueSize=50 useCount=1 protectionCount=0 virtualizedSize=50 tables: [000001]
000002: size=200 refBlobValueSize=100 useCount=2 protectionCount=0 virtualizedSize=90 tables: [000002 000006]
000003: size=150 refBlobValueSize=75 useCount=1 protectionCount=0 virtualizedSize=90 tables: [000003]
000004: size=300 refBlobValueSize=150 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
000005: size=250 refBlobValueSize=125 useCount=0 protectionCount=0 virtualizedSize=0 tables: []
rewrite candidates heap: 000002(30.0%) 000001(33.3%) 000003(40.0%)
unused virtual backings: 000004 000005
Loading