Merge pull request #131 from wildcatdb/pr-061825.1

- db.go minor replacement of options and better comment explainations…

Author: guycipher

blockmanager/fsyncdata.go

@@ -6,6 +6,7 @@ import (
 	"syscall"
 )
 
+// Fdatasync is a Linux-specific implementation of fdatasync.
 func Fdatasync(fd uintptr) error {
 	err := syscall.Fdatasync(int(fd))
 	if err != nil {

blockmanager/fsyncdata_darwin.go

@@ -7,6 +7,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+// // Fdatasync is a Darwin-specific implementation of fdatasync.
 func Fdatasync(fd uintptr) error {
 	// F_FULLFSYNC forces the drive to flush its buffers to stable storage.
 	_, _, errno := unix.Syscall(

blockmanager/openfile.go

@@ -7,6 +7,7 @@ import (
 	"syscall"
 )
 
+// OpenFile opens a file with the specified name and flags, returning a file handle.
 func OpenFile(name string, flags int, perm uint32) (uintptr, error) {
 	fd, err := syscall.Open(name, flags, perm)
 	if err != nil {
@@ -15,6 +16,7 @@ func OpenFile(name string, flags int, perm uint32) (uintptr, error) {
 	return uintptr(fd), nil
 }
 
+// NewFileFromFd creates a new os.File from a file descriptor handle and a name.
 func NewFileFromFd(handle uintptr, name string) *os.File {
 	return os.NewFile(handle, name)
 }

blockmanager/openfile_windows.go

@@ -8,6 +8,7 @@ import (
 	"syscall"
 )
 
+// OpenFile opens a file with the specified name and flags, returning a file handle.
 func OpenFile(name string, flags int, perm uint32) (uintptr, error) {
 	var access uint32
 	var creation uint32
@@ -75,6 +76,7 @@ func OpenFile(name string, flags int, perm uint32) (uintptr, error) {
 	return uintptr(handle), nil
 }
 
+// NewFileFromFd creates a new os.File from a file descriptor handle and name.
 func NewFileFromFd(handle uintptr, name string) *os.File {
 	return os.NewFile(handle, name)
 }

db.go

@@ -32,7 +32,8 @@ import (
 	"unicode/utf8"
 )
 
-// SyncOption is a block manager sync option that can be set by the user.
+// SyncOption is a block manager sync option that can be set for a *DB instance.
+// the sync option will be used for all block managers created by the *DB instance internally.
 type SyncOption int
 
 const (
@@ -100,6 +101,7 @@ type Options struct {
 	BlockManagerLRUAccesWeight           float64       // Access weight for the LRU cache
 	Permission                           os.FileMode   // Permission for created files
 	LogChannel                           chan string   // Channel for logging
+	STDOutLogging                        bool          // Enable logging to standard output (default is false and if set, channel is ignored)
 	BloomFilter                          bool          // Enable Bloom filter for SSTables
 	MaxCompactionConcurrency             int           // Maximum number of concurrent compactions
 	CompactionCooldownPeriod             time.Duration // Cooldown period for compaction
@@ -110,20 +112,19 @@ type Options struct {
 	CompactionScoreCountWeight           float64       // Weight for count-based score
 	CompactionSizeTieredSimilarityRatio  float64       // Similarity ratio for size-tiered compaction.  For grouping SSTables that are "roughly the same size" together for compaction.
 	CompactionActiveSSTReadWaitBackoff   time.Duration // Backoff time for active SSTable read wait during compaction, to avoid busy waiting
-	FlusherTickerInterval                time.Duration // Interval for flusher ticker
+	CompactionPartitionRatio             float64       // How much to move back (0.6 = 60% of data). Used for last level compaction
+	CompactionPartitionDistributionRatio float64       // How to split between L-1 and L-2 (0.7 = 70% to L-1, 30% to L-2)
 	CompactorTickerInterval              time.Duration // Interval for compactor ticker
+	FlusherTickerInterval                time.Duration // Interval for flusher ticker
 	BloomFilterFPR                       float64       // False positive rate for Bloom filter
 	WalAppendRetry                       int           // Number of retries for WAL append
 	WalAppendBackoff                     time.Duration // Backoff duration for WAL append
 	SSTableBTreeOrder                    int           // Order of the B-tree for SSTables
-	STDOutLogging                        bool          // Enable logging to standard output (default is false and if set, channel is ignored)
 	MaxConcurrentTxns                    int           // Maximum concurrent transactions (buffer size)
 	TxnBeginRetry                        int           // Number of retries for Begin() when buffer full
 	TxnBeginBackoff                      time.Duration // Initial backoff duration for Begin() retries
 	TxnBeginMaxBackoff                   time.Duration // Maximum backoff duration for Begin() retries
 	RecoverUncommittedTxns               bool          // Whether to recover uncommitted transactions on startup
-	CompactionPartitionRatio             float64       // How much to move back (0.6 = 60% of data). Used for last level compaction
-	CompactionPartitionDistributionRatio float64       // How to split between L-1 and L-2 (0.7 = 70% to L-1, 30% to L-2)
 }
 
 // DB represents the main Wildcat structure
@@ -167,14 +168,16 @@ func Open(opts *Options) (*DB, error) {
 	}
 
 	// Set default values for what options are not set
-	opts.setDefaults()
+	err := opts.setDefaults()
+	if err != nil {
+		return nil, err
+	}
 
 	buff, err := buffer.New(opts.MaxConcurrentTxns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transaction buffer: %w", err)
 	}
 
-	// We create a db instance with the provided options
 	db := &DB{
 		lru:            lru.New(int64(opts.BlockManagerLRUSize), opts.BlockManagerLRUEvictRatio, opts.BlockManagerLRUAccesWeight), // New block manager LRU cache
 		wg:             &sync.WaitGroup{},                                                                                         // Wait group for background operations
@@ -185,6 +188,7 @@ func Open(opts *Options) (*DB, error) {
 	}
 
 	// Initialize flusher and compactor
+	// The flusher and compactor run in the background and handle flushing memtables to disk and compacting levels respectively.
 	db.flusher = newFlusher(db)
 	db.compactor = newCompactor(db)
 
@@ -242,11 +246,9 @@ func Open(opts *Options) (*DB, error) {
 
 		level.sstables = atomic.Pointer[[]*SSTable]{}                                                       // Atomic pointer to SSTables in this level
 		level.path = fmt.Sprintf("%s%s%d%s", db.opts.Directory, LevelPrefix, i+1, string(os.PathSeparator)) // Path for the level directory
-
-		// Set level
 		levels[i] = level
 
-		db.log(fmt.Sprintf("Creating level %d with capacity %d bytes at path %s", level.id, level.capacity, level.path))
+		db.log(fmt.Sprintf("Initializing level %d with capacity %d bytes at path %s", level.id, level.capacity, level.path))
 
 		// Create or ensure the level directory exists
 		if err := os.MkdirAll(level.path, db.opts.Permission); err != nil {
@@ -275,7 +277,7 @@ func Open(opts *Options) (*DB, error) {
 	db.wg.Add(1)
 	go db.flusher.backgroundProcess()
 
-	// Start the compaction manager
+	// Start the background compactor
 	db.wg.Add(1)
 	go db.compactor.backgroundProcess()
 
@@ -284,7 +286,7 @@ func Open(opts *Options) (*DB, error) {
 }
 
 // setDefaults checks and sets default values for db options
-func (opts *Options) setDefaults() {
+func (opts *Options) setDefaults() error {
 	if opts.WriteBufferSize <= 0 {
 		opts.WriteBufferSize = DefaultWriteBufferSize
 	}
@@ -371,6 +373,12 @@ func (opts *Options) setDefaults() {
 
 	if opts.SSTableBTreeOrder <= 0 {
 		opts.SSTableBTreeOrder = DefaultSSTableBTreeOrder
+	} else {
+		// Ensure the B-tree order is reasonable
+		if opts.SSTableBTreeOrder < 2 {
+			return fmt.Errorf("SSTable B-tree order must be at least 2, got %d", opts.SSTableBTreeOrder)
+		}
+
 	}
 
 	if opts.CompactionSizeTieredSimilarityRatio <= 0 {
@@ -405,6 +413,8 @@ func (opts *Options) setDefaults() {
 		opts.CompactionPartitionDistributionRatio = DefaultCompactionPartitionDistributionRatio
 	}
 
+	return nil
+
 }
 
 // Close closes the database and all open resources

level.go

@@ -91,22 +91,6 @@ func (l *Level) reopen() error {
 		// Get file paths
 		klogPath := fmt.Sprintf("%s%s%d%s", levelPath, SSTablePrefix, id, KLogExtension)
 
-		// Get file sizes to calculate SSTable size
-		klogInfo, err := os.Stat(klogPath)
-		if err != nil {
-			l.db.log(fmt.Sprintf("Warning: Failed to stat KLog file for SSTable %d: %v - skipping", id, err))
-			continue
-		}
-
-		vlogInfo, err := os.Stat(vlogPath)
-		if err != nil {
-			l.db.log(fmt.Sprintf("Warning: Failed to stat VLog file for SSTable %d: %v - skipping", id, err))
-			continue
-		}
-
-		// Calculate total size from file system
-		sstable.Size = klogInfo.Size() + vlogInfo.Size()
-
 		// Open the KLog file to try to get metadata from B-tree
 		klogBm, err := blockmanager.Open(klogPath, os.O_RDONLY, l.db.opts.Permission, blockmanager.SyncOption(l.db.opts.SyncOption))
 		if err != nil {
@@ -156,15 +140,16 @@ func (l *Level) reopen() error {
 
 			default:
 				// This should not occur but if it does log a warning
+				// The system will still function using the sstable but without proper metadata setting nil
 				l.db.log(fmt.Sprintf("Warning: Unknown metadata type %T for SSTable %d - using file system metadata", extraMeta, id))
 
-				sstable.Min = []byte{}
-				sstable.Max = []byte{}
+				sstable.Min = nil
+				sstable.Max = nil
 				sstable.EntryCount = 0
 			}
 		} else {
-			sstable.Min = []byte{}
-			sstable.Max = []byte{}
+			sstable.Min = nil
+			sstable.Max = nil
 			sstable.EntryCount = 0
 		}
 
@@ -229,37 +214,69 @@ func (l *Level) extractSSTableFromBSON(sstable *SSTable, doc primitive.D) {
 			} else if level, ok := elem.Value.(int64); ok {
 				sstable.Level = int(level)
 			}
+		case "timestamp":
+			if timestamp, ok := elem.Value.(int64); ok {
+				sstable.Timestamp = timestamp
+			}
+		case "bloomfilter":
+			if l.db.opts.BloomFilter {
+				if err := sstable.reconstructBloomFilter(); err != nil {
+					l.db.log(fmt.Sprintf("Warning: Failed to reconstruct bloom filter for SSTable %d: %v", sstable.Id, err))
+				}
+			}
 		}
 	}
 }
 
 // extractSSTableFromMap a helper method to extract SSTable metadata from map[string]interface{}
 func (l *Level) extractSSTableFromMap(sstable *SSTable, meta map[string]interface{}) {
-	if id, ok := meta["id"].(int64); ok {
-		sstable.Id = id
-	}
-	if minBytes, ok := meta["min"].([]byte); ok {
-		sstable.Min = minBytes
-	}
-	if maxBytes, ok := meta["max"].([]byte); ok {
-		sstable.Max = maxBytes
-	}
-	if size, ok := meta["size"].(int64); ok && size > 0 {
-		sstable.Size = size
-	}
-	if count, ok := meta["entrycount"].(int); ok {
-		sstable.EntryCount = count
-	} else if count, ok := meta["entrycount"].(int32); ok {
-		sstable.EntryCount = int(count)
-	} else if count, ok := meta["entrycount"].(int64); ok {
-		sstable.EntryCount = int(count)
+	for key, value := range meta {
+		switch key {
+		case "id":
+			if id, ok := value.(int64); ok {
+				sstable.Id = id
+			}
+		case "min":
+			if minBytes, ok := value.([]byte); ok {
+				sstable.Min = minBytes
+			}
+		case "max":
+			if maxBytes, ok := value.([]byte); ok {
+				sstable.Max = maxBytes
+			}
+		case "size":
+			if size, ok := value.(int64); ok && size > 0 {
+				sstable.Size = size
+			}
+		case "entrycount":
+			switch count := value.(type) {
+			case int:
+				sstable.EntryCount = count
+			case int32:
+				sstable.EntryCount = int(count)
+			case int64:
+				sstable.EntryCount = int(count)
+			}
+		case "level":
+			switch level := value.(type) {
+			case int:
+				sstable.Level = level
+			case int32:
+				sstable.Level = int(level)
+			case int64:
+				sstable.Level = int(level)
+			}
+		case "timestamp":
+			if timestamp, ok := value.(int64); ok {
+				sstable.Timestamp = timestamp
+			}
+		}
 	}
-	if level, ok := meta["level"].(int); ok {
-		sstable.Level = level
-	} else if level, ok := meta["level"].(int32); ok {
-		sstable.Level = int(level)
-	} else if level, ok := meta["level"].(int64); ok {
-		sstable.Level = int(level)
+
+	if l.db.opts.BloomFilter {
+		if err := sstable.reconstructBloomFilter(); err != nil {
+			l.db.log(fmt.Sprintf("Warning: Failed to reconstruct bloom filter for SSTable %d: %v", sstable.Id, err))
+		}
 	}
 }
 

readme.md

@@ -23,7 +23,7 @@ Wildcat is a high-performance embedded key-value database (or storage engine) wr
 - Automatic multi-threaded background compaction with configurable concurrency
 - ACID transaction support with configurable durability guarantees
 - Range, prefix, and full iteration support with bidirectional traversal
-- High transactional throughput per second with low latency due to lock-free and non-blocking design.
+- High transactional throughput per second with low latency due to lock-free and non-blocking design from memory to disk
 - Optional Bloom filters per SSTable for improved key lookup performance
 - Key-value separation optimization (`.klog` for keys, `.vlog` for values)
 - Tombstone-aware and version-aware compaction with retention based on active transaction read windows
@@ -134,7 +134,7 @@ The easiest way to interact with Wildcat is through the Update method, which han
 ```go
 // Write a value
 err := db.Update(func(txn *wildcat.Txn) error {
-    return txn.Put([]byte("hello"), []byte("world")) // Put update's existing key's values.
+    return txn.Put([]byte("hello"), []byte("world")) // Put update's existing key's values
 })
 if err != nil {
     // Handle error
@@ -158,7 +158,6 @@ if err != nil {
 ### Manual Transaction Management
 For more complex operations, you can manually manage transactions.
 ```go
-// Begin a transaction
 txn, err := db.Begin()
 if err != nil {
     // Handle error
@@ -221,7 +220,6 @@ err := db.Update(func(txn *wildcat.Txn) error {
 ```go
 // Perform batch operations
 for i := 0; i < 1000; i++ {
-    // Begin a transaction
     txn, err := db.Begin()
     if err != nil {
         // Handle error
@@ -237,7 +235,6 @@ for i := 0; i < 1000; i++ {
         return
     }
 
-    // Commit the transaction
     err = txn.Commit()
     if err != nil {
         // Handle error
@@ -293,7 +290,6 @@ err := db.View(func(txn *wildcat.Txn) error {
 #### Range Iterator (bidirectional)
 ```go
 err := db.View(func(txn *wildcat.Txn) error {
-    // Create range iterator
     iter, err := txn.NewRangeIterator([]byte("start"), []byte("end"), true)
     if err != nil {
         return err
@@ -331,7 +327,6 @@ err := db.View(func(txn *wildcat.Txn) error {
 #### Prefix Iterator (bidirectional)
 ```go
 err := db.View(func(txn *wildcat.Txn) error {
-    // Create prefix iterator
     iter, err := txn.NewPrefixIterator([]byte("prefix"), true)
     if err != nil {
         return err