Cjl/litt integration

go.mod

@@ -262,7 +262,7 @@ require (
 	github.com/tidwall/tinylru v1.1.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.15 // indirect
 	github.com/tklauser/numcpus v0.10.0 // indirect
-	github.com/urfave/cli/v2 v2.27.5 // indirect
+	github.com/urfave/cli/v2 v2.27.5
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect

sei-db/db_engine/litt/Makefile

@@ -1,35 +1,9 @@
 SHELL := /bin/bash
 
-# All .go files in this tree are guarded by `//go:build littdb_wip` so the
-# subtree is invisible to the parent module's default go tooling. Every
-# target in this Makefile therefore passes `-tags littdb_wip` explicitly.
-GO_TAGS := littdb_wip
-
-# `-mod=mod` lets `go build` auto-add missing requires to go.mod as we pull
-# additional upstream packages in-tree. Harmless once `go mod tidy` has been
-# run, and useful during iterative porting.
-GO_MOD_FLAG := -mod=mod
-
-# Resolve + prune go.mod / go.sum against the current set of imports.
-.PHONY: tidy
-tidy:
-	go mod tidy
-
-# Download transitive deps into the module cache + go.sum without requiring
-# tidy to complete. Useful as a one-shot refresh after editing go.mod.
-.PHONY: deps
-deps:
-	go mod download all
-
 # Build the litt CLI tool.
 .PHONY: build
 build:
-	go build $(GO_MOD_FLAG) -tags=$(GO_TAGS) -o ./bin/litt ./cli
-
-# Build every package in the tree (including tests).
-.PHONY: build-all
-build-all:
-	go build $(GO_MOD_FLAG) -tags=$(GO_TAGS) ./...
+	go build -o ./bin/litt ./cli
 
 # Remove the bin directory if it exists.
 .PHONY: clean
@@ -39,14 +13,14 @@ clean:
 # Build the litt CLI tool with debug flags.
 .PHONY: debug-build
 debug-build: clean
-	go build $(GO_MOD_FLAG) -tags=$(GO_TAGS) -gcflags "all=-N -l" -o ./bin/litt ./cli
+	go build -gcflags "all=-N -l" -o ./bin/litt ./cli
 
 # Run all LittDB unit tests.
 .PHONY: test
 test: build
-	go test $(GO_MOD_FLAG) -tags=$(GO_TAGS) ./... -timeout=10m -v -p=1 -parallel=8
+	go test ./... -timeout=10m -v -p=1 -parallel=8
 
 # Run all LittDB unit tests with verbose output.
 .PHONY: test-verbose
 test-verbose: build
-	go test $(GO_MOD_FLAG) -tags=$(GO_TAGS) ./... -v -timeout=10m -p=1 -parallel=8
+	go test ./... -v -timeout=10m -p=1 -parallel=8

sei-db/db_engine/litt/README.md

@@ -1,31 +1,5 @@
 ![](docs/resources/littdb-logo.png)
 
-# Work-in-progress guard
-
-This tree is a raw import from the upstream LittDB project. It has not yet been
-adapted to build inside this module — imports still point at the origin repo
-(`github.com/Layr-Labs/eigenda/...`) and external dependencies have not been
-reconciled with this repo's `go.mod`.
-
-To keep CI green during incremental integration, every `.go` file under
-`sei-db/db_engine/litt/` starts with:
-
-```go
-//go:build littdb_wip
-```
-
-Without `-tags=littdb_wip` (the default in CI and in `make build`), the Go
-toolchain skips the entire tree — `go build ./...`, `go test ./...`, `go vet
-./...`, and `golangci-lint` all treat it as empty.
-
-To see the current (failing) state of the code locally:
-
-```bash
-go build -tags=littdb_wip ./sei-db/db_engine/litt/...
-```
-
-This is expected to fail until each package is adapted to this module.
-
 # Contents
 
 - [License](docs/licenses/README.md)

sei-db/db_engine/litt/benchmark/benchmark_engine.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package benchmark
 
 import (
@@ -12,7 +10,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/docker/go-units"
+	"github.com/sei-protocol/sei-chain/sei-db/common/unit"
 	"github.com/sei-protocol/sei-chain/sei-db/db_engine/litt"
 	"github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/benchmark/config"
 	"github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder"
@@ -68,7 +66,7 @@ func NewBenchmarkEngine(configPath string) (*BenchmarkEngine, error) {
 		cfg.LittConfig.Logger = slog.Default()
 	}
 
-	cfg.LittConfig.ShardingFactor = uint32(len(cfg.LittConfig.Paths))
+	cfg.LittConfig.ShardingFactor = uint32(len(cfg.LittConfig.Paths)) //nolint:gosec // path count bounded
 
 	db, err := littbuilder.NewDB(cfg.LittConfig)
 	if err != nil {
@@ -96,17 +94,17 @@ func NewBenchmarkEngine(configPath string) (*BenchmarkEngine, error) {
 		return nil, fmt.Errorf("failed to create data tracker: %w", err)
 	}
 
-	writeBytesPerSecond := uint64(cfg.MaximumWriteThroughputMB * float64(units.MiB))
-	writeBytesPerSecondPerThread := writeBytesPerSecond / uint64(cfg.WriterParallelism)
+	writeBytesPerSecond := uint64(cfg.MaximumWriteThroughputMB * float64(unit.MB))
+	writeBytesPerSecondPerThread := writeBytesPerSecond / uint64(cfg.WriterParallelism) //nolint:gosec // parallelism positive
 
 	// If we set the write burst size smaller than an individual value, then the rate limiter will never
 	// permit any writes. Ideally, we'd just set the burst size to 0 since we don't want bursty/volatile writes,
 	// but since we are using the rate.Limiter utility, we are required to set a burst size, and a burst size
 	// smaller than an individual value will cause the rate limiter to never permit writes.
-	writeBurstSize := uint64(cfg.ValueSizeMB * float64(units.MiB))
+	writeBurstSize := uint64(cfg.ValueSizeMB * float64(unit.MB))
 
-	readBytesPerSecond := uint64(cfg.MaximumReadThroughputMB * float64(units.MiB))
-	readBytesPerSecondPerThread := readBytesPerSecond / uint64(cfg.ReaderParallelism)
+	readBytesPerSecond := uint64(cfg.MaximumReadThroughputMB * float64(unit.MB))
+	readBytesPerSecondPerThread := readBytesPerSecond / uint64(cfg.ReaderParallelism) //nolint:gosec // parallelism positive
 
 	// If we set the read burst size smaller than an individual value we need to read, then the rate limiter will
 	// never permit us to read that value.
@@ -192,8 +190,8 @@ func (b *BenchmarkEngine) Run() error {
 
 // writer runs on a goroutine and writes data to the database.
 func (b *BenchmarkEngine) writer() {
-	maxBatchSize := uint64(b.config.BatchSizeMB * float64(units.MiB))
-	throttle := rate.NewLimiter(rate.Limit(b.writeBytesPerSecondPerThread), int(b.writeBurstSize))
+	maxBatchSize := uint64(b.config.BatchSizeMB * float64(unit.MB))
+	throttle := rate.NewLimiter(rate.Limit(b.writeBytesPerSecondPerThread), int(b.writeBurstSize)) //nolint:gosec // burst sized by config
 
 	for {
 		select {
@@ -262,7 +260,7 @@ func (b *BenchmarkEngine) verifyValue(expected *ReadInfo, actual []byte) error {
 
 // reader runs on a goroutine and reads data from the database.
 func (b *BenchmarkEngine) reader() {
-	throttle := rate.NewLimiter(rate.Limit(b.readBytesPerSecondPerThread), int(b.readBurstSize))
+	throttle := rate.NewLimiter(rate.Limit(b.readBytesPerSecondPerThread), int(b.readBurstSize)) //nolint:gosec // burst sized by config
 
 	for {
 		select {

sei-db/db_engine/litt/benchmark/benchmark_metrics.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package benchmark
 
 import (
@@ -81,12 +79,12 @@ func newMetrics(
 func (m *metrics) reportWrite(writeDuration time.Duration, bytesWritten uint64) {
 	m.writeCount.Add(1)
 	m.bytesWritten.Add(bytesWritten)
-	m.nanosecondsSpentWriting.Add(uint64(writeDuration.Nanoseconds()))
+	m.nanosecondsSpentWriting.Add(uint64(writeDuration.Nanoseconds())) //nolint:gosec // duration non-negative
 
 	// Update the longest write duration if this one is longer.
 	currentLongest := m.longestWriteDuration.Load()
-	for writeDuration.Nanoseconds() > int64(currentLongest) {
-		swapped := m.longestWriteDuration.CompareAndSwap(currentLongest, uint64(writeDuration.Nanoseconds()))
+	for writeDuration.Nanoseconds() > int64(currentLongest) { //nolint:gosec // durations comfortably fit
+		swapped := m.longestWriteDuration.CompareAndSwap(currentLongest, uint64(writeDuration.Nanoseconds())) //nolint:gosec // duration non-negative
 		if swapped {
 			break
 		}
@@ -98,12 +96,12 @@ func (m *metrics) reportWrite(writeDuration time.Duration, bytesWritten uint64)
 func (m *metrics) reportRead(readDuration time.Duration, bytesRead uint64) {
 	m.readCount.Add(1)
 	m.bytesRead.Add(bytesRead)
-	m.nanosecondsSpentReading.Add(uint64(readDuration.Nanoseconds()))
+	m.nanosecondsSpentReading.Add(uint64(readDuration.Nanoseconds())) //nolint:gosec // duration non-negative
 
 	// Update the longest read duration if this one is longer.
 	currentLongest := m.longestReadDuration.Load()
-	for readDuration.Nanoseconds() > int64(currentLongest) {
-		swapped := m.longestReadDuration.CompareAndSwap(currentLongest, uint64(readDuration.Nanoseconds()))
+	for readDuration.Nanoseconds() > int64(currentLongest) { //nolint:gosec // durations comfortably fit
+		swapped := m.longestReadDuration.CompareAndSwap(currentLongest, uint64(readDuration.Nanoseconds())) //nolint:gosec // duration non-negative
 		if swapped {
 			break
 		}
@@ -114,12 +112,12 @@ func (m *metrics) reportRead(readDuration time.Duration, bytesRead uint64) {
 // reportFlush records a flush operation.
 func (m *metrics) reportFlush(flushDuration time.Duration) {
 	m.flushCount.Add(1)
-	m.nanosecondsSpentFlushing.Add(uint64(flushDuration.Nanoseconds()))
+	m.nanosecondsSpentFlushing.Add(uint64(flushDuration.Nanoseconds())) //nolint:gosec // duration non-negative
 
 	// Update the longest flush duration if this one is longer.
 	currentLongest := m.longestFlushDuration.Load()
-	for flushDuration.Nanoseconds() > int64(currentLongest) {
-		swapped := m.longestFlushDuration.CompareAndSwap(currentLongest, uint64(flushDuration.Nanoseconds()))
+	for flushDuration.Nanoseconds() > int64(currentLongest) { //nolint:gosec // durations comfortably fit
+		swapped := m.longestFlushDuration.CompareAndSwap(currentLongest, uint64(flushDuration.Nanoseconds())) //nolint:gosec // duration non-negative
 		if swapped {
 			break
 		}
@@ -153,24 +151,24 @@ func (m *metrics) logMetrics() {
 	writeCount := m.writeCount.Load()
 	if writeCount > 0 {
 		averageWriteLatency =
-			uint64((time.Duration(m.nanosecondsSpentWriting.Load()) / time.Duration(writeCount)).Nanoseconds())
+			uint64((time.Duration(m.nanosecondsSpentWriting.Load()) / time.Duration(writeCount)).Nanoseconds()) //nolint:gosec // duration non-negative
 	}
 
 	averageReadLatency := uint64(0)
 	readCount := m.readCount.Load()
 	if readCount > 0 {
 		averageReadLatency =
-			uint64((time.Duration(m.nanosecondsSpentReading.Load()) / time.Duration(readCount)).Nanoseconds())
+			uint64((time.Duration(m.nanosecondsSpentReading.Load()) / time.Duration(readCount)).Nanoseconds()) //nolint:gosec // duration non-negative
 	}
 
 	averageFlushLatency := uint64(0)
 	flushCount := m.flushCount.Load()
 	if flushCount > 0 {
 		averageFlushLatency =
-			uint64((time.Duration(m.nanosecondsSpentFlushing.Load()) / time.Duration(flushCount)).Nanoseconds())
+			uint64((time.Duration(m.nanosecondsSpentFlushing.Load()) / time.Duration(flushCount)).Nanoseconds()) //nolint:gosec // duration non-negative
 	}
 
-	elapsedTimeNanoseconds := uint64(time.Since(m.startTime).Nanoseconds())
+	elapsedTimeNanoseconds := uint64(time.Since(m.startTime).Nanoseconds()) //nolint:gosec // duration non-negative
 	elapsedTimeSeconds := float64(elapsedTimeNanoseconds) / float64(time.Second)
 
 	bytesWritten := m.bytesWritten.Load()

sei-db/db_engine/litt/benchmark/cmd/main.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package main
 
 import (

sei-db/db_engine/litt/benchmark/cohort.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package benchmark
 
 import (
@@ -144,12 +142,12 @@ func LoadCohort(path string) (*Cohort, error) {
 		return nil, fmt.Errorf("cohort file does not exist: %s", filePath)
 	}
 
-	file, err := os.Open(filePath)
+	file, err := os.Open(filePath) //nolint:gosec // path within cohort directory
 	if err != nil {
 		return nil, fmt.Errorf("failed to open cohort file: %w", err)
 	}
 
-	data, err := os.ReadFile(filePath)
+	data, err := os.ReadFile(filePath) //nolint:gosec // path within cohort directory
 	if err != nil {
 		return nil, fmt.Errorf("failed to read cohort file: %w", err)
 	}
@@ -318,7 +316,7 @@ func (c *Cohort) serialize() []byte {
 	binary.BigEndian.PutUint64(data[8:16], c.lowKeyIndex)
 	binary.BigEndian.PutUint64(data[16:24], c.highKeyIndex)
 	binary.BigEndian.PutUint64(data[24:32], c.valueSize)
-	binary.BigEndian.PutUint64(data[32:40], uint64(c.firstValueTimestamp.Unix()))
+	binary.BigEndian.PutUint64(data[32:40], uint64(c.firstValueTimestamp.Unix())) //nolint:gosec // wall-clock seconds non-negative
 	if c.allValuesWritten {
 		data[40] = 1
 	} else {
@@ -345,7 +343,7 @@ func (c *Cohort) deserialize(data []byte) error {
 		return fmt.Errorf("invalid index range: %d >= %d", c.lowKeyIndex, c.highKeyIndex)
 	}
 
-	c.firstValueTimestamp = time.Unix(int64(binary.BigEndian.Uint64(data[32:40])), 0)
+	c.firstValueTimestamp = time.Unix(int64(binary.BigEndian.Uint64(data[32:40])), 0) //nolint:gosec // wall-clock seconds fit int64
 	c.allValuesWritten = data[40] == 1
 
 	return nil

sei-db/db_engine/litt/benchmark/cohort_test.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package benchmark
 
 import (

sei-db/db_engine/litt/benchmark/config/benchmark_config.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package config
 
 import (
@@ -8,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/docker/go-units"
+	"github.com/sei-protocol/sei-chain/sei-db/common/unit"
 	"github.com/sei-protocol/sei-chain/sei-db/db_engine/litt"
 	"github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util"
 )
@@ -111,7 +109,7 @@ func DefaultBenchmarkConfig() *BenchmarkConfig {
 		TTLHours:                    1.0,
 		ReadSafetyMarginMinutes:     5.0,
 		Seed:                        1337,
-		RandomPoolSize:              units.GiB,
+		RandomPoolSize:              unit.GB,
 		StartupSleepFactorSeconds:   0.5,
 		MetricsLoggingPeriodSeconds: 60.0,
 		PanicOnReadFailure:          false,
@@ -129,7 +127,7 @@ func LoadConfig(path string) (*BenchmarkConfig, error) {
 	}
 
 	// Read the file
-	data, err := os.ReadFile(path)
+	data, err := os.ReadFile(path) //nolint:gosec // caller-supplied config path
 	if err != nil {
 		return nil, fmt.Errorf("failed to read config file: %w", err)
 	}

sei-db/db_engine/litt/benchmark/config/benchmark_config_test.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package config
 
 import (

sei-db/db_engine/litt/benchmark/data_generator.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package benchmark
 
 import (
@@ -31,7 +29,7 @@ func NewDataGenerator(seed int64, poolSize uint64) *DataGenerator {
 
 	dataPool := make([]byte, poolSize)
 	rng := randPool.Get().(*rand.Rand)
-	rng.Read(dataPool)
+	_, _ = rng.Read(dataPool)
 	randPool.Put(rng)
 
 	return &DataGenerator{
@@ -43,10 +41,10 @@ func NewDataGenerator(seed int64, poolSize uint64) *DataGenerator {
 // Key generates a new key. The value is deterministic for the same index and seed.
 func (g *DataGenerator) Key(index uint64) []byte {
 	rng := g.randPool.Get().(*rand.Rand)
-	rng.Seed(g.seed + int64(index))
+	rng.Seed(g.seed + int64(index)) //nolint:gosec // deterministic test seeding
 
 	key := make([]byte, 32)
-	rng.Read(key)
+	_, _ = rng.Read(key)
 	g.randPool.Put(rng)
 
 	return key
@@ -55,7 +53,7 @@ func (g *DataGenerator) Key(index uint64) []byte {
 // Value generates a new value. The value is deterministic for the same index, seed, and value size.
 func (g *DataGenerator) Value(index uint64, valueLength uint64) []byte {
 	rng := g.randPool.Get().(*rand.Rand)
-	rng.Seed(g.seed + int64(index))
+	rng.Seed(g.seed + int64(index)) //nolint:gosec // deterministic test seeding
 
 	var value []byte
 
@@ -64,10 +62,10 @@ func (g *DataGenerator) Value(index uint64, valueLength uint64) []byte {
 		// For the sake of completeness, just generate the data if this happens.
 		// This shouldn't be encountered for sane configurations (i.e. with a pool size much larger than value sizes).
 		value = make([]byte, valueLength)
-		rng.Read(value)
+		_, _ = rng.Read(value)
 	} else {
-		startIndex := rng.Intn(len(g.dataPool) - int(valueLength))
-		value = g.dataPool[startIndex : startIndex+int(valueLength)]
+		startIndex := rng.Intn(len(g.dataPool) - int(valueLength))   //nolint:gosec // valueLength bounded by len(dataPool)
+		value = g.dataPool[startIndex : startIndex+int(valueLength)] //nolint:gosec // valueLength bounded by len(dataPool)
 	}
 
 	g.randPool.Put(rng)

sei-db/db_engine/litt/benchmark/data_generator_test.go

@@ -1,5 +1,3 @@
-//go:build littdb_wip
-
 package benchmark
 
 import (