From 2c888c6f6d72970faf8956343785d865ab52c005 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 14 May 2026 09:53:10 -0500 Subject: [PATCH 1/2] integrate build --- go.mod | 2 +- sei-db/db_engine/litt/Makefile | 52 - sei-db/db_engine/litt/README.md | 26 - .../litt/benchmark/benchmark_engine.go | 12 +- .../litt/benchmark/benchmark_metrics.go | 2 - sei-db/db_engine/litt/benchmark/cmd/main.go | 2 - sei-db/db_engine/litt/benchmark/cohort.go | 2 - .../db_engine/litt/benchmark/cohort_test.go | 2 - .../litt/benchmark/config/benchmark_config.go | 6 +- .../benchmark/config/benchmark_config_test.go | 2 - .../litt/benchmark/data_generator.go | 2 - .../litt/benchmark/data_generator_test.go | 2 - .../db_engine/litt/benchmark/data_tracker.go | 8 +- .../litt/benchmark/data_tracker_test.go | 18 +- sei-db/db_engine/litt/cli/benchmark.go | 2 - sei-db/db_engine/litt/cli/litt_cli.go | 2 - sei-db/db_engine/litt/cli/ls.go | 2 - sei-db/db_engine/litt/cli/ls_test.go | 2 - sei-db/db_engine/litt/cli/main.go | 2 - sei-db/db_engine/litt/cli/prune.go | 2 - sei-db/db_engine/litt/cli/prune_test.go | 2 - sei-db/db_engine/litt/cli/push.go | 2 - sei-db/db_engine/litt/cli/push_test.go | 1424 ++++++++-------- sei-db/db_engine/litt/cli/rebase.go | 2 - sei-db/db_engine/litt/cli/rebase_test.go | 2 - sei-db/db_engine/litt/cli/sync.go | 2 - sei-db/db_engine/litt/cli/table_info.go | 2 - sei-db/db_engine/litt/cli/table_info_test.go | 2 - sei-db/db_engine/litt/cli/unlock.go | 2 - sei-db/db_engine/litt/db.go | 2 - sei-db/db_engine/litt/dbcache/cached_table.go | 2 - .../db_engine/litt/disktable/boundary_file.go | 2 - .../litt/disktable/boundary_file_test.go | 2 - .../db_engine/litt/disktable/control_loop.go | 2 - .../litt/disktable/control_loop_messages.go | 2 - sei-db/db_engine/litt/disktable/disk_table.go | 2 - .../litt/disktable/disk_table_flush_loop.go | 2 - .../litt/disktable/disk_table_test.go | 2 - .../litt/disktable/flush_coordinator.go | 2 - .../litt/disktable/flush_coordinator_test.go | 2 - sei-db/db_engine/litt/disktable/flush_loop.go | 2 - .../litt/disktable/flush_loop_messages.go | 2 - .../db_engine/litt/disktable/keymap/keymap.go | 2 - .../litt/disktable/keymap/keymap_test.go | 2 - .../litt/disktable/keymap/keymap_type.go | 2 - .../litt/disktable/keymap/keymap_type_file.go | 2 - .../litt/disktable/keymap/mem_keymap.go | 2 - .../litt/disktable/keymap/pebble_db_keymap.go | 2 - .../litt/disktable/segment/address_test.go | 2 - .../litt/disktable/segment/key_file.go | 2 - .../litt/disktable/segment/key_file_test.go | 2 - .../litt/disktable/segment/metadata_file.go | 2 - .../disktable/segment/metadata_file_test.go | 2 - .../litt/disktable/segment/segment.go | 2 - .../litt/disktable/segment/segment_path.go | 2 - .../disktable/segment/segment_path_test.go | 2 - .../litt/disktable/segment/segment_scanner.go | 2 - .../litt/disktable/segment/segment_test.go | 2 - .../litt/disktable/segment/segment_version.go | 2 - .../segment/shard_id_validation_test.go | 2 - .../litt/disktable/segment/value_file.go | 2 - .../litt/disktable/segment/value_file_test.go | 2 - .../litt/disktable/table_metadata.go | 2 - sei-db/db_engine/litt/disktable/unlock.go | 2 - sei-db/db_engine/litt/go.mod | 105 -- sei-db/db_engine/litt/go.sum | 216 --- .../db_engine/litt/littbuilder/build_utils.go | 2 - sei-db/db_engine/litt/littbuilder/db_impl.go | 2 - sei-db/db_engine/litt/littdb_config.go | 6 +- sei-db/db_engine/litt/littdb_config_test.go | 2 - sei-db/db_engine/litt/memtable/mem_table.go | 2 - .../db_engine/litt/metrics/littdb_metrics.go | 2 - sei-db/db_engine/litt/table.go | 2 - sei-db/db_engine/litt/test/cache_test.go | 2 - sei-db/db_engine/litt/test/db_test.go | 2 - .../litt/test/generate_example_tree_test.go | 2 - .../litt/test/keymap_migration_test.go | 2 - sei-db/db_engine/litt/test/lock_test.go | 2 - sei-db/db_engine/litt/test/migration_data.go | 2 - sei-db/db_engine/litt/test/migration_test.go | 2 - sei-db/db_engine/litt/test/snapshot_test.go | 2 - sei-db/db_engine/litt/test/table_test.go | 2 - sei-db/db_engine/litt/test/unlock_test.go | 2 - sei-db/db_engine/litt/types/address.go | 2 - sei-db/db_engine/litt/types/kv_pair.go | 2 - sei-db/db_engine/litt/types/scoped_key.go | 2 - sei-db/db_engine/litt/util/cache.go | 2 - sei-db/db_engine/litt/util/cache_metrics.go | 2 - sei-db/db_engine/litt/util/common.go | 2 - sei-db/db_engine/litt/util/constants.go | 2 - sei-db/db_engine/litt/util/core_utils.go | 2 - sei-db/db_engine/litt/util/enforce.go | 2 - sei-db/db_engine/litt/util/error_monitor.go | 2 - sei-db/db_engine/litt/util/fifo_cache.go | 2 - sei-db/db_engine/litt/util/file_lock.go | 2 - sei-db/db_engine/litt/util/file_lock_test.go | 2 - sei-db/db_engine/litt/util/file_utils.go | 2 - sei-db/db_engine/litt/util/file_utils_test.go | 2 - sei-db/db_engine/litt/util/pprof.go | 2 - sei-db/db_engine/litt/util/queue.go | 2 - .../litt/util/random_access_deque.go | 2 - sei-db/db_engine/litt/util/recursive_move.go | 2 - .../litt/util/recursive_move_test.go | 2 - sei-db/db_engine/litt/util/ssh.go | 2 - .../litt/util/ssh_self_destruct_test.go | 188 ++- sei-db/db_engine/litt/util/ssh_test.go | 418 ++--- sei-db/db_engine/litt/util/ssh_test_utils.go | 1454 +++++++++-------- sei-db/db_engine/litt/util/test_assertions.go | 2 - sei-db/db_engine/litt/util/test_logger.go | 2 - sei-db/db_engine/litt/util/test_random.go | 2 - .../db_engine/litt/util/thread_safe_cache.go | 2 - sei-db/db_engine/litt/util/units.go | 2 - sei-db/db_engine/litt/util/unsafe_string.go | 2 - 113 files changed, 1783 insertions(+), 2350 deletions(-) delete mode 100644 sei-db/db_engine/litt/Makefile delete mode 100644 sei-db/db_engine/litt/go.mod delete mode 100644 sei-db/db_engine/litt/go.sum diff --git a/go.mod b/go.mod index db681f7bf4..009b009979 100644 --- a/go.mod +++ b/go.mod @@ -262,7 +262,7 @@ require ( github.com/tidwall/tinylru v1.1.0 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect - github.com/urfave/cli/v2 v2.27.5 // indirect + github.com/urfave/cli/v2 v2.27.5 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect diff --git a/sei-db/db_engine/litt/Makefile b/sei-db/db_engine/litt/Makefile deleted file mode 100644 index d4c3b0b926..0000000000 --- a/sei-db/db_engine/litt/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -SHELL := /bin/bash - -# All .go files in this tree are guarded by `//go:build littdb_wip` so the -# subtree is invisible to the parent module's default go tooling. Every -# target in this Makefile therefore passes `-tags littdb_wip` explicitly. -GO_TAGS := littdb_wip - -# `-mod=mod` lets `go build` auto-add missing requires to go.mod as we pull -# additional upstream packages in-tree. Harmless once `go mod tidy` has been -# run, and useful during iterative porting. -GO_MOD_FLAG := -mod=mod - -# Resolve + prune go.mod / go.sum against the current set of imports. -.PHONY: tidy -tidy: - go mod tidy - -# Download transitive deps into the module cache + go.sum without requiring -# tidy to complete. Useful as a one-shot refresh after editing go.mod. -.PHONY: deps -deps: - go mod download all - -# Build the litt CLI tool. -.PHONY: build -build: - go build $(GO_MOD_FLAG) -tags=$(GO_TAGS) -o ./bin/litt ./cli - -# Build every package in the tree (including tests). -.PHONY: build-all -build-all: - go build $(GO_MOD_FLAG) -tags=$(GO_TAGS) ./... - -# Remove the bin directory if it exists. -.PHONY: clean -clean: - rm -rf ./bin - -# Build the litt CLI tool with debug flags. -.PHONY: debug-build -debug-build: clean - go build $(GO_MOD_FLAG) -tags=$(GO_TAGS) -gcflags "all=-N -l" -o ./bin/litt ./cli - -# Run all LittDB unit tests. -.PHONY: test -test: build - go test $(GO_MOD_FLAG) -tags=$(GO_TAGS) ./... -timeout=10m -v -p=1 -parallel=8 - -# Run all LittDB unit tests with verbose output. -.PHONY: test-verbose -test-verbose: build - go test $(GO_MOD_FLAG) -tags=$(GO_TAGS) ./... -v -timeout=10m -p=1 -parallel=8 diff --git a/sei-db/db_engine/litt/README.md b/sei-db/db_engine/litt/README.md index 92d967d00c..33a863de69 100644 --- a/sei-db/db_engine/litt/README.md +++ b/sei-db/db_engine/litt/README.md @@ -1,31 +1,5 @@ ![](docs/resources/littdb-logo.png) -# Work-in-progress guard - -This tree is a raw import from the upstream LittDB project. It has not yet been -adapted to build inside this module — imports still point at the origin repo -(`github.com/Layr-Labs/eigenda/...`) and external dependencies have not been -reconciled with this repo's `go.mod`. - -To keep CI green during incremental integration, every `.go` file under -`sei-db/db_engine/litt/` starts with: - -```go -//go:build littdb_wip -``` - -Without `-tags=littdb_wip` (the default in CI and in `make build`), the Go -toolchain skips the entire tree — `go build ./...`, `go test ./...`, `go vet -./...`, and `golangci-lint` all treat it as empty. - -To see the current (failing) state of the code locally: - -```bash -go build -tags=littdb_wip ./sei-db/db_engine/litt/... -``` - -This is expected to fail until each package is adapted to this module. - # Contents - [License](docs/licenses/README.md) diff --git a/sei-db/db_engine/litt/benchmark/benchmark_engine.go b/sei-db/db_engine/litt/benchmark/benchmark_engine.go index 757b66667c..b699e59dbd 100644 --- a/sei-db/db_engine/litt/benchmark/benchmark_engine.go +++ b/sei-db/db_engine/litt/benchmark/benchmark_engine.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( @@ -12,7 +10,7 @@ import ( "syscall" "time" - "github.com/docker/go-units" + "github.com/sei-protocol/sei-chain/sei-db/common/unit" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/benchmark/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" @@ -96,16 +94,16 @@ func NewBenchmarkEngine(configPath string) (*BenchmarkEngine, error) { return nil, fmt.Errorf("failed to create data tracker: %w", err) } - writeBytesPerSecond := uint64(cfg.MaximumWriteThroughputMB * float64(units.MiB)) + writeBytesPerSecond := uint64(cfg.MaximumWriteThroughputMB * float64(unit.MB)) writeBytesPerSecondPerThread := writeBytesPerSecond / uint64(cfg.WriterParallelism) // If we set the write burst size smaller than an individual value, then the rate limiter will never // permit any writes. Ideally, we'd just set the burst size to 0 since we don't want bursty/volatile writes, // but since we are using the rate.Limiter utility, we are required to set a burst size, and a burst size // smaller than an individual value will cause the rate limiter to never permit writes. - writeBurstSize := uint64(cfg.ValueSizeMB * float64(units.MiB)) + writeBurstSize := uint64(cfg.ValueSizeMB * float64(unit.MB)) - readBytesPerSecond := uint64(cfg.MaximumReadThroughputMB * float64(units.MiB)) + readBytesPerSecond := uint64(cfg.MaximumReadThroughputMB * float64(unit.MB)) readBytesPerSecondPerThread := readBytesPerSecond / uint64(cfg.ReaderParallelism) // If we set the read burst size smaller than an individual value we need to read, then the rate limiter will @@ -192,7 +190,7 @@ func (b *BenchmarkEngine) Run() error { // writer runs on a goroutine and writes data to the database. func (b *BenchmarkEngine) writer() { - maxBatchSize := uint64(b.config.BatchSizeMB * float64(units.MiB)) + maxBatchSize := uint64(b.config.BatchSizeMB * float64(unit.MB)) throttle := rate.NewLimiter(rate.Limit(b.writeBytesPerSecondPerThread), int(b.writeBurstSize)) for { diff --git a/sei-db/db_engine/litt/benchmark/benchmark_metrics.go b/sei-db/db_engine/litt/benchmark/benchmark_metrics.go index 1b16e9c353..e7e4c18a4c 100644 --- a/sei-db/db_engine/litt/benchmark/benchmark_metrics.go +++ b/sei-db/db_engine/litt/benchmark/benchmark_metrics.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( diff --git a/sei-db/db_engine/litt/benchmark/cmd/main.go b/sei-db/db_engine/litt/benchmark/cmd/main.go index a35d42bc10..3d19aaafe0 100644 --- a/sei-db/db_engine/litt/benchmark/cmd/main.go +++ b/sei-db/db_engine/litt/benchmark/cmd/main.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/benchmark/cohort.go b/sei-db/db_engine/litt/benchmark/cohort.go index fc1c6dfa00..6cb6cc061d 100644 --- a/sei-db/db_engine/litt/benchmark/cohort.go +++ b/sei-db/db_engine/litt/benchmark/cohort.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( diff --git a/sei-db/db_engine/litt/benchmark/cohort_test.go b/sei-db/db_engine/litt/benchmark/cohort_test.go index bab61ad82f..4a8e97c31d 100644 --- a/sei-db/db_engine/litt/benchmark/cohort_test.go +++ b/sei-db/db_engine/litt/benchmark/cohort_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( diff --git a/sei-db/db_engine/litt/benchmark/config/benchmark_config.go b/sei-db/db_engine/litt/benchmark/config/benchmark_config.go index 09659c71df..5221e1aef2 100644 --- a/sei-db/db_engine/litt/benchmark/config/benchmark_config.go +++ b/sei-db/db_engine/litt/benchmark/config/benchmark_config.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package config import ( @@ -8,7 +6,7 @@ import ( "os" "strings" - "github.com/docker/go-units" + "github.com/sei-protocol/sei-chain/sei-db/common/unit" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" ) @@ -111,7 +109,7 @@ func DefaultBenchmarkConfig() *BenchmarkConfig { TTLHours: 1.0, ReadSafetyMarginMinutes: 5.0, Seed: 1337, - RandomPoolSize: units.GiB, + RandomPoolSize: unit.GB, StartupSleepFactorSeconds: 0.5, MetricsLoggingPeriodSeconds: 60.0, PanicOnReadFailure: false, diff --git a/sei-db/db_engine/litt/benchmark/config/benchmark_config_test.go b/sei-db/db_engine/litt/benchmark/config/benchmark_config_test.go index 0e11eb1f57..425e9a3e05 100644 --- a/sei-db/db_engine/litt/benchmark/config/benchmark_config_test.go +++ b/sei-db/db_engine/litt/benchmark/config/benchmark_config_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package config import ( diff --git a/sei-db/db_engine/litt/benchmark/data_generator.go b/sei-db/db_engine/litt/benchmark/data_generator.go index 1e0705e426..2e53819933 100644 --- a/sei-db/db_engine/litt/benchmark/data_generator.go +++ b/sei-db/db_engine/litt/benchmark/data_generator.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( diff --git a/sei-db/db_engine/litt/benchmark/data_generator_test.go b/sei-db/db_engine/litt/benchmark/data_generator_test.go index eba112d1ed..b5b9418810 100644 --- a/sei-db/db_engine/litt/benchmark/data_generator_test.go +++ b/sei-db/db_engine/litt/benchmark/data_generator_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( diff --git a/sei-db/db_engine/litt/benchmark/data_tracker.go b/sei-db/db_engine/litt/benchmark/data_tracker.go index 5e462e02a3..a255b42351 100644 --- a/sei-db/db_engine/litt/benchmark/data_tracker.go +++ b/sei-db/db_engine/litt/benchmark/data_tracker.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( @@ -12,7 +10,7 @@ import ( "strings" "time" - "github.com/docker/go-units" + "github.com/sei-protocol/sei-chain/sei-db/common/unit" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/benchmark/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" ) @@ -134,7 +132,7 @@ func NewDataTracker( } } - valueSize := uint64(config.ValueSizeMB * float64(units.MiB)) + valueSize := uint64(config.ValueSizeMB * float64(unit.MB)) // Create an initial active cohort. var activeCohort *Cohort @@ -270,7 +268,7 @@ func gatherCohorts(cohortDirPath string) ( // (possibly with different configurations), and values that may be written in the future with the // current configuration. func (t *DataTracker) LargestReadableValueSize() uint64 { - largestValue := uint64(t.config.ValueSizeMB * float64(units.MiB)) + largestValue := uint64(t.config.ValueSizeMB * float64(unit.MB)) if len(t.cohorts) > 0 { for i := t.lowestCohortIndex; i <= t.highestCohortIndex; i++ { diff --git a/sei-db/db_engine/litt/benchmark/data_tracker_test.go b/sei-db/db_engine/litt/benchmark/data_tracker_test.go index bd66c6e697..aaa7f1172c 100644 --- a/sei-db/db_engine/litt/benchmark/data_tracker_test.go +++ b/sei-db/db_engine/litt/benchmark/data_tracker_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package benchmark import ( @@ -7,7 +5,7 @@ import ( "testing" "time" - "github.com/docker/go-units" + "github.com/sei-protocol/sei-chain/sei-db/common/unit" config2 "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/benchmark/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" "github.com/stretchr/testify/require" @@ -19,7 +17,7 @@ func TestTrackerDeterminism(t *testing.T) { directory := t.TempDir() config := config2.DefaultBenchmarkConfig() - config.RandomPoolSize = units.MiB + config.RandomPoolSize = unit.MB config.CohortSize = rand.Uint64Range(10, 20) config.MetadataDirectory = directory config.Seed = rand.Int63() @@ -45,7 +43,7 @@ func TestTrackerDeterminism(t *testing.T) { writeInfo := dataTracker.GetWriteInfo() require.Equal(t, i, writeInfo.KeyIndex) require.Equal(t, 32, len(writeInfo.Key)) - require.Equal(t, units.KiB, len(writeInfo.Value)) + require.Equal(t, unit.KB, len(writeInfo.Value)) expectedKeys[i] = writeInfo.Key expectedValues[i] = writeInfo.Value @@ -65,7 +63,7 @@ func TestTrackerDeterminism(t *testing.T) { writeInfo := dataTracker.GetWriteInfo() require.Equal(t, i, writeInfo.KeyIndex) require.Equal(t, 32, len(writeInfo.Key)) - require.Equal(t, units.KiB, len(writeInfo.Value)) + require.Equal(t, unit.KB, len(writeInfo.Value)) require.Equal(t, expectedKeys[i], writeInfo.Key) require.Equal(t, expectedValues[i], writeInfo.Value) } @@ -84,7 +82,7 @@ func TestTrackerRestart(t *testing.T) { directory := t.TempDir() config := config2.DefaultBenchmarkConfig() - config.RandomPoolSize = units.MiB + config.RandomPoolSize = unit.MB config.CohortSize = rand.Uint64Range(10, 20) config.MetadataDirectory = directory config.Seed = rand.Int63() @@ -105,7 +103,7 @@ func TestTrackerRestart(t *testing.T) { writeInfo := dataTracker.GetWriteInfo() require.Equal(t, i, writeInfo.KeyIndex) require.Equal(t, 32, len(writeInfo.Key)) - require.Equal(t, units.KiB, len(writeInfo.Value)) + require.Equal(t, unit.KB, len(writeInfo.Value)) indexSet[writeInfo.KeyIndex] = struct{}{} } @@ -142,7 +140,7 @@ func TestTrackReads(t *testing.T) { directory := t.TempDir() config := config2.DefaultBenchmarkConfig() - config.RandomPoolSize = units.MiB + config.RandomPoolSize = unit.MB config.CohortSize = rand.Uint64Range(10, 20) config.MetadataDirectory = directory config.Seed = rand.Int63() @@ -168,7 +166,7 @@ func TestTrackReads(t *testing.T) { writeInfo := dataTracker.GetWriteInfo() require.Equal(t, i, writeInfo.KeyIndex) require.Equal(t, 32, len(writeInfo.Key)) - require.Equal(t, units.KiB, len(writeInfo.Value)) + require.Equal(t, unit.KB, len(writeInfo.Value)) keyToIndexMap[string(writeInfo.Key)] = writeInfo.KeyIndex diff --git a/sei-db/db_engine/litt/cli/benchmark.go b/sei-db/db_engine/litt/cli/benchmark.go index 373de27f8b..bdd93d88d6 100644 --- a/sei-db/db_engine/litt/cli/benchmark.go +++ b/sei-db/db_engine/litt/cli/benchmark.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/litt_cli.go b/sei-db/db_engine/litt/cli/litt_cli.go index 69590cf248..b6ada2e830 100644 --- a/sei-db/db_engine/litt/cli/litt_cli.go +++ b/sei-db/db_engine/litt/cli/litt_cli.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/ls.go b/sei-db/db_engine/litt/cli/ls.go index faa7be8975..03b03f77b4 100644 --- a/sei-db/db_engine/litt/cli/ls.go +++ b/sei-db/db_engine/litt/cli/ls.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/ls_test.go b/sei-db/db_engine/litt/cli/ls_test.go index b8cc7a97bd..b35f94f92e 100644 --- a/sei-db/db_engine/litt/cli/ls_test.go +++ b/sei-db/db_engine/litt/cli/ls_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/main.go b/sei-db/db_engine/litt/cli/main.go index 23bc41ddbf..2f11127aac 100644 --- a/sei-db/db_engine/litt/cli/main.go +++ b/sei-db/db_engine/litt/cli/main.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/prune.go b/sei-db/db_engine/litt/cli/prune.go index a6d5d5375a..6142f3203e 100644 --- a/sei-db/db_engine/litt/cli/prune.go +++ b/sei-db/db_engine/litt/cli/prune.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/prune_test.go b/sei-db/db_engine/litt/cli/prune_test.go index 0432dd4938..f82ee9a499 100644 --- a/sei-db/db_engine/litt/cli/prune_test.go +++ b/sei-db/db_engine/litt/cli/prune_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/push.go b/sei-db/db_engine/litt/cli/push.go index 6fa3245765..a1647d1288 100644 --- a/sei-db/db_engine/litt/cli/push.go +++ b/sei-db/db_engine/litt/cli/push.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/push_test.go b/sei-db/db_engine/litt/cli/push_test.go index 8126e24504..f4dcf4b7ca 100644 --- a/sei-db/db_engine/litt/cli/push_test.go +++ b/sei-db/db_engine/litt/cli/push_test.go @@ -1,709 +1,719 @@ -//go:build littdb_wip - package main -import ( - "fmt" - "log/slog" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "testing" - "time" - - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/keymap" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/segment" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" - "github.com/stretchr/testify/require" -) - -func pushTest( - t *testing.T, - sourceDirs uint64, - destDirs uint64, - verbose bool, -) { - logger := slog.Default() - rand := util.NewTestRandom() - testDir := t.TempDir() - sourceRoot := path.Join(testDir, "source") - destRoot := path.Join(testDir, "dest") - - err := os.MkdirAll(sourceRoot, 0755) - require.NoError(t, err) - err = os.MkdirAll(destRoot, 0755) - require.NoError(t, err) - - // Start a container that is running an SSH server. The push() command will communicate with this server. - container := util.SetupSSHTestContainer(t, destRoot) - defer container.Cleanup() - - sourceDirList := make([]string, 0, sourceDirs) - // The destination directories relative to the test's perspective of the filesystem. - destDirList := make([]string, 0, destDirs) - // The destination directories relative to the container's perspective of the filesystem. - dockerDestDirList := make([]string, 0, destDirs) - - for i := uint64(0); i < sourceDirs; i++ { - sourceDirList = append(sourceDirList, path.Join(sourceRoot, fmt.Sprintf("source-%d", i))) - } - for i := uint64(0); i < destDirs; i++ { - dir := fmt.Sprintf("dest-%d", i) - destDirList = append(destDirList, path.Join(destRoot, dir)) - dockerDestDirList = append(dockerDestDirList, path.Join(container.GetDataDir(), dir)) - } - - tableCount := rand.Uint64Range(2, 4) - tableNames := make([]string, 0, tableCount) - for i := uint64(0); i < tableCount; i++ { - tableNames = append(tableNames, rand.String(32)) - } - - shardingFactor := sourceDirs + rand.Uint64Range(0, 4) - - config, err := litt.DefaultConfig(sourceDirList...) - require.NoError(t, err) - config.DoubleWriteProtection = true - config.ShardingFactor = uint32(shardingFactor) - config.Fsync = false - config.TargetSegmentFileSize = 1024 - - db, err := littbuilder.NewDB(config) - require.NoError(t, err) - - expectedData := make(map[string] /*table*/ map[string] /*value*/ []byte) - for _, tableName := range tableNames { - expectedData[tableName] = make(map[string][]byte) - } - - // Insert data into the tables. - keyCount := uint64(1024) - for i := uint64(0); i < keyCount; i++ { - tableIndex := rand.Uint64Range(0, tableCount) - table, err := db.GetTable(tableNames[tableIndex]) - require.NoError(t, err) - key := rand.PrintableBytes(32) - value := rand.PrintableVariableBytes(10, 100) - - expectedData[table.Name()][string(key)] = value - err = table.Put(key, value) - require.NoError(t, err, "failed to put key %s in table %s", key, table.Name()) - } - - // Flush all tables. - for _, tableName := range tableNames { - table, err := db.GetTable(tableName) - require.NoError(t, err) - err = table.Flush() - require.NoError(t, err, "failed to flush table %s", table.Name()) - } - - // Verify the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - // Verify expected directories. - for _, sourceDir := range sourceDirList { - // We should see each source dir. - exists, err := util.Exists(sourceDir) - require.NoError(t, err) - require.True(t, exists, "source directory %s does not exist", sourceDir) - } - for _, destDir := range destDirList { - // We should not see dest dirs yet. - exists, err := util.Exists(destDir) - require.NoError(t, err) - require.False(t, exists, "destination directory %s exists", destDir) - } - - // pushing with the DB still open should fail. - err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, - false, 2, 1, verbose) - require.Error(t, err) - - // None of the source dirs should have been deleted. - for _, sourceDir := range sourceDirList { - // We should see each source dir. - exists, err := util.Exists(sourceDir) - require.NoError(t, err) - require.True(t, exists, "source directory %s does not exist", sourceDir) - } - - // The failed push should not have changed the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - //// Shut down the DB and push it. - err = db.Close() - require.NoError(t, err, "failed to close DB") - - // Deleting after transfer is only support for snapshots (which we are not testing here). - err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", true, - false, 2, 1, verbose) - require.Error(t, err) - - // Actually push it correctly now. - err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, - false, 8, 1, verbose) - require.NoError(t, err, "failed to close DB") - - // Verify the new directories. - for _, sourceDir := range sourceDirList { - exists, err := util.Exists(sourceDir) - require.NoError(t, err) - - // Even if we are deleting after transfer, the source directories should still exist. - require.True(t, exists, "source directory %s does not exist but should", sourceDir) - } - for _, destDir := range destDirList { - // We should see all destination dirs. - exists, err := util.Exists(destDir) - require.NoError(t, err) - require.True(t, exists, "destination directory %s does not exist", destDir) - } - - // Push works when there is nothing at the destination. It also works when some of the files are present or - // corrupted. Let's mess with the files at the destination and make sure that the push command is able to fix - // things afterward. - filesInTree := make([]string, 0) - err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - // Skip directories. - return nil - } - - filesInTree = append(filesInTree, path) - - return nil - }) - require.NoError(t, err) - - for _, segmentFile := range filesInTree { - choice := rand.Float64() - - if choice < 0.3 { - // Delete the file. Push will copy it over again. - err = os.Remove(segmentFile) - require.NoError(t, err, "failed to delete file %s", segmentFile) - } else if choice < 0.6 { - // Overwrite the file with random data. Push will replace it with the correct data. - randomData := rand.Bytes(128) - // use broad file permissions to avoid issues with container user having different UID/GID. - err = os.WriteFile(segmentFile, randomData, 0666) - require.NoError(t, err, "failed to overwrite file %s", segmentFile) - } else if choice < 0.9 { - // Attempt to move the file to another legal location. - - if len(destDirList) == 1 { - // We can't move a file to a different directory if there is only one destination directory. - continue - } - - // Segment files will have the following format: destRoot/dest-N/tableName/segments/segmentFileName - // We want to change the "dest-N" part. This is a legal location for the data, since it doesn't matter - // which destination directory the data is in, as long as it is in one of them. - - parts := strings.Split(segmentFile, string(os.PathSeparator)) - require.Greater(t, len(parts), 3, "unexpected path format: %s", segmentFile) - - oldDir := parts[len(parts)-4] // This is the "dest-N" part. - oldDirIndexString := strings.Replace(oldDir, "dest-", "", 1) - oldDirIndex, err := strconv.Atoi(oldDirIndexString) - require.NoError(t, err) - newDirIndex := (oldDirIndex + 1) % len(destDirList) // Move to the next destination directory. - newPath := strings.Replace(segmentFile, oldDir, fmt.Sprintf("dest-%d", newDirIndex), 1) - - err = os.Rename(segmentFile, newPath) - require.NoError(t, err) - } - } - - // Push again, should fix the messed up files. - err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, - false, 2, 1, verbose) - require.NoError(t, err) - - // Reopen the old DB, verify no data is missing. - db, err = littbuilder.NewDB(config) - require.NoError(t, err, "failed to open DB after rebase") - - // Verify the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - // Fully delete the old DB. The new DB should be a copy of the old one, so this should not affect copied data. - err = db.Destroy() - require.NoError(t, err) - - // Push should NOT copy the keymap. Verify that there is no keymap directory in destRoot. - err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - require.False(t, strings.Contains(path, keymap.KeymapDirectoryName)) - return nil - }) - require.NoError(t, err) - - // Reopen the DB at the new destination directories. - config.Paths = destDirList - db, err = littbuilder.NewDB(config) - require.NoError(t, err, "failed to open DB after rebase") - - // Verify the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - err = db.Close() - require.NoError(t, err, "failed to close DB after rebase") -} - -func TestPush1to1(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - sourceDirs := uint64(1) - destDirs := uint64(1) - - pushTest(t, sourceDirs, destDirs, false) -} - -func TestPush1toN(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - sourceDirs := uint64(1) - destDirs := uint64(4) - - pushTest(t, sourceDirs, destDirs, false) -} - -func TestPushNto1(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - sourceDirs := uint64(4) - destDirs := uint64(1) - - pushTest(t, sourceDirs, destDirs, false) -} - -func TestPushNtoN(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - sourceDirs := uint64(4) - destDirs := uint64(4) - - // This test is run in verbose mode to make sure we don't crash when that is enabled. - // Other tests in this file are not run in verbose mode to reduce log clutter. - pushTest(t, sourceDirs, destDirs, true) -} - -func TestPushSnapshot(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - ctx := t.Context() - logger := slog.Default() - - rand := util.NewTestRandom() - sourceRoot := t.TempDir() - destRoot := t.TempDir() - snapshotDir := path.Join(t.TempDir(), "snapshot") - - sourceDirs := rand.Uint64Range(2, 4) - destDirs := rand.Uint64Range(2, 4) - - // Start a container that is running an SSH server. The push() command will communicate with this server. - container := util.SetupSSHTestContainer(t, destRoot) - defer container.Cleanup() - - sourceDirList := make([]string, 0, sourceDirs) - // The destination directories relative to the test's perspective of the filesystem. - destDirList := make([]string, 0, destDirs) - // The destination directories relative to the container's perspective of the filesystem. - dockerDestDirList := make([]string, 0, destDirs) - - for i := uint64(0); i < sourceDirs; i++ { - sourceDirList = append(sourceDirList, path.Join(sourceRoot, fmt.Sprintf("source-%d", i))) - } - for i := uint64(0); i < destDirs; i++ { - dir := fmt.Sprintf("dest-%d", i) - destDirList = append(destDirList, path.Join(destRoot, dir)) - dockerDestDirList = append(dockerDestDirList, path.Join(container.GetDataDir(), dir)) - } - - tableCount := rand.Uint64Range(2, 4) - tableNames := make([]string, 0, tableCount) - for i := uint64(0); i < tableCount; i++ { - tableNames = append(tableNames, rand.String(32)) - } - - shardingFactor := sourceDirs + rand.Uint64Range(0, 4) - - config, err := litt.DefaultConfig(sourceDirList...) - require.NoError(t, err) - config.DoubleWriteProtection = true - config.ShardingFactor = uint32(shardingFactor) - config.Fsync = false - config.TargetSegmentFileSize = 1024 - config.SnapshotDirectory = snapshotDir - - db, err := littbuilder.NewDB(config) - require.NoError(t, err) - - expectedData := make(map[string] /*table*/ map[string] /*value*/ []byte) - for _, tableName := range tableNames { - expectedData[tableName] = make(map[string][]byte) - } - - // Insert data into the tables. - keyCount := uint64(1024) - for i := uint64(0); i < keyCount; i++ { - tableIndex := rand.Uint64Range(0, tableCount) - table, err := db.GetTable(tableNames[tableIndex]) - require.NoError(t, err) - key := rand.PrintableBytes(32) - value := rand.PrintableVariableBytes(10, 100) - - expectedData[table.Name()][string(key)] = value - err = table.Put(key, value) - require.NoError(t, err, "failed to put key %s in table %s", key, table.Name()) - } - - // Flush all tables. - for _, tableName := range tableNames { - table, err := db.GetTable(tableName) - require.NoError(t, err) - err = table.Flush() - require.NoError(t, err, "failed to flush table %s", table.Name()) - } - - // Verify the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - // Verify expected directories. - for _, sourceDir := range sourceDirList { - // We should see each source dir. - exists, err := util.Exists(sourceDir) - require.NoError(t, err) - require.True(t, exists, "source directory %s does not exist", sourceDir) - } - for _, destDir := range destDirList { - // We should not see dest dirs yet. - exists, err := util.Exists(destDir) - require.NoError(t, err) - require.False(t, exists, "destination directory %s exists", destDir) - } - - // pushing with the DB still open should fail. - err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, - false, 2, 1, false) - require.Error(t, err) - - // None of the source dirs should have been deleted. - for _, sourceDir := range sourceDirList { - // We should see each source dir. - exists, err := util.Exists(sourceDir) - require.NoError(t, err) - require.True(t, exists, "source directory %s does not exist", sourceDir) - } - - // The failed push should not have changed the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - // Power cycle the DB twice. After the first shutdown, the last segment with data will not have been copied - // to the snapshot directory. When the database starts a second time, it will seal the last segment and make - // sure the snapshot directory includes it. - err = db.Close() - require.NoError(t, err, "failed to close DB") - - // Find the highest segment index for each table. We will use it to do verification later. - errorMonitor := util.NewErrorMonitor(ctx, logger, nil) - highestSegmentIndexForTable := make(map[string]uint32) - for tableName := range expectedData { - segmentPaths, err := segment.BuildSegmentPaths(sourceDirList, "", tableName) - require.NoError(t, err, "failed to build segment paths for table %s", tableName) - _, highestSegmentIndex, _, err := segment.GatherSegmentFiles( - logger, - errorMonitor, - segmentPaths, - false, - time.Now(), - false, - false) - require.NoError(t, err) - highestSegmentIndexForTable[tableName] = highestSegmentIndex - } - ok, err := errorMonitor.IsOk() - require.NoError(t, err) - require.True(t, ok) - - // Second power cycle - db, err = littbuilder.NewDB(config) - require.NoError(t, err) - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - err = table.Flush() - require.NoError(t, err, "failed to flush table %s", table.Name()) - } - err = db.Close() - require.NoError(t, err, "failed to close DB after second open") - - // Push the data. Do not delete the snapshot yet. - err = push(logger, []string{snapshotDir}, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, - false, 8, 1, false) - require.NoError(t, err, "failed to close DB") - - // Verify the new directories. - for _, sourceDir := range sourceDirList { - exists, err := util.Exists(sourceDir) - require.NoError(t, err) - - // Even if we are deleting after transfer, the source directories should still exist. - require.True(t, exists, "source directory %s does not exist but should", sourceDir) - } - for _, destDir := range destDirList { - // We should see all destination dirs. - exists, err := util.Exists(destDir) - require.NoError(t, err) - require.True(t, exists, "destination directory %s does not exist", destDir) - } - - // Push works when there is nothing at the destination. It also works when some of the files are present or - // corrupted. Let's mess with the files at the destination and make sure that the push command is able to fix - // things afterward. - filesInTree := make([]string, 0) - err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - // Skip directories. - return nil - } - - filesInTree = append(filesInTree, path) - - return nil - }) - require.NoError(t, err) - - for _, segmentFile := range filesInTree { - choice := rand.Float64() - - if choice < 0.3 { - // Delete the file. Push will copy it over again. - err = os.Remove(segmentFile) - require.NoError(t, err, "failed to delete file %s", segmentFile) - } else if choice < 0.6 { - // Overwrite the file with random data. Push will replace it with the correct data. - randomData := rand.Bytes(128) - err = os.WriteFile(segmentFile, randomData, 0644) - require.NoError(t, err, "failed to overwrite file %s", segmentFile) - } else if choice < 0.9 { - // Attempt to move the file to another legal location. - - if len(destDirList) == 1 { - // We can't move a file to a different directory if there is only one destination directory. - continue - } - - // Segment files will have the following format: destRoot/dest-N/tableName/segments/segmentFileName - // We want to change the "dest-N" part. This is a legal location for the data, since it doesn't matter - // which destination directory the data is in, as long as it is in one of them. - - parts := strings.Split(segmentFile, string(os.PathSeparator)) - require.Greater(t, len(parts), 3, "unexpected path format: %s", segmentFile) - - oldDir := parts[len(parts)-4] // This is the "dest-N" part. - oldDirIndexString := strings.Replace(oldDir, "dest-", "", 1) - oldDirIndex, err := strconv.Atoi(oldDirIndexString) - require.NoError(t, err) - newDirIndex := (oldDirIndex + 1) % len(destDirList) // Move to the next destination directory. - newPath := strings.Replace(segmentFile, oldDir, fmt.Sprintf("dest-%d", newDirIndex), 1) - - err = os.Rename(segmentFile, newPath) - require.NoError(t, err) - } - } - - // Push again, should fix the messed up files. This time, tell the push command to clean up after itself. - err = push(logger, []string{snapshotDir}, dockerDestDirList, container.GetUser(), container.GetHost(), - container.GetSSHPort(), container.GetPrivateKeyPath(), "", true, - false, 2, 1, false) - require.NoError(t, err) - - // We instructed push() to delete files after pushing. For each table, we should observe a "lower bound" file - // with a segment index that matches the expected highest segment index for that table. This boundary file signals - // to LittDB that it shouldn't recreate the snapshot files that have been copied and deleted by push(). - for tableName, highestSegmentIndex := range highestSegmentIndexForTable { - tableSnapshotDir := path.Join(snapshotDir, tableName) - boundaryFile, err := disktable.LoadBoundaryFile(false, tableSnapshotDir) - require.NoError(t, err) - require.True(t, boundaryFile.IsDefined(), "boundary file for table %s is not defined", tableName) - require.Equal(t, highestSegmentIndex, boundaryFile.BoundaryIndex()) - } - - // There should be no segment files remaining in the snapshot directory. - err = filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error { - require.NoError(t, err) - require.False(t, strings.Contains(path, segment.MetadataFileExtension), - "unexpected file: %s", path) - require.False(t, strings.Contains(path, segment.KeyFileExtension), - "unexpected file: %s", path) - require.False(t, strings.Contains(path, segment.ValuesFileExtension), - "unexpected file: %s", path) - return nil - }) - require.NoError(t, err) - - // There should also not be any segment files in the hard link directories. - err = filepath.Walk(sourceRoot, func(path string, info os.FileInfo, err error) error { - require.NoError(t, err) - - inHardLinkDir := strings.Contains(path, segment.HardLinkDirectory) - if !inHardLinkDir { - return nil - } - - require.False(t, strings.Contains(path, segment.MetadataFileExtension), - "unexpected file: %s", path) - require.False(t, strings.Contains(path, segment.KeyFileExtension), - "unexpected file: %s", path) - require.False(t, strings.Contains(path, segment.ValuesFileExtension), - "unexpected file: %s", path) - return nil - }) - require.NoError(t, err) - - // Reopen the old DB, verify no data is missing. - db, err = littbuilder.NewDB(config) - require.NoError(t, err, "failed to open DB after rebase") - - // Verify the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - // Fully delete the old DB. The new DB should be a copy of the old one, so this should not affect copied data. - err = db.Destroy() - require.NoError(t, err) - - // Push should NOT copy the keymap. Verify that there is no keymap directory in destRoot. - err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - require.False(t, strings.Contains(path, keymap.KeymapDirectoryName)) - return nil - }) - require.NoError(t, err) - - // Reopen the DB at the new destination directories. - config.Paths = destDirList - config.SnapshotDirectory = "" - db, err = littbuilder.NewDB(config) - require.NoError(t, err, "failed to open DB after rebase") - - // Verify the data in the DB. - for tableName := range expectedData { - table, err := db.GetTable(tableName) - require.NoError(t, err, "failed to get table %s", tableName) - for key := range expectedData[tableName] { - value, ok, err := table.Get([]byte(key)) - require.NoError(t, err, "failed to get key %s in table %s", key, tableName) - require.True(t, ok, "key %s not found in table %s", key, tableName) - require.Equal(t, expectedData[tableName][key], value, - "value for key %s in table %s does not match expected value", key, tableName) - } - } - - err = db.Close() - require.NoError(t, err, "failed to close DB after rebase") -} +// COMMENTED OUT: depends on github.com/docker/docker Docker SDK, which is +// being removed from sei-chain's dependency closure as part of integrating +// litt into the main build. The functionality these tests exercise (litt +// push-over-SSH against a Docker-managed sshd container) is not currently +// used in production. Revisit when the push/sync feature is reintroduced; +// the original implementation is preserved below as comments. + +// //go:build littdb_wip +// +// package main +// +// import ( +// "fmt" +// "log/slog" +// "os" +// "path" +// "path/filepath" +// "strconv" +// "strings" +// "testing" +// "time" +// +// "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" +// "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable" +// "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/keymap" +// "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/segment" +// "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" +// "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" +// "github.com/stretchr/testify/require" +// ) +// +// func pushTest( +// t *testing.T, +// sourceDirs uint64, +// destDirs uint64, +// verbose bool, +// ) { +// logger := slog.Default() +// rand := util.NewTestRandom() +// testDir := t.TempDir() +// sourceRoot := path.Join(testDir, "source") +// destRoot := path.Join(testDir, "dest") +// +// err := os.MkdirAll(sourceRoot, 0755) +// require.NoError(t, err) +// err = os.MkdirAll(destRoot, 0755) +// require.NoError(t, err) +// +// // Start a container that is running an SSH server. The push() command will communicate with this server. +// container := util.SetupSSHTestContainer(t, destRoot) +// defer container.Cleanup() +// +// sourceDirList := make([]string, 0, sourceDirs) +// // The destination directories relative to the test's perspective of the filesystem. +// destDirList := make([]string, 0, destDirs) +// // The destination directories relative to the container's perspective of the filesystem. +// dockerDestDirList := make([]string, 0, destDirs) +// +// for i := uint64(0); i < sourceDirs; i++ { +// sourceDirList = append(sourceDirList, path.Join(sourceRoot, fmt.Sprintf("source-%d", i))) +// } +// for i := uint64(0); i < destDirs; i++ { +// dir := fmt.Sprintf("dest-%d", i) +// destDirList = append(destDirList, path.Join(destRoot, dir)) +// dockerDestDirList = append(dockerDestDirList, path.Join(container.GetDataDir(), dir)) +// } +// +// tableCount := rand.Uint64Range(2, 4) +// tableNames := make([]string, 0, tableCount) +// for i := uint64(0); i < tableCount; i++ { +// tableNames = append(tableNames, rand.String(32)) +// } +// +// shardingFactor := sourceDirs + rand.Uint64Range(0, 4) +// +// config, err := litt.DefaultConfig(sourceDirList...) +// require.NoError(t, err) +// config.DoubleWriteProtection = true +// config.ShardingFactor = uint32(shardingFactor) +// config.Fsync = false +// config.TargetSegmentFileSize = 1024 +// +// db, err := littbuilder.NewDB(config) +// require.NoError(t, err) +// +// expectedData := make(map[string] /*table*/ map[string] /*value*/ []byte) +// for _, tableName := range tableNames { +// expectedData[tableName] = make(map[string][]byte) +// } +// +// // Insert data into the tables. +// keyCount := uint64(1024) +// for i := uint64(0); i < keyCount; i++ { +// tableIndex := rand.Uint64Range(0, tableCount) +// table, err := db.GetTable(tableNames[tableIndex]) +// require.NoError(t, err) +// key := rand.PrintableBytes(32) +// value := rand.PrintableVariableBytes(10, 100) +// +// expectedData[table.Name()][string(key)] = value +// err = table.Put(key, value) +// require.NoError(t, err, "failed to put key %s in table %s", key, table.Name()) +// } +// +// // Flush all tables. +// for _, tableName := range tableNames { +// table, err := db.GetTable(tableName) +// require.NoError(t, err) +// err = table.Flush() +// require.NoError(t, err, "failed to flush table %s", table.Name()) +// } +// +// // Verify the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// // Verify expected directories. +// for _, sourceDir := range sourceDirList { +// // We should see each source dir. +// exists, err := util.Exists(sourceDir) +// require.NoError(t, err) +// require.True(t, exists, "source directory %s does not exist", sourceDir) +// } +// for _, destDir := range destDirList { +// // We should not see dest dirs yet. +// exists, err := util.Exists(destDir) +// require.NoError(t, err) +// require.False(t, exists, "destination directory %s exists", destDir) +// } +// +// // pushing with the DB still open should fail. +// err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, +// false, 2, 1, verbose) +// require.Error(t, err) +// +// // None of the source dirs should have been deleted. +// for _, sourceDir := range sourceDirList { +// // We should see each source dir. +// exists, err := util.Exists(sourceDir) +// require.NoError(t, err) +// require.True(t, exists, "source directory %s does not exist", sourceDir) +// } +// +// // The failed push should not have changed the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// //// Shut down the DB and push it. +// err = db.Close() +// require.NoError(t, err, "failed to close DB") +// +// // Deleting after transfer is only support for snapshots (which we are not testing here). +// err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", true, +// false, 2, 1, verbose) +// require.Error(t, err) +// +// // Actually push it correctly now. +// err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, +// false, 8, 1, verbose) +// require.NoError(t, err, "failed to close DB") +// +// // Verify the new directories. +// for _, sourceDir := range sourceDirList { +// exists, err := util.Exists(sourceDir) +// require.NoError(t, err) +// +// // Even if we are deleting after transfer, the source directories should still exist. +// require.True(t, exists, "source directory %s does not exist but should", sourceDir) +// } +// for _, destDir := range destDirList { +// // We should see all destination dirs. +// exists, err := util.Exists(destDir) +// require.NoError(t, err) +// require.True(t, exists, "destination directory %s does not exist", destDir) +// } +// +// // Push works when there is nothing at the destination. It also works when some of the files are present or +// // corrupted. Let's mess with the files at the destination and make sure that the push command is able to fix +// // things afterward. +// filesInTree := make([]string, 0) +// err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { +// if err != nil { +// return err +// } +// +// if info.IsDir() { +// // Skip directories. +// return nil +// } +// +// filesInTree = append(filesInTree, path) +// +// return nil +// }) +// require.NoError(t, err) +// +// for _, segmentFile := range filesInTree { +// choice := rand.Float64() +// +// if choice < 0.3 { +// // Delete the file. Push will copy it over again. +// err = os.Remove(segmentFile) +// require.NoError(t, err, "failed to delete file %s", segmentFile) +// } else if choice < 0.6 { +// // Overwrite the file with random data. Push will replace it with the correct data. +// randomData := rand.Bytes(128) +// // use broad file permissions to avoid issues with container user having different UID/GID. +// err = os.WriteFile(segmentFile, randomData, 0666) +// require.NoError(t, err, "failed to overwrite file %s", segmentFile) +// } else if choice < 0.9 { +// // Attempt to move the file to another legal location. +// +// if len(destDirList) == 1 { +// // We can't move a file to a different directory if there is only one destination directory. +// continue +// } +// +// // Segment files will have the following format: destRoot/dest-N/tableName/segments/segmentFileName +// // We want to change the "dest-N" part. This is a legal location for the data, since it doesn't matter +// // which destination directory the data is in, as long as it is in one of them. +// +// parts := strings.Split(segmentFile, string(os.PathSeparator)) +// require.Greater(t, len(parts), 3, "unexpected path format: %s", segmentFile) +// +// oldDir := parts[len(parts)-4] // This is the "dest-N" part. +// oldDirIndexString := strings.Replace(oldDir, "dest-", "", 1) +// oldDirIndex, err := strconv.Atoi(oldDirIndexString) +// require.NoError(t, err) +// newDirIndex := (oldDirIndex + 1) % len(destDirList) // Move to the next destination directory. +// newPath := strings.Replace(segmentFile, oldDir, fmt.Sprintf("dest-%d", newDirIndex), 1) +// +// err = os.Rename(segmentFile, newPath) +// require.NoError(t, err) +// } +// } +// +// // Push again, should fix the messed up files. +// err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, +// false, 2, 1, verbose) +// require.NoError(t, err) +// +// // Reopen the old DB, verify no data is missing. +// db, err = littbuilder.NewDB(config) +// require.NoError(t, err, "failed to open DB after rebase") +// +// // Verify the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// // Fully delete the old DB. The new DB should be a copy of the old one, so this should not affect copied data. +// err = db.Destroy() +// require.NoError(t, err) +// +// // Push should NOT copy the keymap. Verify that there is no keymap directory in destRoot. +// err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { +// if err != nil { +// return err +// } +// require.False(t, strings.Contains(path, keymap.KeymapDirectoryName)) +// return nil +// }) +// require.NoError(t, err) +// +// // Reopen the DB at the new destination directories. +// config.Paths = destDirList +// db, err = littbuilder.NewDB(config) +// require.NoError(t, err, "failed to open DB after rebase") +// +// // Verify the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// err = db.Close() +// require.NoError(t, err, "failed to close DB after rebase") +// } +// +// func TestPush1to1(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// sourceDirs := uint64(1) +// destDirs := uint64(1) +// +// pushTest(t, sourceDirs, destDirs, false) +// } +// +// func TestPush1toN(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// sourceDirs := uint64(1) +// destDirs := uint64(4) +// +// pushTest(t, sourceDirs, destDirs, false) +// } +// +// func TestPushNto1(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// sourceDirs := uint64(4) +// destDirs := uint64(1) +// +// pushTest(t, sourceDirs, destDirs, false) +// } +// +// func TestPushNtoN(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// sourceDirs := uint64(4) +// destDirs := uint64(4) +// +// // This test is run in verbose mode to make sure we don't crash when that is enabled. +// // Other tests in this file are not run in verbose mode to reduce log clutter. +// pushTest(t, sourceDirs, destDirs, true) +// } +// +// func TestPushSnapshot(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// ctx := t.Context() +// logger := slog.Default() +// +// rand := util.NewTestRandom() +// sourceRoot := t.TempDir() +// destRoot := t.TempDir() +// snapshotDir := path.Join(t.TempDir(), "snapshot") +// +// sourceDirs := rand.Uint64Range(2, 4) +// destDirs := rand.Uint64Range(2, 4) +// +// // Start a container that is running an SSH server. The push() command will communicate with this server. +// container := util.SetupSSHTestContainer(t, destRoot) +// defer container.Cleanup() +// +// sourceDirList := make([]string, 0, sourceDirs) +// // The destination directories relative to the test's perspective of the filesystem. +// destDirList := make([]string, 0, destDirs) +// // The destination directories relative to the container's perspective of the filesystem. +// dockerDestDirList := make([]string, 0, destDirs) +// +// for i := uint64(0); i < sourceDirs; i++ { +// sourceDirList = append(sourceDirList, path.Join(sourceRoot, fmt.Sprintf("source-%d", i))) +// } +// for i := uint64(0); i < destDirs; i++ { +// dir := fmt.Sprintf("dest-%d", i) +// destDirList = append(destDirList, path.Join(destRoot, dir)) +// dockerDestDirList = append(dockerDestDirList, path.Join(container.GetDataDir(), dir)) +// } +// +// tableCount := rand.Uint64Range(2, 4) +// tableNames := make([]string, 0, tableCount) +// for i := uint64(0); i < tableCount; i++ { +// tableNames = append(tableNames, rand.String(32)) +// } +// +// shardingFactor := sourceDirs + rand.Uint64Range(0, 4) +// +// config, err := litt.DefaultConfig(sourceDirList...) +// require.NoError(t, err) +// config.DoubleWriteProtection = true +// config.ShardingFactor = uint32(shardingFactor) +// config.Fsync = false +// config.TargetSegmentFileSize = 1024 +// config.SnapshotDirectory = snapshotDir +// +// db, err := littbuilder.NewDB(config) +// require.NoError(t, err) +// +// expectedData := make(map[string] /*table*/ map[string] /*value*/ []byte) +// for _, tableName := range tableNames { +// expectedData[tableName] = make(map[string][]byte) +// } +// +// // Insert data into the tables. +// keyCount := uint64(1024) +// for i := uint64(0); i < keyCount; i++ { +// tableIndex := rand.Uint64Range(0, tableCount) +// table, err := db.GetTable(tableNames[tableIndex]) +// require.NoError(t, err) +// key := rand.PrintableBytes(32) +// value := rand.PrintableVariableBytes(10, 100) +// +// expectedData[table.Name()][string(key)] = value +// err = table.Put(key, value) +// require.NoError(t, err, "failed to put key %s in table %s", key, table.Name()) +// } +// +// // Flush all tables. +// for _, tableName := range tableNames { +// table, err := db.GetTable(tableName) +// require.NoError(t, err) +// err = table.Flush() +// require.NoError(t, err, "failed to flush table %s", table.Name()) +// } +// +// // Verify the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// // Verify expected directories. +// for _, sourceDir := range sourceDirList { +// // We should see each source dir. +// exists, err := util.Exists(sourceDir) +// require.NoError(t, err) +// require.True(t, exists, "source directory %s does not exist", sourceDir) +// } +// for _, destDir := range destDirList { +// // We should not see dest dirs yet. +// exists, err := util.Exists(destDir) +// require.NoError(t, err) +// require.False(t, exists, "destination directory %s exists", destDir) +// } +// +// // pushing with the DB still open should fail. +// err = push(logger, sourceDirList, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, +// false, 2, 1, false) +// require.Error(t, err) +// +// // None of the source dirs should have been deleted. +// for _, sourceDir := range sourceDirList { +// // We should see each source dir. +// exists, err := util.Exists(sourceDir) +// require.NoError(t, err) +// require.True(t, exists, "source directory %s does not exist", sourceDir) +// } +// +// // The failed push should not have changed the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// // Power cycle the DB twice. After the first shutdown, the last segment with data will not have been copied +// // to the snapshot directory. When the database starts a second time, it will seal the last segment and make +// // sure the snapshot directory includes it. +// err = db.Close() +// require.NoError(t, err, "failed to close DB") +// +// // Find the highest segment index for each table. We will use it to do verification later. +// errorMonitor := util.NewErrorMonitor(ctx, logger, nil) +// highestSegmentIndexForTable := make(map[string]uint32) +// for tableName := range expectedData { +// segmentPaths, err := segment.BuildSegmentPaths(sourceDirList, "", tableName) +// require.NoError(t, err, "failed to build segment paths for table %s", tableName) +// _, highestSegmentIndex, _, err := segment.GatherSegmentFiles( +// logger, +// errorMonitor, +// segmentPaths, +// false, +// time.Now(), +// false, +// false) +// require.NoError(t, err) +// highestSegmentIndexForTable[tableName] = highestSegmentIndex +// } +// ok, err := errorMonitor.IsOk() +// require.NoError(t, err) +// require.True(t, ok) +// +// // Second power cycle +// db, err = littbuilder.NewDB(config) +// require.NoError(t, err) +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// err = table.Flush() +// require.NoError(t, err, "failed to flush table %s", table.Name()) +// } +// err = db.Close() +// require.NoError(t, err, "failed to close DB after second open") +// +// // Push the data. Do not delete the snapshot yet. +// err = push(logger, []string{snapshotDir}, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", false, +// false, 8, 1, false) +// require.NoError(t, err, "failed to close DB") +// +// // Verify the new directories. +// for _, sourceDir := range sourceDirList { +// exists, err := util.Exists(sourceDir) +// require.NoError(t, err) +// +// // Even if we are deleting after transfer, the source directories should still exist. +// require.True(t, exists, "source directory %s does not exist but should", sourceDir) +// } +// for _, destDir := range destDirList { +// // We should see all destination dirs. +// exists, err := util.Exists(destDir) +// require.NoError(t, err) +// require.True(t, exists, "destination directory %s does not exist", destDir) +// } +// +// // Push works when there is nothing at the destination. It also works when some of the files are present or +// // corrupted. Let's mess with the files at the destination and make sure that the push command is able to fix +// // things afterward. +// filesInTree := make([]string, 0) +// err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { +// if err != nil { +// return err +// } +// +// if info.IsDir() { +// // Skip directories. +// return nil +// } +// +// filesInTree = append(filesInTree, path) +// +// return nil +// }) +// require.NoError(t, err) +// +// for _, segmentFile := range filesInTree { +// choice := rand.Float64() +// +// if choice < 0.3 { +// // Delete the file. Push will copy it over again. +// err = os.Remove(segmentFile) +// require.NoError(t, err, "failed to delete file %s", segmentFile) +// } else if choice < 0.6 { +// // Overwrite the file with random data. Push will replace it with the correct data. +// randomData := rand.Bytes(128) +// err = os.WriteFile(segmentFile, randomData, 0644) +// require.NoError(t, err, "failed to overwrite file %s", segmentFile) +// } else if choice < 0.9 { +// // Attempt to move the file to another legal location. +// +// if len(destDirList) == 1 { +// // We can't move a file to a different directory if there is only one destination directory. +// continue +// } +// +// // Segment files will have the following format: destRoot/dest-N/tableName/segments/segmentFileName +// // We want to change the "dest-N" part. This is a legal location for the data, since it doesn't matter +// // which destination directory the data is in, as long as it is in one of them. +// +// parts := strings.Split(segmentFile, string(os.PathSeparator)) +// require.Greater(t, len(parts), 3, "unexpected path format: %s", segmentFile) +// +// oldDir := parts[len(parts)-4] // This is the "dest-N" part. +// oldDirIndexString := strings.Replace(oldDir, "dest-", "", 1) +// oldDirIndex, err := strconv.Atoi(oldDirIndexString) +// require.NoError(t, err) +// newDirIndex := (oldDirIndex + 1) % len(destDirList) // Move to the next destination directory. +// newPath := strings.Replace(segmentFile, oldDir, fmt.Sprintf("dest-%d", newDirIndex), 1) +// +// err = os.Rename(segmentFile, newPath) +// require.NoError(t, err) +// } +// } +// +// // Push again, should fix the messed up files. This time, tell the push command to clean up after itself. +// err = push(logger, []string{snapshotDir}, dockerDestDirList, container.GetUser(), container.GetHost(), +// container.GetSSHPort(), container.GetPrivateKeyPath(), "", true, +// false, 2, 1, false) +// require.NoError(t, err) +// +// // We instructed push() to delete files after pushing. For each table, we should observe a "lower bound" file +// // with a segment index that matches the expected highest segment index for that table. This boundary file signals +// // to LittDB that it shouldn't recreate the snapshot files that have been copied and deleted by push(). +// for tableName, highestSegmentIndex := range highestSegmentIndexForTable { +// tableSnapshotDir := path.Join(snapshotDir, tableName) +// boundaryFile, err := disktable.LoadBoundaryFile(false, tableSnapshotDir) +// require.NoError(t, err) +// require.True(t, boundaryFile.IsDefined(), "boundary file for table %s is not defined", tableName) +// require.Equal(t, highestSegmentIndex, boundaryFile.BoundaryIndex()) +// } +// +// // There should be no segment files remaining in the snapshot directory. +// err = filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error { +// require.NoError(t, err) +// require.False(t, strings.Contains(path, segment.MetadataFileExtension), +// "unexpected file: %s", path) +// require.False(t, strings.Contains(path, segment.KeyFileExtension), +// "unexpected file: %s", path) +// require.False(t, strings.Contains(path, segment.ValuesFileExtension), +// "unexpected file: %s", path) +// return nil +// }) +// require.NoError(t, err) +// +// // There should also not be any segment files in the hard link directories. +// err = filepath.Walk(sourceRoot, func(path string, info os.FileInfo, err error) error { +// require.NoError(t, err) +// +// inHardLinkDir := strings.Contains(path, segment.HardLinkDirectory) +// if !inHardLinkDir { +// return nil +// } +// +// require.False(t, strings.Contains(path, segment.MetadataFileExtension), +// "unexpected file: %s", path) +// require.False(t, strings.Contains(path, segment.KeyFileExtension), +// "unexpected file: %s", path) +// require.False(t, strings.Contains(path, segment.ValuesFileExtension), +// "unexpected file: %s", path) +// return nil +// }) +// require.NoError(t, err) +// +// // Reopen the old DB, verify no data is missing. +// db, err = littbuilder.NewDB(config) +// require.NoError(t, err, "failed to open DB after rebase") +// +// // Verify the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// // Fully delete the old DB. The new DB should be a copy of the old one, so this should not affect copied data. +// err = db.Destroy() +// require.NoError(t, err) +// +// // Push should NOT copy the keymap. Verify that there is no keymap directory in destRoot. +// err = filepath.Walk(destRoot, func(path string, info os.FileInfo, err error) error { +// if err != nil { +// return err +// } +// require.False(t, strings.Contains(path, keymap.KeymapDirectoryName)) +// return nil +// }) +// require.NoError(t, err) +// +// // Reopen the DB at the new destination directories. +// config.Paths = destDirList +// config.SnapshotDirectory = "" +// db, err = littbuilder.NewDB(config) +// require.NoError(t, err, "failed to open DB after rebase") +// +// // Verify the data in the DB. +// for tableName := range expectedData { +// table, err := db.GetTable(tableName) +// require.NoError(t, err, "failed to get table %s", tableName) +// for key := range expectedData[tableName] { +// value, ok, err := table.Get([]byte(key)) +// require.NoError(t, err, "failed to get key %s in table %s", key, tableName) +// require.True(t, ok, "key %s not found in table %s", key, tableName) +// require.Equal(t, expectedData[tableName][key], value, +// "value for key %s in table %s does not match expected value", key, tableName) +// } +// } +// +// err = db.Close() +// require.NoError(t, err, "failed to close DB after rebase") +// } +// diff --git a/sei-db/db_engine/litt/cli/rebase.go b/sei-db/db_engine/litt/cli/rebase.go index 3cb8f21615..dbbc4e580a 100644 --- a/sei-db/db_engine/litt/cli/rebase.go +++ b/sei-db/db_engine/litt/cli/rebase.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/rebase_test.go b/sei-db/db_engine/litt/cli/rebase_test.go index f51eb5c6f6..61c92ceca6 100644 --- a/sei-db/db_engine/litt/cli/rebase_test.go +++ b/sei-db/db_engine/litt/cli/rebase_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/sync.go b/sei-db/db_engine/litt/cli/sync.go index 2bf77a8e35..2be8e503a1 100644 --- a/sei-db/db_engine/litt/cli/sync.go +++ b/sei-db/db_engine/litt/cli/sync.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/table_info.go b/sei-db/db_engine/litt/cli/table_info.go index e44f46d46d..560d92766e 100644 --- a/sei-db/db_engine/litt/cli/table_info.go +++ b/sei-db/db_engine/litt/cli/table_info.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/table_info_test.go b/sei-db/db_engine/litt/cli/table_info_test.go index f063aeba02..3dea1b1b26 100644 --- a/sei-db/db_engine/litt/cli/table_info_test.go +++ b/sei-db/db_engine/litt/cli/table_info_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/cli/unlock.go b/sei-db/db_engine/litt/cli/unlock.go index 04dbdfd20b..f1c1c50285 100644 --- a/sei-db/db_engine/litt/cli/unlock.go +++ b/sei-db/db_engine/litt/cli/unlock.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package main import ( diff --git a/sei-db/db_engine/litt/db.go b/sei-db/db_engine/litt/db.go index 9ed1bdcaa4..cee7eb3801 100644 --- a/sei-db/db_engine/litt/db.go +++ b/sei-db/db_engine/litt/db.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package litt // DB is a highly specialized key-value store. It is intentionally very feature poor, sacrificing diff --git a/sei-db/db_engine/litt/dbcache/cached_table.go b/sei-db/db_engine/litt/dbcache/cached_table.go index d16d59c43b..c73755499e 100644 --- a/sei-db/db_engine/litt/dbcache/cached_table.go +++ b/sei-db/db_engine/litt/dbcache/cached_table.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package dbcache import ( diff --git a/sei-db/db_engine/litt/disktable/boundary_file.go b/sei-db/db_engine/litt/disktable/boundary_file.go index 3f477ce68c..cb4ce75f8f 100644 --- a/sei-db/db_engine/litt/disktable/boundary_file.go +++ b/sei-db/db_engine/litt/disktable/boundary_file.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/boundary_file_test.go b/sei-db/db_engine/litt/disktable/boundary_file_test.go index 2e331127a2..eeeb59e3ce 100644 --- a/sei-db/db_engine/litt/disktable/boundary_file_test.go +++ b/sei-db/db_engine/litt/disktable/boundary_file_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/control_loop.go b/sei-db/db_engine/litt/disktable/control_loop.go index 0cdf6c2782..214e9a0973 100644 --- a/sei-db/db_engine/litt/disktable/control_loop.go +++ b/sei-db/db_engine/litt/disktable/control_loop.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/control_loop_messages.go b/sei-db/db_engine/litt/disktable/control_loop_messages.go index 09d1fcb053..c679c443c6 100644 --- a/sei-db/db_engine/litt/disktable/control_loop_messages.go +++ b/sei-db/db_engine/litt/disktable/control_loop_messages.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/types" diff --git a/sei-db/db_engine/litt/disktable/disk_table.go b/sei-db/db_engine/litt/disktable/disk_table.go index 15d478d470..f7f48d33a4 100644 --- a/sei-db/db_engine/litt/disktable/disk_table.go +++ b/sei-db/db_engine/litt/disktable/disk_table.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/disk_table_flush_loop.go b/sei-db/db_engine/litt/disktable/disk_table_flush_loop.go index e036d73e54..512459dec9 100644 --- a/sei-db/db_engine/litt/disktable/disk_table_flush_loop.go +++ b/sei-db/db_engine/litt/disktable/disk_table_flush_loop.go @@ -1,3 +1 @@ -//go:build littdb_wip - package disktable diff --git a/sei-db/db_engine/litt/disktable/disk_table_test.go b/sei-db/db_engine/litt/disktable/disk_table_test.go index ed8960d78d..1f9c9ba0f7 100644 --- a/sei-db/db_engine/litt/disktable/disk_table_test.go +++ b/sei-db/db_engine/litt/disktable/disk_table_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/flush_coordinator.go b/sei-db/db_engine/litt/disktable/flush_coordinator.go index 89d8f2d4cb..0ccc016d7f 100644 --- a/sei-db/db_engine/litt/disktable/flush_coordinator.go +++ b/sei-db/db_engine/litt/disktable/flush_coordinator.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/flush_coordinator_test.go b/sei-db/db_engine/litt/disktable/flush_coordinator_test.go index 8170be2546..e6f80bdefb 100644 --- a/sei-db/db_engine/litt/disktable/flush_coordinator_test.go +++ b/sei-db/db_engine/litt/disktable/flush_coordinator_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/flush_loop.go b/sei-db/db_engine/litt/disktable/flush_loop.go index 192a57fb2f..8062eaab1b 100644 --- a/sei-db/db_engine/litt/disktable/flush_loop.go +++ b/sei-db/db_engine/litt/disktable/flush_loop.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/flush_loop_messages.go b/sei-db/db_engine/litt/disktable/flush_loop_messages.go index 1d6a45fbec..8776f9c78b 100644 --- a/sei-db/db_engine/litt/disktable/flush_loop_messages.go +++ b/sei-db/db_engine/litt/disktable/flush_loop_messages.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/keymap/keymap.go b/sei-db/db_engine/litt/disktable/keymap/keymap.go index 64df44e4ca..3463553ee7 100644 --- a/sei-db/db_engine/litt/disktable/keymap/keymap.go +++ b/sei-db/db_engine/litt/disktable/keymap/keymap.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package keymap import ( diff --git a/sei-db/db_engine/litt/disktable/keymap/keymap_test.go b/sei-db/db_engine/litt/disktable/keymap/keymap_test.go index 9cb0911b73..46acd26a47 100644 --- a/sei-db/db_engine/litt/disktable/keymap/keymap_test.go +++ b/sei-db/db_engine/litt/disktable/keymap/keymap_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package keymap import ( diff --git a/sei-db/db_engine/litt/disktable/keymap/keymap_type.go b/sei-db/db_engine/litt/disktable/keymap/keymap_type.go index dc7be33615..280d690765 100644 --- a/sei-db/db_engine/litt/disktable/keymap/keymap_type.go +++ b/sei-db/db_engine/litt/disktable/keymap/keymap_type.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package keymap // KeymapType represents the type of a keymap. diff --git a/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go b/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go index 353a3766ae..d36962460d 100644 --- a/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go +++ b/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package keymap import ( diff --git a/sei-db/db_engine/litt/disktable/keymap/mem_keymap.go b/sei-db/db_engine/litt/disktable/keymap/mem_keymap.go index ea05782834..030016b29e 100644 --- a/sei-db/db_engine/litt/disktable/keymap/mem_keymap.go +++ b/sei-db/db_engine/litt/disktable/keymap/mem_keymap.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package keymap import ( diff --git a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go index 40e2bb88cb..697c824ab9 100644 --- a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go +++ b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package keymap import ( diff --git a/sei-db/db_engine/litt/disktable/segment/address_test.go b/sei-db/db_engine/litt/disktable/segment/address_test.go index bed7a2d576..fb471f0b87 100644 --- a/sei-db/db_engine/litt/disktable/segment/address_test.go +++ b/sei-db/db_engine/litt/disktable/segment/address_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/key_file.go b/sei-db/db_engine/litt/disktable/segment/key_file.go index e77160e362..f9a610507e 100644 --- a/sei-db/db_engine/litt/disktable/segment/key_file.go +++ b/sei-db/db_engine/litt/disktable/segment/key_file.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/key_file_test.go b/sei-db/db_engine/litt/disktable/segment/key_file_test.go index 7c78a3b16f..0ed106e98e 100644 --- a/sei-db/db_engine/litt/disktable/segment/key_file_test.go +++ b/sei-db/db_engine/litt/disktable/segment/key_file_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/metadata_file.go b/sei-db/db_engine/litt/disktable/segment/metadata_file.go index 0aca7e3eee..ee7111d9da 100644 --- a/sei-db/db_engine/litt/disktable/segment/metadata_file.go +++ b/sei-db/db_engine/litt/disktable/segment/metadata_file.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/metadata_file_test.go b/sei-db/db_engine/litt/disktable/segment/metadata_file_test.go index 897a89e9cc..60233cf784 100644 --- a/sei-db/db_engine/litt/disktable/segment/metadata_file_test.go +++ b/sei-db/db_engine/litt/disktable/segment/metadata_file_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/segment.go b/sei-db/db_engine/litt/disktable/segment/segment.go index 807c92c034..cc314612b9 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment.go +++ b/sei-db/db_engine/litt/disktable/segment/segment.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/segment_path.go b/sei-db/db_engine/litt/disktable/segment/segment_path.go index 89bf01a960..55af2f65c7 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment_path.go +++ b/sei-db/db_engine/litt/disktable/segment/segment_path.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/segment_path_test.go b/sei-db/db_engine/litt/disktable/segment/segment_path_test.go index a33a0cc1e3..d583b4aed3 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment_path_test.go +++ b/sei-db/db_engine/litt/disktable/segment/segment_path_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/segment_scanner.go b/sei-db/db_engine/litt/disktable/segment/segment_scanner.go index a50b15a976..888fbd032f 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment_scanner.go +++ b/sei-db/db_engine/litt/disktable/segment/segment_scanner.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/segment_test.go b/sei-db/db_engine/litt/disktable/segment/segment_test.go index c5ff479832..68cd5ab6a5 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment_test.go +++ b/sei-db/db_engine/litt/disktable/segment/segment_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/segment_version.go b/sei-db/db_engine/litt/disktable/segment/segment_version.go index a7cc21ecbb..0ad9652fb6 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment_version.go +++ b/sei-db/db_engine/litt/disktable/segment/segment_version.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment // SegmentVersion is used to indicate the serialization version of a segment. Whenever serialization formats change diff --git a/sei-db/db_engine/litt/disktable/segment/shard_id_validation_test.go b/sei-db/db_engine/litt/disktable/segment/shard_id_validation_test.go index bb483be8e7..cd53ff15cb 100644 --- a/sei-db/db_engine/litt/disktable/segment/shard_id_validation_test.go +++ b/sei-db/db_engine/litt/disktable/segment/shard_id_validation_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/value_file.go b/sei-db/db_engine/litt/disktable/segment/value_file.go index 3f2e349bc2..a44559e3a7 100644 --- a/sei-db/db_engine/litt/disktable/segment/value_file.go +++ b/sei-db/db_engine/litt/disktable/segment/value_file.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/segment/value_file_test.go b/sei-db/db_engine/litt/disktable/segment/value_file_test.go index de648a4145..5405776ace 100644 --- a/sei-db/db_engine/litt/disktable/segment/value_file_test.go +++ b/sei-db/db_engine/litt/disktable/segment/value_file_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package segment import ( diff --git a/sei-db/db_engine/litt/disktable/table_metadata.go b/sei-db/db_engine/litt/disktable/table_metadata.go index d710dd6ed2..b19a5bdc5c 100644 --- a/sei-db/db_engine/litt/disktable/table_metadata.go +++ b/sei-db/db_engine/litt/disktable/table_metadata.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/disktable/unlock.go b/sei-db/db_engine/litt/disktable/unlock.go index 4b8b803c1d..8b509b38d5 100644 --- a/sei-db/db_engine/litt/disktable/unlock.go +++ b/sei-db/db_engine/litt/disktable/unlock.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package disktable import ( diff --git a/sei-db/db_engine/litt/go.mod b/sei-db/db_engine/litt/go.mod deleted file mode 100644 index 669427e11a..0000000000 --- a/sei-db/db_engine/litt/go.mod +++ /dev/null @@ -1,105 +0,0 @@ -// Nested Go module boundary for the LittDB subtree. -// -// The sei-db/db_engine/litt/ tree is a raw import from the upstream LittDB -// project (originally vendored from github.com/Layr-Labs/eigenda/litt). It -// has not yet been adapted to this repo's dependency set — all files are -// guarded by `//go:build littdb_wip`. -// -// Declaring this subtree as a separate module hides it from the parent -// module's `go mod tidy`, `go test ./...`, `go build ./...`, `go vet ./...`, -// and `golangci-lint run` — none of which cross module boundaries. See -// `sei-db/db_engine/litt/README.md` ("Work-in-progress guard") for the -// incremental integration policy. -// -// Integration status -// ------------------ -// Internal cross-package imports have been rewritten from the upstream -// `github.com/Layr-Labs/eigenda/litt/...` prefix to this repo's native -// `github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/...` prefix, and -// the helpers originally imported from `github.com/Layr-Labs/eigenda/common` -// (cache, structures, enforce, pprof), `github.com/Layr-Labs/eigenda/core` -// and `github.com/Layr-Labs/eigenda/test{,/random}` have been pulled in-tree -// under `./util/`. The subtree currently builds under `-tags littdb_wip`. -// -// Logging has been migrated from `github.com/Layr-Labs/eigensdk-go/logging` -// to the standard library's `log/slog`; no external logger dependency -// remains in this subtree. -// -// This file (and this module boundary) can be removed once the litt package -// fully compiles and passes lint inside the parent sei-chain module. -module github.com/sei-protocol/sei-chain/sei-db/db_engine/litt - -go 1.25.6 - -require ( - github.com/cockroachdb/pebble/v2 v2.1.3 - github.com/docker/docker v28.2.2+incompatible - github.com/docker/go-connections v0.5.0 - github.com/docker/go-units v0.5.0 - github.com/prometheus/client_golang v1.23.2 - github.com/stretchr/testify v1.11.1 - github.com/urfave/cli/v2 v2.27.7 - golang.org/x/crypto v0.50.0 - golang.org/x/time v0.15.0 -) - -require ( - github.com/DataDog/zstd v1.5.7 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect - github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect - github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect - github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/getsentry/sentry-go v0.27.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/atomicwriter v0.1.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.5.2 // indirect -) diff --git a/sei-db/db_engine/litt/go.sum b/sei-db/db_engine/litt/go.sum deleted file mode 100644 index 08fb3195ec..0000000000 --- a/sei-db/db_engine/litt/go.sum +++ /dev/null @@ -1,216 +0,0 @@ -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= -github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= -github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= -github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= -github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= -github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= -github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= -github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= -github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= -github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= -github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= -github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.1.3 h1:irU503OnjRoJBrkZQIJvwv9c4WvpUeOJxhRApojB8D8= -github.com/cockroachdb/pebble/v2 v2.1.3/go.mod h1:B1UgWsyR+L+UvZXNgpxw+WqsUKA8VQ/bb//FXOHghB8= -github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= -github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI= -github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= -github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= -github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= -github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= -github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= -github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/sei-db/db_engine/litt/littbuilder/build_utils.go b/sei-db/db_engine/litt/littbuilder/build_utils.go index 2ed53b4d93..3eac031985 100644 --- a/sei-db/db_engine/litt/littbuilder/build_utils.go +++ b/sei-db/db_engine/litt/littbuilder/build_utils.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package littbuilder import ( diff --git a/sei-db/db_engine/litt/littbuilder/db_impl.go b/sei-db/db_engine/litt/littbuilder/db_impl.go index 24e425283e..949aa1de53 100644 --- a/sei-db/db_engine/litt/littbuilder/db_impl.go +++ b/sei-db/db_engine/litt/littbuilder/db_impl.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package littbuilder import ( diff --git a/sei-db/db_engine/litt/littdb_config.go b/sei-db/db_engine/litt/littdb_config.go index 5b3471b12c..b273f8255d 100644 --- a/sei-db/db_engine/litt/littdb_config.go +++ b/sei-db/db_engine/litt/littdb_config.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package litt import ( @@ -9,8 +7,8 @@ import ( "math" "time" - "github.com/docker/go-units" "github.com/prometheus/client_golang/prometheus" + "github.com/sei-protocol/sei-chain/sei-db/common/unit" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/keymap" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" ) @@ -190,7 +188,7 @@ func DefaultConfigNoPaths() *Config { ControlChannelSize: 64, TargetSegmentFileSize: math.MaxUint32, MaxSegmentKeyCount: 50_000, - TargetSegmentKeyFileSize: 2 * units.MiB, + TargetSegmentKeyFileSize: 2 * unit.MB, Fsync: true, DoubleWriteProtection: false, MetricsEnabled: false, diff --git a/sei-db/db_engine/litt/littdb_config_test.go b/sei-db/db_engine/litt/littdb_config_test.go index 3a0c4994b0..e184e4d44a 100644 --- a/sei-db/db_engine/litt/littdb_config_test.go +++ b/sei-db/db_engine/litt/littdb_config_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package litt import ( diff --git a/sei-db/db_engine/litt/memtable/mem_table.go b/sei-db/db_engine/litt/memtable/mem_table.go index 47b72b8ccf..29db131ef3 100644 --- a/sei-db/db_engine/litt/memtable/mem_table.go +++ b/sei-db/db_engine/litt/memtable/mem_table.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package memtable import ( diff --git a/sei-db/db_engine/litt/metrics/littdb_metrics.go b/sei-db/db_engine/litt/metrics/littdb_metrics.go index 5bb50ef570..6e41b6c44a 100644 --- a/sei-db/db_engine/litt/metrics/littdb_metrics.go +++ b/sei-db/db_engine/litt/metrics/littdb_metrics.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package metrics import ( diff --git a/sei-db/db_engine/litt/table.go b/sei-db/db_engine/litt/table.go index 72b43dfb3f..41bb8cff72 100644 --- a/sei-db/db_engine/litt/table.go +++ b/sei-db/db_engine/litt/table.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package litt import ( diff --git a/sei-db/db_engine/litt/test/cache_test.go b/sei-db/db_engine/litt/test/cache_test.go index efb83bbdd8..c5c4f1dea3 100644 --- a/sei-db/db_engine/litt/test/cache_test.go +++ b/sei-db/db_engine/litt/test/cache_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/db_test.go b/sei-db/db_engine/litt/test/db_test.go index 1d9cbf71d2..70be8a5089 100644 --- a/sei-db/db_engine/litt/test/db_test.go +++ b/sei-db/db_engine/litt/test/db_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/generate_example_tree_test.go b/sei-db/db_engine/litt/test/generate_example_tree_test.go index e1b59eb4f9..09c77bac73 100644 --- a/sei-db/db_engine/litt/test/generate_example_tree_test.go +++ b/sei-db/db_engine/litt/test/generate_example_tree_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/keymap_migration_test.go b/sei-db/db_engine/litt/test/keymap_migration_test.go index 4c5ebca425..7d8cbab6e3 100644 --- a/sei-db/db_engine/litt/test/keymap_migration_test.go +++ b/sei-db/db_engine/litt/test/keymap_migration_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/lock_test.go b/sei-db/db_engine/litt/test/lock_test.go index 86417e8cf4..66abf325ad 100644 --- a/sei-db/db_engine/litt/test/lock_test.go +++ b/sei-db/db_engine/litt/test/lock_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/migration_data.go b/sei-db/db_engine/litt/test/migration_data.go index a06b6f11ce..321f3f9664 100644 --- a/sei-db/db_engine/litt/test/migration_data.go +++ b/sei-db/db_engine/litt/test/migration_data.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test // This map is used for migration tests. This data is written to a table at the old version, and used to verify that diff --git a/sei-db/db_engine/litt/test/migration_test.go b/sei-db/db_engine/litt/test/migration_test.go index dbac3cd3c2..3648155996 100644 --- a/sei-db/db_engine/litt/test/migration_test.go +++ b/sei-db/db_engine/litt/test/migration_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/snapshot_test.go b/sei-db/db_engine/litt/test/snapshot_test.go index 27917a2213..c1ecb0a6b1 100644 --- a/sei-db/db_engine/litt/test/snapshot_test.go +++ b/sei-db/db_engine/litt/test/snapshot_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/table_test.go b/sei-db/db_engine/litt/test/table_test.go index f157385a31..de4e8ed0c9 100644 --- a/sei-db/db_engine/litt/test/table_test.go +++ b/sei-db/db_engine/litt/test/table_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/test/unlock_test.go b/sei-db/db_engine/litt/test/unlock_test.go index fa5f56e2ff..5bbd3685f4 100644 --- a/sei-db/db_engine/litt/test/unlock_test.go +++ b/sei-db/db_engine/litt/test/unlock_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package test import ( diff --git a/sei-db/db_engine/litt/types/address.go b/sei-db/db_engine/litt/types/address.go index b9fe6ed1f8..e8a65ebf94 100644 --- a/sei-db/db_engine/litt/types/address.go +++ b/sei-db/db_engine/litt/types/address.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package types import ( diff --git a/sei-db/db_engine/litt/types/kv_pair.go b/sei-db/db_engine/litt/types/kv_pair.go index f7e703f4c6..7fbc8bf74e 100644 --- a/sei-db/db_engine/litt/types/kv_pair.go +++ b/sei-db/db_engine/litt/types/kv_pair.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package types // KVPair represents a key-value pair. diff --git a/sei-db/db_engine/litt/types/scoped_key.go b/sei-db/db_engine/litt/types/scoped_key.go index 0ccb471a61..b785248f40 100644 --- a/sei-db/db_engine/litt/types/scoped_key.go +++ b/sei-db/db_engine/litt/types/scoped_key.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package types // ScopedKey is a key paired with the Address that locates its value on disk. diff --git a/sei-db/db_engine/litt/util/cache.go b/sei-db/db_engine/litt/util/cache.go index 79dd728c40..867c894fad 100644 --- a/sei-db/db_engine/litt/util/cache.go +++ b/sei-db/db_engine/litt/util/cache.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util // WeightCalculator is a function that calculates the weight of a key-value pair in a Cache. diff --git a/sei-db/db_engine/litt/util/cache_metrics.go b/sei-db/db_engine/litt/util/cache_metrics.go index 9886c24492..572f8e3327 100644 --- a/sei-db/db_engine/litt/util/cache_metrics.go +++ b/sei-db/db_engine/litt/util/cache_metrics.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/common.go b/sei-db/db_engine/litt/util/common.go index e9a9805c67..e6ee459578 100644 --- a/sei-db/db_engine/litt/util/common.go +++ b/sei-db/db_engine/litt/util/common.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "time" diff --git a/sei-db/db_engine/litt/util/constants.go b/sei-db/db_engine/litt/util/constants.go index 217b9e0c7a..bf5704be48 100644 --- a/sei-db/db_engine/litt/util/constants.go +++ b/sei-db/db_engine/litt/util/constants.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util // The name of the LittDB lockfile. Protects against DBs in multiple processes from accessing the same data directory. diff --git a/sei-db/db_engine/litt/util/core_utils.go b/sei-db/db_engine/litt/util/core_utils.go index 37ee634ba3..dfe1b2c167 100644 --- a/sei-db/db_engine/litt/util/core_utils.go +++ b/sei-db/db_engine/litt/util/core_utils.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/enforce.go b/sei-db/db_engine/litt/util/enforce.go index 1a9dec5f05..23f646de87 100644 --- a/sei-db/db_engine/litt/util/enforce.go +++ b/sei-db/db_engine/litt/util/enforce.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "fmt" diff --git a/sei-db/db_engine/litt/util/error_monitor.go b/sei-db/db_engine/litt/util/error_monitor.go index a5a3314c94..833503a285 100644 --- a/sei-db/db_engine/litt/util/error_monitor.go +++ b/sei-db/db_engine/litt/util/error_monitor.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/fifo_cache.go b/sei-db/db_engine/litt/util/fifo_cache.go index b4c0a631ee..9b4d99536d 100644 --- a/sei-db/db_engine/litt/util/fifo_cache.go +++ b/sei-db/db_engine/litt/util/fifo_cache.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "time" diff --git a/sei-db/db_engine/litt/util/file_lock.go b/sei-db/db_engine/litt/util/file_lock.go index 62efaa69f7..538887f5e7 100644 --- a/sei-db/db_engine/litt/util/file_lock.go +++ b/sei-db/db_engine/litt/util/file_lock.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/file_lock_test.go b/sei-db/db_engine/litt/util/file_lock_test.go index 388df2c9bd..3ca1d85342 100644 --- a/sei-db/db_engine/litt/util/file_lock_test.go +++ b/sei-db/db_engine/litt/util/file_lock_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/file_utils.go b/sei-db/db_engine/litt/util/file_utils.go index bdf69eee88..14b538ce6e 100644 --- a/sei-db/db_engine/litt/util/file_utils.go +++ b/sei-db/db_engine/litt/util/file_utils.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/file_utils_test.go b/sei-db/db_engine/litt/util/file_utils_test.go index 8afc91aad7..455d3a21e4 100644 --- a/sei-db/db_engine/litt/util/file_utils_test.go +++ b/sei-db/db_engine/litt/util/file_utils_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/pprof.go b/sei-db/db_engine/litt/util/pprof.go index e1624de94d..0951b8e0e0 100644 --- a/sei-db/db_engine/litt/util/pprof.go +++ b/sei-db/db_engine/litt/util/pprof.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/queue.go b/sei-db/db_engine/litt/util/queue.go index 1f2b891ef2..d1d4439ee5 100644 --- a/sei-db/db_engine/litt/util/queue.go +++ b/sei-db/db_engine/litt/util/queue.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util // A standard generic queue. diff --git a/sei-db/db_engine/litt/util/random_access_deque.go b/sei-db/db_engine/litt/util/random_access_deque.go index 7e7ef111dd..4aa4e609d4 100644 --- a/sei-db/db_engine/litt/util/random_access_deque.go +++ b/sei-db/db_engine/litt/util/random_access_deque.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "math" diff --git a/sei-db/db_engine/litt/util/recursive_move.go b/sei-db/db_engine/litt/util/recursive_move.go index 31cdff9aa1..a02291d56f 100644 --- a/sei-db/db_engine/litt/util/recursive_move.go +++ b/sei-db/db_engine/litt/util/recursive_move.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/recursive_move_test.go b/sei-db/db_engine/litt/util/recursive_move_test.go index 25f795c189..2b7282f3f2 100644 --- a/sei-db/db_engine/litt/util/recursive_move_test.go +++ b/sei-db/db_engine/litt/util/recursive_move_test.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/ssh.go b/sei-db/db_engine/litt/util/ssh.go index e199d5739a..1517288f1b 100644 --- a/sei-db/db_engine/litt/util/ssh.go +++ b/sei-db/db_engine/litt/util/ssh.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/ssh_self_destruct_test.go b/sei-db/db_engine/litt/util/ssh_self_destruct_test.go index 40bc37b5c9..bd1c9e9b05 100644 --- a/sei-db/db_engine/litt/util/ssh_self_destruct_test.go +++ b/sei-db/db_engine/litt/util/ssh_self_destruct_test.go @@ -1,91 +1,101 @@ -//go:build littdb_wip - package util -import ( - "context" - "os" - "testing" - "time" - - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - "github.com/stretchr/testify/require" -) - -func TestSSHContainerSelfDestruct(t *testing.T) { - t.Skip("This test takes 5+ minutes to run - only enable for manual testing") - - ctx := context.Background() - - // Create Docker client - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - require.NoError(t, err) - - // Generate SSH key pair - tempDir := t.TempDir() - privateKeyPath := tempDir + "/test_ssh_key" - publicKeyPath := tempDir + "/test_ssh_key.pub" - - err = GenerateSSHKeyPair(privateKeyPath, publicKeyPath) - require.NoError(t, err) - - publicKeyContent, err := os.ReadFile(publicKeyPath) - require.NoError(t, err) - - // Create mount directory for file operations - mountDir := tempDir + "/ssh_mount" - err = os.MkdirAll(mountDir, 0755) - require.NoError(t, err) - - // Build Docker image - imageName := "ssh-test-selfdestruct:latest" - // Get current user's UID/GID for the container - uid, err := getCurrentUserUID() - require.NoError(t, err) - gid, err := getCurrentUserGID() - require.NoError(t, err) - err = BuildSSHTestImage(ctx, cli, tempDir, imageName, string(publicKeyContent), uid, gid) - require.NoError(t, err) - - // Start container - containerID, sshPort, err := StartSSHContainer(ctx, cli, imageName, mountDir, t.Name()) - require.NoError(t, err) - - // Verify container is running - containerInfo, err := cli.ContainerInspect(ctx, containerID) - require.NoError(t, err) - require.True(t, containerInfo.State.Running) - - // Wait for SSH to be ready - WaitForSSH(t, sshPort, privateKeyPath) - - t.Logf("Container %s is running and SSH is ready. Waiting for self-destruct...", containerID[:12]) - - // Wait for 6 minutes (container should self-destruct after 5 minutes) - timeout := time.After(6 * time.Minute) - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - containerStopped := false - for !containerStopped { - select { - case <-timeout: - t.Fatal("Container did not self-destruct within 6 minutes") - case <-ticker.C: - containerInfo, err := cli.ContainerInspect(ctx, containerID) - require.NoError(t, err) - - if !containerInfo.State.Running { - containerStopped = true - t.Logf("Container self-destructed successfully") - } else { - t.Logf("Container still running...") - } - } - } - - // Clean up the stopped container - err = cli.ContainerRemove(ctx, containerID, container.RemoveOptions{}) - require.NoError(t, err) -} +// COMMENTED OUT: depends on github.com/docker/docker Docker SDK, which is +// being removed from sei-chain's dependency closure as part of integrating +// litt into the main build. The functionality these tests exercise (litt +// push-over-SSH against a Docker-managed sshd container) is not currently +// used in production. Revisit when the push/sync feature is reintroduced; +// the original implementation is preserved below as comments. + +// //go:build littdb_wip +// +// package util +// +// import ( +// "context" +// "os" +// "testing" +// "time" +// +// "github.com/docker/docker/api/types/container" +// "github.com/docker/docker/client" +// "github.com/stretchr/testify/require" +// ) +// +// func TestSSHContainerSelfDestruct(t *testing.T) { +// t.Skip("This test takes 5+ minutes to run - only enable for manual testing") +// +// ctx := context.Background() +// +// // Create Docker client +// cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) +// require.NoError(t, err) +// +// // Generate SSH key pair +// tempDir := t.TempDir() +// privateKeyPath := tempDir + "/test_ssh_key" +// publicKeyPath := tempDir + "/test_ssh_key.pub" +// +// err = GenerateSSHKeyPair(privateKeyPath, publicKeyPath) +// require.NoError(t, err) +// +// publicKeyContent, err := os.ReadFile(publicKeyPath) +// require.NoError(t, err) +// +// // Create mount directory for file operations +// mountDir := tempDir + "/ssh_mount" +// err = os.MkdirAll(mountDir, 0755) +// require.NoError(t, err) +// +// // Build Docker image +// imageName := "ssh-test-selfdestruct:latest" +// // Get current user's UID/GID for the container +// uid, err := getCurrentUserUID() +// require.NoError(t, err) +// gid, err := getCurrentUserGID() +// require.NoError(t, err) +// err = BuildSSHTestImage(ctx, cli, tempDir, imageName, string(publicKeyContent), uid, gid) +// require.NoError(t, err) +// +// // Start container +// containerID, sshPort, err := StartSSHContainer(ctx, cli, imageName, mountDir, t.Name()) +// require.NoError(t, err) +// +// // Verify container is running +// containerInfo, err := cli.ContainerInspect(ctx, containerID) +// require.NoError(t, err) +// require.True(t, containerInfo.State.Running) +// +// // Wait for SSH to be ready +// WaitForSSH(t, sshPort, privateKeyPath) +// +// t.Logf("Container %s is running and SSH is ready. Waiting for self-destruct...", containerID[:12]) +// +// // Wait for 6 minutes (container should self-destruct after 5 minutes) +// timeout := time.After(6 * time.Minute) +// ticker := time.NewTicker(10 * time.Second) +// defer ticker.Stop() +// +// containerStopped := false +// for !containerStopped { +// select { +// case <-timeout: +// t.Fatal("Container did not self-destruct within 6 minutes") +// case <-ticker.C: +// containerInfo, err := cli.ContainerInspect(ctx, containerID) +// require.NoError(t, err) +// +// if !containerInfo.State.Running { +// containerStopped = true +// t.Logf("Container self-destructed successfully") +// } else { +// t.Logf("Container still running...") +// } +// } +// } +// +// // Clean up the stopped container +// err = cli.ContainerRemove(ctx, containerID, container.RemoveOptions{}) +// require.NoError(t, err) +// } +// diff --git a/sei-db/db_engine/litt/util/ssh_test.go b/sei-db/db_engine/litt/util/ssh_test.go index 2a2169ec69..5d57943dcd 100644 --- a/sei-db/db_engine/litt/util/ssh_test.go +++ b/sei-db/db_engine/litt/util/ssh_test.go @@ -1,206 +1,216 @@ -//go:build littdb_wip - package util -import ( - "fmt" - "log/slog" - "os" - "path" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSSHSession_NewSSHSession(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - container := SetupSSHTestContainer(t, "") - defer container.Cleanup() - - logger := slog.Default() - - // Test successful connection - session, err := NewSSHSession( - logger, - container.GetUser(), - container.GetHost(), - container.GetSSHPort(), - container.GetPrivateKeyPath(), - "", - true) - require.NoError(t, err) - require.NotNil(t, session) - defer func() { _ = session.Close() }() - - // Test with non-existent key - _, err = NewSSHSession( - logger, - container.GetUser(), - container.GetHost(), - container.GetSSHPort(), - "/nonexistent/key", - "", - false) - require.Error(t, err) - require.Contains(t, err.Error(), "private key does not exist") - - // Test with wrong user - _, err = NewSSHSession( - logger, - "wronguser", - container.GetHost(), - container.GetSSHPort(), - container.GetPrivateKeyPath(), - "", - false) - require.Error(t, err) -} - -func TestSSHSession_Mkdirs(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - dataDir := t.TempDir() - - container := SetupSSHTestContainer(t, dataDir) - defer container.Cleanup() - - logger := slog.Default() - - session, err := NewSSHSession( - logger, - container.GetUser(), - container.GetHost(), - container.GetSSHPort(), - container.GetPrivateKeyPath(), - "", - true) - require.NoError(t, err) - defer func() { _ = session.Close() }() - - // Test creating directory - testDir := path.Join(container.GetDataDir(), "foo", "bar", "baz") - err = session.Mkdirs(testDir) - require.NoError(t, err) - - // Verify directories were created in the container workspace - exists, err := Exists(path.Join(dataDir, "foo", "bar", "baz")) - require.NoError(t, err) - require.True(t, exists) - - // Recreating the same directory should not error. - err = session.Mkdirs(testDir) - require.NoError(t, err) -} - -func TestSSHSession_FindFiles(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - dataDir := t.TempDir() - - container := SetupSSHTestContainer(t, dataDir) - defer container.Cleanup() - - logger := slog.Default() - - session, err := NewSSHSession( - logger, - container.GetUser(), - container.GetHost(), - container.GetSSHPort(), - container.GetPrivateKeyPath(), - "", - true) - require.NoError(t, err) - defer func() { _ = session.Close() }() - - // Create a test subdirectory in the container's data directory - testDir := path.Join(container.GetDataDir(), "search") - err = session.Mkdirs(testDir) - require.NoError(t, err) - - // Create test files via SSH instead of host filesystem to avoid permission issues - // This ensures all files are created with proper container ownership - _, _, err = session.Exec(fmt.Sprintf("echo 'test content' > %s/test.txt", testDir)) - require.NoError(t, err) - _, _, err = session.Exec(fmt.Sprintf("echo 'log content' > %s/test.log", testDir)) - require.NoError(t, err) - _, _, err = session.Exec(fmt.Sprintf("echo 'data content' > %s/other.dat", testDir)) - require.NoError(t, err) - - // Test finding files with specific extensions - files, err := session.FindFiles(testDir, []string{".txt", ".log"}) - require.NoError(t, err) - require.Len(t, files, 2) - require.Contains(t, files, path.Join(testDir, "test.txt")) - require.Contains(t, files, path.Join(testDir, "test.log")) - - // Test with non-existent directory - files, err = session.FindFiles("/nonexistent", []string{".txt"}) - require.NoError(t, err) - require.Empty(t, files) -} - -func TestSSHSession_Rsync(t *testing.T) { - t.Skip() // Docker build is flaky, need to fix prior to re-enabling - - t.Parallel() - - // Create a temporary data directory for testing - dataDir := t.TempDir() - container := SetupSSHTestContainer(t, dataDir) - defer container.Cleanup() - - logger := slog.Default() - - session, err := NewSSHSession( - logger, - container.GetUser(), - container.GetHost(), - container.GetSSHPort(), - container.GetPrivateKeyPath(), - "", - true) - require.NoError(t, err) - defer func() { _ = session.Close() }() - - // Create local test file - localFile := filepath.Join(container.GetTempDir(), "test_rsync.txt") - testContent := []byte("This is test content for rsync") - err = os.WriteFile(localFile, testContent, 0644) - require.NoError(t, err) - - // Test rsync without throttling - sync to data directory - remoteFile := filepath.Join(container.GetDataDir(), "remote_file.txt") - err = session.Rsync(localFile, remoteFile, 0) - require.NoError(t, err) - - // Verify file was transferred via the container workspace directory - transferredFile := filepath.Join(dataDir, "remote_file.txt") - transferredContent, err := os.ReadFile(transferredFile) - require.NoError(t, err) - require.Equal(t, testContent, transferredContent) - - // Test rsync with throttling - localFile2 := filepath.Join(container.GetTempDir(), "test_rsync2.txt") - throttledContent := []byte("throttled content") - err = os.WriteFile(localFile2, throttledContent, 0644) - require.NoError(t, err) - - remoteFile2 := filepath.Join(container.GetDataDir(), "throttled_file.txt") - err = session.Rsync(localFile2, remoteFile2, 1.0) // 1MB/s throttle - require.NoError(t, err) - - // Verify throttled file was transferred via the container workspace directory - transferredFile2 := filepath.Join(dataDir, "throttled_file.txt") - transferredContent2, err := os.ReadFile(transferredFile2) - require.NoError(t, err) - require.Equal(t, throttledContent, transferredContent2) -} +// COMMENTED OUT: depends on github.com/docker/docker Docker SDK, which is +// being removed from sei-chain's dependency closure as part of integrating +// litt into the main build. The functionality these tests exercise (litt +// push-over-SSH against a Docker-managed sshd container) is not currently +// used in production. Revisit when the push/sync feature is reintroduced; +// the original implementation is preserved below as comments. + +// //go:build littdb_wip +// +// package util +// +// import ( +// "fmt" +// "log/slog" +// "os" +// "path" +// "path/filepath" +// "testing" +// +// "github.com/stretchr/testify/require" +// ) +// +// func TestSSHSession_NewSSHSession(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// container := SetupSSHTestContainer(t, "") +// defer container.Cleanup() +// +// logger := slog.Default() +// +// // Test successful connection +// session, err := NewSSHSession( +// logger, +// container.GetUser(), +// container.GetHost(), +// container.GetSSHPort(), +// container.GetPrivateKeyPath(), +// "", +// true) +// require.NoError(t, err) +// require.NotNil(t, session) +// defer func() { _ = session.Close() }() +// +// // Test with non-existent key +// _, err = NewSSHSession( +// logger, +// container.GetUser(), +// container.GetHost(), +// container.GetSSHPort(), +// "/nonexistent/key", +// "", +// false) +// require.Error(t, err) +// require.Contains(t, err.Error(), "private key does not exist") +// +// // Test with wrong user +// _, err = NewSSHSession( +// logger, +// "wronguser", +// container.GetHost(), +// container.GetSSHPort(), +// container.GetPrivateKeyPath(), +// "", +// false) +// require.Error(t, err) +// } +// +// func TestSSHSession_Mkdirs(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// dataDir := t.TempDir() +// +// container := SetupSSHTestContainer(t, dataDir) +// defer container.Cleanup() +// +// logger := slog.Default() +// +// session, err := NewSSHSession( +// logger, +// container.GetUser(), +// container.GetHost(), +// container.GetSSHPort(), +// container.GetPrivateKeyPath(), +// "", +// true) +// require.NoError(t, err) +// defer func() { _ = session.Close() }() +// +// // Test creating directory +// testDir := path.Join(container.GetDataDir(), "foo", "bar", "baz") +// err = session.Mkdirs(testDir) +// require.NoError(t, err) +// +// // Verify directories were created in the container workspace +// exists, err := Exists(path.Join(dataDir, "foo", "bar", "baz")) +// require.NoError(t, err) +// require.True(t, exists) +// +// // Recreating the same directory should not error. +// err = session.Mkdirs(testDir) +// require.NoError(t, err) +// } +// +// func TestSSHSession_FindFiles(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// dataDir := t.TempDir() +// +// container := SetupSSHTestContainer(t, dataDir) +// defer container.Cleanup() +// +// logger := slog.Default() +// +// session, err := NewSSHSession( +// logger, +// container.GetUser(), +// container.GetHost(), +// container.GetSSHPort(), +// container.GetPrivateKeyPath(), +// "", +// true) +// require.NoError(t, err) +// defer func() { _ = session.Close() }() +// +// // Create a test subdirectory in the container's data directory +// testDir := path.Join(container.GetDataDir(), "search") +// err = session.Mkdirs(testDir) +// require.NoError(t, err) +// +// // Create test files via SSH instead of host filesystem to avoid permission issues +// // This ensures all files are created with proper container ownership +// _, _, err = session.Exec(fmt.Sprintf("echo 'test content' > %s/test.txt", testDir)) +// require.NoError(t, err) +// _, _, err = session.Exec(fmt.Sprintf("echo 'log content' > %s/test.log", testDir)) +// require.NoError(t, err) +// _, _, err = session.Exec(fmt.Sprintf("echo 'data content' > %s/other.dat", testDir)) +// require.NoError(t, err) +// +// // Test finding files with specific extensions +// files, err := session.FindFiles(testDir, []string{".txt", ".log"}) +// require.NoError(t, err) +// require.Len(t, files, 2) +// require.Contains(t, files, path.Join(testDir, "test.txt")) +// require.Contains(t, files, path.Join(testDir, "test.log")) +// +// // Test with non-existent directory +// files, err = session.FindFiles("/nonexistent", []string{".txt"}) +// require.NoError(t, err) +// require.Empty(t, files) +// } +// +// func TestSSHSession_Rsync(t *testing.T) { +// t.Skip() // Docker build is flaky, need to fix prior to re-enabling +// +// t.Parallel() +// +// // Create a temporary data directory for testing +// dataDir := t.TempDir() +// container := SetupSSHTestContainer(t, dataDir) +// defer container.Cleanup() +// +// logger := slog.Default() +// +// session, err := NewSSHSession( +// logger, +// container.GetUser(), +// container.GetHost(), +// container.GetSSHPort(), +// container.GetPrivateKeyPath(), +// "", +// true) +// require.NoError(t, err) +// defer func() { _ = session.Close() }() +// +// // Create local test file +// localFile := filepath.Join(container.GetTempDir(), "test_rsync.txt") +// testContent := []byte("This is test content for rsync") +// err = os.WriteFile(localFile, testContent, 0644) +// require.NoError(t, err) +// +// // Test rsync without throttling - sync to data directory +// remoteFile := filepath.Join(container.GetDataDir(), "remote_file.txt") +// err = session.Rsync(localFile, remoteFile, 0) +// require.NoError(t, err) +// +// // Verify file was transferred via the container workspace directory +// transferredFile := filepath.Join(dataDir, "remote_file.txt") +// transferredContent, err := os.ReadFile(transferredFile) +// require.NoError(t, err) +// require.Equal(t, testContent, transferredContent) +// +// // Test rsync with throttling +// localFile2 := filepath.Join(container.GetTempDir(), "test_rsync2.txt") +// throttledContent := []byte("throttled content") +// err = os.WriteFile(localFile2, throttledContent, 0644) +// require.NoError(t, err) +// +// remoteFile2 := filepath.Join(container.GetDataDir(), "throttled_file.txt") +// err = session.Rsync(localFile2, remoteFile2, 1.0) // 1MB/s throttle +// require.NoError(t, err) +// +// // Verify throttled file was transferred via the container workspace directory +// transferredFile2 := filepath.Join(dataDir, "throttled_file.txt") +// transferredContent2, err := os.ReadFile(transferredFile2) +// require.NoError(t, err) +// require.Equal(t, throttledContent, transferredContent2) +// } +// diff --git a/sei-db/db_engine/litt/util/ssh_test_utils.go b/sei-db/db_engine/litt/util/ssh_test_utils.go index e6dc49beea..7178eb9ae4 100644 --- a/sei-db/db_engine/litt/util/ssh_test_utils.go +++ b/sei-db/db_engine/litt/util/ssh_test_utils.go @@ -1,724 +1,734 @@ -//go:build littdb_wip - package util -import ( - "archive/tar" - "compress/gzip" - "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/base64" - "encoding/pem" - "fmt" - "hash/fnv" - "io" - "log/slog" - "net" - "os" - "os/user" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/client" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "golang.org/x/crypto/ssh" -) - -// SSHTestPortBase is the base port used for SSH testing to avoid port collisions in CI -const SSHTestPortBase = 22022 - -const containerDataDir = "/mnt/data" -const username = "testuser" - -// Global variables for shared SSH test image -var ( - sharedImageName string - imageMutex sync.Mutex -) - -// getCurrentUserUID returns the current user's UID -func getCurrentUserUID() (int, error) { - currentUser, err := user.Current() - if err != nil { - return 0, fmt.Errorf("failed to get current user: %w", err) - } - uid, err := strconv.Atoi(currentUser.Uid) - if err != nil { - return 0, fmt.Errorf("failed to convert UID to int: %w", err) - } - return uid, nil -} - -// getCurrentUserGID returns the current user's GID -func getCurrentUserGID() (int, error) { - currentUser, err := user.Current() - if err != nil { - return 0, fmt.Errorf("failed to get current user: %w", err) - } - gid, err := strconv.Atoi(currentUser.Gid) - if err != nil { - return 0, fmt.Errorf("failed to convert GID to int: %w", err) - } - return gid, nil -} - -// GetFreeSSHTestPort returns a free port starting from SSHTestPortBase -func GetFreeSSHTestPort() (int, error) { - // Try ports starting from the base port - for port := SSHTestPortBase; port < SSHTestPortBase+100; port++ { - addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) - listener, err := net.Listen("tcp", addr) - if err != nil { - continue // Port is in use, try next one - } - _ = listener.Close() - return port, nil - } - return 0, fmt.Errorf("no free port found in range %d-%d", SSHTestPortBase, SSHTestPortBase+100) -} - -// GetUniqueSSHTestPort returns a unique port based on test name hash to avoid collisions -func GetUniqueSSHTestPort(testName string) (int, error) { - // Create a hash of the test name to get a deterministic port offset - h := fnv.New32a() - _, _ = h.Write([]byte(testName)) - hash := h.Sum32() - - // Try multiple ports starting from the hash-based offset - for i := 0; i < 10; i++ { - portOffset := int((hash + uint32(i)) % 100) - port := SSHTestPortBase + portOffset - - // Check if this port is free with a short timeout - addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) - conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) - if err != nil { - // Port is free (connection failed) - return port, nil - } - _ = conn.Close() - } - - // If no port found in the hash range, fall back to free port finder - return GetFreeSSHTestPort() -} - -// SSHTestContainer manages a Docker container with SSH server for testing -type SSHTestContainer struct { - t *testing.T - client *client.Client - containerID string - sshPort uint64 - tempDir string - privateKey string - publicKey string - host string - uid int - gid int -} - -// GetSSHPort returns the SSH port of the test container -func (c *SSHTestContainer) GetSSHPort() uint64 { - return c.sshPort -} - -// GetPrivateKeyPath returns the path to the private key file -func (c *SSHTestContainer) GetPrivateKeyPath() string { - return c.privateKey -} - -// GetPublicKeyPath returns the path to the public key file -func (c *SSHTestContainer) GetPublicKeyPath() string { - return c.publicKey -} - -// GetTempDir returns the temporary directory used by the container -func (c *SSHTestContainer) GetTempDir() string { - return c.tempDir -} - -// GetUser returns the SSH user for the test container -func (c *SSHTestContainer) GetUser() string { - return username -} - -// Get the UID of the user inside the container. -func (c *SSHTestContainer) GetUID() int { - return c.uid -} - -// Get the GID of the user inside the container. -func (c *SSHTestContainer) GetGID() int { - return c.gid -} - -// GetHost returns the host address for the SSH connection -func (c *SSHTestContainer) GetHost() string { - return c.host -} - -// GetDataDir returns the path to the container-controlled workspace directory -func (c *SSHTestContainer) GetDataDir() string { - return containerDataDir -} - -// delete the mounted data dir from within the container to avoid permission issues -func (c *SSHTestContainer) cleanupDataDir() error { - - logger := slog.Default() - - session, err := NewSSHSession( - logger, - c.GetUser(), - c.host, - c.sshPort, - c.privateKey, - "", - false) // Don't log connection errors during cleanup - if err != nil { - return fmt.Errorf("failed to create SSH session: %w", err) - } - defer func() { _ = session.Close() }() - - require.NotEqual(c.t, "", containerDataDir, - "if this is an empty string then we will attempt to 'rm -rf /*'... let's not do that") - - // Remove the entire workspace directory tree from inside the container - // This ensures container-owned files are removed by the container user - cleanupCmd := fmt.Sprintf("rm -rf %s/*", containerDataDir) - stdout, stderr, err := session.Exec(cleanupCmd) - if err != nil { - return fmt.Errorf("failed to cleanup workspace: %w\nstdout: %s\nstderr: %s", err, stdout, stderr) - } - - return nil -} - -// Cleanup removes the Docker container and cleans up resources -func (c *SSHTestContainer) Cleanup() { - err := c.cleanupDataDir() - require.NoError(c.t, err) - - // Use a context with timeout for cleanup operations - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // Stop and remove container with timeout - stopTimeout := 10 // seconds - err = c.client.ContainerStop(ctx, c.containerID, container.StopOptions{ - Timeout: &stopTimeout, - }) - if err != nil { - // Log the error but continue with removal - fmt.Printf("Warning: failed to stop container %s: %v\n", c.containerID, err) - } - - // Remove container even if stop failed - err = c.client.ContainerRemove(ctx, c.containerID, container.RemoveOptions{ - Force: true, // Force removal even if container is still running - }) - require.NoError(c.t, err) -} - -// GenerateSSHKeyPair creates an RSA key pair for testing -func GenerateSSHKeyPair(privateKeyPath string, publicKeyPath string) error { - privateKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return fmt.Errorf("failed to generate private key: %w", err) - } - - // Save private key - privateKeyPEM := &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(privateKey), - } - - privateKeyFile, err := os.Create(privateKeyPath) - if err != nil { - return fmt.Errorf("failed to create private key file: %w", err) - } - defer func() { _ = privateKeyFile.Close() }() - - err = pem.Encode(privateKeyFile, privateKeyPEM) - if err != nil { - return fmt.Errorf("failed to encode private key: %w", err) - } - - err = os.Chmod(privateKeyPath, 0600) - if err != nil { - return fmt.Errorf("failed to set private key permissions: %w", err) - } - - // Save public key - publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) - if err != nil { - return fmt.Errorf("failed to create SSH public key: %w", err) - } - - publicKeyBytes := ssh.MarshalAuthorizedKey(publicKey) - err = os.WriteFile(publicKeyPath, publicKeyBytes, 0644) - if err != nil { - return fmt.Errorf("failed to write public key: %w", err) - } - - return nil -} - -// configureContainerSSHKey updates the container's SSH authorized_keys file with the test-specific public key -func configureContainerSSHKey(ctx context.Context, cli *client.Client, containerID string, publicKeyPath string) error { - publicKeyContent, err := os.ReadFile(publicKeyPath) - if err != nil { - return fmt.Errorf("failed to read public key: %w", err) - } - - // Use base64 encoding to safely pass the SSH key content without shell escaping issues - // Base64 encoding ensures no shell metacharacters can cause problems - encodedKey := base64.StdEncoding.EncodeToString(publicKeyContent) - - execConfig := container.ExecOptions{ - Cmd: []string{ - "sh", "-c", - fmt.Sprintf( - "echo '%s' | base64 -d > /home/%s/.ssh/authorized_keys && chmod 600 /home/%s/.ssh/authorized_keys", - encodedKey, username, username), - }, - } - - // Create the exec instance - execIDResp, err := cli.ContainerExecCreate(ctx, containerID, execConfig) - if err != nil { - return fmt.Errorf("failed to create exec instance: %w", err) - } - - // Start the exec instance with Detach: false to ensure it blocks until completion - err = cli.ContainerExecStart(ctx, execIDResp.ID, container.ExecStartOptions{ - Detach: false, // Explicitly set to false to block until completion - }) - if err != nil { - return fmt.Errorf("failed to start exec instance: %w", err) - } - - // With Detach: false, ContainerExecStart should block until completion. - // However, to be absolutely certain, we'll add a brief polling loop. - for i := 0; i < 10; i++ { // Max 10 attempts with 100ms intervals = 1 second max wait - execInspect, err := cli.ContainerExecInspect(ctx, execIDResp.ID) - if err != nil { - return fmt.Errorf("failed to inspect exec instance: %w", err) - } - - // If the command is no longer running, we can check the exit code - if !execInspect.Running { - // Check if the command was successful - if execInspect.ExitCode != 0 { - return fmt.Errorf("SSH key configuration command failed with exit code %d", execInspect.ExitCode) - } - return nil // Success! - } - - // Brief sleep before checking again - time.Sleep(10 * time.Millisecond) - } - - // If still running after polling, something is wrong - return fmt.Errorf("SSH key configuration command is still running after timeout") -} - -// WaitForSSH waits for the SSH server to be ready -func WaitForSSH(t *testing.T, sshPort uint64, privateKeyPath string) { - logger := slog.Default() - - // Use a context with timeout to prevent indefinite hanging - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - require.Fail(t, "SSH server did not become ready within 30 seconds") - return - case <-ticker.C: - session, err := NewSSHSession( - logger, - username, - "localhost", - sshPort, - privateKeyPath, - "", - false) - if err == nil { - _ = session.Close() - return - } - // Continue trying on error - } - } -} - -// getOrBuildSharedSSHImage returns the name of the shared SSH test image. -// If the image doesn't exist, it builds it. This method is thread-safe. -func getOrBuildSharedSSHImage(ctx context.Context, cli *client.Client, t *testing.T) (string, error) { - imageMutex.Lock() - defer imageMutex.Unlock() - - // If we already have a cached image name, verify it still exists - if sharedImageName != "" { - _, err := cli.ImageInspect(ctx, sharedImageName) - if err == nil { - return sharedImageName, nil - } - // Image no longer exists, reset and rebuild - sharedImageName = "" - } - - // Get current user's UID/GID for the shared image - uid, err := getCurrentUserUID() - if err != nil { - return "", fmt.Errorf("failed to get current user UID: %w", err) - } - gid, err := getCurrentUserGID() - if err != nil { - return "", fmt.Errorf("failed to get current user GID: %w", err) - } - - // Generate a unique image name based on UID/GID and current time to avoid conflicts - imageName := fmt.Sprintf("ssh-test-shared:%d-%d-%d", uid, gid, time.Now().Unix()) - - // Create a temporary directory for building the image - tempDir := t.TempDir() - privateKeyPath := filepath.Join(tempDir, "shared_ssh_key") - publicKeyPath := filepath.Join(tempDir, "shared_ssh_key.pub") - - // Generate SSH key pair for the shared image - err = GenerateSSHKeyPair(privateKeyPath, publicKeyPath) - if err != nil { - return "", fmt.Errorf("failed to generate SSH key pair: %w", err) - } - - publicKeyContent, err := os.ReadFile(publicKeyPath) - if err != nil { - return "", fmt.Errorf("failed to read public key: %w", err) - } - - // Build the shared image - t.Logf("Building shared SSH test Docker image: %s", imageName) - err = BuildSSHTestImage(ctx, cli, tempDir, imageName, string(publicKeyContent), uid, gid) - if err != nil { - return "", fmt.Errorf("failed to build shared SSH image: %w", err) - } - - // Cache the image name for future use - sharedImageName = imageName - return sharedImageName, nil -} - -// SetupSSHTestContainer creates and starts a Docker container with SSH server -// If dataDir is not empty, it will be mounted in the container at /mnt/data -func SetupSSHTestContainer(t *testing.T, dataDir string) *SSHTestContainer { - // Use a longer timeout for the entire setup process to handle slow CI environments - ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) - defer cancel() - - // Get current user's UID/GID - uid, err := getCurrentUserUID() - require.NoError(t, err) - gid, err := getCurrentUserGID() - require.NoError(t, err) - - // Create Docker client - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - require.NoError(t, err) - - // Generate SSH key pair for this specific test - tempDir := t.TempDir() - privateKeyPath := filepath.Join(tempDir, "test_ssh_key") - publicKeyPath := filepath.Join(tempDir, "test_ssh_key.pub") - - err = GenerateSSHKeyPair(privateKeyPath, publicKeyPath) - require.NoError(t, err) - - // Get or build the shared SSH test image - imageName, err := getOrBuildSharedSSHImage(ctx, cli, t) - require.NoError(t, err) - - if dataDir != "" { - // we have to grant broad permissions here because the container may have a different UID - err = os.Chmod(dataDir, 0777) - require.NoError(t, err, "failed to set permissions on data directory") - } - - // Start container and configure it with the test-specific SSH key - containerID, sshPort, err := StartSSHContainer(ctx, cli, imageName, dataDir, t.Name()) - require.NoError(t, err) - - // Configure the container to use the test-specific SSH key - err = configureContainerSSHKey(ctx, cli, containerID, publicKeyPath) - require.NoError(t, err) - - // Wait for SSH to be ready - WaitForSSH(t, sshPort, privateKeyPath) - - return &SSHTestContainer{ - t: t, - client: cli, - containerID: containerID, - sshPort: sshPort, - tempDir: tempDir, - privateKey: privateKeyPath, - publicKey: publicKeyPath, - host: "localhost", - uid: uid, - gid: gid, - } -} - -// BuildSSHTestImage builds the SSH test image with the provided public key and user IDs -func BuildSSHTestImage( - ctx context.Context, - cli *client.Client, - tempDir string, - imageName string, - publicKey string, - uid int, - gid int, -) error { - - // Get the Dockerfile path - _, currentFile, _, ok := runtime.Caller(0) - if !ok { - return fmt.Errorf("failed to get current file path") - } - dockerfilePath := filepath.Join(filepath.Dir(currentFile), "testdata", "ssh-test.Dockerfile") - - // Create build context directory - buildContext := filepath.Join(tempDir, "docker_build") - err := os.MkdirAll(buildContext, 0755) - if err != nil { - return fmt.Errorf("failed to create build context: %w", err) - } - - // Copy Dockerfile to build context - dockerfileContent, err := os.ReadFile(dockerfilePath) - if err != nil { - return fmt.Errorf("failed to read Dockerfile: %w", err) - } - - // Copy start.sh script to build context - startScriptPath := filepath.Join(filepath.Dir(currentFile), "testdata", "start.sh") - startScriptContent, err := os.ReadFile(startScriptPath) - if err != nil { - return fmt.Errorf("failed to read start.sh script: %w", err) - } - err = os.WriteFile(filepath.Join(buildContext, "start.sh"), startScriptContent, 0755) - if err != nil { - return fmt.Errorf("failed to copy start.sh to build context: %w", err) - } - - // Add the public key setup to the Dockerfile - publicKeySetup := fmt.Sprintf( - "\n# Add test SSH public key\n"+ - "RUN echo '%s' > /home/testuser/.ssh/authorized_keys\n"+ - "RUN chmod 600 /home/testuser/.ssh/authorized_keys\n"+ - "RUN chown %d:%d /home/testuser/.ssh/authorized_keys\n", strings.TrimSpace(publicKey), uid, gid) - modifiedDockerfile := string(dockerfileContent) + publicKeySetup - - err = os.WriteFile(filepath.Join(buildContext, "Dockerfile"), []byte(modifiedDockerfile), 0644) - if err != nil { - return fmt.Errorf("failed to write modified Dockerfile: %w", err) - } - - // Create tar archive for build context - buildCtx, err := ArchiveDirectory(buildContext) - if err != nil { - return fmt.Errorf("failed to create build context archive: %w", err) - } - defer func() { _ = buildCtx.Close() }() - - // Build the image with optimized settings for CI - buildOptions := types.ImageBuildOptions{ - Tags: []string{imageName}, - Dockerfile: "Dockerfile", - Remove: true, - ForceRemove: true, - NoCache: false, // Allow caching to speed up builds - BuildArgs: map[string]*string{ - "USER_UID": &[]string{strconv.Itoa(uid)}[0], - "USER_GID": &[]string{strconv.Itoa(gid)}[0], - }, - } - - response, err := cli.ImageBuild(ctx, buildCtx, buildOptions) - if err != nil { - return fmt.Errorf("failed to build image: %w", err) - } - defer func() { _ = response.Body.Close() }() - - // Read build output with proper error handling for timeouts - // Create a buffer to capture build output for debugging on failure - var buildOutput strings.Builder - reader := io.TeeReader(response.Body, &buildOutput) - - _, err = io.Copy(io.Discard, reader) - if err != nil { - // Include build output in error for debugging - buildOutputStr := buildOutput.String() - if len(buildOutputStr) > 1000 { - buildOutputStr = buildOutputStr[:1000] + "... (truncated)" - } - return fmt.Errorf("failed to read build response: %w\nBuild output: %s", err, buildOutputStr) - } - - // After the build finishes, verify the image actually exists - _, err = cli.ImageInspect(ctx, imageName) - if err != nil { - buildOutputStr := buildOutput.String() - if len(buildOutputStr) > 2000 { - buildOutputStr = buildOutputStr[:2000] + "... (truncated)" - } - return fmt.Errorf("docker image build failed - image not found after build: %w\nBuild output: %s", - err, buildOutputStr) - } - - return nil -} - -// StartSSHContainer starts the SSH container and returns container ID and SSH port -// If dataDir is not empty, it will be mounted at /mnt/data in the container -func StartSSHContainer( - ctx context.Context, - cli *client.Client, - imageName string, - dataDir string, - testName string, -) (string, uint64, error) { - - // Get a unique port for this test based on test name hash - sshPort, err := GetUniqueSSHTestPort(testName) - if err != nil { - return "", 0, fmt.Errorf("failed to get unique SSH port: %w", err) - } - - containerConfig := &container.Config{ - Image: imageName, - ExposedPorts: nat.PortSet{ - "22/tcp": struct{}{}, - }, - } - - hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - "22/tcp": []nat.PortBinding{ - { - HostIP: "127.0.0.1", - HostPort: strconv.Itoa(sshPort), // Use custom port to avoid collisions in CI - }, - }, - }, - Mounts: func() []mount.Mount { - var mounts []mount.Mount - if dataDir != "" { - mounts = append(mounts, mount.Mount{ - Type: mount.TypeBind, - Source: dataDir, - Target: "/mnt/data", - }) - } - return mounts - }(), - } - - // Create a container name that includes the test name for easier debugging - containerName := fmt.Sprintf("ssh-test-%s-%d", - strings.ReplaceAll(testName, "/", "-"), time.Now().Unix()) - - resp, err := cli.ContainerCreate( - ctx, - containerConfig, - hostConfig, - nil, - nil, - containerName) - if err != nil { - return "", 0, fmt.Errorf("failed to create container: %w", err) - } - - err = cli.ContainerStart(ctx, resp.ID, container.StartOptions{}) - if err != nil { - return "", 0, fmt.Errorf("failed to start container: %w", err) - } - - // Use the custom SSH port (convert to uint64 for compatibility) - return resp.ID, uint64(sshPort), nil -} - -// ArchiveDirectory creates a tar.gz archive of a directory for Docker build context -func ArchiveDirectory(srcDir string) (io.ReadCloser, error) { - pr, pw := io.Pipe() - - go func() { - defer func() { _ = pw.Close() }() - - gw := gzip.NewWriter(pw) - defer func() { _ = gw.Close() }() - - tw := tar.NewWriter(gw) - defer func() { _ = tw.Close() }() - - _ = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(srcDir, path) - if err != nil { - return fmt.Errorf("failed to get relative path: %w", err) - } - - // Skip the root directory itself - if relPath == "." { - return nil - } - - header, err := tar.FileInfoHeader(info, "") - if err != nil { - return fmt.Errorf("failed to create tar header: %w", err) - } - header.Name = relPath - - if err := tw.WriteHeader(header); err != nil { - return fmt.Errorf("failed to write tar header for %s: %w", relPath, err) - } - - if info.IsDir() { - return nil - } - - file, err := os.Open(path) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", path, err) - } - defer func() { _ = file.Close() }() - - _, err = io.Copy(tw, file) - if err != nil { - return fmt.Errorf("failed to copy file %s to tar: %w", path, err) - } - return nil - }) - }() - - return pr, nil -} +// COMMENTED OUT: depends on github.com/docker/docker Docker SDK, which is +// being removed from sei-chain's dependency closure as part of integrating +// litt into the main build. The functionality these tests exercise (litt +// push-over-SSH against a Docker-managed sshd container) is not currently +// used in production. Revisit when the push/sync feature is reintroduced; +// the original implementation is preserved below as comments. + +// //go:build littdb_wip +// +// package util +// +// import ( +// "archive/tar" +// "compress/gzip" +// "context" +// "crypto/rand" +// "crypto/rsa" +// "crypto/x509" +// "encoding/base64" +// "encoding/pem" +// "fmt" +// "hash/fnv" +// "io" +// "log/slog" +// "net" +// "os" +// "os/user" +// "path/filepath" +// "runtime" +// "strconv" +// "strings" +// "sync" +// "testing" +// "time" +// +// "github.com/docker/docker/api/types" +// "github.com/docker/docker/api/types/container" +// "github.com/docker/docker/api/types/mount" +// "github.com/docker/docker/client" +// "github.com/docker/go-connections/nat" +// "github.com/stretchr/testify/require" +// "golang.org/x/crypto/ssh" +// ) +// +// // SSHTestPortBase is the base port used for SSH testing to avoid port collisions in CI +// const SSHTestPortBase = 22022 +// +// const containerDataDir = "/mnt/data" +// const username = "testuser" +// +// // Global variables for shared SSH test image +// var ( +// sharedImageName string +// imageMutex sync.Mutex +// ) +// +// // getCurrentUserUID returns the current user's UID +// func getCurrentUserUID() (int, error) { +// currentUser, err := user.Current() +// if err != nil { +// return 0, fmt.Errorf("failed to get current user: %w", err) +// } +// uid, err := strconv.Atoi(currentUser.Uid) +// if err != nil { +// return 0, fmt.Errorf("failed to convert UID to int: %w", err) +// } +// return uid, nil +// } +// +// // getCurrentUserGID returns the current user's GID +// func getCurrentUserGID() (int, error) { +// currentUser, err := user.Current() +// if err != nil { +// return 0, fmt.Errorf("failed to get current user: %w", err) +// } +// gid, err := strconv.Atoi(currentUser.Gid) +// if err != nil { +// return 0, fmt.Errorf("failed to convert GID to int: %w", err) +// } +// return gid, nil +// } +// +// // GetFreeSSHTestPort returns a free port starting from SSHTestPortBase +// func GetFreeSSHTestPort() (int, error) { +// // Try ports starting from the base port +// for port := SSHTestPortBase; port < SSHTestPortBase+100; port++ { +// addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) +// listener, err := net.Listen("tcp", addr) +// if err != nil { +// continue // Port is in use, try next one +// } +// _ = listener.Close() +// return port, nil +// } +// return 0, fmt.Errorf("no free port found in range %d-%d", SSHTestPortBase, SSHTestPortBase+100) +// } +// +// // GetUniqueSSHTestPort returns a unique port based on test name hash to avoid collisions +// func GetUniqueSSHTestPort(testName string) (int, error) { +// // Create a hash of the test name to get a deterministic port offset +// h := fnv.New32a() +// _, _ = h.Write([]byte(testName)) +// hash := h.Sum32() +// +// // Try multiple ports starting from the hash-based offset +// for i := 0; i < 10; i++ { +// portOffset := int((hash + uint32(i)) % 100) +// port := SSHTestPortBase + portOffset +// +// // Check if this port is free with a short timeout +// addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) +// conn, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) +// if err != nil { +// // Port is free (connection failed) +// return port, nil +// } +// _ = conn.Close() +// } +// +// // If no port found in the hash range, fall back to free port finder +// return GetFreeSSHTestPort() +// } +// +// // SSHTestContainer manages a Docker container with SSH server for testing +// type SSHTestContainer struct { +// t *testing.T +// client *client.Client +// containerID string +// sshPort uint64 +// tempDir string +// privateKey string +// publicKey string +// host string +// uid int +// gid int +// } +// +// // GetSSHPort returns the SSH port of the test container +// func (c *SSHTestContainer) GetSSHPort() uint64 { +// return c.sshPort +// } +// +// // GetPrivateKeyPath returns the path to the private key file +// func (c *SSHTestContainer) GetPrivateKeyPath() string { +// return c.privateKey +// } +// +// // GetPublicKeyPath returns the path to the public key file +// func (c *SSHTestContainer) GetPublicKeyPath() string { +// return c.publicKey +// } +// +// // GetTempDir returns the temporary directory used by the container +// func (c *SSHTestContainer) GetTempDir() string { +// return c.tempDir +// } +// +// // GetUser returns the SSH user for the test container +// func (c *SSHTestContainer) GetUser() string { +// return username +// } +// +// // Get the UID of the user inside the container. +// func (c *SSHTestContainer) GetUID() int { +// return c.uid +// } +// +// // Get the GID of the user inside the container. +// func (c *SSHTestContainer) GetGID() int { +// return c.gid +// } +// +// // GetHost returns the host address for the SSH connection +// func (c *SSHTestContainer) GetHost() string { +// return c.host +// } +// +// // GetDataDir returns the path to the container-controlled workspace directory +// func (c *SSHTestContainer) GetDataDir() string { +// return containerDataDir +// } +// +// // delete the mounted data dir from within the container to avoid permission issues +// func (c *SSHTestContainer) cleanupDataDir() error { +// +// logger := slog.Default() +// +// session, err := NewSSHSession( +// logger, +// c.GetUser(), +// c.host, +// c.sshPort, +// c.privateKey, +// "", +// false) // Don't log connection errors during cleanup +// if err != nil { +// return fmt.Errorf("failed to create SSH session: %w", err) +// } +// defer func() { _ = session.Close() }() +// +// require.NotEqual(c.t, "", containerDataDir, +// "if this is an empty string then we will attempt to 'rm -rf /*'... let's not do that") +// +// // Remove the entire workspace directory tree from inside the container +// // This ensures container-owned files are removed by the container user +// cleanupCmd := fmt.Sprintf("rm -rf %s/*", containerDataDir) +// stdout, stderr, err := session.Exec(cleanupCmd) +// if err != nil { +// return fmt.Errorf("failed to cleanup workspace: %w\nstdout: %s\nstderr: %s", err, stdout, stderr) +// } +// +// return nil +// } +// +// // Cleanup removes the Docker container and cleans up resources +// func (c *SSHTestContainer) Cleanup() { +// err := c.cleanupDataDir() +// require.NoError(c.t, err) +// +// // Use a context with timeout for cleanup operations +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// +// // Stop and remove container with timeout +// stopTimeout := 10 // seconds +// err = c.client.ContainerStop(ctx, c.containerID, container.StopOptions{ +// Timeout: &stopTimeout, +// }) +// if err != nil { +// // Log the error but continue with removal +// fmt.Printf("Warning: failed to stop container %s: %v\n", c.containerID, err) +// } +// +// // Remove container even if stop failed +// err = c.client.ContainerRemove(ctx, c.containerID, container.RemoveOptions{ +// Force: true, // Force removal even if container is still running +// }) +// require.NoError(c.t, err) +// } +// +// // GenerateSSHKeyPair creates an RSA key pair for testing +// func GenerateSSHKeyPair(privateKeyPath string, publicKeyPath string) error { +// privateKey, err := rsa.GenerateKey(rand.Reader, 2048) +// if err != nil { +// return fmt.Errorf("failed to generate private key: %w", err) +// } +// +// // Save private key +// privateKeyPEM := &pem.Block{ +// Type: "RSA PRIVATE KEY", +// Bytes: x509.MarshalPKCS1PrivateKey(privateKey), +// } +// +// privateKeyFile, err := os.Create(privateKeyPath) +// if err != nil { +// return fmt.Errorf("failed to create private key file: %w", err) +// } +// defer func() { _ = privateKeyFile.Close() }() +// +// err = pem.Encode(privateKeyFile, privateKeyPEM) +// if err != nil { +// return fmt.Errorf("failed to encode private key: %w", err) +// } +// +// err = os.Chmod(privateKeyPath, 0600) +// if err != nil { +// return fmt.Errorf("failed to set private key permissions: %w", err) +// } +// +// // Save public key +// publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) +// if err != nil { +// return fmt.Errorf("failed to create SSH public key: %w", err) +// } +// +// publicKeyBytes := ssh.MarshalAuthorizedKey(publicKey) +// err = os.WriteFile(publicKeyPath, publicKeyBytes, 0644) +// if err != nil { +// return fmt.Errorf("failed to write public key: %w", err) +// } +// +// return nil +// } +// +// // configureContainerSSHKey updates the container's SSH authorized_keys file with the test-specific public key +// func configureContainerSSHKey(ctx context.Context, cli *client.Client, containerID string, publicKeyPath string) error { +// publicKeyContent, err := os.ReadFile(publicKeyPath) +// if err != nil { +// return fmt.Errorf("failed to read public key: %w", err) +// } +// +// // Use base64 encoding to safely pass the SSH key content without shell escaping issues +// // Base64 encoding ensures no shell metacharacters can cause problems +// encodedKey := base64.StdEncoding.EncodeToString(publicKeyContent) +// +// execConfig := container.ExecOptions{ +// Cmd: []string{ +// "sh", "-c", +// fmt.Sprintf( +// "echo '%s' | base64 -d > /home/%s/.ssh/authorized_keys && chmod 600 /home/%s/.ssh/authorized_keys", +// encodedKey, username, username), +// }, +// } +// +// // Create the exec instance +// execIDResp, err := cli.ContainerExecCreate(ctx, containerID, execConfig) +// if err != nil { +// return fmt.Errorf("failed to create exec instance: %w", err) +// } +// +// // Start the exec instance with Detach: false to ensure it blocks until completion +// err = cli.ContainerExecStart(ctx, execIDResp.ID, container.ExecStartOptions{ +// Detach: false, // Explicitly set to false to block until completion +// }) +// if err != nil { +// return fmt.Errorf("failed to start exec instance: %w", err) +// } +// +// // With Detach: false, ContainerExecStart should block until completion. +// // However, to be absolutely certain, we'll add a brief polling loop. +// for i := 0; i < 10; i++ { // Max 10 attempts with 100ms intervals = 1 second max wait +// execInspect, err := cli.ContainerExecInspect(ctx, execIDResp.ID) +// if err != nil { +// return fmt.Errorf("failed to inspect exec instance: %w", err) +// } +// +// // If the command is no longer running, we can check the exit code +// if !execInspect.Running { +// // Check if the command was successful +// if execInspect.ExitCode != 0 { +// return fmt.Errorf("SSH key configuration command failed with exit code %d", execInspect.ExitCode) +// } +// return nil // Success! +// } +// +// // Brief sleep before checking again +// time.Sleep(10 * time.Millisecond) +// } +// +// // If still running after polling, something is wrong +// return fmt.Errorf("SSH key configuration command is still running after timeout") +// } +// +// // WaitForSSH waits for the SSH server to be ready +// func WaitForSSH(t *testing.T, sshPort uint64, privateKeyPath string) { +// logger := slog.Default() +// +// // Use a context with timeout to prevent indefinite hanging +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// +// ticker := time.NewTicker(500 * time.Millisecond) +// defer ticker.Stop() +// +// for { +// select { +// case <-ctx.Done(): +// require.Fail(t, "SSH server did not become ready within 30 seconds") +// return +// case <-ticker.C: +// session, err := NewSSHSession( +// logger, +// username, +// "localhost", +// sshPort, +// privateKeyPath, +// "", +// false) +// if err == nil { +// _ = session.Close() +// return +// } +// // Continue trying on error +// } +// } +// } +// +// // getOrBuildSharedSSHImage returns the name of the shared SSH test image. +// // If the image doesn't exist, it builds it. This method is thread-safe. +// func getOrBuildSharedSSHImage(ctx context.Context, cli *client.Client, t *testing.T) (string, error) { +// imageMutex.Lock() +// defer imageMutex.Unlock() +// +// // If we already have a cached image name, verify it still exists +// if sharedImageName != "" { +// _, err := cli.ImageInspect(ctx, sharedImageName) +// if err == nil { +// return sharedImageName, nil +// } +// // Image no longer exists, reset and rebuild +// sharedImageName = "" +// } +// +// // Get current user's UID/GID for the shared image +// uid, err := getCurrentUserUID() +// if err != nil { +// return "", fmt.Errorf("failed to get current user UID: %w", err) +// } +// gid, err := getCurrentUserGID() +// if err != nil { +// return "", fmt.Errorf("failed to get current user GID: %w", err) +// } +// +// // Generate a unique image name based on UID/GID and current time to avoid conflicts +// imageName := fmt.Sprintf("ssh-test-shared:%d-%d-%d", uid, gid, time.Now().Unix()) +// +// // Create a temporary directory for building the image +// tempDir := t.TempDir() +// privateKeyPath := filepath.Join(tempDir, "shared_ssh_key") +// publicKeyPath := filepath.Join(tempDir, "shared_ssh_key.pub") +// +// // Generate SSH key pair for the shared image +// err = GenerateSSHKeyPair(privateKeyPath, publicKeyPath) +// if err != nil { +// return "", fmt.Errorf("failed to generate SSH key pair: %w", err) +// } +// +// publicKeyContent, err := os.ReadFile(publicKeyPath) +// if err != nil { +// return "", fmt.Errorf("failed to read public key: %w", err) +// } +// +// // Build the shared image +// t.Logf("Building shared SSH test Docker image: %s", imageName) +// err = BuildSSHTestImage(ctx, cli, tempDir, imageName, string(publicKeyContent), uid, gid) +// if err != nil { +// return "", fmt.Errorf("failed to build shared SSH image: %w", err) +// } +// +// // Cache the image name for future use +// sharedImageName = imageName +// return sharedImageName, nil +// } +// +// // SetupSSHTestContainer creates and starts a Docker container with SSH server +// // If dataDir is not empty, it will be mounted in the container at /mnt/data +// func SetupSSHTestContainer(t *testing.T, dataDir string) *SSHTestContainer { +// // Use a longer timeout for the entire setup process to handle slow CI environments +// ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) +// defer cancel() +// +// // Get current user's UID/GID +// uid, err := getCurrentUserUID() +// require.NoError(t, err) +// gid, err := getCurrentUserGID() +// require.NoError(t, err) +// +// // Create Docker client +// cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) +// require.NoError(t, err) +// +// // Generate SSH key pair for this specific test +// tempDir := t.TempDir() +// privateKeyPath := filepath.Join(tempDir, "test_ssh_key") +// publicKeyPath := filepath.Join(tempDir, "test_ssh_key.pub") +// +// err = GenerateSSHKeyPair(privateKeyPath, publicKeyPath) +// require.NoError(t, err) +// +// // Get or build the shared SSH test image +// imageName, err := getOrBuildSharedSSHImage(ctx, cli, t) +// require.NoError(t, err) +// +// if dataDir != "" { +// // we have to grant broad permissions here because the container may have a different UID +// err = os.Chmod(dataDir, 0777) +// require.NoError(t, err, "failed to set permissions on data directory") +// } +// +// // Start container and configure it with the test-specific SSH key +// containerID, sshPort, err := StartSSHContainer(ctx, cli, imageName, dataDir, t.Name()) +// require.NoError(t, err) +// +// // Configure the container to use the test-specific SSH key +// err = configureContainerSSHKey(ctx, cli, containerID, publicKeyPath) +// require.NoError(t, err) +// +// // Wait for SSH to be ready +// WaitForSSH(t, sshPort, privateKeyPath) +// +// return &SSHTestContainer{ +// t: t, +// client: cli, +// containerID: containerID, +// sshPort: sshPort, +// tempDir: tempDir, +// privateKey: privateKeyPath, +// publicKey: publicKeyPath, +// host: "localhost", +// uid: uid, +// gid: gid, +// } +// } +// +// // BuildSSHTestImage builds the SSH test image with the provided public key and user IDs +// func BuildSSHTestImage( +// ctx context.Context, +// cli *client.Client, +// tempDir string, +// imageName string, +// publicKey string, +// uid int, +// gid int, +// ) error { +// +// // Get the Dockerfile path +// _, currentFile, _, ok := runtime.Caller(0) +// if !ok { +// return fmt.Errorf("failed to get current file path") +// } +// dockerfilePath := filepath.Join(filepath.Dir(currentFile), "testdata", "ssh-test.Dockerfile") +// +// // Create build context directory +// buildContext := filepath.Join(tempDir, "docker_build") +// err := os.MkdirAll(buildContext, 0755) +// if err != nil { +// return fmt.Errorf("failed to create build context: %w", err) +// } +// +// // Copy Dockerfile to build context +// dockerfileContent, err := os.ReadFile(dockerfilePath) +// if err != nil { +// return fmt.Errorf("failed to read Dockerfile: %w", err) +// } +// +// // Copy start.sh script to build context +// startScriptPath := filepath.Join(filepath.Dir(currentFile), "testdata", "start.sh") +// startScriptContent, err := os.ReadFile(startScriptPath) +// if err != nil { +// return fmt.Errorf("failed to read start.sh script: %w", err) +// } +// err = os.WriteFile(filepath.Join(buildContext, "start.sh"), startScriptContent, 0755) +// if err != nil { +// return fmt.Errorf("failed to copy start.sh to build context: %w", err) +// } +// +// // Add the public key setup to the Dockerfile +// publicKeySetup := fmt.Sprintf( +// "\n# Add test SSH public key\n"+ +// "RUN echo '%s' > /home/testuser/.ssh/authorized_keys\n"+ +// "RUN chmod 600 /home/testuser/.ssh/authorized_keys\n"+ +// "RUN chown %d:%d /home/testuser/.ssh/authorized_keys\n", strings.TrimSpace(publicKey), uid, gid) +// modifiedDockerfile := string(dockerfileContent) + publicKeySetup +// +// err = os.WriteFile(filepath.Join(buildContext, "Dockerfile"), []byte(modifiedDockerfile), 0644) +// if err != nil { +// return fmt.Errorf("failed to write modified Dockerfile: %w", err) +// } +// +// // Create tar archive for build context +// buildCtx, err := ArchiveDirectory(buildContext) +// if err != nil { +// return fmt.Errorf("failed to create build context archive: %w", err) +// } +// defer func() { _ = buildCtx.Close() }() +// +// // Build the image with optimized settings for CI +// buildOptions := types.ImageBuildOptions{ +// Tags: []string{imageName}, +// Dockerfile: "Dockerfile", +// Remove: true, +// ForceRemove: true, +// NoCache: false, // Allow caching to speed up builds +// BuildArgs: map[string]*string{ +// "USER_UID": &[]string{strconv.Itoa(uid)}[0], +// "USER_GID": &[]string{strconv.Itoa(gid)}[0], +// }, +// } +// +// response, err := cli.ImageBuild(ctx, buildCtx, buildOptions) +// if err != nil { +// return fmt.Errorf("failed to build image: %w", err) +// } +// defer func() { _ = response.Body.Close() }() +// +// // Read build output with proper error handling for timeouts +// // Create a buffer to capture build output for debugging on failure +// var buildOutput strings.Builder +// reader := io.TeeReader(response.Body, &buildOutput) +// +// _, err = io.Copy(io.Discard, reader) +// if err != nil { +// // Include build output in error for debugging +// buildOutputStr := buildOutput.String() +// if len(buildOutputStr) > 1000 { +// buildOutputStr = buildOutputStr[:1000] + "... (truncated)" +// } +// return fmt.Errorf("failed to read build response: %w\nBuild output: %s", err, buildOutputStr) +// } +// +// // After the build finishes, verify the image actually exists +// _, err = cli.ImageInspect(ctx, imageName) +// if err != nil { +// buildOutputStr := buildOutput.String() +// if len(buildOutputStr) > 2000 { +// buildOutputStr = buildOutputStr[:2000] + "... (truncated)" +// } +// return fmt.Errorf("docker image build failed - image not found after build: %w\nBuild output: %s", +// err, buildOutputStr) +// } +// +// return nil +// } +// +// // StartSSHContainer starts the SSH container and returns container ID and SSH port +// // If dataDir is not empty, it will be mounted at /mnt/data in the container +// func StartSSHContainer( +// ctx context.Context, +// cli *client.Client, +// imageName string, +// dataDir string, +// testName string, +// ) (string, uint64, error) { +// +// // Get a unique port for this test based on test name hash +// sshPort, err := GetUniqueSSHTestPort(testName) +// if err != nil { +// return "", 0, fmt.Errorf("failed to get unique SSH port: %w", err) +// } +// +// containerConfig := &container.Config{ +// Image: imageName, +// ExposedPorts: nat.PortSet{ +// "22/tcp": struct{}{}, +// }, +// } +// +// hostConfig := &container.HostConfig{ +// PortBindings: nat.PortMap{ +// "22/tcp": []nat.PortBinding{ +// { +// HostIP: "127.0.0.1", +// HostPort: strconv.Itoa(sshPort), // Use custom port to avoid collisions in CI +// }, +// }, +// }, +// Mounts: func() []mount.Mount { +// var mounts []mount.Mount +// if dataDir != "" { +// mounts = append(mounts, mount.Mount{ +// Type: mount.TypeBind, +// Source: dataDir, +// Target: "/mnt/data", +// }) +// } +// return mounts +// }(), +// } +// +// // Create a container name that includes the test name for easier debugging +// containerName := fmt.Sprintf("ssh-test-%s-%d", +// strings.ReplaceAll(testName, "/", "-"), time.Now().Unix()) +// +// resp, err := cli.ContainerCreate( +// ctx, +// containerConfig, +// hostConfig, +// nil, +// nil, +// containerName) +// if err != nil { +// return "", 0, fmt.Errorf("failed to create container: %w", err) +// } +// +// err = cli.ContainerStart(ctx, resp.ID, container.StartOptions{}) +// if err != nil { +// return "", 0, fmt.Errorf("failed to start container: %w", err) +// } +// +// // Use the custom SSH port (convert to uint64 for compatibility) +// return resp.ID, uint64(sshPort), nil +// } +// +// // ArchiveDirectory creates a tar.gz archive of a directory for Docker build context +// func ArchiveDirectory(srcDir string) (io.ReadCloser, error) { +// pr, pw := io.Pipe() +// +// go func() { +// defer func() { _ = pw.Close() }() +// +// gw := gzip.NewWriter(pw) +// defer func() { _ = gw.Close() }() +// +// tw := tar.NewWriter(gw) +// defer func() { _ = tw.Close() }() +// +// _ = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { +// if err != nil { +// return err +// } +// +// relPath, err := filepath.Rel(srcDir, path) +// if err != nil { +// return fmt.Errorf("failed to get relative path: %w", err) +// } +// +// // Skip the root directory itself +// if relPath == "." { +// return nil +// } +// +// header, err := tar.FileInfoHeader(info, "") +// if err != nil { +// return fmt.Errorf("failed to create tar header: %w", err) +// } +// header.Name = relPath +// +// if err := tw.WriteHeader(header); err != nil { +// return fmt.Errorf("failed to write tar header for %s: %w", relPath, err) +// } +// +// if info.IsDir() { +// return nil +// } +// +// file, err := os.Open(path) +// if err != nil { +// return fmt.Errorf("failed to open file %s: %w", path, err) +// } +// defer func() { _ = file.Close() }() +// +// _, err = io.Copy(tw, file) +// if err != nil { +// return fmt.Errorf("failed to copy file %s to tar: %w", path, err) +// } +// return nil +// }) +// }() +// +// return pr, nil +// } +// diff --git a/sei-db/db_engine/litt/util/test_assertions.go b/sei-db/db_engine/litt/util/test_assertions.go index 8fc2c719db..beec336d3d 100644 --- a/sei-db/db_engine/litt/util/test_assertions.go +++ b/sei-db/db_engine/litt/util/test_assertions.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/test_logger.go b/sei-db/db_engine/litt/util/test_logger.go index 065a6c0947..3c6a2e578f 100644 --- a/sei-db/db_engine/litt/util/test_logger.go +++ b/sei-db/db_engine/litt/util/test_logger.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import ( diff --git a/sei-db/db_engine/litt/util/test_random.go b/sei-db/db_engine/litt/util/test_random.go index fb4968b6b9..7ef7d02608 100644 --- a/sei-db/db_engine/litt/util/test_random.go +++ b/sei-db/db_engine/litt/util/test_random.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - // TODO: this file is duplicated at sei-db/common/utils/test_random.go (without // the build tag) so that non-litt packages can reuse it without depending on // littdb_wip. When the littdb_wip build tag is removed, dedup by deleting this diff --git a/sei-db/db_engine/litt/util/thread_safe_cache.go b/sei-db/db_engine/litt/util/thread_safe_cache.go index 5d4bca572f..77bf498093 100644 --- a/sei-db/db_engine/litt/util/thread_safe_cache.go +++ b/sei-db/db_engine/litt/util/thread_safe_cache.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "sync" diff --git a/sei-db/db_engine/litt/util/units.go b/sei-db/db_engine/litt/util/units.go index c037672495..95f424d389 100644 --- a/sei-db/db_engine/litt/util/units.go +++ b/sei-db/db_engine/litt/util/units.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "fmt" diff --git a/sei-db/db_engine/litt/util/unsafe_string.go b/sei-db/db_engine/litt/util/unsafe_string.go index e5b05f078f..238dfd2892 100644 --- a/sei-db/db_engine/litt/util/unsafe_string.go +++ b/sei-db/db_engine/litt/util/unsafe_string.go @@ -1,5 +1,3 @@ -//go:build littdb_wip - package util import "unsafe" From 1722a7c2c763bae46322a761ecefaa97ede188b5 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 14 May 2026 10:44:21 -0500 Subject: [PATCH 2/2] fix lint --- sei-db/db_engine/litt/Makefile | 26 +++++++++++++++++++ .../litt/benchmark/benchmark_engine.go | 10 +++---- .../litt/benchmark/benchmark_metrics.go | 26 +++++++++---------- sei-db/db_engine/litt/benchmark/cohort.go | 8 +++--- .../litt/benchmark/config/benchmark_config.go | 2 +- .../litt/benchmark/data_generator.go | 14 +++++----- .../db_engine/litt/benchmark/data_tracker.go | 14 +++++----- sei-db/db_engine/litt/cli/prune.go | 2 +- sei-db/db_engine/litt/cli/table_info.go | 4 +-- .../db_engine/litt/disktable/boundary_file.go | 2 +- sei-db/db_engine/litt/disktable/disk_table.go | 8 +++--- .../litt/disktable/keymap/keymap_type_file.go | 4 +-- .../litt/disktable/keymap/pebble_db_keymap.go | 2 +- .../litt/disktable/segment/key_file.go | 10 +++---- .../litt/disktable/segment/metadata_file.go | 6 ++--- .../litt/disktable/segment/segment.go | 8 +++--- .../litt/disktable/segment/segment_scanner.go | 9 ++++--- .../litt/disktable/segment/value_file.go | 16 ++++++------ .../litt/disktable/table_metadata.go | 6 ++--- .../db_engine/litt/littbuilder/build_utils.go | 14 +++++----- sei-db/db_engine/litt/util/file_lock.go | 10 +++---- sei-db/db_engine/litt/util/file_utils.go | 12 ++++----- sei-db/db_engine/litt/util/pprof.go | 9 +++++-- .../litt/util/random_access_deque.go | 4 +-- sei-db/db_engine/litt/util/ssh.go | 6 ++--- sei-db/db_engine/litt/util/unsafe_string.go | 2 +- 26 files changed, 134 insertions(+), 100 deletions(-) create mode 100644 sei-db/db_engine/litt/Makefile diff --git a/sei-db/db_engine/litt/Makefile b/sei-db/db_engine/litt/Makefile new file mode 100644 index 0000000000..d1dee01953 --- /dev/null +++ b/sei-db/db_engine/litt/Makefile @@ -0,0 +1,26 @@ +SHELL := /bin/bash + +# Build the litt CLI tool. +.PHONY: build +build: + go build -o ./bin/litt ./cli + +# Remove the bin directory if it exists. +.PHONY: clean +clean: + rm -rf ./bin + +# Build the litt CLI tool with debug flags. +.PHONY: debug-build +debug-build: clean + go build -gcflags "all=-N -l" -o ./bin/litt ./cli + +# Run all LittDB unit tests. +.PHONY: test +test: build + go test ./... -timeout=10m -v -p=1 -parallel=8 + +# Run all LittDB unit tests with verbose output. +.PHONY: test-verbose +test-verbose: build + go test ./... -v -timeout=10m -p=1 -parallel=8 diff --git a/sei-db/db_engine/litt/benchmark/benchmark_engine.go b/sei-db/db_engine/litt/benchmark/benchmark_engine.go index b699e59dbd..8e3f955996 100644 --- a/sei-db/db_engine/litt/benchmark/benchmark_engine.go +++ b/sei-db/db_engine/litt/benchmark/benchmark_engine.go @@ -66,7 +66,7 @@ func NewBenchmarkEngine(configPath string) (*BenchmarkEngine, error) { cfg.LittConfig.Logger = slog.Default() } - cfg.LittConfig.ShardingFactor = uint32(len(cfg.LittConfig.Paths)) + cfg.LittConfig.ShardingFactor = uint32(len(cfg.LittConfig.Paths)) //nolint:gosec // path count bounded db, err := littbuilder.NewDB(cfg.LittConfig) if err != nil { @@ -95,7 +95,7 @@ func NewBenchmarkEngine(configPath string) (*BenchmarkEngine, error) { } writeBytesPerSecond := uint64(cfg.MaximumWriteThroughputMB * float64(unit.MB)) - writeBytesPerSecondPerThread := writeBytesPerSecond / uint64(cfg.WriterParallelism) + writeBytesPerSecondPerThread := writeBytesPerSecond / uint64(cfg.WriterParallelism) //nolint:gosec // parallelism positive // If we set the write burst size smaller than an individual value, then the rate limiter will never // permit any writes. Ideally, we'd just set the burst size to 0 since we don't want bursty/volatile writes, @@ -104,7 +104,7 @@ func NewBenchmarkEngine(configPath string) (*BenchmarkEngine, error) { writeBurstSize := uint64(cfg.ValueSizeMB * float64(unit.MB)) readBytesPerSecond := uint64(cfg.MaximumReadThroughputMB * float64(unit.MB)) - readBytesPerSecondPerThread := readBytesPerSecond / uint64(cfg.ReaderParallelism) + readBytesPerSecondPerThread := readBytesPerSecond / uint64(cfg.ReaderParallelism) //nolint:gosec // parallelism positive // If we set the read burst size smaller than an individual value we need to read, then the rate limiter will // never permit us to read that value. @@ -191,7 +191,7 @@ func (b *BenchmarkEngine) Run() error { // writer runs on a goroutine and writes data to the database. func (b *BenchmarkEngine) writer() { maxBatchSize := uint64(b.config.BatchSizeMB * float64(unit.MB)) - throttle := rate.NewLimiter(rate.Limit(b.writeBytesPerSecondPerThread), int(b.writeBurstSize)) + throttle := rate.NewLimiter(rate.Limit(b.writeBytesPerSecondPerThread), int(b.writeBurstSize)) //nolint:gosec // burst sized by config for { select { @@ -260,7 +260,7 @@ func (b *BenchmarkEngine) verifyValue(expected *ReadInfo, actual []byte) error { // reader runs on a goroutine and reads data from the database. func (b *BenchmarkEngine) reader() { - throttle := rate.NewLimiter(rate.Limit(b.readBytesPerSecondPerThread), int(b.readBurstSize)) + throttle := rate.NewLimiter(rate.Limit(b.readBytesPerSecondPerThread), int(b.readBurstSize)) //nolint:gosec // burst sized by config for { select { diff --git a/sei-db/db_engine/litt/benchmark/benchmark_metrics.go b/sei-db/db_engine/litt/benchmark/benchmark_metrics.go index e7e4c18a4c..034c6eeb85 100644 --- a/sei-db/db_engine/litt/benchmark/benchmark_metrics.go +++ b/sei-db/db_engine/litt/benchmark/benchmark_metrics.go @@ -79,12 +79,12 @@ func newMetrics( func (m *metrics) reportWrite(writeDuration time.Duration, bytesWritten uint64) { m.writeCount.Add(1) m.bytesWritten.Add(bytesWritten) - m.nanosecondsSpentWriting.Add(uint64(writeDuration.Nanoseconds())) + m.nanosecondsSpentWriting.Add(uint64(writeDuration.Nanoseconds())) //nolint:gosec // duration non-negative // Update the longest write duration if this one is longer. currentLongest := m.longestWriteDuration.Load() - for writeDuration.Nanoseconds() > int64(currentLongest) { - swapped := m.longestWriteDuration.CompareAndSwap(currentLongest, uint64(writeDuration.Nanoseconds())) + for writeDuration.Nanoseconds() > int64(currentLongest) { //nolint:gosec // durations comfortably fit + swapped := m.longestWriteDuration.CompareAndSwap(currentLongest, uint64(writeDuration.Nanoseconds())) //nolint:gosec // duration non-negative if swapped { break } @@ -96,12 +96,12 @@ func (m *metrics) reportWrite(writeDuration time.Duration, bytesWritten uint64) func (m *metrics) reportRead(readDuration time.Duration, bytesRead uint64) { m.readCount.Add(1) m.bytesRead.Add(bytesRead) - m.nanosecondsSpentReading.Add(uint64(readDuration.Nanoseconds())) + m.nanosecondsSpentReading.Add(uint64(readDuration.Nanoseconds())) //nolint:gosec // duration non-negative // Update the longest read duration if this one is longer. currentLongest := m.longestReadDuration.Load() - for readDuration.Nanoseconds() > int64(currentLongest) { - swapped := m.longestReadDuration.CompareAndSwap(currentLongest, uint64(readDuration.Nanoseconds())) + for readDuration.Nanoseconds() > int64(currentLongest) { //nolint:gosec // durations comfortably fit + swapped := m.longestReadDuration.CompareAndSwap(currentLongest, uint64(readDuration.Nanoseconds())) //nolint:gosec // duration non-negative if swapped { break } @@ -112,12 +112,12 @@ func (m *metrics) reportRead(readDuration time.Duration, bytesRead uint64) { // reportFlush records a flush operation. func (m *metrics) reportFlush(flushDuration time.Duration) { m.flushCount.Add(1) - m.nanosecondsSpentFlushing.Add(uint64(flushDuration.Nanoseconds())) + m.nanosecondsSpentFlushing.Add(uint64(flushDuration.Nanoseconds())) //nolint:gosec // duration non-negative // Update the longest flush duration if this one is longer. currentLongest := m.longestFlushDuration.Load() - for flushDuration.Nanoseconds() > int64(currentLongest) { - swapped := m.longestFlushDuration.CompareAndSwap(currentLongest, uint64(flushDuration.Nanoseconds())) + for flushDuration.Nanoseconds() > int64(currentLongest) { //nolint:gosec // durations comfortably fit + swapped := m.longestFlushDuration.CompareAndSwap(currentLongest, uint64(flushDuration.Nanoseconds())) //nolint:gosec // duration non-negative if swapped { break } @@ -151,24 +151,24 @@ func (m *metrics) logMetrics() { writeCount := m.writeCount.Load() if writeCount > 0 { averageWriteLatency = - uint64((time.Duration(m.nanosecondsSpentWriting.Load()) / time.Duration(writeCount)).Nanoseconds()) + uint64((time.Duration(m.nanosecondsSpentWriting.Load()) / time.Duration(writeCount)).Nanoseconds()) //nolint:gosec // duration non-negative } averageReadLatency := uint64(0) readCount := m.readCount.Load() if readCount > 0 { averageReadLatency = - uint64((time.Duration(m.nanosecondsSpentReading.Load()) / time.Duration(readCount)).Nanoseconds()) + uint64((time.Duration(m.nanosecondsSpentReading.Load()) / time.Duration(readCount)).Nanoseconds()) //nolint:gosec // duration non-negative } averageFlushLatency := uint64(0) flushCount := m.flushCount.Load() if flushCount > 0 { averageFlushLatency = - uint64((time.Duration(m.nanosecondsSpentFlushing.Load()) / time.Duration(flushCount)).Nanoseconds()) + uint64((time.Duration(m.nanosecondsSpentFlushing.Load()) / time.Duration(flushCount)).Nanoseconds()) //nolint:gosec // duration non-negative } - elapsedTimeNanoseconds := uint64(time.Since(m.startTime).Nanoseconds()) + elapsedTimeNanoseconds := uint64(time.Since(m.startTime).Nanoseconds()) //nolint:gosec // duration non-negative elapsedTimeSeconds := float64(elapsedTimeNanoseconds) / float64(time.Second) bytesWritten := m.bytesWritten.Load() diff --git a/sei-db/db_engine/litt/benchmark/cohort.go b/sei-db/db_engine/litt/benchmark/cohort.go index 6cb6cc061d..1f37932a1c 100644 --- a/sei-db/db_engine/litt/benchmark/cohort.go +++ b/sei-db/db_engine/litt/benchmark/cohort.go @@ -142,12 +142,12 @@ func LoadCohort(path string) (*Cohort, error) { return nil, fmt.Errorf("cohort file does not exist: %s", filePath) } - file, err := os.Open(filePath) + file, err := os.Open(filePath) //nolint:gosec // path within cohort directory if err != nil { return nil, fmt.Errorf("failed to open cohort file: %w", err) } - data, err := os.ReadFile(filePath) + data, err := os.ReadFile(filePath) //nolint:gosec // path within cohort directory if err != nil { return nil, fmt.Errorf("failed to read cohort file: %w", err) } @@ -316,7 +316,7 @@ func (c *Cohort) serialize() []byte { binary.BigEndian.PutUint64(data[8:16], c.lowKeyIndex) binary.BigEndian.PutUint64(data[16:24], c.highKeyIndex) binary.BigEndian.PutUint64(data[24:32], c.valueSize) - binary.BigEndian.PutUint64(data[32:40], uint64(c.firstValueTimestamp.Unix())) + binary.BigEndian.PutUint64(data[32:40], uint64(c.firstValueTimestamp.Unix())) //nolint:gosec // wall-clock seconds non-negative if c.allValuesWritten { data[40] = 1 } else { @@ -343,7 +343,7 @@ func (c *Cohort) deserialize(data []byte) error { return fmt.Errorf("invalid index range: %d >= %d", c.lowKeyIndex, c.highKeyIndex) } - c.firstValueTimestamp = time.Unix(int64(binary.BigEndian.Uint64(data[32:40])), 0) + c.firstValueTimestamp = time.Unix(int64(binary.BigEndian.Uint64(data[32:40])), 0) //nolint:gosec // wall-clock seconds fit int64 c.allValuesWritten = data[40] == 1 return nil diff --git a/sei-db/db_engine/litt/benchmark/config/benchmark_config.go b/sei-db/db_engine/litt/benchmark/config/benchmark_config.go index 5221e1aef2..df2e6bf91f 100644 --- a/sei-db/db_engine/litt/benchmark/config/benchmark_config.go +++ b/sei-db/db_engine/litt/benchmark/config/benchmark_config.go @@ -127,7 +127,7 @@ func LoadConfig(path string) (*BenchmarkConfig, error) { } // Read the file - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // caller-supplied config path if err != nil { return nil, fmt.Errorf("failed to read config file: %w", err) } diff --git a/sei-db/db_engine/litt/benchmark/data_generator.go b/sei-db/db_engine/litt/benchmark/data_generator.go index 2e53819933..9984323a81 100644 --- a/sei-db/db_engine/litt/benchmark/data_generator.go +++ b/sei-db/db_engine/litt/benchmark/data_generator.go @@ -29,7 +29,7 @@ func NewDataGenerator(seed int64, poolSize uint64) *DataGenerator { dataPool := make([]byte, poolSize) rng := randPool.Get().(*rand.Rand) - rng.Read(dataPool) + _, _ = rng.Read(dataPool) randPool.Put(rng) return &DataGenerator{ @@ -41,10 +41,10 @@ func NewDataGenerator(seed int64, poolSize uint64) *DataGenerator { // Key generates a new key. The value is deterministic for the same index and seed. func (g *DataGenerator) Key(index uint64) []byte { rng := g.randPool.Get().(*rand.Rand) - rng.Seed(g.seed + int64(index)) + rng.Seed(g.seed + int64(index)) //nolint:gosec // deterministic test seeding key := make([]byte, 32) - rng.Read(key) + _, _ = rng.Read(key) g.randPool.Put(rng) return key @@ -53,7 +53,7 @@ func (g *DataGenerator) Key(index uint64) []byte { // Value generates a new value. The value is deterministic for the same index, seed, and value size. func (g *DataGenerator) Value(index uint64, valueLength uint64) []byte { rng := g.randPool.Get().(*rand.Rand) - rng.Seed(g.seed + int64(index)) + rng.Seed(g.seed + int64(index)) //nolint:gosec // deterministic test seeding var value []byte @@ -62,10 +62,10 @@ func (g *DataGenerator) Value(index uint64, valueLength uint64) []byte { // For the sake of completeness, just generate the data if this happens. // This shouldn't be encountered for sane configurations (i.e. with a pool size much larger than value sizes). value = make([]byte, valueLength) - rng.Read(value) + _, _ = rng.Read(value) } else { - startIndex := rng.Intn(len(g.dataPool) - int(valueLength)) - value = g.dataPool[startIndex : startIndex+int(valueLength)] + startIndex := rng.Intn(len(g.dataPool) - int(valueLength)) //nolint:gosec // valueLength bounded by len(dataPool) + value = g.dataPool[startIndex : startIndex+int(valueLength)] //nolint:gosec // valueLength bounded by len(dataPool) } g.randPool.Put(rng) diff --git a/sei-db/db_engine/litt/benchmark/data_tracker.go b/sei-db/db_engine/litt/benchmark/data_tracker.go index a255b42351..e72e8e7478 100644 --- a/sei-db/db_engine/litt/benchmark/data_tracker.go +++ b/sei-db/db_engine/litt/benchmark/data_tracker.go @@ -185,8 +185,8 @@ func NewDataTracker( activeCohort: activeCohort, lowestCohortIndex: lowestCohortIndex, highestCohortIndex: highestCohortIndex, - highestWrittenKeyIndex: int64(activeCohort.LowKeyIndex()) - 1, - highestWrittenCohortIndex: int64(highestCohortIndex) - 1, + highestWrittenKeyIndex: int64(activeCohort.LowKeyIndex()) - 1, //nolint:gosec // indices fit int64 + highestWrittenCohortIndex: int64(highestCohortIndex) - 1, //nolint:gosec // indices fit int64 safeTTL: safeTTL, valueSize: valueSize, generator: NewDataGenerator(config.Seed, config.RandomPoolSize), @@ -405,10 +405,10 @@ func (t *DataTracker) handleWrittenKey(keyIndex uint64) { // Determine the highest key index written so far that also has all lower key indices written. for { - nextKeyIndex := uint64(t.highestWrittenKeyIndex + 1) + nextKeyIndex := uint64(t.highestWrittenKeyIndex + 1) //nolint:gosec // index non-negative if _, ok := t.writtenKeysSet[nextKeyIndex]; ok { // The next key has been written, mark it as such. - t.highestWrittenKeyIndex = int64(nextKeyIndex) + t.highestWrittenKeyIndex = int64(nextKeyIndex) //nolint:gosec // index fits int64 delete(t.writtenKeysSet, nextKeyIndex) } else { // Once we find the first key that has not been written, we can stop checking. @@ -420,15 +420,15 @@ func (t *DataTracker) handleWrittenKey(keyIndex uint64) { // Determine the highest cohort index written so far that also has all lower cohorts written. for { - nextCohortIndex := uint64(t.highestWrittenCohortIndex + 1) + nextCohortIndex := uint64(t.highestWrittenCohortIndex + 1) //nolint:gosec // index non-negative if nextCohortIndex >= t.activeCohort.CohortIndex() { // Don't ever mark the active cohort as complete. break } nextCohort := t.cohorts[nextCohortIndex] - if int64(nextCohort.HighKeyIndex()) <= t.highestWrittenKeyIndex { + if int64(nextCohort.HighKeyIndex()) <= t.highestWrittenKeyIndex { //nolint:gosec // index fits int64 // We've found a cohort that has all keys written. - t.highestWrittenCohortIndex = int64(nextCohort.CohortIndex()) + t.highestWrittenCohortIndex = int64(nextCohort.CohortIndex()) //nolint:gosec // index fits int64 t.completeCohortSet[nextCohort.CohortIndex()] = struct{}{} err := nextCohort.MarkComplete() if err != nil { diff --git a/sei-db/db_engine/litt/cli/prune.go b/sei-db/db_engine/litt/cli/prune.go index 6142f3203e..240874a4ab 100644 --- a/sei-db/db_engine/litt/cli/prune.go +++ b/sei-db/db_engine/litt/cli/prune.go @@ -147,7 +147,7 @@ func pruneTable( seg := segments[segmentIndex] segmentAge := time.Since(seg.GetSealTime()) - if segmentAge < time.Duration(maxAgeSeconds)*time.Second { + if segmentAge < time.Duration(maxAgeSeconds)*time.Second { //nolint:gosec // CLI flag bounded // We've pruned all segments that we can. break } diff --git a/sei-db/db_engine/litt/cli/table_info.go b/sei-db/db_engine/litt/cli/table_info.go index 560d92766e..db90f4ebd5 100644 --- a/sei-db/db_engine/litt/cli/table_info.go +++ b/sei-db/db_engine/litt/cli/table_info.go @@ -64,8 +64,8 @@ func tableInfoCommand(ctx *cli.Context) error { return fmt.Errorf("failed to get table info for table %s at paths %v: %w", tableName, sources, err) } - oldestSegmentAge := uint64(time.Since(info.OldestSegmentSealTime).Nanoseconds()) - newestSegmentAge := uint64(time.Since(info.NewestSegmentSealTime).Nanoseconds()) + oldestSegmentAge := uint64(time.Since(info.OldestSegmentSealTime).Nanoseconds()) //nolint:gosec // duration non-negative + newestSegmentAge := uint64(time.Since(info.NewestSegmentSealTime).Nanoseconds()) //nolint:gosec // duration non-negative segmentSpan := oldestSegmentAge - newestSegmentAge logger.Info("Table", "name", tableName) diff --git a/sei-db/db_engine/litt/disktable/boundary_file.go b/sei-db/db_engine/litt/disktable/boundary_file.go index cb4ce75f8f..76a645d6f2 100644 --- a/sei-db/db_engine/litt/disktable/boundary_file.go +++ b/sei-db/db_engine/litt/disktable/boundary_file.go @@ -137,7 +137,7 @@ func (b *BoundaryFile) deserialize(data []byte) error { if err != nil { return fmt.Errorf("failed to parse boundary index from data: %v", err) } - b.boundaryIndex = uint32(boundaryIndex) + b.boundaryIndex = uint32(boundaryIndex) //nolint:gosec // boundary index fits uint32 return nil } diff --git a/sei-db/db_engine/litt/disktable/disk_table.go b/sei-db/db_engine/litt/disktable/disk_table.go index f7f48d33a4..8f1c239236 100644 --- a/sei-db/db_engine/litt/disktable/disk_table.go +++ b/sei-db/db_engine/litt/disktable/disk_table.go @@ -314,7 +314,7 @@ func NewDiskTable( } func (d *DiskTable) KeyCount() uint64 { - return uint64(d.keyCount.Load()) + return uint64(d.keyCount.Load()) //nolint:gosec // key count non-negative } func (d *DiskTable) Size() uint64 { @@ -366,7 +366,7 @@ func (d *DiskTable) repairSnapshot( } } - err = os.MkdirAll(symlinkSegmentsDirectory, 0755) + err = os.MkdirAll(symlinkSegmentsDirectory, 0750) if err != nil { return nil, fmt.Errorf("failed to create symlink segments directory: %w", err) } @@ -457,12 +457,12 @@ func (d *DiskTable) reloadKeymap( // Now that the keymap is loaded, write the marker file that indicates that the keymap is fully loaded. // If we crash prior to writing this file, the keymap will reload from the segments again. keymapInitializedFile := path.Join(d.keymapPath, keymap.KeymapInitializedFileName) - err := os.MkdirAll(d.keymapPath, 0755) + err := os.MkdirAll(d.keymapPath, 0750) if err != nil { return fmt.Errorf("failed to create keymap directory: %w", err) } - f, err := os.Create(keymapInitializedFile) + f, err := os.Create(keymapInitializedFile) //nolint:gosec // path within keymap directory if err != nil { return fmt.Errorf("failed to create keymap initialized file after reload: %w", err) } diff --git a/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go b/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go index d36962460d..05b3a3ad02 100644 --- a/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go +++ b/sei-db/db_engine/litt/disktable/keymap/keymap_type_file.go @@ -42,7 +42,7 @@ func LoadKeymapTypeFile(keymapPath string) (*KeymapTypeFile, error) { return nil, fmt.Errorf("keymap type file does not exist: %v", filePath) } - fileContents, err := os.ReadFile(filePath) + fileContents, err := os.ReadFile(filePath) //nolint:gosec // path within keymap directory if err != nil { return nil, fmt.Errorf("unable to read keymap type file: %v", err) } @@ -83,7 +83,7 @@ func (k *KeymapTypeFile) Write() error { return fmt.Errorf("keymap type file already exists: %v", filePath) } - keymapFile, err := os.Create(filePath) + keymapFile, err := os.Create(filePath) //nolint:gosec // path within keymap directory if err != nil { return fmt.Errorf("unable to create keymap type file: %v", err) } diff --git a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go index 697c824ab9..b7f6abf281 100644 --- a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go +++ b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go @@ -63,7 +63,7 @@ func newPebbleDBKeymap( } if !exists { - err = os.MkdirAll(keymapPath, 0755) + err = os.MkdirAll(keymapPath, 0750) if err != nil { return nil, false, fmt.Errorf("error creating keymap directory: %w", err) } diff --git a/sei-db/db_engine/litt/disktable/segment/key_file.go b/sei-db/db_engine/litt/disktable/segment/key_file.go index f9a610507e..7bc4684abe 100644 --- a/sei-db/db_engine/litt/disktable/segment/key_file.go +++ b/sei-db/db_engine/litt/disktable/segment/key_file.go @@ -77,7 +77,7 @@ func createKeyFile( } flags := os.O_RDWR | os.O_CREATE - file, err := os.OpenFile(filePath, flags, 0644) + file, err := os.OpenFile(filePath, flags, 0600) //nolint:gosec // path validated by segment manager if err != nil { return nil, fmt.Errorf("failed to open key file: %w", err) } @@ -121,7 +121,7 @@ func loadKeyFile( } if exists { - keys.size = uint64(size) + keys.size = uint64(size) //nolint:gosec // file size is non-negative } if !exists { @@ -176,7 +176,7 @@ func (k *keyFile) write(scopedKey *types.ScopedKey) error { } // Write the length of the key. - err := binary.Write(k.writer, binary.BigEndian, uint32(len(scopedKey.Key))) + err := binary.Write(k.writer, binary.BigEndian, uint32(len(scopedKey.Key))) //nolint:gosec // key length fits uint32 if err != nil { return fmt.Errorf("failed to write key length to key file: %w", err) } @@ -193,7 +193,7 @@ func (k *keyFile) write(scopedKey *types.ScopedKey) error { return fmt.Errorf("failed to write address to key file: %w", err) } - k.size += uint64( + k.size += uint64( //nolint:gosec // sizes are non-negative 4 /* uint32 size of key */ + len(scopedKey.Key) + types.AddressSerializedSize) @@ -211,7 +211,7 @@ func getKeyFileIndex(fileName string) (uint32, error) { return 0, fmt.Errorf("failed to parse index from file name %s: %w", fileName, err) } - return uint32(index), nil + return uint32(index), nil //nolint:gosec // segment index fits uint32 } // flush flushes the key file to disk. diff --git a/sei-db/db_engine/litt/disktable/segment/metadata_file.go b/sei-db/db_engine/litt/disktable/segment/metadata_file.go index ee7111d9da..7773226106 100644 --- a/sei-db/db_engine/litt/disktable/segment/metadata_file.go +++ b/sei-db/db_engine/litt/disktable/segment/metadata_file.go @@ -112,7 +112,7 @@ func loadMetadataFile(index uint32, segmentPaths []*SegmentPath, fsync bool) (*m filePath := file.path() - data, err := os.ReadFile(filePath) + data, err := os.ReadFile(filePath) //nolint:gosec // path within segment directory if err != nil { return nil, fmt.Errorf("failed to read metadata file %s: %v", filePath, err) } @@ -133,7 +133,7 @@ func getMetadataFileIndex(fileName string) (uint32, error) { return 0, fmt.Errorf("failed to parse index from file name %s: %v", fileName, err) } - return uint32(index), nil + return uint32(index), nil //nolint:gosec // segment index fits uint32 } // Size returns the size of the metadata file in bytes. @@ -155,7 +155,7 @@ func (m *metadataFile) path() string { // and should only be performed when all data that will be written to the key/value files has been made durable. func (m *metadataFile) seal(now time.Time, keyCount uint32) error { m.sealed = true - m.lastValueTimestamp = uint64(now.UnixNano()) + m.lastValueTimestamp = uint64(now.UnixNano()) //nolint:gosec // wall-clock nanos non-negative m.keyCount = keyCount err := m.write() if err != nil { diff --git a/sei-db/db_engine/litt/disktable/segment/segment.go b/sei-db/db_engine/litt/disktable/segment/segment.go index cc314612b9..54113d031e 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment.go +++ b/sei-db/db_engine/litt/disktable/segment/segment.go @@ -326,11 +326,11 @@ func (s *Segment) sealLoadedSegment(now time.Time) error { s.keys = swapFile } - err = s.metadata.seal(now, uint32(len(goodKeys))) + err = s.metadata.seal(now, uint32(len(goodKeys))) //nolint:gosec // key count fits uint32 if err != nil { return fmt.Errorf("failed to seal metadata file: %w", err) } - s.keyCount = uint32(len(goodKeys)) + s.keyCount = uint32(len(goodKeys)) //nolint:gosec // key count fits uint32 return nil } @@ -444,7 +444,7 @@ func (s *Segment) Write(data *types.KVPair) (keyCount uint32, keyFileSize uint64 // Forward the value to the key and its address file control loop, which asynchronously writes it to the key file. keyRequest := &types.ScopedKey{ Key: data.Key, - Address: types.NewAddress(s.index, firstByteIndex, uint8(shard), uint32(len(data.Value))), + Address: types.NewAddress(s.index, firstByteIndex, uint8(shard), uint32(len(data.Value))), //nolint:gosec // shard <= 255, value len fits uint32 } err = util.Send(s.errorMonitor, s.keyFileChannel, keyRequest) @@ -632,7 +632,7 @@ func (s *Segment) IsSealed() bool { // GetSealTime returns the time at which the segment was sealed. If the file is not sealed, this method will return // the zero time. func (s *Segment) GetSealTime() time.Time { - return time.Unix(0, int64(s.metadata.lastValueTimestamp)) + return time.Unix(0, int64(s.metadata.lastValueTimestamp)) //nolint:gosec // wall-clock nanos fit int64 } // Reserve reserves the segment, preventing it from being deleted. Returns true if the reservation was successful, and diff --git a/sei-db/db_engine/litt/disktable/segment/segment_scanner.go b/sei-db/db_engine/litt/disktable/segment/segment_scanner.go index 888fbd032f..eb79e44a87 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment_scanner.go +++ b/sei-db/db_engine/litt/disktable/segment/segment_scanner.go @@ -116,21 +116,22 @@ func diagnoseMissingFile( fileType string, damagedSegments map[uint32]struct{}) error { - if index == highestFileIndex { + switch index { + case highestFileIndex: // This can happen if we crash while creating a new segment. Recoverable. logger.Warn("Missing file for last segment", "file", fileType, "segment", index, ) damagedSegments[index] = struct{}{} - } else if index == lowestFileIndex { + case lowestFileIndex: // This can happen when deleting the oldest segment. Recoverable. logger.Warn("Missing file for first segment", "file", fileType, "segment", index, ) damagedSegments[index] = struct{}{} - } else { + default: // Database is missing internal files. Catastrophic failure. return fmt.Errorf("missing %s file for segment %d", fileType, index) } @@ -219,7 +220,7 @@ func lookForMissingFiles( fmt.Errorf("failed to load metadata file: %v", err) } - if uint32(len(valueFiles[segment])) > metadata.shardingFactor { + if uint32(len(valueFiles[segment])) > metadata.shardingFactor { //nolint:gosec // shard count bounded by 256 return nil, nil, fmt.Errorf("too many value files for segment %d, expected at most %d, got %d", segment, metadata.shardingFactor, len(valueFiles[segment])) diff --git a/sei-db/db_engine/litt/disktable/segment/value_file.go b/sei-db/db_engine/litt/disktable/segment/value_file.go index a44559e3a7..c65b05483c 100644 --- a/sei-db/db_engine/litt/disktable/segment/value_file.go +++ b/sei-db/db_engine/litt/disktable/segment/value_file.go @@ -82,7 +82,7 @@ func createValueFile( } // Open the file for writing. - file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0644) + file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0600) //nolint:gosec // path validated by segment manager if err != nil { return nil, fmt.Errorf("failed to open value file %s: %v", filePath, err) } @@ -128,7 +128,7 @@ func loadValueFile( return nil, fmt.Errorf("value file %s does not exist", filePath) } - values.size = uint64(size) + values.size = uint64(size) //nolint:gosec // file size is non-negative values.flushedSize.Store(values.size) return values, nil @@ -151,7 +151,7 @@ func getValueFileIndex(fileName string) (uint32, error) { return 0, fmt.Errorf("failed to parse index from file name %s: %v", fileName, err) } - return uint32(index), nil + return uint32(index), nil //nolint:gosec // segment index fits uint32 } // getValueFileShard returns the shard number of the value file from the file name. Value file names have the form @@ -171,7 +171,7 @@ func getValueFileShard(fileName string) (uint32, error) { return 0, fmt.Errorf("failed to parse shard from file name %s: %v", fileName, err) } - return uint32(shard), nil + return uint32(shard), nil //nolint:gosec // shard index fits uint32 } // Size returns the size of the value file in bytes. @@ -197,7 +197,7 @@ func (v *valueFile) read(firstByteIndex uint32) ([]byte, error) { firstByteIndex, flushedSize) } - file, err := os.OpenFile(v.path(), os.O_RDONLY, 0644) + file, err := os.OpenFile(v.path(), os.O_RDONLY, 0600) //nolint:gosec // path validated by segment manager if err != nil { return nil, fmt.Errorf("failed to open value file: %v", err) } @@ -225,7 +225,7 @@ func (v *valueFile) read(firstByteIndex uint32) ([]byte, error) { return nil, fmt.Errorf("failed to read value from value file: %v", err) } - if uint32(bytesRead) != length { + if uint32(bytesRead) != length { //nolint:gosec // bytesRead bounded by length return nil, fmt.Errorf("failed to read value from value file: read %d bytes, expected %d", bytesRead, length) } @@ -247,7 +247,7 @@ func (v *valueFile) write(value []byte) (uint32, error) { firstByteIndex := uint32(v.size) // First, write the length of the value. - err := binary.Write(v.writer, binary.BigEndian, uint32(len(value))) + err := binary.Write(v.writer, binary.BigEndian, uint32(len(value))) //nolint:gosec // value length fits uint32 if err != nil { return 0, fmt.Errorf("failed to write value length to value file: %v", err) } @@ -258,7 +258,7 @@ func (v *valueFile) write(value []byte) (uint32, error) { return 0, fmt.Errorf("failed to write value to value file: %v", err) } - v.size += uint64(len(value) + 4) + v.size += uint64(len(value) + 4) //nolint:gosec // value length non-negative return firstByteIndex, nil } diff --git a/sei-db/db_engine/litt/disktable/table_metadata.go b/sei-db/db_engine/litt/disktable/table_metadata.go index b19a5bdc5c..ab2690ad10 100644 --- a/sei-db/db_engine/litt/disktable/table_metadata.go +++ b/sei-db/db_engine/litt/disktable/table_metadata.go @@ -65,7 +65,7 @@ func loadTableMetadata(logger *slog.Logger, tableDirectory string) (*tableMetada return nil, fmt.Errorf("table metadata file does not exist: %s", mPath) } - data, err := os.ReadFile(mPath) + data, err := os.ReadFile(mPath) //nolint:gosec // path within table directory if err != nil { return nil, fmt.Errorf("failed to read table metadata file %s: %v", mPath, err) } @@ -137,7 +137,7 @@ func (t *tableMetadata) serialize() []byte { // Write the TTL ttlNanoseconds := t.GetTTL().Nanoseconds() - binary.BigEndian.PutUint64(data[4:12], uint64(ttlNanoseconds)) + binary.BigEndian.PutUint64(data[4:12], uint64(ttlNanoseconds)) //nolint:gosec // serialized as time.Duration // Write the sharding factor binary.BigEndian.PutUint32(data[12:16], t.GetShardingFactor()) @@ -159,7 +159,7 @@ func deserialize(data []byte) (*tableMetadata, error) { return nil, fmt.Errorf("unsupported serialization version: %d", serializationVersion) } - ttl := time.Duration(binary.BigEndian.Uint64(data[4:12])) + ttl := time.Duration(binary.BigEndian.Uint64(data[4:12])) //nolint:gosec // serialized as time.Duration shardingFactor := binary.BigEndian.Uint32(data[12:16]) metadata := &tableMetadata{} diff --git a/sei-db/db_engine/litt/littbuilder/build_utils.go b/sei-db/db_engine/litt/littbuilder/build_utils.go index 3eac031985..844e788eba 100644 --- a/sei-db/db_engine/litt/littbuilder/build_utils.go +++ b/sei-db/db_engine/litt/littbuilder/build_utils.go @@ -7,6 +7,7 @@ import ( "os" "path" "strings" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -28,7 +29,7 @@ var keymapBuilders = map[keymap.KeymapType]keymap.BuildKeymap{ // cacheWeight is a function that calculates the weight of a cache entry. func cacheWeight(key string, value []byte) uint64 { - return uint64(len(key) + len(value)) + return uint64(len(key) + len(value)) //nolint:gosec // lengths non-negative } // Look for a table's keymap directory in the provided segment paths. @@ -124,7 +125,7 @@ func buildKeymap( keymapTypeFile = keymap.NewKeymapTypeFile(keymapDirectory, config.KeymapType) // create the keymap directory - err := os.MkdirAll(keymapDirectory, 0755) + err := os.MkdirAll(keymapDirectory, 0750) if err != nil { return nil, "", nil, false, fmt.Errorf("error creating keymap directory: %w", err) @@ -152,7 +153,7 @@ func buildKeymap( } // write the new keymap type file - err = os.MkdirAll(keymapDirectory, 0755) + err = os.MkdirAll(keymapDirectory, 0750) if err != nil { return nil, "", nil, false, fmt.Errorf("error creating keymap directory: %w", err) @@ -176,7 +177,7 @@ func buildKeymap( if !requiresReload { // If the keymap does not need to be reloaded, then it is already fully initialized. keymapInitializedFile := path.Join(keymapDirectory, keymap.KeymapInitializedFileName) - f, err := os.Create(keymapInitializedFile) + f, err := os.Create(keymapInitializedFile) //nolint:gosec // path within keymap directory if err != nil { return nil, "", nil, false, fmt.Errorf("failed to create keymap initialized file: %v", err) @@ -270,8 +271,9 @@ func buildMetrics(config *litt.Config, logger *slog.Logger) (*metrics.LittDBMetr promhttp.HandlerOpts{}, )) server = &http.Server{ - Addr: addr, - Handler: mux, + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, } go func() { diff --git a/sei-db/db_engine/litt/util/file_lock.go b/sei-db/db_engine/litt/util/file_lock.go index 538887f5e7..7636aa1b01 100644 --- a/sei-db/db_engine/litt/util/file_lock.go +++ b/sei-db/db_engine/litt/util/file_lock.go @@ -54,7 +54,7 @@ func IsProcessAlive(pid int) bool { // parseLockFile parses a lock file and returns the PID if valid func parseLockFile(path string) (int, error) { - content, err := os.ReadFile(path) + content, err := os.ReadFile(path) //nolint:gosec // caller-supplied lock file path if err != nil { return 0, fmt.Errorf("failed to read lock file: %w", err) } @@ -85,7 +85,7 @@ func NewFileLock(logger *slog.Logger, path string, fsync bool) (*FileLock, error } // Try to create the lock file exclusively (O_EXCL ensures it fails if file exists) - file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) //nolint:gosec // caller-supplied lock file path if err != nil { if os.IsExist(err) { // Lock file exists, check if it's stale @@ -97,7 +97,7 @@ func NewFileLock(logger *slog.Logger, path string, fsync bool) (*FileLock, error } // Try to create the lock file again - file, err = os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + file, err = os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) //nolint:gosec // caller-supplied lock file path if err != nil { return nil, fmt.Errorf("failed to create lock file after removing stale lock %s: %w", path, err) @@ -105,7 +105,7 @@ func NewFileLock(logger *slog.Logger, path string, fsync bool) (*FileLock, error } else { // Process is still alive, cannot acquire lock debugInfo := "" - content, readErr := os.ReadFile(path) + content, readErr := os.ReadFile(path) //nolint:gosec // caller-supplied lock file path if readErr == nil { debugInfo = fmt.Sprintf(" (existing lock info: %s)", strings.TrimSpace(string(content))) } else { @@ -117,7 +117,7 @@ func NewFileLock(logger *slog.Logger, path string, fsync bool) (*FileLock, error } else { // Cannot parse lock file, treat as existing lock with debug info debugInfo := "" - if content, readErr := os.ReadFile(path); readErr == nil { + if content, readErr := os.ReadFile(path); readErr == nil { //nolint:gosec // caller-supplied lock file path debugInfo = fmt.Sprintf(" (existing lock info: %s)", strings.TrimSpace(string(content))) } return nil, fmt.Errorf("lock file already exists: %s%s", path, debugInfo) diff --git a/sei-db/db_engine/litt/util/file_utils.go b/sei-db/db_engine/litt/util/file_utils.go index 14b538ce6e..22bb77cf6d 100644 --- a/sei-db/db_engine/litt/util/file_utils.go +++ b/sei-db/db_engine/litt/util/file_utils.go @@ -103,7 +103,7 @@ func AtomicWrite(destination string, data []byte, fsync bool) error { swapPath := destination + SwapFileExtension // Write the data into the swap file. - swapFile, err := os.Create(swapPath) + swapFile, err := os.Create(swapPath) //nolint:gosec // caller-supplied destination path if err != nil { return fmt.Errorf("failed to create swap file: %v", err) } @@ -145,7 +145,7 @@ func AtomicRename(oldPath string, newPath string, fsync bool) error { parentDirectory := filepath.Dir(newPath) // Ensure that the rename is committed to disk. - dirFile, err := os.Open(parentDirectory) + dirFile, err := os.Open(parentDirectory) //nolint:gosec // derived from caller-supplied path if err != nil { return fmt.Errorf("failed to open parent directory %s: %w", parentDirectory, err) } @@ -278,7 +278,7 @@ func Exists(path string) (bool, error) { // SyncFile syncs a file/directory func SyncPath(path string) error { - file, err := os.Open(path) + file, err := os.Open(path) //nolint:gosec // caller-supplied path if err != nil { return fmt.Errorf("failed to open path for sync: %w", err) } @@ -307,7 +307,7 @@ func CopyRegularFile(src string, dst string, fsync bool) error { } // Open source file - in, err := os.Open(src) + in, err := os.Open(src) //nolint:gosec // caller-supplied source path if err != nil { return fmt.Errorf("failed to open source file %s: %w", src, err) } @@ -327,7 +327,7 @@ func CopyRegularFile(src string, dst string, fsync bool) error { } // Create destination file - out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) //nolint:gosec // caller-supplied destination path if err != nil { return fmt.Errorf("failed to create destination file %s: %w", dst, err) } @@ -403,7 +403,7 @@ func EnsureDirectoryExists(dirPath string, fsync bool) error { dirToCreate := pathsToCreate[i] // Create the directory - if err := os.Mkdir(dirToCreate, 0755); err != nil { + if err := os.Mkdir(dirToCreate, 0750); err != nil { return fmt.Errorf("failed to create directory %s: %w", dirToCreate, err) } diff --git a/sei-db/db_engine/litt/util/pprof.go b/sei-db/db_engine/litt/util/pprof.go index 0951b8e0e0..496fb685ee 100644 --- a/sei-db/db_engine/litt/util/pprof.go +++ b/sei-db/db_engine/litt/util/pprof.go @@ -4,8 +4,9 @@ import ( "fmt" "log/slog" "net/http" + "time" - _ "net/http/pprof" + _ "net/http/pprof" //nolint:gosec // pprof endpoint is intentional for profiling ) type PprofProfiler struct { @@ -24,7 +25,11 @@ func NewPprofProfiler(httpPort string, logger *slog.Logger) *PprofProfiler { func (p *PprofProfiler) Start() { pprofAddr := fmt.Sprintf("%s:%s", "0.0.0.0", p.httpPort) - if err := http.ListenAndServe(pprofAddr, nil); err != nil { + server := &http.Server{ + Addr: pprofAddr, + ReadHeaderTimeout: 10 * time.Second, + } + if err := server.ListenAndServe(); err != nil { p.logger.Error("pprof server failed", "error", err, "pprofAddr", pprofAddr) } } diff --git a/sei-db/db_engine/litt/util/random_access_deque.go b/sei-db/db_engine/litt/util/random_access_deque.go index 4aa4e609d4..412af3cd9b 100644 --- a/sei-db/db_engine/litt/util/random_access_deque.go +++ b/sei-db/db_engine/litt/util/random_access_deque.go @@ -110,7 +110,7 @@ func (s *RandomAccessDeque[T]) TryPopFront() (value T, ok bool) { var zero T s.data[s.startIndex] = zero - if s.startIndex == uint64(len(s.data)-1) { + if s.startIndex == uint64(len(s.data)-1) { //nolint:gosec // slice length non-negative // wrap around s.startIndex = 0 } else { @@ -130,7 +130,7 @@ func (s *RandomAccessDeque[T]) PushBack(value T) { s.data[s.endIndex] = value - if s.endIndex == uint64(len(s.data)-1) { + if s.endIndex == uint64(len(s.data)-1) { //nolint:gosec // slice length non-negative // wrap around s.endIndex = 0 } else { diff --git a/sei-db/db_engine/litt/util/ssh.go b/sei-db/db_engine/litt/util/ssh.go index 1517288f1b..77aa0510c3 100644 --- a/sei-db/db_engine/litt/util/ssh.go +++ b/sei-db/db_engine/litt/util/ssh.go @@ -40,7 +40,7 @@ func NewSSHSession( var err error - hostKeyCallback := ssh.InsecureIgnoreHostKey() + hostKeyCallback := ssh.InsecureIgnoreHostKey() //nolint:gosec // overridden when knownHosts provided if knownHosts != "" { knownHosts, err = SanitizePath(knownHosts) if err != nil { @@ -61,7 +61,7 @@ func NewSSHSession( return nil, fmt.Errorf("private key does not exist at path: %s", keyPath) } - keyData, err := os.ReadFile(keyPath) + keyData, err := os.ReadFile(keyPath) //nolint:gosec // caller-supplied key path if err != nil { return nil, fmt.Errorf("failed to read private key: %w", err) } @@ -193,7 +193,7 @@ func (s *SSHSession) Rsync(sourceFile string, destFile string, throttleMB float6 s.logger.Info("Executing", "command", strings.Join(arguments, " ")) } - cmd := exec.Command(arguments[0], arguments[1:]...) + cmd := exec.Command(arguments[0], arguments[1:]...) //nolint:gosec // arguments built from caller-trusted config cmd.Stderr = os.Stderr err = cmd.Run() diff --git a/sei-db/db_engine/litt/util/unsafe_string.go b/sei-db/db_engine/litt/util/unsafe_string.go index 238dfd2892..018195c2d8 100644 --- a/sei-db/db_engine/litt/util/unsafe_string.go +++ b/sei-db/db_engine/litt/util/unsafe_string.go @@ -8,5 +8,5 @@ func UnsafeBytesToString(b []byte) string { if len(b) == 0 { return "" } - return unsafe.String(&b[0], len(b)) + return unsafe.String(&b[0], len(b)) //nolint:gosec // documented zero-copy conversion }