Files
brianmcgee e82eb75681 state: separate file storage by network
Moves away from a callback-based API for the file and signature storage.
2026-02-25 14:21:52 +00:00

961 lines
21 KiB
Go

package state_test
import (
"fmt"
"os"
"strings"
"testing"
"time"
"git.clan.lol/clan/data-mesher/pkg/config"
"git.clan.lol/clan/data-mesher/pkg/crypto"
"git.clan.lol/clan/data-mesher/pkg/model"
"git.clan.lol/clan/data-mesher/pkg/state"
"git.clan.lol/clan/data-mesher/test"
"github.com/stretchr/testify/require"
)
func newTestSignatures(t *testing.T) (*state.Signatures, *crypto.PublicKey, func()) {
t.Helper()
tmpDir := t.TempDir()
cfg := &config.Config{
StateDirectory: tmpDir,
}
fs, err := state.NewSignatures(cfg)
require.NoError(t, err, "failed to create Signatures")
networkKeys := test.GenerateKeys(t, 1)
networkID := networkKeys[0].Public
cleanup := func() {
_ = fs.Close()
}
return fs, networkID, cleanup
}
func TestNewSignatures(t *testing.T) {
t.Parallel()
t.Run("create database and bucket", func(t *testing.T) {
t.Parallel()
fs, _, cleanup := newTestSignatures(t)
defer cleanup()
require.NotNil(t, fs, "expected Signatures to be non-nil")
})
t.Run("fail with invalid path", func(t *testing.T) {
t.Parallel()
cfg := &config.Config{
StateDirectory: "/foo/bar/baz/db.bolt",
}
_, err := state.NewSignatures(cfg)
require.Error(t, err, "expected error for invalid path")
})
}
func TestSignatures_PutAndGet(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
// Create a test signing key
keys := test.GenerateKeys(t, 1)
signingKey := keys[0]
// Create a test signature
testSig := &model.Signature{
Name: "test-file.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC().Truncate(time.Second),
SignedBy: signingKey.Public,
Signature: []byte("test-signature"),
}
// Put signature
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, testSig)
as.NoError(err, "failed to put signature")
as.NoError(tx.Commit())
// Get signature
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, testSig.Name, &sig)
as.NoError(err, "failed to get signature")
as.NoError(tx.Rollback())
as.Equal(testSig, &sig, "signatures do not match")
}
func TestSignatures_Get_NotFound(t *testing.T) {
t.Parallel()
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
// Seed the network bucket so the network exists
seedSig := &model.Signature{
Name: "seed.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC(),
SignedBy: networkID,
Signature: []byte("seed"),
}
tx, err := fs.BeginTx(true)
require.NoError(t, err)
require.NoError(t, fs.Put(tx, seedSig))
require.NoError(t, tx.Commit())
// Query a nonexistent name
var sig model.Signature
tx, err = fs.BeginTx(false)
require.NoError(t, err)
err = fs.Get(tx, networkID, "nonexistent-sig", &sig)
require.NoError(t, tx.Rollback())
require.ErrorIs(t, err, state.ErrSignatureNotFound)
}
func TestSignatures_Delete(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
// Create a test key
keys := test.GenerateKeys(t, 1)
key := keys[0]
// Create a test signature
testSig := &model.Signature{
Name: "to-delete.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC(),
SignedBy: key.Public,
Signature: []byte("sig"),
}
// Put signature first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, testSig)
as.NoError(err, "failed to put signature")
as.NoError(tx.Commit())
// Delete it
tx, err = fs.BeginTx(true)
as.NoError(err)
err = fs.Delete(tx, networkID, testSig.Name)
as.NoError(err, "failed to delete signature")
as.NoError(tx.Commit())
// Verify it's gone
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, testSig.Name, &sig)
as.NoError(tx.Rollback())
as.ErrorIs(err, state.ErrSignatureNotFound)
}
func TestSignatures_Delete_NonExistent(t *testing.T) {
t.Parallel()
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
// Seed the network bucket so the network exists
seedSig := &model.Signature{
Name: "seed.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC(),
SignedBy: networkID,
Signature: []byte("seed"),
}
tx, err := fs.BeginTx(true)
require.NoError(t, err)
require.NoError(t, fs.Put(tx, seedSig))
require.NoError(t, tx.Commit())
// Deleting a non-existent file should not error (BoltDB behavior)
tx, err = fs.BeginTx(true)
require.NoError(t, err)
err = fs.Delete(tx, networkID, "nonexistent-sig")
require.NoError(t, err, "unexpected error deleting non-existent signature")
require.NoError(t, tx.Commit())
}
func TestSignatures_List(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
// Create some test keys
keys := test.GenerateKeys(t, 3)
// Add multiple signatures
sigs := []*model.Signature{
{
Name: "file1.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC(),
SignedBy: keys[0].Public,
Signature: []byte("sig1"),
},
{
Name: "file2.txt",
NetworkID: networkID,
SignedAt: time.Now().Add(time.Second).UTC(),
SignedBy: keys[1].Public,
Signature: []byte("sig1"),
},
{
Name: "file3.txt",
NetworkID: networkID,
SignedAt: time.Now().Add(time.Hour).UTC(),
SignedBy: keys[2].Public,
Signature: []byte("sig1"),
},
}
tx, err := fs.BeginTx(true)
as.NoError(err)
for _, sig := range sigs {
err = fs.Put(tx, sig)
as.NoError(err, "failed to put signature %s", sig.Name)
}
as.NoError(tx.Commit())
// List files
tx, err = fs.BeginTx(false)
as.NoError(err)
reader, err := fs.List(tx)
as.NoError(err, "failed to list signatures")
buf := make([]model.Signature, 16)
var results []model.Signature
for {
n, readErr := reader.Read(buf)
for i := range n {
results = append(results, buf[i])
}
if readErr != nil {
break
}
}
as.NoError(tx.Rollback())
as.Len(results, len(sigs))
// Check all files are present (BoltDB returns in key order)
resultNames := make(map[string]bool)
for _, r := range results {
resultNames[r.Name] = true
}
for _, sig := range sigs {
as.True(resultNames[sig.Name], "signature %s not found in results", sig.Name)
}
}
func TestSignatures_List_Empty(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, _, cleanup := newTestSignatures(t)
defer cleanup()
tx, err := fs.BeginTx(false)
as.NoError(err)
reader, err := fs.List(tx)
as.NoError(err, "failed to list signatures")
var count int
buf := make([]model.Signature, 16)
for {
n, readErr := reader.Read(buf)
count += n
if readErr != nil {
break
}
}
as.NoError(tx.Rollback())
as.Zero(count, "expected 0 files")
}
func TestSignatures_ListFilenames(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
// Create some test keys
keys := test.GenerateKeys(t, 2)
// Add multiple files
sigs := []*model.Signature{
{
Name: "alpha.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC(),
SignedBy: keys[0].Public,
Signature: []byte("sig1"),
},
{
Name: "beta.txt",
NetworkID: networkID,
SignedAt: time.Now().Add(time.Hour).UTC(),
SignedBy: keys[1].Public,
Signature: []byte("sig2"),
},
}
tx, err := fs.BeginTx(true)
as.NoError(err)
for _, sig := range sigs {
err = fs.Put(tx, sig)
as.NoError(err, "failed to put signature %s", sig.Name)
}
as.NoError(tx.Commit())
// List via reader and extract names
tx, err = fs.BeginTx(false)
as.NoError(err)
reader, err := fs.List(tx)
as.NoError(err, "failed to list signatures")
var names []string
buf := make([]model.Signature, 16)
for {
n, readErr := reader.Read(buf)
for i := range n {
names = append(names, buf[i].Name)
}
if readErr != nil {
break
}
}
as.NoError(tx.Rollback())
require.Len(t, names, len(sigs))
}
func TestSignatures_PutIfLater(t *testing.T) {
t.Parallel()
t.Run("write when signature does not exist", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
testSig := &model.Signature{
Name: "new-file.txt",
NetworkID: networkID,
SignedAt: time.Now().UTC().Truncate(time.Second),
SignedBy: keys[0].Public,
Signature: []byte("sig"),
}
tx, err := fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, testSig)
as.NoError(err)
as.True(written, "expected PutIfLater to write new signature")
as.NoError(tx.Commit())
// Verify the signature was stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, testSig.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(testSig, &sig)
})
t.Run("write when new signature is later", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
oldSig := &model.Signature{
Name: "existing-file.txt",
NetworkID: networkID,
SignedAt: baseTime,
SignedBy: keys[0].Public,
Signature: []byte("old-sig"),
}
newSig := &model.Signature{
Name: "existing-file.txt",
NetworkID: networkID,
SignedAt: baseTime.Add(time.Hour),
SignedBy: keys[0].Public,
Signature: []byte("new-sig"),
}
// Put the old signature first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, oldSig)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with a later signature
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, newSig)
as.NoError(err)
as.True(written, "expected PutIfLater to write later signature")
as.NoError(tx.Commit())
// Verify the new signature was stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, oldSig.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(newSig, &sig)
})
t.Run("skip write when new signature is earlier", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
existingSig := &model.Signature{
Name: "existing-file.txt",
NetworkID: networkID,
SignedAt: baseTime,
SignedBy: keys[0].Public,
Signature: []byte("existing-sig"),
}
earlierSig := &model.Signature{
Name: "existing-file.txt",
NetworkID: networkID,
SignedAt: baseTime.Add(-time.Hour),
SignedBy: keys[0].Public,
Signature: []byte("earlier-sig"),
}
// Put the existing signature first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, existingSig)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with an earlier signature
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, earlierSig)
as.NoError(err)
as.False(written, "expected PutIfLater to skip earlier signature")
as.NoError(tx.Commit())
// Verify the original signature is still stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, existingSig.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(existingSig, &sig)
})
t.Run("skip write when signatures have equal timestamps", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
existingSig := &model.Signature{
Name: "existing-file.txt",
NetworkID: networkID,
SignedAt: baseTime,
SignedBy: keys[0].Public,
Signature: []byte("existing-sig"),
}
equalTimeSig := &model.Signature{
Name: "existing-file.txt",
NetworkID: networkID,
SignedAt: baseTime,
SignedBy: keys[0].Public,
Signature: []byte("equal-time-sig"),
}
// Put the existing signature first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, existingSig)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with an equal timestamp signature
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, equalTimeSig)
as.NoError(err)
as.False(written, "expected PutIfLater to skip equal timestamp signature")
as.NoError(tx.Commit())
// Verify the original signature is still stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, existingSig.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(existingSig, &sig)
})
}
func TestSignatures_List_MultipleNetworks(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, _, cleanup := newTestSignatures(t)
defer cleanup()
// Generate 5 distinct network IDs and 1 signing key
networkKeys := test.GenerateKeys(t, 5)
signerKeys := test.GenerateKeys(t, 1)
signer := signerKeys[0]
const (
networksCount = 5
sigsPerNetwork = 20
totalSigs = networksCount * sigsPerNetwork
)
baseTime := time.Now().UTC().Truncate(time.Second)
// Insert 20 signatures per network in a single write transaction
tx, err := fs.BeginTx(true)
as.NoError(err)
for i := range networksCount {
networkID := networkKeys[i].Public
for j := range sigsPerNetwork {
sig := &model.Signature{
Name: fmt.Sprintf("net-%d/file-%d.txt", i, j),
NetworkID: networkID,
SignedAt: baseTime.Add(time.Duration(i*sigsPerNetwork+j) * time.Second),
SignedBy: signer.Public,
Signature: fmt.Appendf(nil, "sig-%d-%d", i, j),
}
err = fs.Put(tx, sig)
as.NoError(err, "failed to put signature %s", sig.Name)
}
}
as.NoError(tx.Commit())
// Read back with a small buffer of size 3 to force bucket transitions mid-read
tx, err = fs.BeginTx(false)
as.NoError(err)
reader, err := fs.List(tx)
as.NoError(err, "failed to list signatures")
buf := make([]model.Signature, 3)
results := make(map[string]model.Signature)
for {
// Reset buffer entries so the decoder allocates fresh pointer fields
// instead of reusing (and mutating) previously decoded pointers.
clear(buf)
n, readErr := reader.Read(buf)
for i := range n {
results[buf[i].Name] = buf[i]
}
if readErr != nil {
break
}
}
as.NoError(tx.Rollback())
// Assertions
as.Len(results, totalSigs, "expected %d total signatures", totalSigs)
for i := range networksCount {
networkID := networkKeys[i].Public
for j := range sigsPerNetwork {
name := fmt.Sprintf("net-%d/file-%d.txt", i, j)
sig, ok := results[name]
as.True(ok, "signature %s not found", name)
as.Equal(networkID, sig.NetworkID, "wrong NetworkID for %s", name)
as.NotNil(sig.SignedBy, "SignedBy should be set for %s", name)
as.False(sig.SignedAt.IsZero(), "SignedAt should be non-zero for %s", name)
as.NotEmpty(sig.Signature, "Signature should be non-empty for %s", name)
}
}
}
func TestSignatures_Close(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
cfg := &config.Config{
StateDirectory: tmpDir,
}
fs, err := state.NewSignatures(cfg)
require.NoError(t, err, "failed to create Signatures")
// Close should succeed
err = fs.Close()
require.NoError(t, err, "failed to close")
// Verify the file exists (db was written)
dbPath := tmpDir + "/signatures.db"
_, err = os.Stat(dbPath)
require.False(t, os.IsNotExist(err), "database file should exist after close")
}
func TestSignatures_PutIfLater_Tombstone(t *testing.T) {
t.Parallel()
t.Run("tombstone replaces older file signature", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
// Create a file signature
fileSig, err := model.SignFile("test-file.txt", networkID, keys[0], test.Clock(baseTime), strings.NewReader("test"))
as.NoError(err, "failed to sign file")
// Create a later tombstone
tombstone, err := model.SignTombstone("test-file.txt", networkID, keys[0], test.Clock(baseTime.Add(time.Nanosecond)))
as.NoError(err, "failed to sign tombstone")
// Put the file signature first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, fileSig)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with the tombstone
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, tombstone)
as.NoError(err)
as.True(written, "expected tombstone to replace older file signature")
as.NoError(tx.Commit())
// Verify the tombstone was stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, fileSig.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(model.SignatureTypeTombstone, sig.Type, "expected stored signature to be a tombstone")
as.Equal(tombstone.SignedAt, sig.SignedAt)
})
t.Run("newer file signature replaces older tombstone (resurrection)", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
// Create a tombstone
tombstone, err := model.SignTombstone("test-file.txt", networkID, keys[0], test.Clock(baseTime))
as.NoError(err, "failed to sign tombstone")
// Create a later file signature
fileSig, err := model.SignFile(
"test-file.txt",
networkID,
keys[0],
test.Clock(baseTime.Add(time.Nanosecond)),
strings.NewReader("test"),
)
as.NoError(err, "failed to sign file")
// Put the tombstone first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, tombstone)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with the newer file signature
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, fileSig)
as.NoError(err)
as.True(written, "expected newer file signature to replace tombstone")
as.NoError(tx.Commit())
// Verify the file signature was stored (resurrection)
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, fileSig.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(model.SignatureTypeFile, sig.Type, "expected stored signature to be a file signature")
as.Equal(fileSig.SignedAt, sig.SignedAt)
})
t.Run("older file signature does not replace tombstone", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
// Create a tombstone
tombstone, err := model.SignTombstone("test-file.txt", networkID, keys[0], test.Clock(baseTime))
as.NoError(err, "failed to sign tombstone")
// Create an older file signature
fileSig, err := model.SignFile(
"test-file.txt",
networkID,
keys[0],
test.Clock(baseTime.Add(time.Nanosecond*-1)),
strings.NewReader("test"),
)
as.NoError(err, "failed to sign file")
// Put the tombstone first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, tombstone)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with the older file signature
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, fileSig)
as.NoError(err)
as.False(written, "expected older file signature to not replace tombstone")
as.NoError(tx.Commit())
// Verify the tombstone is still stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, tombstone.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(model.SignatureTypeTombstone, sig.Type, "expected stored signature to be a tombstone")
})
t.Run("newer tombstone replaces older tombstone", func(t *testing.T) {
t.Parallel()
as := require.New(t)
fs, networkID, cleanup := newTestSignatures(t)
defer cleanup()
keys := test.GenerateKeys(t, 1)
baseTime := time.Now().UTC().Truncate(time.Second)
// Create an old tombstone
oldTombstone, err := model.SignTombstone(
"test-file.txt",
networkID,
keys[0],
test.Clock(baseTime.Add(time.Nanosecond*-1)),
)
as.NoError(err, "failed to sign tombstone")
// Create a newer tombstone
newTombstone, err := model.SignTombstone("test-file.txt", networkID, keys[0], test.Clock(baseTime))
as.NoError(err, "failed to sign tombstone")
// Put the older tombstone first
tx, err := fs.BeginTx(true)
as.NoError(err)
err = fs.Put(tx, oldTombstone)
as.NoError(err)
as.NoError(tx.Commit())
// PutIfLater with the newer tombstone
tx, err = fs.BeginTx(true)
as.NoError(err)
written, err := fs.PutIfLater(tx, newTombstone)
as.NoError(err)
as.True(written, "expected newer tombstone to replace older tombstone")
as.NoError(tx.Commit())
// Verify the newer tombstone was stored
var sig model.Signature
tx, err = fs.BeginTx(false)
as.NoError(err)
err = fs.Get(tx, networkID, oldTombstone.Name, &sig)
as.NoError(err)
as.NoError(tx.Rollback())
as.Equal(model.SignatureTypeTombstone, sig.Type, "expected stored signature to be a tombstone")
as.Equal(newTombstone.SignedAt, sig.SignedAt)
})
}