WIP: Clsuter support (cross node delivery, leader election)

util/bloom.go

@@ -0,0 +1,103 @@
+package util
+
+import (
+	"errors"
+	"hash/fnv"
+	"math"
+)
+
+// BloomFilter is a fixed-size probabilistic set: Contains may return false positives (bounded by
+// the target rate the filter was sized for) but never false negatives. Elements cannot be
+// removed; rebuild the filter from scratch instead.
+type BloomFilter struct {
+	bits []uint64
+	k    int // number of hash probes per element, derived via double hashing
+}
+
+// NewBloomFilter creates a filter sized for n elements at the given false-positive rate.
+func NewBloomFilter(n int, fpRate float64) *BloomFilter {
+	if n < 1 {
+		n = 1
+	}
+	if fpRate <= 0 || fpRate >= 1 {
+		fpRate = 0.01
+	}
+	m := int(math.Ceil(-float64(n) * math.Log(fpRate) / (math.Ln2 * math.Ln2))) // bits
+	k := int(math.Round(float64(m) / float64(n) * math.Ln2))                    // probes
+	if k < 1 {
+		k = 1
+	}
+	return &BloomFilter{
+		bits: make([]uint64, (m+63)/64),
+		k:    k,
+	}
+}
+
+// Add inserts an element into the filter.
+func (b *BloomFilter) Add(s string) {
+	h1, h2 := hashPair(s)
+	m := uint64(len(b.bits)) * 64
+	for i := 0; i < b.k; i++ {
+		bit := (h1 + uint64(i)*h2) % m
+		b.bits[bit/64] |= 1 << (bit % 64)
+	}
+}
+
+// Contains reports whether the element may be in the set. A false result is definitive: the
+// element was never added.
+func (b *BloomFilter) Contains(s string) bool {
+	h1, h2 := hashPair(s)
+	m := uint64(len(b.bits)) * 64
+	for i := 0; i < b.k; i++ {
+		bit := (h1 + uint64(i)*h2) % m
+		if b.bits[bit/64]&(1<<(bit%64)) == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// MarshalBinary serializes the filter as [k][bits...], with 64-bit little-endian words.
+func (b *BloomFilter) MarshalBinary() ([]byte, error) {
+	data := make([]byte, 1+len(b.bits)*8)
+	data[0] = byte(b.k)
+	for i, word := range b.bits {
+		for j := 0; j < 8; j++ {
+			data[1+i*8+j] = byte(word >> (8 * j))
+		}
+	}
+	return data, nil
+}
+
+// UnmarshalBloomFilter deserializes a filter produced by MarshalBinary.
+func UnmarshalBloomFilter(data []byte) (*BloomFilter, error) {
+	if len(data) < 9 || (len(data)-1)%8 != 0 {
+		return nil, errors.New("invalid bloom filter data")
+	}
+	b := &BloomFilter{
+		bits: make([]uint64, (len(data)-1)/8),
+		k:    int(data[0]),
+	}
+	if b.k < 1 {
+		return nil, errors.New("invalid bloom filter hash count")
+	}
+	for i := range b.bits {
+		var word uint64
+		for j := 0; j < 8; j++ {
+			word |= uint64(data[1+i*8+j]) << (8 * j)
+		}
+		b.bits[i] = word
+	}
+	return b, nil
+}
+
+// hashPair derives the two independent hash values used for double hashing (probe i uses
+// h1 + i*h2), from FNV-1a over the element and a domain-separated variant of it.
+func hashPair(s string) (uint64, uint64) {
+	f := fnv.New64a()
+	f.Write([]byte(s))
+	h1 := f.Sum64()
+	f.Write([]byte{0xff}) // Domain separation for the second hash
+	h2 := f.Sum64() | 1   // Odd, so probes cycle through all bit positions
+	return h1, h2
+}