From bfd715dcc672264b204935a62be901c98eb3e956 Mon Sep 17 00:00:00 2001 From: Jannik Luhn Date: Tue, 2 Jun 2026 00:34:25 +0200 Subject: [PATCH 1/2] fix: backfill historical keyper sets on startup The keyperset syncer only fetched the currently-active keyper set and any future ones, silently skipping all sets that had been active before the keyper started. On a fresh install or after extended downtime, missed keyper sets were never stored. On restart, most keyper sets are already in the DB. The syncer now reads existing indices at startup and only fetches the gaps, avoiding redundant contract calls. --- .../db/keyper/keyper.sqlc.gen.go | 25 ++++ .../db/keyper/sql/queries/keyper.sql | 6 +- rolling-shutter/keyperimpl/gnosis/keyper.go | 7 ++ rolling-shutter/keyperimpl/optimism/keyper.go | 6 + rolling-shutter/keyperimpl/primev/keyper.go | 7 ++ .../keyperimpl/shutterservice/keyper.go | 7 ++ rolling-shutter/medley/chainsync/options.go | 27 +++-- .../medley/chainsync/syncer/keyperset.go | 39 +++---- .../medley/chainsync/syncer/range.go | 48 ++++++++ .../medley/chainsync/syncer/range_test.go | 108 ++++++++++++++++++ 10 files changed, 245 insertions(+), 35 deletions(-) create mode 100644 rolling-shutter/medley/chainsync/syncer/range.go create mode 100644 rolling-shutter/medley/chainsync/syncer/range_test.go diff --git a/rolling-shutter/chainobserver/db/keyper/keyper.sqlc.gen.go b/rolling-shutter/chainobserver/db/keyper/keyper.sqlc.gen.go index fea52e4a4..fd0f0fe68 100644 --- a/rolling-shutter/chainobserver/db/keyper/keyper.sqlc.gen.go +++ b/rolling-shutter/chainobserver/db/keyper/keyper.sqlc.gen.go @@ -43,6 +43,31 @@ func (q *Queries) GetKeyperSetByKeyperConfigIndex(ctx context.Context, keyperCon return i, err } +const getKeyperSetIndices = `-- name: GetKeyperSetIndices :many +SELECT keyper_config_index FROM keyper_set +ORDER BY keyper_config_index ASC +` + +func (q *Queries) GetKeyperSetIndices(ctx context.Context) ([]int64, error) { + rows, err := q.db.Query(ctx, getKeyperSetIndices) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int64 + for rows.Next() { + var keyper_config_index int64 + if err := rows.Scan(&keyper_config_index); err != nil { + return nil, err + } + items = append(items, keyper_config_index) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getKeyperSets = `-- name: GetKeyperSets :many SELECT keyper_config_index, activation_block_number, keypers, threshold FROM keyper_set ORDER BY activation_block_number ASC diff --git a/rolling-shutter/chainobserver/db/keyper/sql/queries/keyper.sql b/rolling-shutter/chainobserver/db/keyper/sql/queries/keyper.sql index 90b386425..fd822cdd0 100644 --- a/rolling-shutter/chainobserver/db/keyper/sql/queries/keyper.sql +++ b/rolling-shutter/chainobserver/db/keyper/sql/queries/keyper.sql @@ -18,4 +18,8 @@ ORDER BY activation_block_number DESC LIMIT 1; -- name: GetKeyperSets :many SELECT * FROM keyper_set -ORDER BY activation_block_number ASC; \ No newline at end of file +ORDER BY activation_block_number ASC; + +-- name: GetKeyperSetIndices :many +SELECT keyper_config_index FROM keyper_set +ORDER BY keyper_config_index ASC; \ No newline at end of file diff --git a/rolling-shutter/keyperimpl/gnosis/keyper.go b/rolling-shutter/keyperimpl/gnosis/keyper.go index 70d8fbd8f..f36cb5a8f 100644 --- a/rolling-shutter/keyperimpl/gnosis/keyper.go +++ b/rolling-shutter/keyperimpl/gnosis/keyper.go @@ -13,6 +13,7 @@ import ( sequencerBindings "github.com/shutter-network/gnosh-contracts/gnoshcontracts/sequencer" validatorRegistryBindings "github.com/shutter-network/gnosh-contracts/gnoshcontracts/validatorregistry" + obskeyper "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper" "github.com/shutter-network/rolling-shutter/rolling-shutter/eonkeypublisher" "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper" "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/epochkghandler" @@ -22,6 +23,7 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" + chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/slotticker" @@ -109,6 +111,10 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { return errors.Wrap(err, "can't instantiate keyper core") } + ksIndices, err := obskeyper.New(kpr.dbpool).GetKeyperSetIndices(ctx) + if err != nil { + return errors.Wrap(err, "failed to load keyper set indices") + } kpr.chainSyncClient, err = chainsync.NewClient( ctx, chainsync.WithClientURL(kpr.config.Gnosis.Node.EthereumURL), @@ -118,6 +124,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewKeyperSet(kpr.channelNewKeyperSet), chainsync.WithPrivateKey(kpr.config.Gnosis.Node.PrivateKey.Key), chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())), + chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), ) if err != nil { return err diff --git a/rolling-shutter/keyperimpl/optimism/keyper.go b/rolling-shutter/keyperimpl/optimism/keyper.go index 07ec6191e..0c48aeacc 100644 --- a/rolling-shutter/keyperimpl/optimism/keyper.go +++ b/rolling-shutter/keyperimpl/optimism/keyper.go @@ -19,6 +19,7 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" + chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/identitypreimage" @@ -78,6 +79,10 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { if err != nil { return errors.Wrap(err, "can't instantiate keyper core") } + ksIndices, err := obskeyper.New(kpr.dbpool).GetKeyperSetIndices(ctx) + if err != nil { + return errors.Wrap(err, "failed to load keyper set indices") + } // TODO: wrap the logger and pass in kpr.l2Client, err = chainsync.NewClient( ctx, @@ -85,6 +90,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewBlock(kpr.newBlock), chainsync.WithSyncNewKeyperSet(kpr.newKeyperSet), chainsync.WithPrivateKey(kpr.config.Optimism.PrivateKey.Key), + chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), ) if err != nil { return err diff --git a/rolling-shutter/keyperimpl/primev/keyper.go b/rolling-shutter/keyperimpl/primev/keyper.go index 609a35672..f058d8b64 100644 --- a/rolling-shutter/keyperimpl/primev/keyper.go +++ b/rolling-shutter/keyperimpl/primev/keyper.go @@ -11,6 +11,7 @@ import ( providerregistry "github.com/primev/mev-commit/contracts-abi/clients/ProviderRegistry" "github.com/rs/zerolog/log" + obskeyper "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper" "github.com/shutter-network/rolling-shutter/rolling-shutter/eonkeypublisher" "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper" "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/epochkghandler" @@ -19,6 +20,7 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" + chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service" "github.com/shutter-network/rolling-shutter/rolling-shutter/p2p" @@ -77,6 +79,10 @@ func (k *Keyper) Start(ctx context.Context, runner service.Runner) error { return errors.Wrap(err, "can't instantiate keyper core") } + ksIndices, err := obskeyper.New(k.dbpool).GetKeyperSetIndices(ctx) + if err != nil { + return errors.Wrap(err, "failed to load keyper set indices") + } k.chainSyncClient, err = chainsync.NewClient( ctx, chainsync.WithClientURL(k.config.Chain.Node.EthereumURL), @@ -86,6 +92,7 @@ func (k *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewBlock(k.channelNewBlock), chainsync.WithPrivateKey(k.config.Chain.Node.PrivateKey.Key), chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())), + chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), ) if err != nil { return err diff --git a/rolling-shutter/keyperimpl/shutterservice/keyper.go b/rolling-shutter/keyperimpl/shutterservice/keyper.go index 4d5960c41..4e6e7c3b7 100644 --- a/rolling-shutter/keyperimpl/shutterservice/keyper.go +++ b/rolling-shutter/keyperimpl/shutterservice/keyper.go @@ -14,6 +14,7 @@ import ( triggerRegistryV1Bindings "github.com/shutter-network/contracts/v2/bindings/shuttereventtriggerregistryv1" registryBindings "github.com/shutter-network/contracts/v2/bindings/shutterregistry" + obskeyper "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper" "github.com/shutter-network/rolling-shutter/rolling-shutter/eonkeypublisher" "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper" "github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/epochkghandler" @@ -22,6 +23,7 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" + chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service" "github.com/shutter-network/rolling-shutter/rolling-shutter/p2p" @@ -83,6 +85,10 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { if err != nil { return errors.Wrap(err, "can't instantiate keyper core") } + ksIndices, err := obskeyper.New(kpr.dbpool).GetKeyperSetIndices(ctx) + if err != nil { + return errors.Wrap(err, "failed to load keyper set indices") + } kpr.chainSyncClient, err = chainsync.NewClient( ctx, chainsync.WithClientURL(kpr.config.Chain.Node.EthereumURL), @@ -92,6 +98,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewKeyperSet(kpr.channelNewKeyperSet), chainsync.WithPrivateKey(kpr.config.Chain.Node.PrivateKey.Key), chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())), + chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), ) if err != nil { return err diff --git a/rolling-shutter/medley/chainsync/options.go b/rolling-shutter/medley/chainsync/options.go index 580876243..066544bf1 100644 --- a/rolling-shutter/medley/chainsync/options.go +++ b/rolling-shutter/medley/chainsync/options.go @@ -30,10 +30,11 @@ type options struct { syncStart *number.BlockNumber privKey *ecdsa.PrivateKey - handlerShutterState event.ShutterStateHandler - handlerKeyperSet event.KeyperSetHandler - handlerEonPublicKey event.EonPublicKeyHandler - handlerBlock event.BlockHandler + handlerShutterState event.ShutterStateHandler + handlerKeyperSet event.KeyperSetHandler + handlerEonPublicKey event.EonPublicKeyHandler + handlerBlock event.BlockHandler + knownKeyperSetRanges []syncer.IndexRange } func (o *options) verify() error { @@ -85,11 +86,12 @@ func (o *options) apply(ctx context.Context, c *Client) error { return err } c.kssync = &syncer.KeyperSetSyncer{ - Client: client, - Contract: c.KeyperSetManager, - Log: c.log, - StartBlock: o.syncStart, - Handler: o.handlerKeyperSet, + Client: client, + Contract: c.KeyperSetManager, + Log: c.log, + StartBlock: o.syncStart, + Handler: o.handlerKeyperSet, + KnownRanges: o.knownKeyperSetRanges, } if o.handlerKeyperSet != nil { c.services = append(c.services, c.kssync) @@ -214,6 +216,13 @@ func WithSyncNewKeyperSet(handler event.KeyperSetHandler) Option { } } +func WithKeyperSetKnownRanges(ranges []syncer.IndexRange) Option { + return func(o *options) error { + o.knownKeyperSetRanges = ranges + return nil + } +} + func WithSyncNewBlock(handler event.BlockHandler) Option { return func(o *options) error { o.handlerBlock = handler diff --git a/rolling-shutter/medley/chainsync/syncer/keyperset.go b/rolling-shutter/medley/chainsync/syncer/keyperset.go index 9daada16e..7a058a93c 100644 --- a/rolling-shutter/medley/chainsync/syncer/keyperset.go +++ b/rolling-shutter/medley/chainsync/syncer/keyperset.go @@ -22,11 +22,12 @@ func makeCallError(attrName string, err error) error { const channelSize = 10 type KeyperSetSyncer struct { - Client client.Client - Contract *bindings.KeyperSetManager - Log log.Logger - StartBlock *number.BlockNumber - Handler event.KeyperSetHandler + Client client.Client + Contract *bindings.KeyperSetManager + Log log.Logger + StartBlock *number.BlockNumber + Handler event.KeyperSetHandler + KnownRanges []IndexRange keyperAddedCh chan *bindings.KeyperSetManagerKeyperSetAdded } @@ -89,34 +90,22 @@ func (s *KeyperSetSyncer) getInitialKeyperSets(ctx context.Context) ([]*event.Ke if err := guardCallOpts(opts, false); err != nil { return nil, err } - bn := s.StartBlock.ToUInt64Ptr() - if bn == nil { - // this should not be the case - return nil, errors.New("start block is 'latest'") - } - - initialKeyperSets := []*event.KeyperSet{} - // this blocknumber specifies the argument to the contract - // getter - ks, err := s.GetKeyperSetForBlock(ctx, opts, s.StartBlock) - if err != nil { - return nil, err - } - initialKeyperSets = append(initialKeyperSets, ks) numKS, err := s.Contract.GetNumKeyperSets(opts) if err != nil { return nil, err } - for i := ks.Eon + 1; i < numKS; i++ { - ks, err = s.GetKeyperSetByIndex(ctx, opts, i) - if err != nil { - return nil, err + var initialKeyperSets []*event.KeyperSet + for _, r := range complementRanges(s.KnownRanges, numKS) { + for i := r.Start; i <= r.End; i++ { + ks, err := s.GetKeyperSetByIndex(ctx, opts, i) + if err != nil { + return nil, err + } + initialKeyperSets = append(initialKeyperSets, ks) } - initialKeyperSets = append(initialKeyperSets, ks) } - return initialKeyperSets, nil } diff --git a/rolling-shutter/medley/chainsync/syncer/range.go b/rolling-shutter/medley/chainsync/syncer/range.go new file mode 100644 index 000000000..75f6a9087 --- /dev/null +++ b/rolling-shutter/medley/chainsync/syncer/range.go @@ -0,0 +1,48 @@ +package syncer + +// IndexRange is a closed interval [Start, End] of keyper set indices. +type IndexRange struct { + Start uint64 + End uint64 +} + +// IndicesToRanges converts a sorted slice of int64 indices into contiguous ranges. +// Indices must be non-negative; this matches the int64 type returned by sqlc for bigint columns. +func IndicesToRanges(indices []int64) []IndexRange { + if len(indices) == 0 { + return nil + } + var ranges []IndexRange + start := uint64(indices[0]) //nolint:gosec + end := start + for _, idx := range indices[1:] { + v := uint64(idx) //nolint:gosec + if v == end+1 { + end = v + } else { + ranges = append(ranges, IndexRange{Start: start, End: end}) + start = v + end = v + } + } + return append(ranges, IndexRange{Start: start, End: end}) +} + +// complementRanges returns the ranges in [0, total) not covered by known. +// known must be sorted and non-overlapping. +func complementRanges(known []IndexRange, total uint64) []IndexRange { + var result []IndexRange + cursor := uint64(0) + for _, r := range known { + if r.Start > cursor { + result = append(result, IndexRange{Start: cursor, End: r.Start - 1}) + } + if r.End+1 > cursor { + cursor = r.End + 1 + } + } + if cursor < total { + result = append(result, IndexRange{Start: cursor, End: total - 1}) + } + return result +} diff --git a/rolling-shutter/medley/chainsync/syncer/range_test.go b/rolling-shutter/medley/chainsync/syncer/range_test.go new file mode 100644 index 000000000..5d1d920a1 --- /dev/null +++ b/rolling-shutter/medley/chainsync/syncer/range_test.go @@ -0,0 +1,108 @@ +package syncer + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func TestIndicesToRanges(t *testing.T) { + tests := []struct { + name string + indices []int64 + want []IndexRange + }{ + { + name: "empty", + indices: nil, + want: nil, + }, + { + name: "single", + indices: []int64{3}, + want: []IndexRange{{3, 3}}, + }, + { + name: "contiguous", + indices: []int64{0, 1, 2, 3}, + want: []IndexRange{{0, 3}}, + }, + { + name: "gap in middle", + indices: []int64{0, 1, 3, 4}, + want: []IndexRange{{0, 1}, {3, 4}}, + }, + { + name: "all separate", + indices: []int64{1, 3, 5}, + want: []IndexRange{{1, 1}, {3, 3}, {5, 5}}, + }, + { + name: "starts above zero", + indices: []int64{5, 6, 7}, + want: []IndexRange{{5, 7}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.DeepEqual(t, IndicesToRanges(tc.indices), tc.want) + }) + } +} + +func TestComplementRanges(t *testing.T) { + tests := []struct { + name string + known []IndexRange + total uint64 + want []IndexRange + }{ + { + name: "nothing known, fetch all", + known: nil, + total: 4, + want: []IndexRange{{0, 3}}, + }, + { + name: "nothing known, empty contract", + known: nil, + total: 0, + want: nil, + }, + { + name: "all known", + known: []IndexRange{{0, 7}}, + total: 8, + want: nil, + }, + { + name: "historical gap only", + known: []IndexRange{{5, 7}}, + total: 8, + want: []IndexRange{{0, 4}}, + }, + { + name: "historical gap plus new entries", + known: []IndexRange{{5, 7}}, + total: 10, + want: []IndexRange{{0, 4}, {8, 9}}, + }, + { + name: "gap in middle", + known: []IndexRange{{0, 2}, {5, 6}}, + total: 9, + want: []IndexRange{{3, 4}, {7, 8}}, + }, + { + name: "new entries only", + known: []IndexRange{{0, 5}}, + total: 8, + want: []IndexRange{{6, 7}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.DeepEqual(t, complementRanges(tc.known, tc.total), tc.want) + }) + } +} From 574b7b4791513ae5ebd0a9bcd5e4264e9a7c8d5a Mon Sep 17 00:00:00 2001 From: Jannik Luhn Date: Tue, 2 Jun 2026 16:55:08 +0200 Subject: [PATCH 2/2] fixup! fix: backfill historical keyper sets on startup --- rolling-shutter/keyperimpl/gnosis/keyper.go | 3 +- rolling-shutter/keyperimpl/optimism/keyper.go | 3 +- rolling-shutter/keyperimpl/primev/keyper.go | 3 +- .../keyperimpl/shutterservice/keyper.go | 3 +- rolling-shutter/medley/chainsync/options.go | 26 ++--- .../medley/chainsync/syncer/keyperset.go | 38 +++--- .../medley/chainsync/syncer/keyperset_test.go | 76 ++++++++++++ .../medley/chainsync/syncer/range.go | 48 -------- .../medley/chainsync/syncer/range_test.go | 108 ------------------ 9 files changed, 118 insertions(+), 190 deletions(-) create mode 100644 rolling-shutter/medley/chainsync/syncer/keyperset_test.go delete mode 100644 rolling-shutter/medley/chainsync/syncer/range.go delete mode 100644 rolling-shutter/medley/chainsync/syncer/range_test.go diff --git a/rolling-shutter/keyperimpl/gnosis/keyper.go b/rolling-shutter/keyperimpl/gnosis/keyper.go index f36cb5a8f..1ab0876b3 100644 --- a/rolling-shutter/keyperimpl/gnosis/keyper.go +++ b/rolling-shutter/keyperimpl/gnosis/keyper.go @@ -23,7 +23,6 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" - chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/slotticker" @@ -124,7 +123,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewKeyperSet(kpr.channelNewKeyperSet), chainsync.WithPrivateKey(kpr.config.Gnosis.Node.PrivateKey.Key), chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())), - chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), + chainsync.WithKnownKeyperSetIndices(ksIndices), ) if err != nil { return err diff --git a/rolling-shutter/keyperimpl/optimism/keyper.go b/rolling-shutter/keyperimpl/optimism/keyper.go index 0c48aeacc..235a69cb8 100644 --- a/rolling-shutter/keyperimpl/optimism/keyper.go +++ b/rolling-shutter/keyperimpl/optimism/keyper.go @@ -19,7 +19,6 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" - chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/identitypreimage" @@ -90,7 +89,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewBlock(kpr.newBlock), chainsync.WithSyncNewKeyperSet(kpr.newKeyperSet), chainsync.WithPrivateKey(kpr.config.Optimism.PrivateKey.Key), - chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), + chainsync.WithKnownKeyperSetIndices(ksIndices), ) if err != nil { return err diff --git a/rolling-shutter/keyperimpl/primev/keyper.go b/rolling-shutter/keyperimpl/primev/keyper.go index f058d8b64..dedded73d 100644 --- a/rolling-shutter/keyperimpl/primev/keyper.go +++ b/rolling-shutter/keyperimpl/primev/keyper.go @@ -20,7 +20,6 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" - chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service" "github.com/shutter-network/rolling-shutter/rolling-shutter/p2p" @@ -92,7 +91,7 @@ func (k *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewBlock(k.channelNewBlock), chainsync.WithPrivateKey(k.config.Chain.Node.PrivateKey.Key), chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())), - chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), + chainsync.WithKnownKeyperSetIndices(ksIndices), ) if err != nil { return err diff --git a/rolling-shutter/keyperimpl/shutterservice/keyper.go b/rolling-shutter/keyperimpl/shutterservice/keyper.go index 4e6e7c3b7..73d93dabb 100644 --- a/rolling-shutter/keyperimpl/shutterservice/keyper.go +++ b/rolling-shutter/keyperimpl/shutterservice/keyper.go @@ -23,7 +23,6 @@ import ( "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync" syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event" - chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db" "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service" "github.com/shutter-network/rolling-shutter/rolling-shutter/p2p" @@ -98,7 +97,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error { chainsync.WithSyncNewKeyperSet(kpr.channelNewKeyperSet), chainsync.WithPrivateKey(kpr.config.Chain.Node.PrivateKey.Key), chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())), - chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)), + chainsync.WithKnownKeyperSetIndices(ksIndices), ) if err != nil { return err diff --git a/rolling-shutter/medley/chainsync/options.go b/rolling-shutter/medley/chainsync/options.go index 066544bf1..46a861117 100644 --- a/rolling-shutter/medley/chainsync/options.go +++ b/rolling-shutter/medley/chainsync/options.go @@ -30,11 +30,11 @@ type options struct { syncStart *number.BlockNumber privKey *ecdsa.PrivateKey - handlerShutterState event.ShutterStateHandler - handlerKeyperSet event.KeyperSetHandler - handlerEonPublicKey event.EonPublicKeyHandler - handlerBlock event.BlockHandler - knownKeyperSetRanges []syncer.IndexRange + handlerShutterState event.ShutterStateHandler + handlerKeyperSet event.KeyperSetHandler + handlerEonPublicKey event.EonPublicKeyHandler + handlerBlock event.BlockHandler + knownKeyperSetIndices []int64 } func (o *options) verify() error { @@ -86,12 +86,12 @@ func (o *options) apply(ctx context.Context, c *Client) error { return err } c.kssync = &syncer.KeyperSetSyncer{ - Client: client, - Contract: c.KeyperSetManager, - Log: c.log, - StartBlock: o.syncStart, - Handler: o.handlerKeyperSet, - KnownRanges: o.knownKeyperSetRanges, + Client: client, + Contract: c.KeyperSetManager, + Log: c.log, + StartBlock: o.syncStart, + Handler: o.handlerKeyperSet, + KnownIndices: o.knownKeyperSetIndices, } if o.handlerKeyperSet != nil { c.services = append(c.services, c.kssync) @@ -216,9 +216,9 @@ func WithSyncNewKeyperSet(handler event.KeyperSetHandler) Option { } } -func WithKeyperSetKnownRanges(ranges []syncer.IndexRange) Option { +func WithKnownKeyperSetIndices(indices []int64) Option { return func(o *options) error { - o.knownKeyperSetRanges = ranges + o.knownKeyperSetIndices = indices return nil } } diff --git a/rolling-shutter/medley/chainsync/syncer/keyperset.go b/rolling-shutter/medley/chainsync/syncer/keyperset.go index 7a058a93c..e0832f02e 100644 --- a/rolling-shutter/medley/chainsync/syncer/keyperset.go +++ b/rolling-shutter/medley/chainsync/syncer/keyperset.go @@ -22,12 +22,12 @@ func makeCallError(attrName string, err error) error { const channelSize = 10 type KeyperSetSyncer struct { - Client client.Client - Contract *bindings.KeyperSetManager - Log log.Logger - StartBlock *number.BlockNumber - Handler event.KeyperSetHandler - KnownRanges []IndexRange + Client client.Client + Contract *bindings.KeyperSetManager + Log log.Logger + StartBlock *number.BlockNumber + Handler event.KeyperSetHandler + KnownIndices []int64 keyperAddedCh chan *bindings.KeyperSetManagerKeyperSetAdded } @@ -82,6 +82,20 @@ func (s *KeyperSetSyncer) Start(ctx context.Context, runner service.Runner) erro return nil } +func missingIndices(known []int64, total uint64) []uint64 { + knownSet := make(map[uint64]struct{}, len(known)) + for _, k := range known { + knownSet[uint64(k)] = struct{}{} //nolint:gosec + } + var result []uint64 + for i := uint64(0); i < total; i++ { + if _, ok := knownSet[i]; !ok { + result = append(result, i) + } + } + return result +} + func (s *KeyperSetSyncer) getInitialKeyperSets(ctx context.Context) ([]*event.KeyperSet, error) { opts := &bind.CallOpts{ Context: ctx, @@ -97,14 +111,12 @@ func (s *KeyperSetSyncer) getInitialKeyperSets(ctx context.Context) ([]*event.Ke } var initialKeyperSets []*event.KeyperSet - for _, r := range complementRanges(s.KnownRanges, numKS) { - for i := r.Start; i <= r.End; i++ { - ks, err := s.GetKeyperSetByIndex(ctx, opts, i) - if err != nil { - return nil, err - } - initialKeyperSets = append(initialKeyperSets, ks) + for _, i := range missingIndices(s.KnownIndices, numKS) { + ks, err := s.GetKeyperSetByIndex(ctx, opts, i) + if err != nil { + return nil, err } + initialKeyperSets = append(initialKeyperSets, ks) } return initialKeyperSets, nil } diff --git a/rolling-shutter/medley/chainsync/syncer/keyperset_test.go b/rolling-shutter/medley/chainsync/syncer/keyperset_test.go new file mode 100644 index 000000000..ca4e881d0 --- /dev/null +++ b/rolling-shutter/medley/chainsync/syncer/keyperset_test.go @@ -0,0 +1,76 @@ +package syncer + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func TestMissingIndices(t *testing.T) { + tests := []struct { + name string + known []int64 + total uint64 + want []uint64 + }{ + { + name: "nothing known, fetch all", + known: nil, + total: 4, + want: []uint64{0, 1, 2, 3}, + }, + { + name: "nothing known, empty contract", + known: nil, + total: 0, + want: nil, + }, + { + name: "all known", + known: []int64{0, 1, 2, 3}, + total: 4, + want: nil, + }, + { + name: "gap at the start", + known: []int64{3, 4, 5}, + total: 6, + want: []uint64{0, 1, 2}, + }, + { + name: "gap at the end", + known: []int64{0, 1, 2}, + total: 6, + want: []uint64{3, 4, 5}, + }, + { + name: "gap in the middle", + known: []int64{0, 1, 5, 6}, + total: 7, + want: []uint64{2, 3, 4}, + }, + { + name: "known indices beyond total are ignored", + known: []int64{0, 1, 10, 20}, + total: 4, + want: []uint64{2, 3}, + }, + { + name: "duplicate known indices", + known: []int64{1, 1, 2, 2}, + total: 4, + want: []uint64{0, 3}, + }, + { + name: "unordered known indices", + known: []int64{4, 1, 3}, + total: 5, + want: []uint64{0, 2}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.DeepEqual(t, missingIndices(tc.known, tc.total), tc.want) + }) + } +} diff --git a/rolling-shutter/medley/chainsync/syncer/range.go b/rolling-shutter/medley/chainsync/syncer/range.go deleted file mode 100644 index 75f6a9087..000000000 --- a/rolling-shutter/medley/chainsync/syncer/range.go +++ /dev/null @@ -1,48 +0,0 @@ -package syncer - -// IndexRange is a closed interval [Start, End] of keyper set indices. -type IndexRange struct { - Start uint64 - End uint64 -} - -// IndicesToRanges converts a sorted slice of int64 indices into contiguous ranges. -// Indices must be non-negative; this matches the int64 type returned by sqlc for bigint columns. -func IndicesToRanges(indices []int64) []IndexRange { - if len(indices) == 0 { - return nil - } - var ranges []IndexRange - start := uint64(indices[0]) //nolint:gosec - end := start - for _, idx := range indices[1:] { - v := uint64(idx) //nolint:gosec - if v == end+1 { - end = v - } else { - ranges = append(ranges, IndexRange{Start: start, End: end}) - start = v - end = v - } - } - return append(ranges, IndexRange{Start: start, End: end}) -} - -// complementRanges returns the ranges in [0, total) not covered by known. -// known must be sorted and non-overlapping. -func complementRanges(known []IndexRange, total uint64) []IndexRange { - var result []IndexRange - cursor := uint64(0) - for _, r := range known { - if r.Start > cursor { - result = append(result, IndexRange{Start: cursor, End: r.Start - 1}) - } - if r.End+1 > cursor { - cursor = r.End + 1 - } - } - if cursor < total { - result = append(result, IndexRange{Start: cursor, End: total - 1}) - } - return result -} diff --git a/rolling-shutter/medley/chainsync/syncer/range_test.go b/rolling-shutter/medley/chainsync/syncer/range_test.go deleted file mode 100644 index 5d1d920a1..000000000 --- a/rolling-shutter/medley/chainsync/syncer/range_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package syncer - -import ( - "testing" - - "gotest.tools/v3/assert" -) - -func TestIndicesToRanges(t *testing.T) { - tests := []struct { - name string - indices []int64 - want []IndexRange - }{ - { - name: "empty", - indices: nil, - want: nil, - }, - { - name: "single", - indices: []int64{3}, - want: []IndexRange{{3, 3}}, - }, - { - name: "contiguous", - indices: []int64{0, 1, 2, 3}, - want: []IndexRange{{0, 3}}, - }, - { - name: "gap in middle", - indices: []int64{0, 1, 3, 4}, - want: []IndexRange{{0, 1}, {3, 4}}, - }, - { - name: "all separate", - indices: []int64{1, 3, 5}, - want: []IndexRange{{1, 1}, {3, 3}, {5, 5}}, - }, - { - name: "starts above zero", - indices: []int64{5, 6, 7}, - want: []IndexRange{{5, 7}}, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.DeepEqual(t, IndicesToRanges(tc.indices), tc.want) - }) - } -} - -func TestComplementRanges(t *testing.T) { - tests := []struct { - name string - known []IndexRange - total uint64 - want []IndexRange - }{ - { - name: "nothing known, fetch all", - known: nil, - total: 4, - want: []IndexRange{{0, 3}}, - }, - { - name: "nothing known, empty contract", - known: nil, - total: 0, - want: nil, - }, - { - name: "all known", - known: []IndexRange{{0, 7}}, - total: 8, - want: nil, - }, - { - name: "historical gap only", - known: []IndexRange{{5, 7}}, - total: 8, - want: []IndexRange{{0, 4}}, - }, - { - name: "historical gap plus new entries", - known: []IndexRange{{5, 7}}, - total: 10, - want: []IndexRange{{0, 4}, {8, 9}}, - }, - { - name: "gap in middle", - known: []IndexRange{{0, 2}, {5, 6}}, - total: 9, - want: []IndexRange{{3, 4}, {7, 8}}, - }, - { - name: "new entries only", - known: []IndexRange{{0, 5}}, - total: 8, - want: []IndexRange{{6, 7}}, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.DeepEqual(t, complementRanges(tc.known, tc.total), tc.want) - }) - } -}