Merge tag 'v3.5.0' into epociask--feat-v3.5.0-rc2-merge

cmd/chaininfo/arbitrum_chain_info.json

@@ -211,7 +211,6 @@
     "chain-name": "sepolia-rollup",
     "sequencer-url": "https://sepolia-rollup-sequencer.arbitrum.io/rpc",
     "feed-url": "wss://sepolia-rollup.arbitrum.io/feed",
-    "track-block-metadata-from": 121300000,
     "chain-config": {
       "chainId": 421614,
       "homesteadBlock": 0,

cmd/nitro/nitro.go

@@ -239,7 +239,7 @@ func mainImpl() int {
 		log.Error("Sequencer coordinator must be enabled with parent chain reader, try starting node with --parent-chain.connection.url")
 		return 1
 	}
-	if nodeConfig.Execution.Sequencer.Enable && !nodeConfig.Execution.Sequencer.Timeboost.Enable && nodeConfig.Node.TransactionStreamer.TrackBlockMetadataFrom != 0 {
+	if nodeConfig.Execution.Sequencer.Enable && !nodeConfig.Execution.Sequencer.Dangerous.Timeboost.Enable && nodeConfig.Node.TransactionStreamer.TrackBlockMetadataFrom != 0 {
 		log.Warn("Sequencer node's track-block-metadata-from is set but timeboost is not enabled")
 	}
 
@@ -710,13 +710,13 @@ func mainImpl() int {
 	}
 
 	execNodeConfig := execNode.ConfigFetcher()
-	if execNodeConfig.Sequencer.Enable && execNodeConfig.Sequencer.Timeboost.Enable {
+	if execNodeConfig.Sequencer.Enable && execNodeConfig.Sequencer.Dangerous.Timeboost.Enable {
 		err := execNode.Sequencer.InitializeExpressLaneService(
 			execNode.Backend.APIBackend(),
 			execNode.FilterSystem,
-			common.HexToAddress(execNodeConfig.Sequencer.Timeboost.AuctionContractAddress),
-			common.HexToAddress(execNodeConfig.Sequencer.Timeboost.AuctioneerAddress),
-			execNodeConfig.Sequencer.Timeboost.EarlySubmissionGrace,
+			common.HexToAddress(execNodeConfig.Sequencer.Dangerous.Timeboost.AuctionContractAddress),
+			common.HexToAddress(execNodeConfig.Sequencer.Dangerous.Timeboost.AuctioneerAddress),
+			execNodeConfig.Sequencer.Dangerous.Timeboost.EarlySubmissionGrace,
 		)
 		if err != nil {
 			log.Error("failed to create express lane service", "err", err)

execution/gethexec/express_lane_service.go

@@ -104,8 +104,8 @@ pending:
 	}
 
 	var redisCoordinator *timeboost.RedisCoordinator
-	if seqConfig().Timeboost.RedisUrl != "" {
-		redisCoordinator, err = timeboost.NewRedisCoordinator(seqConfig().Timeboost.RedisUrl, roundTimingInfo.Round)
+	if seqConfig().Dangerous.Timeboost.RedisUrl != "" {
+		redisCoordinator, err = timeboost.NewRedisCoordinator(seqConfig().Dangerous.Timeboost.RedisUrl, roundTimingInfo.Round)
 		if err != nil {
 			return nil, fmt.Errorf("error initializing expressLaneService redis: %w", err)
 		}
@@ -367,11 +367,14 @@ func (es *expressLaneService) sequenceExpressLaneSubmission(
 
 	// Log an informational warning if the message's sequence number is in the future.
 	if msg.SequenceNumber > roundInfo.sequence {
-		if seqConfig.Timeboost.MaxQueuedTxCount != 0 &&
+		if seqConfig.Dangerous.Timeboost.MaxQueuedTxCount != 0 &&
 			// Pending msgs count=(total msgs present in the map)-(number of processed messages=roundInfo.Sequence)
 			// #nosec G115
-			len(roundInfo.msgAndResultBySequenceNumber)-int(roundInfo.sequence) >= seqConfig.Timeboost.MaxQueuedTxCount {
-			return fmt.Errorf("reached limit for queuing of future sequence number transactions, please try again with the correct sequence number. Limit: %d, Current sequence number: %d", seqConfig.Timeboost.MaxQueuedTxCount, roundInfo.sequence)
+			len(roundInfo.msgAndResultBySequenceNumber)-int(roundInfo.sequence) >= seqConfig.Dangerous.Timeboost.MaxQueuedTxCount {
+			return fmt.Errorf("reached limit for queuing of future sequence number transactions, please try again with the correct sequence number. Limit: %d, Current sequence number: %d", seqConfig.Dangerous.Timeboost.MaxQueuedTxCount, roundInfo.sequence)
+		}
+		if msg.SequenceNumber > roundInfo.sequence+seqConfig.Dangerous.Timeboost.MaxFutureSequenceDistance {
+			return fmt.Errorf("message sequence number has reached max allowed limit. SequenceNumber: %d, Limit: %d", msg.SequenceNumber, roundInfo.sequence+seqConfig.Dangerous.Timeboost.MaxFutureSequenceDistance)
 		}
 		log.Info("Received express lane submission with future sequence number", "SequenceNumber", msg.SequenceNumber)
 	}
@@ -505,9 +508,11 @@ func (es *expressLaneService) syncFromRedis() {
 		roundInfo.sequence = redisSeqCount
 	}
 	es.roundInfo.Add(currentRound, roundInfo)
+	sequenceCount := roundInfo.sequence
 	es.roundInfoMutex.Unlock()
 
-	pendingMsgs := es.redisCoordinator.GetAcceptedTxs(currentRound, roundInfo.sequence)
+	pendingMsgs := es.redisCoordinator.GetAcceptedTxs(currentRound, sequenceCount, sequenceCount+es.seqConfig().Dangerous.Timeboost.MaxFutureSequenceDistance)
+	log.Info("Attempting to sequence pending expressLane transactions from redis", "count", len(pendingMsgs))
 	for _, msg := range pendingMsgs {
 		es.LaunchThread(func(ctx context.Context) {
 			if err := es.sequenceExpressLaneSubmission(ctx, msg); err != nil {

execution/gethexec/express_lane_service_test.go

@@ -311,7 +311,7 @@ func Test_expressLaneService_sequenceExpressLaneSubmission_duplicateNonce(t *tes
 	els := &expressLaneService{
 		roundInfo:       containers.NewLruCache[uint64, *expressLaneRoundInfo](8),
 		roundTimingInfo: defaultTestRoundTimingInfo(time.Now()),
-		seqConfig:       func() *SequencerConfig { return &SequencerConfig{} },
+		seqConfig:       func() *SequencerConfig { return &DefaultSequencerConfig },
 	}
 	var err error
 	els.redisCoordinator, err = timeboost.NewRedisCoordinator(redisUrl, els.roundTimingInfo.Round)
@@ -357,7 +357,7 @@ func Test_expressLaneService_sequenceExpressLaneSubmission_outOfOrder(t *testing
 	els := &expressLaneService{
 		roundInfo:       containers.NewLruCache[uint64, *expressLaneRoundInfo](8),
 		roundTimingInfo: defaultTestRoundTimingInfo(time.Now()),
-		seqConfig:       func() *SequencerConfig { return &SequencerConfig{} },
+		seqConfig:       func() *SequencerConfig { return &DefaultSequencerConfig },
 	}
 	var err error
 	els.redisCoordinator, err = timeboost.NewRedisCoordinator(redisUrl, els.roundTimingInfo.Round)
@@ -454,7 +454,7 @@ func Test_expressLaneService_syncFromRedis(t *testing.T) {
 	els1 := &expressLaneService{
 		roundInfo:       containers.NewLruCache[uint64, *expressLaneRoundInfo](8),
 		roundTimingInfo: defaultTestRoundTimingInfo(time.Now()),
-		seqConfig:       func() *SequencerConfig { return &SequencerConfig{} },
+		seqConfig:       func() *SequencerConfig { return &DefaultSequencerConfig },
 	}
 	var err error
 	els1.redisCoordinator, err = timeboost.NewRedisCoordinator(redisUrl, els1.roundTimingInfo.Round)
@@ -496,7 +496,7 @@ func Test_expressLaneService_syncFromRedis(t *testing.T) {
 	els2 := &expressLaneService{
 		roundInfo:       containers.NewLruCache[uint64, *expressLaneRoundInfo](8),
 		roundTimingInfo: defaultTestRoundTimingInfo(time.Now()),
-		seqConfig:       func() *SequencerConfig { return &SequencerConfig{} },
+		seqConfig:       func() *SequencerConfig { return &DefaultSequencerConfig },
 	}
 	els2.redisCoordinator, err = timeboost.NewRedisCoordinator(redisUrl, els2.roundTimingInfo.Round)
 	require.NoError(t, err)

execution/gethexec/node.go

@@ -191,7 +191,7 @@ func CreateExecutionNode(
 	configFetcher ConfigFetcher,
 ) (*ExecutionNode, error) {
 	config := configFetcher()
-	execEngine, err := NewExecutionEngine(l2BlockChain, config.Sequencer.Timeboost.Enable)
+	execEngine, err := NewExecutionEngine(l2BlockChain, config.Sequencer.Dangerous.Timeboost.Enable)
 	if config.EnablePrefetchBlock {
 		execEngine.EnablePrefetchBlock()
 	}

execution/gethexec/sequencer.go

@@ -79,31 +79,37 @@ type SequencerConfig struct {
 	ExpectedSurplusSoftThreshold string          `koanf:"expected-surplus-soft-threshold" reload:"hot"`
 	ExpectedSurplusHardThreshold string          `koanf:"expected-surplus-hard-threshold" reload:"hot"`
 	EnableProfiling              bool            `koanf:"enable-profiling" reload:"hot"`
-	Timeboost                    TimeboostConfig `koanf:"timeboost"`
+	Dangerous                    DangerousConfig `koanf:"dangerous"`
 	expectedSurplusSoftThreshold int
 	expectedSurplusHardThreshold int
 }
 
+type DangerousConfig struct {
+	Timeboost TimeboostConfig `koanf:"timeboost"`
+}
+
 type TimeboostConfig struct {
-	Enable                 bool          `koanf:"enable"`
-	AuctionContractAddress string        `koanf:"auction-contract-address"`
-	AuctioneerAddress      string        `koanf:"auctioneer-address"`
-	ExpressLaneAdvantage   time.Duration `koanf:"express-lane-advantage"`
-	SequencerHTTPEndpoint  string        `koanf:"sequencer-http-endpoint"`
-	EarlySubmissionGrace   time.Duration `koanf:"early-submission-grace"`
-	MaxQueuedTxCount       int           `koanf:"max-queued-tx-count"`
-	RedisUrl               string        `koanf:"redis-url"`
+	Enable                    bool          `koanf:"enable"`
+	AuctionContractAddress    string        `koanf:"auction-contract-address"`
+	AuctioneerAddress         string        `koanf:"auctioneer-address"`
+	ExpressLaneAdvantage      time.Duration `koanf:"express-lane-advantage"`
+	SequencerHTTPEndpoint     string        `koanf:"sequencer-http-endpoint"`
+	EarlySubmissionGrace      time.Duration `koanf:"early-submission-grace"`
+	MaxQueuedTxCount          int           `koanf:"max-queued-tx-count"`
+	MaxFutureSequenceDistance uint64        `koanf:"max-future-sequence-distance"`
+	RedisUrl                  string        `koanf:"redis-url"`
 }
 
 var DefaultTimeboostConfig = TimeboostConfig{
-	Enable:                 false,
-	AuctionContractAddress: "",
-	AuctioneerAddress:      "",
-	ExpressLaneAdvantage:   time.Millisecond * 200,
-	SequencerHTTPEndpoint:  "http://localhost:8547",
-	EarlySubmissionGrace:   time.Second * 2,
-	MaxQueuedTxCount:       10,
-	RedisUrl:               "unset",
+	Enable:                    false,
+	AuctionContractAddress:    "",
+	AuctioneerAddress:         "",
+	ExpressLaneAdvantage:      time.Millisecond * 200,
+	SequencerHTTPEndpoint:     "http://localhost:8547",
+	EarlySubmissionGrace:      time.Second * 2,
+	MaxQueuedTxCount:          10,
+	MaxFutureSequenceDistance: 100,
+	RedisUrl:                  "unset",
 }
 
 func (c *SequencerConfig) Validate() error {
@@ -132,7 +138,7 @@ func (c *SequencerConfig) Validate() error {
 	if c.MaxTxDataSize > arbostypes.MaxL2MessageSize-50000 {
 		return errors.New("max-tx-data-size too large for MaxL2MessageSize")
 	}
-	return c.Timeboost.Validate()
+	return c.Dangerous.Timeboost.Validate()
 }
 
 func (c *TimeboostConfig) Validate() error {
@@ -148,6 +154,9 @@ func (c *TimeboostConfig) Validate() error {
 	if len(c.AuctioneerAddress) > 0 && !common.IsHexAddress(c.AuctioneerAddress) {
 		return fmt.Errorf("invalid timeboost.auctioneer-address \"%v\"", c.AuctioneerAddress)
 	}
+	if c.MaxFutureSequenceDistance == 0 {
+		return errors.New("timeboost max-future-sequence-distance option cannot be zero, it should be set to a positive value")
+	}
 	return nil
 }
 
@@ -171,7 +180,11 @@ var DefaultSequencerConfig = SequencerConfig{
 	ExpectedSurplusSoftThreshold: "default",
 	ExpectedSurplusHardThreshold: "default",
 	EnableProfiling:              false,
-	Timeboost:                    DefaultTimeboostConfig,
+	Dangerous:                    DefaultDangerousConfig,
+}
+
+var DefaultDangerousConfig = DangerousConfig{
+	Timeboost: DefaultTimeboostConfig,
 }
 
 func SequencerConfigAddOptions(prefix string, f *flag.FlagSet) {
@@ -181,7 +194,7 @@ func SequencerConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.Duration(prefix+".max-acceptable-timestamp-delta", DefaultSequencerConfig.MaxAcceptableTimestampDelta, "maximum acceptable time difference between the local time and the latest L1 block's timestamp")
 	f.StringSlice(prefix+".sender-whitelist", DefaultSequencerConfig.SenderWhitelist, "comma separated whitelist of authorized senders (if empty, everyone is allowed)")
 	AddOptionsForSequencerForwarderConfig(prefix+".forwarder", f)
-	TimeboostAddOptions(prefix+".timeboost", f)
+	DangerousAddOptions(prefix+".dangerous", f)
 
 	f.Int(prefix+".queue-size", DefaultSequencerConfig.QueueSize, "size of the pending tx queue")
 	f.Duration(prefix+".queue-timeout", DefaultSequencerConfig.QueueTimeout, "maximum amount of time transaction can wait in queue")
@@ -202,9 +215,14 @@ func TimeboostAddOptions(prefix string, f *flag.FlagSet) {
 	f.String(prefix+".sequencer-http-endpoint", DefaultTimeboostConfig.SequencerHTTPEndpoint, "this sequencer's http endpoint")
 	f.Duration(prefix+".early-submission-grace", DefaultTimeboostConfig.EarlySubmissionGrace, "period of time before the next round where submissions for the next round will be queued")
 	f.Int(prefix+".max-queued-tx-count", DefaultTimeboostConfig.MaxQueuedTxCount, "maximum allowed number of express lane txs with future sequence number to be queued. Set 0 to disable this check and a negative value to prevent queuing of any future sequence number transactions")
+	f.Uint64(prefix+".max-future-sequence-distance", DefaultTimeboostConfig.MaxFutureSequenceDistance, "maximum allowed difference (in terms of sequence numbers) between a future express lane tx and the current sequence count of a round")
 	f.String(prefix+".redis-url", DefaultTimeboostConfig.RedisUrl, "the Redis URL for expressLaneService to coordinate via")
 }
 
+func DangerousAddOptions(prefix string, f *flag.FlagSet) {
+	TimeboostAddOptions(prefix+".timeboost", f)
+}
+
 type txQueueItem struct {
 	tx              *types.Transaction
 	txSize          int // size in bytes of the marshalled transaction
@@ -481,7 +499,7 @@ func (s *Sequencer) PublishTransaction(parentCtx context.Context, tx *types.Tran
 
 	config := s.config()
 	queueTimeout := config.QueueTimeout
-	queueCtx, cancelFunc := ctxWithTimeout(parentCtx, queueTimeout+config.Timeboost.ExpressLaneAdvantage) // Include timeboost delay in ctx timeout
+	queueCtx, cancelFunc := ctxWithTimeout(parentCtx, queueTimeout+config.Dangerous.Timeboost.ExpressLaneAdvantage) // Include timeboost delay in ctx timeout
 	defer cancelFunc()
 
 	resultChan := make(chan error, 1)
@@ -511,7 +529,7 @@ func (s *Sequencer) PublishTransaction(parentCtx context.Context, tx *types.Tran
 }
 
 func (s *Sequencer) PublishAuctionResolutionTransaction(ctx context.Context, tx *types.Transaction) error {
-	if !s.config().Timeboost.Enable {
+	if !s.config().Dangerous.Timeboost.Enable {
 		return errors.New("timeboost not enabled")
 	}
 
@@ -567,7 +585,7 @@ func (s *Sequencer) PublishAuctionResolutionTransaction(ctx context.Context, tx
 }
 
 func (s *Sequencer) PublishExpressLaneTransaction(ctx context.Context, msg *timeboost.ExpressLaneSubmission) error {
-	if !s.config().Timeboost.Enable {
+	if !s.config().Dangerous.Timeboost.Enable {
 		return errors.New("timeboost not enabled")
 	}
 
@@ -640,9 +658,9 @@ func (s *Sequencer) publishTransactionToQueue(queueCtx context.Context, tx *type
 		return err
 	}
 
-	if s.config().Timeboost.Enable && s.expressLaneService != nil {
+	if s.config().Dangerous.Timeboost.Enable && s.expressLaneService != nil {
 		if !isExpressLaneController && s.expressLaneService.currentRoundHasController() {
-			time.Sleep(s.config().Timeboost.ExpressLaneAdvantage)
+			time.Sleep(s.config().Dangerous.Timeboost.ExpressLaneAdvantage)
 		}
 	}
 
@@ -1089,7 +1107,7 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) {
 	queueItems = s.precheckNonces(queueItems, totalBlockSize)
 	txes := make([]*types.Transaction, len(queueItems))
 	var timeboostedTxs map[common.Hash]struct{}
-	if config.Timeboost.Enable {
+	if config.Dangerous.Timeboost.Enable {
 		timeboostedTxs = make(map[common.Hash]struct{})
 	}
 	hooks := s.makeSequencingHooks()
@@ -1402,7 +1420,7 @@ func (s *Sequencer) Start(ctxIn context.Context) error {
 
 func (s *Sequencer) StopAndWait() {
 	s.StopWaiter.StopAndWait()
-	if s.config().Timeboost.Enable && s.expressLaneService != nil {
+	if s.config().Dangerous.Timeboost.Enable && s.expressLaneService != nil {
 		s.expressLaneService.StopAndWait()
 	}
 	if s.txRetryQueue.Len() == 0 &&

system_tests/timeboost_test.go

@@ -1292,10 +1292,11 @@ func setupExpressLaneAuction(
 	builderSeq.nodeConfig.SeqCoordinator.MyUrl = nodeNames[0]
 	builderSeq.nodeConfig.SeqCoordinator.DeleteFinalizedMsgs = false
 	builderSeq.execConfig.Sequencer.Enable = true
-	builderSeq.execConfig.Sequencer.Timeboost = gethexec.TimeboostConfig{
-		Enable:               false, // We need to start without timeboost initially to create the auction contract
-		ExpressLaneAdvantage: time.Second * 5,
-		RedisUrl:             expressLaneRedisURL,
+	builderSeq.execConfig.Sequencer.Dangerous.Timeboost = gethexec.TimeboostConfig{
+		Enable:                    false, // We need to start without timeboost initially to create the auction contract
+		ExpressLaneAdvantage:      time.Second * 5,
+		RedisUrl:                  expressLaneRedisURL,
+		MaxFutureSequenceDistance: 1500, // Required for TestExpressLaneTransactionHandlingComplex
 	}
 	builderSeq.nodeConfig.TransactionStreamer.TrackBlockMetadataFrom = 1
 	cleanupSeq := builderSeq.Build(t)
@@ -1475,7 +1476,7 @@ func setupExpressLaneAuction(
 
 	// This is hacky- we are manually starting the ExpressLaneService here instead of letting it be started
 	// by the sequencer. This is due to needing to deploy the auction contract first.
-	builderSeq.execConfig.Sequencer.Timeboost.Enable = true
+	builderSeq.execConfig.Sequencer.Dangerous.Timeboost.Enable = true
 	err = builderSeq.L2.ExecNode.Sequencer.InitializeExpressLaneService(builderSeq.L2.ExecNode.Backend.APIBackend(), builderSeq.L2.ExecNode.FilterSystem, proxyAddr, seqInfo.GetAddress("AuctionContract"), gethexec.DefaultTimeboostConfig.EarlySubmissionGrace)
 	Require(t, err)
 	builderSeq.L2.ExecNode.Sequencer.StartExpressLaneService(ctx)

timeboost/redis_coordinator.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -89,50 +87,31 @@ func acceptedTxKeyFor(round, seqNum uint64) string {
 	return fmt.Sprintf("%s%d.%d", EXPRESS_LANE_ACCEPTED_TX_KEY_PREFIX, round, seqNum)
 }
 
-func (rc *RedisCoordinator) GetAcceptedTxs(round, startSeqNum uint64) []*ExpressLaneSubmission {
+func (rc *RedisCoordinator) GetAcceptedTxs(round, startSeqNum, endSeqNum uint64) []*ExpressLaneSubmission {
 	ctx := rc.GetContext()
 	fetchMsg := func(key string) *ExpressLaneSubmission {
 		msgBytes, err := rc.client.Get(ctx, key).Bytes()
 		if err != nil {
-			log.Error("Error fetching accepted expressLane tx", "err", err)
+			log.Error("Error fetching accepted expressLane tx", "key", key, "err", err)
 			return nil
 		}
 		msgJson := JsonExpressLaneSubmission{}
 		if err := json.Unmarshal(msgBytes, &msgJson); err != nil {
-			log.Error("Error unmarshalling", "err", err)
+			log.Error("Error unmarshalling", "key", key, "err", err)
 			return nil
 		}
 		msg, err := JsonSubmissionToGo(&msgJson)
 		if err != nil {
-			log.Error("Error converting JsonExpressLaneSubmission to ExpressLaneSubmission", "err", err)
+			log.Error("Error converting JsonExpressLaneSubmission to ExpressLaneSubmission", "key", key, "err", err)
 			return nil
 		}
 		return msg
 	}
 
 	var msgs []*ExpressLaneSubmission
-	prefix := fmt.Sprintf("%s%d.", EXPRESS_LANE_ACCEPTED_TX_KEY_PREFIX, round)
-	cursor := uint64(0)
-	for {
-		keys, cursor, err := rc.client.Scan(ctx, cursor, prefix+"*", 0).Result()
-		if err != nil {
-			break // Best effort
-		}
-		for _, key := range keys {
-			seq, err := strconv.Atoi(strings.TrimPrefix(key, prefix))
-			if err != nil {
-				log.Error("Error getting sequence number from the redis key of accepted timeboost Tx", "key", key, "error", err)
-				continue
-			}
-			// #nosec G115
-			if uint64(seq) >= startSeqNum {
-				if msg := fetchMsg(key); msg != nil {
-					msgs = append(msgs, msg)
-				}
-			}
-		}
-		if cursor == 0 {
-			break
+	for seq := startSeqNum; seq <= endSeqNum; seq++ {
+		if msg := fetchMsg(acceptedTxKeyFor(round, seq)); msg != nil {
+			msgs = append(msgs, msg)
 		}
 	}
 	return msgs

timeboost/redis_coordinator_test.go

@@ -64,7 +64,7 @@ func TestRedisSeqCoordinatorAtomic(t *testing.T) {
 	}
 
 	checkCorrectness := func(startSeqNum uint64) {
-		fetchedMsgs := redisCoordinator.GetAcceptedTxs(round, startSeqNum)
+		fetchedMsgs := redisCoordinator.GetAcceptedTxs(round, startSeqNum, startSeqNum+5)
 		if len(fetchedMsgs) != len(addedMsgs[startSeqNum:]) {
 			t.Fatal("mismatch in number of fetched msgs")
 		}