From c22b5010ffadfbfe641bf13006ec0495f2a55f13 Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Tue, 7 May 2024 08:35:39 +0000 Subject: [PATCH 1/7] chore: update rpc server --- x/evm/keeper/keeper.go | 6 ++ x/evm/keeper/rpc_client.go | 78 ++++++++++++++------- x/evm/keeper/state_transition.go | 117 +++++++++++++++++++++++-------- x/evm/types/errors.go | 4 ++ 4 files changed, 147 insertions(+), 58 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 299f8de643..bee901ac64 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -78,6 +78,11 @@ type Keeper struct { // a set of store keys that should cover all the precompile use cases, // or ideally just pass the application's all stores. keys map[string]storetypes.StoreKey + + // sdkCtxs maps a unique handler ID to a sdk.Context. It is used to track + // requests coming from the SGX binary, and handle them using the + // correct sdk.Context. + sdkCtxs map[uint64]*sdk.Context } // NewKeeper generates new evm module keeper @@ -118,6 +123,7 @@ func NewKeeper( tracer: tracer, customContractFns: customContractFns, keys: keys, + sdkCtxs: make(map[uint64]*sdk.Context), } } diff --git a/x/evm/keeper/rpc_client.go b/x/evm/keeper/rpc_client.go index af604eddf9..510fd1e2ad 100644 --- a/x/evm/keeper/rpc_client.go +++ b/x/evm/keeper/rpc_client.go @@ -41,8 +41,12 @@ func (c *sgxRPCClient) doCall(method string, args, reply any) error { return err } -func (c *sgxRPCClient) PrepareTx(args PrepareTxArgs, reply *PrepareTxReply) error { - return c.doCall("SgxRpcServer.PrepareTx", args, reply) +func (c *sgxRPCClient) StartEVM(args StartEVMArgs, reply *StartEVMReply) error { + return c.doCall("SgxRpcServer.StartEVM", args, reply) +} + +func (c *sgxRPCClient) InitFhevm(args InitFhevmArgs, reply *InitFhevmReply) error { + return c.doCall("SgxRpcServer.InitFhevm", args, reply) } func (c *sgxRPCClient) Call(args CallArgs, reply *CallReply) error { @@ -85,10 +89,10 @@ func (c *sgxRPCClient) StateDBGetLogs(args StateDBGetLogsArgs, reply *StateDBGet return c.doCall("SgxRpcServer.StateDBGetLogs", args, reply) } -// PrepareTxEVMConfig only contains the fields from EVMConfig that are needed +// StartEVMTxEVMConfig only contains the fields from EVMConfig that are needed // to create a new EVM instance. This is used to pass the EVM configuration // over RPC to the SGX binary. -type PrepareTxEVMConfig struct { +type StartEVMTxEVMConfig struct { // ChainConfig is the EVM chain configuration in JSON format. Since the // underlying params.ChainConfig struct contains pointer fields, they are // not serializable over RPC with gob. Instead, the JSON representation is @@ -111,8 +115,8 @@ type PrepareTxEVMConfig struct { Overrides string } -// PrepareTxArgs is the argument struct for the SgxRpcServer.PrepareTx RPC method. -type PrepareTxArgs struct { +// StartEVMArgs is the argument struct for the SgxRpcServer.StartEVM RPC method. +type StartEVMArgs struct { TxHash []byte // Header is the Tendermint header of the block in which the transaction // will be executed. @@ -120,20 +124,31 @@ type PrepareTxArgs struct { // Msg is the EVM transaction message to run on the EVM. Msg core.Message // EvmConfig is the EVM configuration to set. - EvmConfig PrepareTxEVMConfig + EvmConfig StartEVMTxEVMConfig +} + +// StartEVMReply is the reply struct for the SgxRpcServer.StartEVM RPC method. +type StartEVMReply struct { + HandlerId uint64 +} + +// InitFhevmArgs is the arg struct for the SgxRpcServer.InitFhevm RPC method. +type InitFhevmArgs struct { + HandlerId uint64 } -// PrepareTxArgs is the reply struct for the SgxRpcServer.PrepareTx RPC method. -type PrepareTxReply struct { +// InitFhevmReply is the reply struct for the SgxRpcServer.InitFhevm RPC method. +type InitFhevmReply struct { } // CallArgs is the argument struct for the SgxRpcServer.Call RPC method. type CallArgs struct { - Caller vm.AccountRef - Addr common.Address - Input []byte - Gas uint64 - Value *big.Int + HandlerId uint64 + Caller vm.AccountRef + Addr common.Address + Input []byte + Gas uint64 + Value *big.Int } // CallReply is the reply struct for the SgxRpcServer.Call RPC method. @@ -144,10 +159,11 @@ type CallReply struct { // CreateArgs is the argument struct for the SgxRpcServer.Create RPC method. type CreateArgs struct { - Caller vm.AccountRef - Code []byte - Gas uint64 - Value *big.Int + HandlerId uint64 + Caller vm.AccountRef + Code []byte + Gas uint64 + Value *big.Int } // CreateReply is the reply struct for the SgxRpcServer.Create RPC method. @@ -159,7 +175,7 @@ type CreateReply struct { // CommitArgs is the argument struct for the SgxRpcServer.Commit RPC method. type CommitArgs struct { - Commit bool + HandlerId uint64 } // CommitReply is the reply struct for the SgxRpcServer.Commit RPC method. @@ -168,8 +184,9 @@ type CommitReply struct { // CommitArgs is the argument struct for the SgxRpcServer.StateDBSubBalance RPC method. type StateDBSubBalanceArgs struct { - Caller vm.AccountRef - Msg core.Message + HandlerId uint64 + Caller vm.AccountRef + Msg core.Message } // CommitReply is the reply struct for the SgxRpcServer.StateDBSubBalance RPC method. @@ -178,8 +195,9 @@ type StateDBSubBalanceReply struct { // CommitArgs is the argument struct for the SgxRpcServer.StateDSetNonce RPC method. type StateDBSetNonceArgs struct { - Caller vm.AccountRef - Nonce uint64 + HandlerId uint64 + Caller vm.AccountRef + Nonce uint64 } // CommitReply is the reply struct for the SgxRpcServer.StateDSetNonce RPC method. @@ -188,6 +206,7 @@ type StateDBSetNonceReply struct { // StateDBAddBalanceArgs is the argument struct for the SgxRpcServer.StateDBAddBalance RPC method. type StateDBAddBalanceArgs struct { + HandlerId uint64 Caller vm.AccountRef Msg core.Message LeftoverGas uint64 @@ -198,8 +217,10 @@ type StateDBAddBalanceReply struct { } type StateDBPrepareArgs struct { - Msg core.Message - Rules params.Rules + HandlerId uint64 + Msg core.Message + Rules params.Rules + CoinBase common.Address } type StateDBPrepareReply struct { @@ -207,8 +228,9 @@ type StateDBPrepareReply struct { // StateDBIncreaseNonceArgs is the argument struct for the SgxRpcServer.StateDBIncreaseNonce RPC method. type StateDBIncreaseNonceArgs struct { - Caller vm.AccountRef - Msg core.Message + HandlerId uint64 + Caller vm.AccountRef + Msg core.Message } // StateDBIncreaseNonceReply is the reply struct for the SgxRpcServer.StateDBIncreaseNonce RPC method. @@ -216,6 +238,7 @@ type StateDBIncreaseNonceReply struct { } type StateDBGetRefundArgs struct { + HandlerId uint64 } type StateDBGetRefundReply struct { @@ -223,6 +246,7 @@ type StateDBGetRefundReply struct { } type StateDBGetLogsArgs struct { + HandlerId uint64 } type StateDBGetLogsReply struct { diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index a2acdccc75..42ca349803 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -22,6 +22,7 @@ import ( "net" "net/http" "net/rpc" + "strings" cmttypes "github.com/cometbft/cometbft/types" @@ -305,7 +306,7 @@ func (k *Keeper) ApplyMessageWithConfig( return nil, errorsmod.Wrap(err, "failed to create new SGX rpc client") } - err = k.prepareTxForSgx(ctx, msg, cfg, sgxRPCClient) + handlerId, err := k.startEVM(ctx, msg, cfg, sgxRPCClient) if err != nil { return nil, errorsmod.Wrap(err, "failed to create new RPC server") } @@ -322,8 +323,9 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.SubBalance(sender.Address(), new(big.Int).Mul(msg.GasPrice, new(big.Int).SetUint64(msg.GasLimit))) var reply StateDBSubBalanceReply err := sgxRPCClient.StateDBSubBalance(StateDBSubBalanceArgs{ - Caller: sender, - Msg: msg, + HandlerId: handlerId, + Caller: sender, + Msg: msg, }, &reply) if err != nil { return nil, err @@ -333,8 +335,9 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.SetNonce(sender.Address(), stateDB.GetNonce(sender.Address())+1) var replyNonce StateDBIncreaseNonceReply err = sgxRPCClient.StateDBIncreaseNonce(StateDBIncreaseNonceArgs{ - Caller: sender, - Msg: msg, + HandlerId: handlerId, + Caller: sender, + Msg: msg, }, &replyNonce) if err != nil { return nil, err @@ -347,6 +350,7 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.AddBalance(sender.Address(), new(big.Int).Mul(msg.GasPrice, new(big.Int).SetUint64(leftoverGas))) var reply StateDBAddBalanceReply err := sgxRPCClient.StateDBAddBalance(StateDBAddBalanceArgs{ + HandlerId: handlerId, Caller: sender, Msg: msg, LeftoverGas: leftoverGas, @@ -391,8 +395,10 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.Prepare(rules, msg.From, cfg.CoinBase, msg.To, vm.ActivePrecompiles(rules), msg.AccessList) var replyPrepare StateDBPrepareReply err = sgxRPCClient.StateDBPrepare(StateDBPrepareArgs{ - Msg: msg, - Rules: rules, + HandlerId: handlerId, + Msg: msg, + Rules: rules, + CoinBase: cfg.CoinBase, }, &replyPrepare) if err != nil { return nil, err @@ -407,8 +413,9 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.SetNonce(sender.Address(), msg.Nonce) var replyNonce StateDBSetNonceReply err := sgxRPCClient.StateDBSetNonce(StateDBSetNonceArgs{ - Caller: sender, - Nonce: msg.Nonce, + HandlerId: handlerId, + Caller: sender, + Nonce: msg.Nonce, }, &replyNonce) if err != nil { return nil, err @@ -418,10 +425,11 @@ func (k *Keeper) ApplyMessageWithConfig( // ret, _, leftoverGas, vmErr = evm.Create(sender, msg.Data, leftoverGas, msg.Value) var reply CreateReply vmErr = sgxRPCClient.Create(CreateArgs{ - Caller: sender, - Code: msg.Data, - Gas: leftoverGas, - Value: msg.Value, + HandlerId: handlerId, + Caller: sender, + Code: msg.Data, + Gas: leftoverGas, + Value: msg.Value, }, &reply) ret = reply.Ret leftoverGas = reply.LeftOverGas @@ -429,19 +437,21 @@ func (k *Keeper) ApplyMessageWithConfig( // Ethermint original code: // stateDB.SetNonce(sender.Address(), msg.Nonce+1) sgxRPCClient.StateDBSetNonce(StateDBSetNonceArgs{ - Caller: sender, - Nonce: msg.Nonce + 1, + HandlerId: handlerId, + Caller: sender, + Nonce: msg.Nonce + 1, }, &replyNonce) } else { // Ethermint original code: // ret, leftoverGas, vmErr = evm.Call(sender, *msg.To, msg.Data, leftoverGas, msg.Value) var reply CallReply vmErr = sgxRPCClient.Call(CallArgs{ - Caller: sender, - Addr: *msg.To, - Input: msg.Data, - Gas: leftoverGas, - Value: msg.Value, + HandlerId: handlerId, + Caller: sender, + Addr: *msg.To, + Input: msg.Data, + Gas: leftoverGas, + Value: msg.Value, }, &reply) ret = reply.Ret leftoverGas = reply.LeftOverGas @@ -465,7 +475,9 @@ func (k *Keeper) ApplyMessageWithConfig( // Ethermint original code: // leftoverGas += GasToRefund(stateDB.GetRefund(), temporaryGasUsed, refundQuotient) var replyRefund StateDBGetRefundReply - err = sgxRPCClient.StateDBGetRefund(StateDBGetRefundArgs{}, &replyRefund) + err = sgxRPCClient.StateDBGetRefund(StateDBGetRefundArgs{ + HandlerId: handlerId, + }, &replyRefund) if err != nil { return nil, err } @@ -487,7 +499,7 @@ func (k *Keeper) ApplyMessageWithConfig( // } var reply CommitReply err := sgxRPCClient.Commit(CommitArgs{ - Commit: true, + HandlerId: handlerId, }, &reply) if err != nil { return nil, errorsmod.Wrap(err, "failed to commit sgx stateDB") @@ -516,7 +528,9 @@ func (k *Keeper) ApplyMessageWithConfig( // Ethermint original code: // Logs: types.NewLogsFromEth(stateDB.Logs()), var replyLog StateDBGetLogsReply - err = sgxRPCClient.StateDBGetLogs(StateDBGetLogsArgs{}, &replyLog) + err = sgxRPCClient.StateDBGetLogs(StateDBGetLogsArgs{ + HandlerId: handlerId, + }, &replyLog) if err != nil { return nil, err } @@ -531,28 +545,28 @@ func (k *Keeper) ApplyMessageWithConfig( }, nil } -// prepareTxForSgx prepares the transaction for the SGX enclave. It: +// startEVM prepares the transaction for the SGX enclave. It: // - creates an RPC server around the keeper to receive requests sent by the // SGX -// - sends a "PrepareTx" request to the SGX enclave with the relevant tx and +// - sends a "StartEVM" request to the SGX enclave with the relevant tx and // block info -func (k *Keeper) prepareTxForSgx(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgxRPCClient *sgxRPCClient) error { +func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgxRPCClient *sgxRPCClient) (uint64, error) { // Step 1. Create an RPC server to receive requests from the SGX enclave. err := k.runRPCServer(ctx, msg, cfg) if err != nil { - return err + return 0, err } // Step 2. Send a "PrepareTx" request to the SGX enclave. ChainConfigJson, err := json.Marshal(cfg.ChainConfig) if err != nil { - return err + return 0, err } ctx.HeaderHash() - args := PrepareTxArgs{ + args := StartEVMArgs{ Header: ctx.BlockHeader(), Msg: msg, - EvmConfig: PrepareTxEVMConfig{ + EvmConfig: StartEVMTxEVMConfig{ ChainConfigJson: ChainConfigJson, CoinBase: cfg.CoinBase, BaseFee: cfg.BaseFee, @@ -564,7 +578,43 @@ func (k *Keeper) prepareTxForSgx(ctx sdk.Context, msg core.Message, cfg *EVMConf }, } - return sgxRPCClient.PrepareTx(args, &PrepareTxReply{}) + reply := StartEVMReply{} + err = sgxRPCClient.StartEVM(args, &reply) + if err != nil { + // panic cosmos if sgx isn't available. + if isSgxDownError(err) { + panic("sgx rpc server is down") + } + return 0, err + } + + // Snapshot the sdk ctx with this handlerId + k.sdkCtxs[reply.HandlerId] = &ctx + + // We unfortunately can't call InitFhevm on the EVM instance during the + // StartEVM method (inside newEVM), because InitFhevm needs access to the + // stateDB (via a gRPC calls), but the stateDB (i.e. its associated + // sdk.Context) is not yet initialized at that point. + // + // To solve this, we separate the fhEVM initialization in two phases: + // 1. StartEVM: Initialize the EVM instance and store it in the evms map + // (doesn't need access to stateDB). + // 2. InitFhevm: Initialize the fhEVM instance (needs access to stateDB). + // ref: https://github.com/Inco-fhevm/zbc-go-ethereum/pull/2 + // + // We are doing step 2 here. + err = sgxRPCClient.InitFhevm(InitFhevmArgs{ + HandlerId: reply.HandlerId, + }, &InitFhevmReply{}) + if err != nil { + // panic cosmos if sgx isn't available. + if isSgxDownError(err) { + panic("sgx rpc server is down") + } + return 0, err + } + + return reply.HandlerId, err } func (k *Keeper) runRPCServer(ctx sdk.Context, msg core.Message, cfg *EVMConfig) error { @@ -593,3 +643,8 @@ func (k *Keeper) runRPCServer(ctx sdk.Context, msg core.Message, cfg *EVMConfig) return nil } + +// isSgxDownError checks if the error is related with RPC server down +func isSgxDownError(err error) bool { + return strings.Contains(err.Error(), types.ErrTeeConnDown.Error()) +} diff --git a/x/evm/types/errors.go b/x/evm/types/errors.go index 87179c50f6..e94f241530 100644 --- a/x/evm/types/errors.go +++ b/x/evm/types/errors.go @@ -49,6 +49,7 @@ const ( codeErrGasOverflow codeErrInvalidAccount codeErrInvalidGasLimit + codeErrTeeConnDown ) var ErrPostTxProcessing = errors.New("failed to execute post processing") @@ -116,6 +117,9 @@ var ( // ErrInvalidGasLimit returns an error if gas limit value is invalid ErrInvalidGasLimit = errorsmod.Register(ModuleName, codeErrInvalidGasLimit, "invalid gas limit") + + // TEE isn't responding + ErrTeeConnDown = errorsmod.Register(ModuleName, codeErrTeeConnDown, "tee connection down") ) // NewExecErrorWithReason unpacks the revert return bytes and returns a wrapped error From 7586a155cd4260c5873ab6d73290d1b558683711 Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Tue, 7 May 2024 08:55:38 +0000 Subject: [PATCH 2/7] chore: remove ctx map in keeper as ctx is being handled in rpc server --- x/evm/keeper/keeper.go | 6 ---- x/evm/keeper/rpc_server.go | 49 +++++++++++++++++++------------- x/evm/keeper/state_transition.go | 3 -- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index bee901ac64..299f8de643 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -78,11 +78,6 @@ type Keeper struct { // a set of store keys that should cover all the precompile use cases, // or ideally just pass the application's all stores. keys map[string]storetypes.StoreKey - - // sdkCtxs maps a unique handler ID to a sdk.Context. It is used to track - // requests coming from the SGX binary, and handle them using the - // correct sdk.Context. - sdkCtxs map[uint64]*sdk.Context } // NewKeeper generates new evm module keeper @@ -123,7 +118,6 @@ func NewKeeper( tracer: tracer, customContractFns: customContractFns, keys: keys, - sdkCtxs: make(map[uint64]*sdk.Context), } } diff --git a/x/evm/keeper/rpc_server.go b/x/evm/keeper/rpc_server.go index 6aff116e38..1a65a7f930 100644 --- a/x/evm/keeper/rpc_server.go +++ b/x/evm/keeper/rpc_server.go @@ -71,8 +71,9 @@ func (s *EthmRpcServer) DeleteAccount(args *DeleteAccountArgs, reply *DeleteAcco // AddBalanceArgs is the argument struct for the statedb.Keeper#AddBalance method. type AddBalanceArgs struct { - Addr sdk.AccAddress - Amount sdk.Coins + HandlerId uint64 + Addr sdk.AccAddress + Amount sdk.Coins } // AddBalanceReply is the reply struct for the statedb.Keeper#AddBalance method. @@ -81,8 +82,9 @@ type AddBalanceReply struct { // SubBalanceArgs is the argument struct for the statedb.Keeper#SubBalance method. type SubBalanceArgs struct { - Addr sdk.AccAddress - Amount sdk.Coins + HandlerId uint64 + Addr sdk.AccAddress + Amount sdk.Coins } // SubBalanceReply is the reply struct for the statedb.Keeper#SubBalance method. @@ -91,8 +93,9 @@ type SubBalanceReply struct { // GetBalanceArgs is the argument struct for the statedb.Keeper#GetBalance method. type GetBalanceArgs struct { - Addr sdk.AccAddress - Denom string + HandlerId uint64 + Addr sdk.AccAddress + Denom string } // GetBalanceReply is the reply struct for the statedb.Keeper#GetBalance method. @@ -102,7 +105,8 @@ type GetBalanceReply struct { // GetAccountArgs is the argument struct for the statedb.Keeper#GetAccount method. type GetAccountArgs struct { - Addr common.Address + HandlerId uint64 + Addr common.Address } // GetAccountReply is the reply struct for the statedb.Keeper#GetAccount method. @@ -112,18 +116,21 @@ type GetAccountReply struct { // GetStateArgs is the argument struct for the statedb.Keeper#GetState method. type GetStateArgs struct { - Addr common.Address - Key common.Hash + HandlerId uint64 + Addr common.Address + Key common.Hash } // GetStateReply is the reply struct for the statedb.Keeper#GetState method. type GetStateReply struct { - Hash common.Hash + HandlerId uint64 + Hash common.Hash } // GetCodeArgs is the argument struct for the statedb.Keeper#GetCode method. type GetCodeArgs struct { - CodeHash common.Hash + HandlerId uint64 + CodeHash common.Hash } // GetCodeReply is the reply struct for the statedb.Keeper#GetCode method. @@ -133,8 +140,9 @@ type GetCodeReply struct { // SetAccountArgs is the argument struct for the statedb.Keeper#SetAccount method. type SetAccountArgs struct { - Addr common.Address - Account statedb.Account + HandlerId uint64 + Addr common.Address + Account statedb.Account } // SetAccountReply is the reply struct for the statedb.Keeper#SetAccount method. @@ -143,9 +151,10 @@ type SetAccountReply struct { // SetStateArgs is the argument struct for the statedb.Keeper#SetState method. type SetStateArgs struct { - Addr common.Address - Key common.Hash - Value []byte + HandlerId uint64 + Addr common.Address + Key common.Hash + Value []byte } // SetStateReply is the reply struct for the statedb.Keeper#SetState method. @@ -154,8 +163,9 @@ type SetStateReply struct { // SetCodeArgs is the argument struct for the statedb.Keeper#SetCode method. type SetCodeArgs struct { - CodeHash []byte - Code []byte + HandlerId uint64 + CodeHash []byte + Code []byte } // SetCodeReply is the reply struct for the statedb.Keeper#SetCode method. @@ -164,7 +174,8 @@ type SetCodeReply struct { // DeleteAccountArgs is the argument struct for the statedb.Keeper#DeleteAccount method. type DeleteAccountArgs struct { - Addr common.Address + HandlerId uint64 + Addr common.Address } // DeleteAccountReply is the reply struct for the statedb.Keeper#DeleteAccount method. diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index 42ca349803..f1b711f710 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -588,9 +588,6 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx return 0, err } - // Snapshot the sdk ctx with this handlerId - k.sdkCtxs[reply.HandlerId] = &ctx - // We unfortunately can't call InitFhevm on the EVM instance during the // StartEVM method (inside newEVM), because InitFhevm needs access to the // stateDB (via a gRPC calls), but the stateDB (i.e. its associated From 2ebadfe062fed8b6dee4549a5298101f149708ac Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Mon, 13 May 2024 06:16:33 +0000 Subject: [PATCH 3/7] chore: remove rpc server start --- x/evm/keeper/keeper.go | 10 ++++ x/evm/keeper/rpc_server.go | 82 +++++++++++++++++++++++++------- x/evm/keeper/state_transition.go | 40 ++-------------- 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 299f8de643..db2f60cb9d 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -78,6 +78,11 @@ type Keeper struct { // a set of store keys that should cover all the precompile use cases, // or ideally just pass the application's all stores. keys map[string]storetypes.StoreKey + + // sdkCtxs maps a unique handler ID to a sdk.Context. It is used to track + // requests coming from the SGX binary, and handle them using the + // correct sdk.Context. + sdkCtxs map[uint64]*sdk.Context } // NewKeeper generates new evm module keeper @@ -118,6 +123,7 @@ func NewKeeper( tracer: tracer, customContractFns: customContractFns, keys: keys, + sdkCtxs: make(map[uint64]*sdk.Context), } } @@ -391,3 +397,7 @@ func (k Keeper) AddTransientGasUsed(ctx sdk.Context, gasUsed uint64) (uint64, er k.SetTransientGasUsed(ctx, result) return result, nil } + +func (k Keeper) GetSdkCtx(handlerId uint64) *sdk.Context { + return k.sdkCtxs[handlerId] +} diff --git a/x/evm/keeper/rpc_server.go b/x/evm/keeper/rpc_server.go index 1a65a7f930..a0e7a0d48e 100644 --- a/x/evm/keeper/rpc_server.go +++ b/x/evm/keeper/rpc_server.go @@ -5,68 +5,116 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/evmos/ethermint/x/evm/statedb" ) // EthmRpcServer is a RPC server wrapper around the keeper. It is updated on // each new sdk.Message with the latest context and Ethereum core.Message. type EthmRpcServer struct { - ctx sdk.Context - msg core.Message - evmCfg *EVMConfig - k *Keeper + Keeper *Keeper } -func (s *EthmRpcServer) GetHash(height *uint64, hash *common.Hash) error { - *hash = s.k.GetHashFn(s.ctx)(*height) +func (s *EthmRpcServer) GetHash(handlerId uint64, height *uint64, hash *common.Hash) error { + ctx := s.Keeper.GetSdkCtx(handlerId) + if ctx == nil { + panic("context is invalid") + } + + *hash = s.Keeper.GetHashFn(*ctx)(*height) return nil } func (s *EthmRpcServer) AddBalance(args *AddBalanceArgs, reply *AddBalanceReply) error { - return s.k.AddBalance(s.ctx, args.Addr, args.Amount) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + return s.Keeper.AddBalance(*ctx, args.Addr, args.Amount) } func (s *EthmRpcServer) SubBalance(args *SubBalanceArgs, reply *SubBalanceReply) error { - return s.k.SubBalance(s.ctx, args.Addr, args.Amount) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + return s.Keeper.SubBalance(*ctx, args.Addr, args.Amount) } func (s *EthmRpcServer) GetBalance(args *GetBalanceArgs, reply *GetBalanceReply) error { - reply.Balance = s.k.GetBalance(s.ctx, args.Addr, args.Denom) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + reply.Balance = s.Keeper.GetBalance(*ctx, args.Addr, args.Denom) return nil } func (s *EthmRpcServer) GetAccount(args *GetAccountArgs, reply *GetAccountReply) error { - reply.Account = s.k.GetAccount(s.ctx, args.Addr) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + reply.Account = s.Keeper.GetAccount(*ctx, args.Addr) return nil } func (s *EthmRpcServer) GetState(args *GetStateArgs, reply *GetStateReply) error { - reply.Hash = s.k.GetState(s.ctx, args.Addr, args.Key) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + reply.Hash = s.Keeper.GetState(*ctx, args.Addr, args.Key) return nil } func (s *EthmRpcServer) GetCode(args *GetCodeArgs, reply *GetCodeReply) error { - reply.Code = s.k.GetCode(s.ctx, args.CodeHash) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + reply.Code = s.Keeper.GetCode(*ctx, args.CodeHash) return nil } func (s *EthmRpcServer) SetAccount(args *SetAccountArgs, reply *SetAccountReply) error { - return s.k.SetAccount(s.ctx, args.Addr, args.Account) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + + return s.Keeper.SetAccount(*ctx, args.Addr, args.Account) } func (s *EthmRpcServer) SetState(args *SetStateArgs, reply *SetStateReply) error { - s.k.SetState(s.ctx, args.Addr, args.Key, args.Value) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + s.Keeper.SetState(*ctx, args.Addr, args.Key, args.Value) return nil } func (s *EthmRpcServer) SetCode(args *SetCodeArgs, reply *SetCodeReply) error { - s.k.SetCode(s.ctx, args.CodeHash, args.Code) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + s.Keeper.SetCode(*ctx, args.CodeHash, args.Code) return nil } func (s *EthmRpcServer) DeleteAccount(args *DeleteAccountArgs, reply *DeleteAccountReply) error { - return s.k.DeleteAccount(s.ctx, args.Addr) + ctx := s.Keeper.GetSdkCtx(args.HandlerId) + if ctx == nil { + panic("context is invalid") + } + return s.Keeper.DeleteAccount(*ctx, args.Addr) } // AddBalanceArgs is the argument struct for the statedb.Keeper#AddBalance method. diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index f1b711f710..9b8608bec2 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -19,9 +19,6 @@ import ( "encoding/json" "fmt" "math/big" - "net" - "net/http" - "net/rpc" "strings" cmttypes "github.com/cometbft/cometbft/types" @@ -551,13 +548,7 @@ func (k *Keeper) ApplyMessageWithConfig( // - sends a "StartEVM" request to the SGX enclave with the relevant tx and // block info func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgxRPCClient *sgxRPCClient) (uint64, error) { - // Step 1. Create an RPC server to receive requests from the SGX enclave. - err := k.runRPCServer(ctx, msg, cfg) - if err != nil { - return 0, err - } - - // Step 2. Send a "PrepareTx" request to the SGX enclave. + // Step 1. Send a "PrepareTx" request to the SGX enclave. ChainConfigJson, err := json.Marshal(cfg.ChainConfig) if err != nil { return 0, err @@ -587,6 +578,8 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx } return 0, err } + // Snapshot the sdk ctx with this handlerId + k.sdkCtxs[reply.HandlerId] = &ctx // We unfortunately can't call InitFhevm on the EVM instance during the // StartEVM method (inside newEVM), because InitFhevm needs access to the @@ -614,33 +607,6 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx return reply.HandlerId, err } -func (k *Keeper) runRPCServer(ctx sdk.Context, msg core.Message, cfg *EVMConfig) error { - defer func() { - if r := recover(); r != nil { - ctx.Logger().Debug("recovered from panic", "error", r) - } - }() - - // TODO Think about whether the RPC server should be persistent or ephemeral - // - If it's persistent, we need to handle the lifecycle of the RPC server - // - If it's ephemeral, we need to create a new RPC server for each message - // The current implementation is ephemeral. - srv := &EthmRpcServer{k: k, ctx: ctx, msg: msg, evmCfg: cfg} - rpc.Register(srv) - rpc.HandleHTTP() - - // TODO handle port customization - l, err := net.Listen("tcp", ":9093") - if err != nil { - // TODO handle error - panic(err) - } - // TODO Handle shutdown - go http.Serve(l, nil) - - return nil -} - // isSgxDownError checks if the error is related with RPC server down func isSgxDownError(err error) bool { return strings.Contains(err.Error(), types.ErrTeeConnDown.Error()) From dc44609a8ed4691b40f46845002962d521fae699 Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Mon, 13 May 2024 14:56:24 +0000 Subject: [PATCH 4/7] chore: rpc run in etheremint --- server/start.go | 64 +++++++++++++++++++++ x/evm/keeper/keeper.go | 10 ++-- x/evm/keeper/rpc_client.go | 60 ++++++++++---------- x/evm/keeper/rpc_server.go | 84 ++++++++++++++-------------- x/evm/keeper/state_transition.go | 96 ++++++++++++++++++-------------- 5 files changed, 196 insertions(+), 118 deletions(-) diff --git a/server/start.go b/server/start.go index 9342874eee..f624582a09 100644 --- a/server/start.go +++ b/server/start.go @@ -21,6 +21,7 @@ import ( "io" "net" "net/http" + "net/rpc" "os" "path/filepath" "runtime/pprof" @@ -69,8 +70,14 @@ import ( "github.com/evmos/ethermint/server/config" srvflags "github.com/evmos/ethermint/server/flags" ethermint "github.com/evmos/ethermint/types" + evmKeeper "github.com/evmos/ethermint/x/evm/keeper" ) +type IncoApp interface { + // Return evm keeper which is used in ethermint side to get evm keeper access + GetEvmKeeper() *evmKeeper.Keeper +} + // DBOpener is a function to open `application.db`, potentially with customized options. type DBOpener func(opts types.AppOptions, rootDir string, backend dbm.BackendType) (dbm.DB, error) @@ -488,9 +495,66 @@ func startInProcess(svrCtx *server.Context, clientCtx client.Context, opts Start return err } + // Start running rpc server for sgx binary access on the evm Keeper statedb + listener, err := startRpcServer(svrCtx, g, app) + if err != nil { + return err + } + + defer func() { + // TODO: Graceful stop + listener.Close() + }() + return g.Wait() } +func startRpcServer( + svrCtx *server.Context, + g *errgroup.Group, + app types.Application, +) (listener net.Listener, err error) { + ethApp := app.(IncoApp) + if ethApp == nil { + return nil, nil + } + + evmKeeper := ethApp.GetEvmKeeper() + if evmKeeper == nil { + return nil, nil + } + + g.Go(func() error { + listener, err = runRPCServer(svrCtx, evmKeeper) + return err + }) + + return +} + +func runRPCServer(svrCtx *server.Context, keeper *evmKeeper.Keeper) (net.Listener, error) { + defer func() { + if r := recover(); r != nil { + svrCtx.Logger.Error("recovered from panic", "error", r) + } + }() + + // Run a persistent RPC server for sgx binary can access to evm keeper statedb + srv := &evmKeeper.EthmRpcServer{Keeper: keeper} + rpc.Register(srv) + rpc.HandleHTTP() + + // TODO handle port customization + l, err := net.Listen("tcp", ":9093") + if err != nil { + return nil, err + } + + go http.Serve(l, nil) + + return l, nil +} + func openDB(_ types.AppOptions, rootDir string, backendType dbm.BackendType) (dbm.DB, error) { dataDir := filepath.Join(rootDir, "data") return dbm.NewDB("application", backendType, dataDir) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index db2f60cb9d..8d81c669e4 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -79,9 +79,9 @@ type Keeper struct { // or ideally just pass the application's all stores. keys map[string]storetypes.StoreKey - // sdkCtxs maps a unique handler ID to a sdk.Context. It is used to track - // requests coming from the SGX binary, and handle them using the - // correct sdk.Context. + // sdkCtxs maps a unique evmID to a sdk.Context. It is used to track + // requests coming from the SGX binary, and handle them using the correct sdk.Context. + // Each evmID is mapped to a unique EVM instance on the TEE side. sdkCtxs map[uint64]*sdk.Context } @@ -398,6 +398,6 @@ func (k Keeper) AddTransientGasUsed(ctx sdk.Context, gasUsed uint64) (uint64, er return result, nil } -func (k Keeper) GetSdkCtx(handlerId uint64) *sdk.Context { - return k.sdkCtxs[handlerId] +func (k Keeper) getSdkCtx(evmId uint64) *sdk.Context { + return k.sdkCtxs[evmId] } diff --git a/x/evm/keeper/rpc_client.go b/x/evm/keeper/rpc_client.go index 510fd1e2ad..225bd153d8 100644 --- a/x/evm/keeper/rpc_client.go +++ b/x/evm/keeper/rpc_client.go @@ -129,12 +129,12 @@ type StartEVMArgs struct { // StartEVMReply is the reply struct for the SgxRpcServer.StartEVM RPC method. type StartEVMReply struct { - HandlerId uint64 + EvmId uint64 } // InitFhevmArgs is the arg struct for the SgxRpcServer.InitFhevm RPC method. type InitFhevmArgs struct { - HandlerId uint64 + EvmId uint64 } // InitFhevmReply is the reply struct for the SgxRpcServer.InitFhevm RPC method. @@ -143,12 +143,12 @@ type InitFhevmReply struct { // CallArgs is the argument struct for the SgxRpcServer.Call RPC method. type CallArgs struct { - HandlerId uint64 - Caller vm.AccountRef - Addr common.Address - Input []byte - Gas uint64 - Value *big.Int + EvmId uint64 + Caller vm.AccountRef + Addr common.Address + Input []byte + Gas uint64 + Value *big.Int } // CallReply is the reply struct for the SgxRpcServer.Call RPC method. @@ -159,11 +159,11 @@ type CallReply struct { // CreateArgs is the argument struct for the SgxRpcServer.Create RPC method. type CreateArgs struct { - HandlerId uint64 - Caller vm.AccountRef - Code []byte - Gas uint64 - Value *big.Int + EvmId uint64 + Caller vm.AccountRef + Code []byte + Gas uint64 + Value *big.Int } // CreateReply is the reply struct for the SgxRpcServer.Create RPC method. @@ -175,7 +175,7 @@ type CreateReply struct { // CommitArgs is the argument struct for the SgxRpcServer.Commit RPC method. type CommitArgs struct { - HandlerId uint64 + EvmId uint64 } // CommitReply is the reply struct for the SgxRpcServer.Commit RPC method. @@ -184,9 +184,9 @@ type CommitReply struct { // CommitArgs is the argument struct for the SgxRpcServer.StateDBSubBalance RPC method. type StateDBSubBalanceArgs struct { - HandlerId uint64 - Caller vm.AccountRef - Msg core.Message + EvmId uint64 + Caller vm.AccountRef + Msg core.Message } // CommitReply is the reply struct for the SgxRpcServer.StateDBSubBalance RPC method. @@ -195,9 +195,9 @@ type StateDBSubBalanceReply struct { // CommitArgs is the argument struct for the SgxRpcServer.StateDSetNonce RPC method. type StateDBSetNonceArgs struct { - HandlerId uint64 - Caller vm.AccountRef - Nonce uint64 + EvmId uint64 + Caller vm.AccountRef + Nonce uint64 } // CommitReply is the reply struct for the SgxRpcServer.StateDSetNonce RPC method. @@ -206,7 +206,7 @@ type StateDBSetNonceReply struct { // StateDBAddBalanceArgs is the argument struct for the SgxRpcServer.StateDBAddBalance RPC method. type StateDBAddBalanceArgs struct { - HandlerId uint64 + EvmId uint64 Caller vm.AccountRef Msg core.Message LeftoverGas uint64 @@ -217,10 +217,10 @@ type StateDBAddBalanceReply struct { } type StateDBPrepareArgs struct { - HandlerId uint64 - Msg core.Message - Rules params.Rules - CoinBase common.Address + EvmId uint64 + Msg core.Message + Rules params.Rules + CoinBase common.Address } type StateDBPrepareReply struct { @@ -228,9 +228,9 @@ type StateDBPrepareReply struct { // StateDBIncreaseNonceArgs is the argument struct for the SgxRpcServer.StateDBIncreaseNonce RPC method. type StateDBIncreaseNonceArgs struct { - HandlerId uint64 - Caller vm.AccountRef - Msg core.Message + EvmId uint64 + Caller vm.AccountRef + Msg core.Message } // StateDBIncreaseNonceReply is the reply struct for the SgxRpcServer.StateDBIncreaseNonce RPC method. @@ -238,7 +238,7 @@ type StateDBIncreaseNonceReply struct { } type StateDBGetRefundArgs struct { - HandlerId uint64 + EvmId uint64 } type StateDBGetRefundReply struct { @@ -246,7 +246,7 @@ type StateDBGetRefundReply struct { } type StateDBGetLogsArgs struct { - HandlerId uint64 + EvmId uint64 } type StateDBGetLogsReply struct { diff --git a/x/evm/keeper/rpc_server.go b/x/evm/keeper/rpc_server.go index a0e7a0d48e..1d63da76b2 100644 --- a/x/evm/keeper/rpc_server.go +++ b/x/evm/keeper/rpc_server.go @@ -14,8 +14,8 @@ type EthmRpcServer struct { Keeper *Keeper } -func (s *EthmRpcServer) GetHash(handlerId uint64, height *uint64, hash *common.Hash) error { - ctx := s.Keeper.GetSdkCtx(handlerId) +func (s *EthmRpcServer) GetHash(evmId uint64, height *uint64, hash *common.Hash) error { + ctx := s.Keeper.getSdkCtx(evmId) if ctx == nil { panic("context is invalid") } @@ -25,7 +25,7 @@ func (s *EthmRpcServer) GetHash(handlerId uint64, height *uint64, hash *common.H } func (s *EthmRpcServer) AddBalance(args *AddBalanceArgs, reply *AddBalanceReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -34,7 +34,7 @@ func (s *EthmRpcServer) AddBalance(args *AddBalanceArgs, reply *AddBalanceReply) } func (s *EthmRpcServer) SubBalance(args *SubBalanceArgs, reply *SubBalanceReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -43,7 +43,7 @@ func (s *EthmRpcServer) SubBalance(args *SubBalanceArgs, reply *SubBalanceReply) } func (s *EthmRpcServer) GetBalance(args *GetBalanceArgs, reply *GetBalanceReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -53,7 +53,7 @@ func (s *EthmRpcServer) GetBalance(args *GetBalanceArgs, reply *GetBalanceReply) } func (s *EthmRpcServer) GetAccount(args *GetAccountArgs, reply *GetAccountReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -63,7 +63,7 @@ func (s *EthmRpcServer) GetAccount(args *GetAccountArgs, reply *GetAccountReply) } func (s *EthmRpcServer) GetState(args *GetStateArgs, reply *GetStateReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -73,7 +73,7 @@ func (s *EthmRpcServer) GetState(args *GetStateArgs, reply *GetStateReply) error } func (s *EthmRpcServer) GetCode(args *GetCodeArgs, reply *GetCodeReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -83,7 +83,7 @@ func (s *EthmRpcServer) GetCode(args *GetCodeArgs, reply *GetCodeReply) error { } func (s *EthmRpcServer) SetAccount(args *SetAccountArgs, reply *SetAccountReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -92,7 +92,7 @@ func (s *EthmRpcServer) SetAccount(args *SetAccountArgs, reply *SetAccountReply) } func (s *EthmRpcServer) SetState(args *SetStateArgs, reply *SetStateReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -101,7 +101,7 @@ func (s *EthmRpcServer) SetState(args *SetStateArgs, reply *SetStateReply) error } func (s *EthmRpcServer) SetCode(args *SetCodeArgs, reply *SetCodeReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -110,7 +110,7 @@ func (s *EthmRpcServer) SetCode(args *SetCodeArgs, reply *SetCodeReply) error { } func (s *EthmRpcServer) DeleteAccount(args *DeleteAccountArgs, reply *DeleteAccountReply) error { - ctx := s.Keeper.GetSdkCtx(args.HandlerId) + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } @@ -119,9 +119,9 @@ func (s *EthmRpcServer) DeleteAccount(args *DeleteAccountArgs, reply *DeleteAcco // AddBalanceArgs is the argument struct for the statedb.Keeper#AddBalance method. type AddBalanceArgs struct { - HandlerId uint64 - Addr sdk.AccAddress - Amount sdk.Coins + EvmId uint64 + Addr sdk.AccAddress + Amount sdk.Coins } // AddBalanceReply is the reply struct for the statedb.Keeper#AddBalance method. @@ -130,9 +130,9 @@ type AddBalanceReply struct { // SubBalanceArgs is the argument struct for the statedb.Keeper#SubBalance method. type SubBalanceArgs struct { - HandlerId uint64 - Addr sdk.AccAddress - Amount sdk.Coins + EvmId uint64 + Addr sdk.AccAddress + Amount sdk.Coins } // SubBalanceReply is the reply struct for the statedb.Keeper#SubBalance method. @@ -141,9 +141,9 @@ type SubBalanceReply struct { // GetBalanceArgs is the argument struct for the statedb.Keeper#GetBalance method. type GetBalanceArgs struct { - HandlerId uint64 - Addr sdk.AccAddress - Denom string + EvmId uint64 + Addr sdk.AccAddress + Denom string } // GetBalanceReply is the reply struct for the statedb.Keeper#GetBalance method. @@ -153,8 +153,8 @@ type GetBalanceReply struct { // GetAccountArgs is the argument struct for the statedb.Keeper#GetAccount method. type GetAccountArgs struct { - HandlerId uint64 - Addr common.Address + EvmId uint64 + Addr common.Address } // GetAccountReply is the reply struct for the statedb.Keeper#GetAccount method. @@ -164,21 +164,21 @@ type GetAccountReply struct { // GetStateArgs is the argument struct for the statedb.Keeper#GetState method. type GetStateArgs struct { - HandlerId uint64 - Addr common.Address - Key common.Hash + EvmId uint64 + Addr common.Address + Key common.Hash } // GetStateReply is the reply struct for the statedb.Keeper#GetState method. type GetStateReply struct { - HandlerId uint64 - Hash common.Hash + EvmId uint64 + Hash common.Hash } // GetCodeArgs is the argument struct for the statedb.Keeper#GetCode method. type GetCodeArgs struct { - HandlerId uint64 - CodeHash common.Hash + EvmId uint64 + CodeHash common.Hash } // GetCodeReply is the reply struct for the statedb.Keeper#GetCode method. @@ -188,9 +188,9 @@ type GetCodeReply struct { // SetAccountArgs is the argument struct for the statedb.Keeper#SetAccount method. type SetAccountArgs struct { - HandlerId uint64 - Addr common.Address - Account statedb.Account + EvmId uint64 + Addr common.Address + Account statedb.Account } // SetAccountReply is the reply struct for the statedb.Keeper#SetAccount method. @@ -199,10 +199,10 @@ type SetAccountReply struct { // SetStateArgs is the argument struct for the statedb.Keeper#SetState method. type SetStateArgs struct { - HandlerId uint64 - Addr common.Address - Key common.Hash - Value []byte + EvmId uint64 + Addr common.Address + Key common.Hash + Value []byte } // SetStateReply is the reply struct for the statedb.Keeper#SetState method. @@ -211,9 +211,9 @@ type SetStateReply struct { // SetCodeArgs is the argument struct for the statedb.Keeper#SetCode method. type SetCodeArgs struct { - HandlerId uint64 - CodeHash []byte - Code []byte + EvmId uint64 + CodeHash []byte + Code []byte } // SetCodeReply is the reply struct for the statedb.Keeper#SetCode method. @@ -222,8 +222,8 @@ type SetCodeReply struct { // DeleteAccountArgs is the argument struct for the statedb.Keeper#DeleteAccount method. type DeleteAccountArgs struct { - HandlerId uint64 - Addr common.Address + EvmId uint64 + Addr common.Address } // DeleteAccountReply is the reply struct for the statedb.Keeper#DeleteAccount method. diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index 9b8608bec2..5870626d02 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -303,7 +303,7 @@ func (k *Keeper) ApplyMessageWithConfig( return nil, errorsmod.Wrap(err, "failed to create new SGX rpc client") } - handlerId, err := k.startEVM(ctx, msg, cfg, sgxRPCClient) + evmId, err := k.startEVM(ctx, msg, cfg, sgxRPCClient) if err != nil { return nil, errorsmod.Wrap(err, "failed to create new RPC server") } @@ -320,9 +320,9 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.SubBalance(sender.Address(), new(big.Int).Mul(msg.GasPrice, new(big.Int).SetUint64(msg.GasLimit))) var reply StateDBSubBalanceReply err := sgxRPCClient.StateDBSubBalance(StateDBSubBalanceArgs{ - HandlerId: handlerId, - Caller: sender, - Msg: msg, + EvmId: evmId, + Caller: sender, + Msg: msg, }, &reply) if err != nil { return nil, err @@ -332,9 +332,9 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.SetNonce(sender.Address(), stateDB.GetNonce(sender.Address())+1) var replyNonce StateDBIncreaseNonceReply err = sgxRPCClient.StateDBIncreaseNonce(StateDBIncreaseNonceArgs{ - HandlerId: handlerId, - Caller: sender, - Msg: msg, + EvmId: evmId, + Caller: sender, + Msg: msg, }, &replyNonce) if err != nil { return nil, err @@ -347,7 +347,7 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.AddBalance(sender.Address(), new(big.Int).Mul(msg.GasPrice, new(big.Int).SetUint64(leftoverGas))) var reply StateDBAddBalanceReply err := sgxRPCClient.StateDBAddBalance(StateDBAddBalanceArgs{ - HandlerId: handlerId, + EvmId: evmId, Caller: sender, Msg: msg, LeftoverGas: leftoverGas, @@ -392,10 +392,10 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.Prepare(rules, msg.From, cfg.CoinBase, msg.To, vm.ActivePrecompiles(rules), msg.AccessList) var replyPrepare StateDBPrepareReply err = sgxRPCClient.StateDBPrepare(StateDBPrepareArgs{ - HandlerId: handlerId, - Msg: msg, - Rules: rules, - CoinBase: cfg.CoinBase, + EvmId: evmId, + Msg: msg, + Rules: rules, + CoinBase: cfg.CoinBase, }, &replyPrepare) if err != nil { return nil, err @@ -410,9 +410,9 @@ func (k *Keeper) ApplyMessageWithConfig( // stateDB.SetNonce(sender.Address(), msg.Nonce) var replyNonce StateDBSetNonceReply err := sgxRPCClient.StateDBSetNonce(StateDBSetNonceArgs{ - HandlerId: handlerId, - Caller: sender, - Nonce: msg.Nonce, + EvmId: evmId, + Caller: sender, + Nonce: msg.Nonce, }, &replyNonce) if err != nil { return nil, err @@ -422,11 +422,11 @@ func (k *Keeper) ApplyMessageWithConfig( // ret, _, leftoverGas, vmErr = evm.Create(sender, msg.Data, leftoverGas, msg.Value) var reply CreateReply vmErr = sgxRPCClient.Create(CreateArgs{ - HandlerId: handlerId, - Caller: sender, - Code: msg.Data, - Gas: leftoverGas, - Value: msg.Value, + EvmId: evmId, + Caller: sender, + Code: msg.Data, + Gas: leftoverGas, + Value: msg.Value, }, &reply) ret = reply.Ret leftoverGas = reply.LeftOverGas @@ -434,21 +434,21 @@ func (k *Keeper) ApplyMessageWithConfig( // Ethermint original code: // stateDB.SetNonce(sender.Address(), msg.Nonce+1) sgxRPCClient.StateDBSetNonce(StateDBSetNonceArgs{ - HandlerId: handlerId, - Caller: sender, - Nonce: msg.Nonce + 1, + EvmId: evmId, + Caller: sender, + Nonce: msg.Nonce + 1, }, &replyNonce) } else { // Ethermint original code: // ret, leftoverGas, vmErr = evm.Call(sender, *msg.To, msg.Data, leftoverGas, msg.Value) var reply CallReply vmErr = sgxRPCClient.Call(CallArgs{ - HandlerId: handlerId, - Caller: sender, - Addr: *msg.To, - Input: msg.Data, - Gas: leftoverGas, - Value: msg.Value, + EvmId: evmId, + Caller: sender, + Addr: *msg.To, + Input: msg.Data, + Gas: leftoverGas, + Value: msg.Value, }, &reply) ret = reply.Ret leftoverGas = reply.LeftOverGas @@ -473,7 +473,7 @@ func (k *Keeper) ApplyMessageWithConfig( // leftoverGas += GasToRefund(stateDB.GetRefund(), temporaryGasUsed, refundQuotient) var replyRefund StateDBGetRefundReply err = sgxRPCClient.StateDBGetRefund(StateDBGetRefundArgs{ - HandlerId: handlerId, + EvmId: evmId, }, &replyRefund) if err != nil { return nil, err @@ -496,7 +496,7 @@ func (k *Keeper) ApplyMessageWithConfig( // } var reply CommitReply err := sgxRPCClient.Commit(CommitArgs{ - HandlerId: handlerId, + EvmId: evmId, }, &reply) if err != nil { return nil, errorsmod.Wrap(err, "failed to commit sgx stateDB") @@ -526,12 +526,14 @@ func (k *Keeper) ApplyMessageWithConfig( // Logs: types.NewLogsFromEth(stateDB.Logs()), var replyLog StateDBGetLogsReply err = sgxRPCClient.StateDBGetLogs(StateDBGetLogsArgs{ - HandlerId: handlerId, + EvmId: evmId, }, &replyLog) if err != nil { return nil, err } + err = k.stopEVM(ctx, evmId, sgxRPCClient) + return &types.MsgEthereumTxResponse{ GasUsed: gasUsed, VmError: vmError, @@ -543,12 +545,10 @@ func (k *Keeper) ApplyMessageWithConfig( } // startEVM prepares the transaction for the SGX enclave. It: -// - creates an RPC server around the keeper to receive requests sent by the -// SGX -// - sends a "StartEVM" request to the SGX enclave with the relevant tx and -// block info +// - sends a "StartEVM" request to the SGX enclave with the relevant tx and block info +// - sends a "InitFhevm" request to the SGX enclave to initialize the fhEVM instance. (More comments are in the below.) func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgxRPCClient *sgxRPCClient) (uint64, error) { - // Step 1. Send a "PrepareTx" request to the SGX enclave. + // Step 1. Send a "StartEVM" request to the SGX enclave. ChainConfigJson, err := json.Marshal(cfg.ChainConfig) if err != nil { return 0, err @@ -578,8 +578,8 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx } return 0, err } - // Snapshot the sdk ctx with this handlerId - k.sdkCtxs[reply.HandlerId] = &ctx + // Snapshot the sdk ctx with this evmId + k.sdkCtxs[reply.EvmId] = &ctx // We unfortunately can't call InitFhevm on the EVM instance during the // StartEVM method (inside newEVM), because InitFhevm needs access to the @@ -594,7 +594,7 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx // // We are doing step 2 here. err = sgxRPCClient.InitFhevm(InitFhevmArgs{ - HandlerId: reply.HandlerId, + EvmId: reply.EvmId, }, &InitFhevmReply{}) if err != nil { // panic cosmos if sgx isn't available. @@ -604,7 +604,21 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx return 0, err } - return reply.HandlerId, err + return reply.EvmId, err +} +func (k *Keeper) stopEVM(ctx sdk.Context, handlerId uint64, sgxRPCClient sgxRPCClient) error { + _, err := sgxRPCClient.StopEVM(ctx, &sgx.StopEVMRequest{HandlerId: handlerId}) + if err != nil { + // TODO handle error + panic(err) + // panic cosmos if sgx isn't available. + if isSgxDownError(err) { + panic("sgx rpc server is down") + } + } + delete(k.sdkCtxs, handlerId) + + return err } // isSgxDownError checks if the error is related with RPC server down From 54f35e7f55824129d2ed236250396bea8747ecb9 Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Mon, 13 May 2024 15:08:36 +0000 Subject: [PATCH 5/7] chore: add stopEVM --- x/evm/keeper/rpc_client.go | 11 +++++++++++ x/evm/keeper/state_transition.go | 13 +++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/x/evm/keeper/rpc_client.go b/x/evm/keeper/rpc_client.go index 225bd153d8..8e4548c3e7 100644 --- a/x/evm/keeper/rpc_client.go +++ b/x/evm/keeper/rpc_client.go @@ -89,6 +89,10 @@ func (c *sgxRPCClient) StateDBGetLogs(args StateDBGetLogsArgs, reply *StateDBGet return c.doCall("SgxRpcServer.StateDBGetLogs", args, reply) } +func (c *sgxRPCClient) StopEVM(args StopEVMArgs, reply *StopEVMReply) error { + return c.doCall("SgxRpcServer.StopEVM", args, reply) +} + // StartEVMTxEVMConfig only contains the fields from EVMConfig that are needed // to create a new EVM instance. This is used to pass the EVM configuration // over RPC to the SGX binary. @@ -252,3 +256,10 @@ type StateDBGetLogsArgs struct { type StateDBGetLogsReply struct { Logs []*ethtypes.Log } + +type StopEVMArgs struct { + EvmId uint64 +} + +type StopEVMReply struct { +} diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index 5870626d02..916b441d95 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -532,7 +532,7 @@ func (k *Keeper) ApplyMessageWithConfig( return nil, err } - err = k.stopEVM(ctx, evmId, sgxRPCClient) + err = k.stopEVM(evmId, sgxRPCClient) return &types.MsgEthereumTxResponse{ GasUsed: gasUsed, @@ -606,17 +606,18 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx return reply.EvmId, err } -func (k *Keeper) stopEVM(ctx sdk.Context, handlerId uint64, sgxRPCClient sgxRPCClient) error { - _, err := sgxRPCClient.StopEVM(ctx, &sgx.StopEVMRequest{HandlerId: handlerId}) + +func (k *Keeper) stopEVM(evmId uint64, sgxRPCClient *sgxRPCClient) error { + err := sgxRPCClient.StopEVM(StopEVMArgs{ + EvmId: evmId}, + &StopEVMReply{}) if err != nil { - // TODO handle error - panic(err) // panic cosmos if sgx isn't available. if isSgxDownError(err) { panic("sgx rpc server is down") } } - delete(k.sdkCtxs, handlerId) + delete(k.sdkCtxs, evmId) return err } From fceaefb6fefaf580890e719aa073056fa623596f Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Wed, 15 May 2024 08:39:05 +0000 Subject: [PATCH 6/7] chore: code refactor --- server/start.go | 13 +++++++------ x/evm/keeper/rpc_server.go | 17 ++++++++++++++--- x/evm/keeper/state_transition.go | 5 ++++- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/server/start.go b/server/start.go index f624582a09..eca455e1d2 100644 --- a/server/start.go +++ b/server/start.go @@ -17,6 +17,7 @@ package server import ( "context" + "errors" "fmt" "io" "net" @@ -515,13 +516,9 @@ func startRpcServer( app types.Application, ) (listener net.Listener, err error) { ethApp := app.(IncoApp) - if ethApp == nil { - return nil, nil - } - evmKeeper := ethApp.GetEvmKeeper() if evmKeeper == nil { - return nil, nil + return nil, errors.New("evm keeper is invalid") } g.Go(func() error { @@ -541,7 +538,11 @@ func runRPCServer(svrCtx *server.Context, keeper *evmKeeper.Keeper) (net.Listene // Run a persistent RPC server for sgx binary can access to evm keeper statedb srv := &evmKeeper.EthmRpcServer{Keeper: keeper} - rpc.Register(srv) + err := rpc.Register(srv) + if err != nil { + return nil, err + } + rpc.HandleHTTP() // TODO handle port customization diff --git a/x/evm/keeper/rpc_server.go b/x/evm/keeper/rpc_server.go index 1d63da76b2..6c7e6b9d1c 100644 --- a/x/evm/keeper/rpc_server.go +++ b/x/evm/keeper/rpc_server.go @@ -14,13 +14,13 @@ type EthmRpcServer struct { Keeper *Keeper } -func (s *EthmRpcServer) GetHash(evmId uint64, height *uint64, hash *common.Hash) error { - ctx := s.Keeper.getSdkCtx(evmId) +func (s *EthmRpcServer) GetHash(args *GetHashArgs, reply *GetHashReply) error { + ctx := s.Keeper.getSdkCtx(args.EvmId) if ctx == nil { panic("context is invalid") } - *hash = s.Keeper.GetHashFn(*ctx)(*height) + reply.Hash = s.Keeper.GetHashFn(*ctx)(args.Height) return nil } @@ -229,3 +229,14 @@ type DeleteAccountArgs struct { // DeleteAccountReply is the reply struct for the statedb.Keeper#DeleteAccount method. type DeleteAccountReply struct { } + +// GetHashArgs is the argument struct for the statedb.Keeper#GetHash method. +type GetHashArgs struct { + EvmId uint64 + Height uint64 +} + +// GetHashReply is the reply struct for the statedb.Keeper#GetHash method. +type GetHashReply struct { + Hash common.Hash +} diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index 916b441d95..1e1c96e4d4 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -433,11 +433,14 @@ func (k *Keeper) ApplyMessageWithConfig( // Ethermint original code: // stateDB.SetNonce(sender.Address(), msg.Nonce+1) - sgxRPCClient.StateDBSetNonce(StateDBSetNonceArgs{ + err = sgxRPCClient.StateDBSetNonce(StateDBSetNonceArgs{ EvmId: evmId, Caller: sender, Nonce: msg.Nonce + 1, }, &replyNonce) + if err != nil { + return nil, err + } } else { // Ethermint original code: // ret, leftoverGas, vmErr = evm.Call(sender, *msg.To, msg.Data, leftoverGas, msg.Value) From edc8ab47989bb12cba6631d166097941a20900e2 Mon Sep 17 00:00:00 2001 From: kenta-mori3322 Date: Wed, 15 May 2024 22:39:01 +0000 Subject: [PATCH 7/7] chore: code refactor --- server/start.go | 8 +++++--- x/evm/keeper/rpc_client.go | 6 +++--- x/evm/keeper/state_transition.go | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/start.go b/server/start.go index eca455e1d2..951c72eadc 100644 --- a/server/start.go +++ b/server/start.go @@ -74,8 +74,10 @@ import ( evmKeeper "github.com/evmos/ethermint/x/evm/keeper" ) -type IncoApp interface { - // Return evm keeper which is used in ethermint side to get evm keeper access +// incoApp is a local interface to get the EVM keeper from the Inco chain app, +// without introducing any circular dependencies. +type incoApp interface { + // GetEvmKeeper returns evm keeper which is used in ethermint side to get evm keeper access GetEvmKeeper() *evmKeeper.Keeper } @@ -515,7 +517,7 @@ func startRpcServer( g *errgroup.Group, app types.Application, ) (listener net.Listener, err error) { - ethApp := app.(IncoApp) + ethApp := app.(incoApp) evmKeeper := ethApp.GetEvmKeeper() if evmKeeper == nil { return nil, errors.New("evm keeper is invalid") diff --git a/x/evm/keeper/rpc_client.go b/x/evm/keeper/rpc_client.go index 8e4548c3e7..f4bbb38e61 100644 --- a/x/evm/keeper/rpc_client.go +++ b/x/evm/keeper/rpc_client.go @@ -93,10 +93,10 @@ func (c *sgxRPCClient) StopEVM(args StopEVMArgs, reply *StopEVMReply) error { return c.doCall("SgxRpcServer.StopEVM", args, reply) } -// StartEVMTxEVMConfig only contains the fields from EVMConfig that are needed +// StartEVMConfig only contains the fields from EVMConfig that are needed // to create a new EVM instance. This is used to pass the EVM configuration // over RPC to the SGX binary. -type StartEVMTxEVMConfig struct { +type StartEVMConfig struct { // ChainConfig is the EVM chain configuration in JSON format. Since the // underlying params.ChainConfig struct contains pointer fields, they are // not serializable over RPC with gob. Instead, the JSON representation is @@ -128,7 +128,7 @@ type StartEVMArgs struct { // Msg is the EVM transaction message to run on the EVM. Msg core.Message // EvmConfig is the EVM configuration to set. - EvmConfig StartEVMTxEVMConfig + EvmConfig StartEVMConfig } // StartEVMReply is the reply struct for the SgxRpcServer.StartEVM RPC method. diff --git a/x/evm/keeper/state_transition.go b/x/evm/keeper/state_transition.go index 1e1c96e4d4..040889829c 100644 --- a/x/evm/keeper/state_transition.go +++ b/x/evm/keeper/state_transition.go @@ -560,7 +560,7 @@ func (k *Keeper) startEVM(ctx sdk.Context, msg core.Message, cfg *EVMConfig, sgx args := StartEVMArgs{ Header: ctx.BlockHeader(), Msg: msg, - EvmConfig: StartEVMTxEVMConfig{ + EvmConfig: StartEVMConfig{ ChainConfigJson: ChainConfigJson, CoinBase: cfg.CoinBase, BaseFee: cfg.BaseFee,