diff --git a/anchor/BASE.txt b/anchor/BASE.txt index 2707fd6..55be9da 100644 --- a/anchor/BASE.txt +++ b/anchor/BASE.txt @@ -5,6 +5,11 @@ anything else they overwrite, and silently delete, whatever upstream added in be against a newer base, 26.7.1: 97 lines would be lost, among them the registration of the /liveness and /readiness probes in BesuCommand.java, and nothing would fail. +Since 2026-10-02 this directory holds every source file that the published patches (patches/0001 and +0003 to 0008, in the order run-node.sh applies them) change or add, taken from the recipe itself: +the base tree with those patches applied. It is a reading copy; the patches remain the authoritative +form, and a check in the operator repository compares the two file by file. + IF YOU MOVE THE BASE, do not take whole files from here. Re-apply the same changes as a patch against your own base: a patch fails loudly when upstream has moved, a whole file does not. diff --git a/anchor/MANIFEST-sha256.txt b/anchor/MANIFEST-sha256.txt index d69b3c7..f4e5679 100644 --- a/anchor/MANIFEST-sha256.txt +++ b/anchor/MANIFEST-sha256.txt @@ -16,6 +16,7 @@ ed2e46aaf2e74661ad831dc31df0b6f795f28f4a94496a7d226feedf2a770fde consensus/comm a6c2bbf05eab4173a749c67ef8b7bce6d184d8eac1b445acb847efd082dc6d62 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/AerePqFinalityTags.java 290d94bb50f23666b131ec19659982c15f1e6cbaf8f8bfe128df9acb2f2f79fd consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/AerePqHeaderSeal.java 348135b2e4d0a6a8ce91489c94fbfa77da2536196c13823676b33ea9d08f8fb9 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/AerePqIdentity.java +dfdb28fb2d5016f0cbb7e8ba2cd9ababf18c6530349ddad3bf1e615cb25cc703 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftBlockHashing.java c531f9e719c48d3d5da2800fdef4506bd167af4dad4e25af96b4353a4390067d consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftBlockInterface.java 9fcf44789ddf6d5b80be7d44f0bec58af6ba8316733d49817e890f0e17b71223 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftExtraData.java 70fa067469a084ac225765da6a01619dec51130acc255b7194a1ad274d327141 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BlockTimer.java @@ -37,16 +38,22 @@ f67a205036550f7d0f1bfed2abe7b89abd73e80614adff8d696b33c580269a6f consensus/comm 660ba13ad60c49e86cf9ee4ced2b6932a5422d988f85c5c23098a92b029f4d25 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqRegistryHashTool.java 11f1c2a97a40df999471977cb3eab9af4e816081b100e345e8c962ef3bbf76f5 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqRegistryV3.java 2ee139d21f73ff3417f6ad61b233e5ca3a95f92e68a4284eb99a47b1f8f4b58b consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSchemeSchedule.java -c0c93a096f4eceab6bf92cc3618ce7cee1e54d2d20b6ac781850a390ec6e514f consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java +80104994f8e5f02353c4bcb25084c451fd422c51c49e6891b574d76c7adf135f consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java 73abbaddcec1533a7a758f1c4a26ae68a83ddbb96fd98ad147e74d7386bbbf3f consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealStore.java +776d9a422bcd306ad92d561ddebbbc1835d4ef5b147aa19d781556298bb27be4 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java fad72fb52b6f6198113b80f5f2729002066a10decf596cfe26e602b058d9c603 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealed.java a235b3ec8c69665c525d7c74a084d589904879f5cd33cb87e1d1f38eabba8a15 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSignerRegistry.java addba40c0d931a3ecfa3b2f0179311dacaa604884e3ebb1d79f958bd913f72e2 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/SchemeSeal.java 285f6a7c1188a387fb1ac21f5ab809595ef48d936b40aced94047525e6def0a6 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/SealScheme.java 3b57840dc103063b3797697feb78b90af93ca123587e9278cfb43b84edbd995a consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/SealSchemes.java 444961503d515cb9f045729577e190941e59baa4bd90438a2da05f41fed7750a consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/SlhDsaSealScheme.java +9021d608f62e1330cf9b421013a06f2141bae469ba9162a24ce66f749aa4b21a consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java 454eeca487505e70c7f0000f5401b6bf4530da1d5ce6bcb803427f1d08e3cb1c consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreatorFactory.java +610275a18c2519d84ffb91982d7b9006a42605bbb330071572859df49f183c2a consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftMiningCoordinator.java e89deb6f346c221affa717eafa464d497401b7b5a01c84c144dace42032879a9 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/PqAnchorProducer.java +d8efb4d625233e18ef4764eaa29650a97b56e75af18d07607eda1dd484187f76 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/messagewrappers/BftMessage.java +4b0731ec053e313a9ea9afc78592f25b6e97e6d230ebb273532437f08da92e5a consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/network/ValidatorPeers.java +478f9be60c1259481bfd7b5a687e23ada29b4b07d4cdf6f9d3bbc13584d384f8 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/payload/SignedData.java 1e0113154d79a91a07104bcc67a9a843847751b921c5f9b7cc96d7bc30a1452f consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/slhdsa/ADRS.java 15ab42221f35aeb0dfd468bcb840254f08ccfccb5ff36fba559ac1017beca5ee consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/slhdsa/Fors.java 51b7d889188793a307bccb297d983f9318548e6066ad6228e37170585eda665f consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/slhdsa/HT.java @@ -71,6 +78,8 @@ fa85313f90f2e92b338d9c3f6df4cb57a7583fe8c82417f19928a133932dfd18 consensus/comm 2cd85f1517fe9770558ffef6251600bce76609a96beb21f0a8f120412b2a935f consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/slhdsa/WotsPlus.java 6ace00e18914a1558563e689b7427654b5e6226a31447bef1d0416d993f78840 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/tools/PqRegistryHashTool.java ed1cc6f617dc5a2f7801b6e8f6f039a035a2ffc9542b116262dac39bb70563dd consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/tools/PqRegistryV3Tool.java +8aff8079c93d1463111e303831dbef886568790e2cff8ade77801a2864ae355b consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTally.java +5a1014676efeea6e29d0603b4e7dac2d7334840c2f1ba3885bdc1b13e337cd7c consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTallyUpdater.java cf0dcc008ce2d040590329088abdcbfc0829bf529445422deee8022d4856e08a consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/AereAnchorProposalDelayTest.java f5d6d26b9cd3babad8f7d16b9595497908b1f854d166837fbecd485d721ae856 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/AereKeysFromV3Test.java 1d7e3a30d3f4653b778a1bcbb81203ed8855d904017fa438a06e26baad8958c3 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/AereLocalKeysV3Test.java @@ -108,6 +117,7 @@ f260d284f936307ac4c31142cbea0359ded1197880b8a960ac27dd7beb5c3b53 consensus/comm 05423fc69ce0e4e94cda0d56a181a996320bd2166af3415f2d88102cf3b0ae21 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqRegistryV3Test.java d7d675be2a5ee598ce74f58156383589a05b8e977c2cc790671e3a39c7728309 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSchemeScheduleTest.java e73a075b2acfce7a6d2de63130ed55a3d8f422bdc72c51325a010852064d406d consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealPersistenceTest.java +45f25de86a064b8fd8e16a2c2f2194fabe55b3fefaadfcdaad6e58e96c72c800 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java e0bc7a465dbb2dd7efa5f1830211a1a45c6554c300b2088dd70bb9677b0ac217 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSignedHeightTest.java 9a371ec367cda846640c6da18c8bf7d921a6882cfc024dc0c86d4f965cae0c63 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqStartupHistoryTest.java e65ba92e288aebe768909b1fee5c0b8850be74b3658c6242e6abdb91409a9521 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqV2Fixture.java @@ -120,6 +130,8 @@ ae37d3525697bf84defd796bb8813193c78f81e7fa24a72f8f048285d52399e2 consensus/comm d0ba00d95427ae4fb1c0a1eeb37bcd039fbfa5412957dc4b2da0725c0dea5028 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/slhdsa/SlhDsaParallelSigningTest.java bcdd5c3c267f38eee9700bc6ba07d41b487c7889224820868e1839044108451f consensus/common/src/test/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTallyIdentityTest.java 6b6ecfcfb9b25bb009cb5d4df74648c729abf62626e5034af126771825b7418e consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/Commit.java +5ed47d976d4aafbb55369e1b3e96e6ad5efc3bf7c68be99ddb0e0ef9dbbc2f32 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/Proposal.java +394c557de273fa9fa9a5ad024e8a1982575ad756736aa045ffeb0bdc7dc8016c consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/RoundChange.java a1405402e891189909a436abd9f83c863349fdcf01d3011542d805e2bb3911ed consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/network/QbftMessageTransmitter.java 35d38accf0d22b6fdadeab4f2bce0cdac708f8115591594b1bb1fa385b227c5c consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/payload/CommitPayload.java 536375fa2d9f27107d9907f34391413e052f2a8cd89611033fefe29b9edc7891 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/payload/MessageFactory.java @@ -128,7 +140,8 @@ a1405402e891189909a436abd9f83c863349fdcf01d3011542d805e2bb3911ed consensus/qbft e19b12e6db33cfb9bb1c2f22e321815130dfbcc5730bb505320f08a45a371093 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/payload/RoundChangePayload.java 51eddf3cb8e37489abd35e6f4c36294a13ec76337bbbbfd54c07584085dceaac consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftBlockHeightManager.java 0a450699c702104dcef6bf8912b4d180f05d647288c75a9f78da6af3bd3d90f1 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftController.java -80613a8ac11ab48dfcecce5690a28da7d66dd8332584fd221f2875cb33d8c9d0 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRound.java +4a87188f209904c19f2b393fd50559a05b2be3161b4582364bc1641002bb4dff consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRound.java +324a5607313f25adb377d633be1f221588f8dafaa47a3c9d1e84fc5697323090 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRoundFactory.java 4494f33ca028b60eaa093e6c24dd13706461e228f7f5a29e40e8917d1ef3b373 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/RoundState.java eadc4a14e2f0fdf7c1d9667d67c59f709af7a86fccf51923de0248b73e223c21 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/types/QbftBlockCreator.java 4c0811d182f5252bdde31560f514a518db58c8d0c82c748a6c0f5d68472470c6 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/validation/CommitValidator.java @@ -141,6 +154,7 @@ afb487f8c1dd87e323fdc1153ecfdf8ec6cb5a10e0726783d4214c07a3fa9a95 consensus/qbft 3aaa572c81641ad9a3e9cef9460b9a82dc4faff8080b933ef0e835881cf30bb4 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/validation/PrepareValidator.java 2f0001ca7b302b38b81e51fee14120c385070290afc2754a75ec581d170b4943 consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/validation/ProposalPayloadValidator.java afa427cea95acd4d1f54f6824d39fae4125a7d4e23a84785f10b84ae9fe756ad consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/validation/RoundChangePayloadValidator.java +a32033398fd0a680f5d47930bbc7910e9eb178cdbc091419c35cf2a17e04ec27 consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/AereD434ListBoundsTest.java e2107737579c16b73cbd2c9b50e90abcd6c910309286fb862cef780555ca43a5 consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/network/ProposalSealPlumbingTest.java 6da7d76a2ddc3dcab2a6d54d60615bb54182594f230e858cfd0c13b2626b76e0 consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/network/RoundChangeSealPlumbingTest.java 36dd77cdea980e4de0090e86ae2ca75d1af62a9477fe056c7fdde06422617898 consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/payload/CommitPayloadHybridTest.java @@ -176,6 +190,7 @@ dc9f9e862a11f0135d26976176a1d1adac3e84fd5f1e06572d8727262cb3374f consensus/qbft 24ee993a415eea57c80fe2c9b839ce20140bfcc580e269e7b1182f71e960c1f0 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/QbftBlockHeaderValidationRulesetFactory.java ebf6da56130720ea95c64505959d64bfba46540ebf2139c3f71a7051d88837a1 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/QbftExtraDataCodec.java c870d39a19174328bc394658f4a365a61bd57fe7b0e28e6c5e4aaeb28bb509e5 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/adaptor/QbftBlockCreatorAdaptor.java +1238bdf6968781d12c7c7da8d31e5cf8b281a3e09612d450dc8ebae208542b31 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/adaptor/QbftFinalStateImpl.java cfb0aad408ab620f28b09914ece7d8d3d09f9e6bbb503070e28d9ef39c023dee consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/QbftBlockCreatorFactory.java cbec6f4e280250ddefe5c79c453b59a07be6435e6f88d5614e92168c855e4693 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/AereBaseFeeImportRule.java e4e75c28ff15d058176145b1ae32606bf91b22112d1668a718f1a6ae4add0e71 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/FalconSealValidationRule.java @@ -186,6 +201,8 @@ a76a87987050ea6765e2a1f43d8f828ca8aec218c951e395e7acb62465e8b2fd consensus/qbft 2add2a7733668e11b50ddd64b05bb37db26a18284a636bbe720c2cb1f6aa28ed consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/PqEmergencyShoutRule.java e0f8afc69fef826c5ee8a95745a967b1049ce10b9d591696a4fc095fb28062cb consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/PqHeaderSealRule.java 8ac99e94c89f61f0281199cf369e282fc4f9fb1a3414a1650766e1d7548e76f5 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/PqRegistryBindingRule.java +99cc2db4894a35c1df3c10d52d0deeffc4d6bf2a219f07249f5baccedf3889e7 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java +24b52131993fc93c5f146cab8588067cdb6201e308c36eef455d0ea56460a13b consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java 5dcf5c22389c063e12363b83366b39bd3b2e79f36c68e777fb3ef11080f53ea9 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/AereIdentityHeaderTest.java 52b56f157500ae3527b7e55c51786915bb5d6980065668f1e7b8e297bf7125ef consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/QbftAnchorRuleWiringTest.java 5a18c7fee308654d9557fc507ca7a64704bdac5d13835e7df47a3dfb41519902 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/AereBaseFeeImportRuleTest.java @@ -201,12 +218,72 @@ a81da71dd34dd111a0da43c22d22345dcc595c084a2c2d36aec1f45d8b762f80 consensus/qbft 484bc74ff3d1e52d25631264d83af70c2b695e973c7360b0f3db471ab5c6613a consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/PqEmergencyShoutRuleTest.java 49c3fd97e7985966531928af173b6f38ac48ff9dc8b5be1106e5b7bfb9c7b2b6 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/PqForkGateFeedTest.java 8d0113eec652dc5e08600fb7fb07de3e08524a500ea6bdcf3fd0f46f3f1b38e6 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/headervalidationrules/PqHeaderSealStage5Test.java -7ea99f24bc1c9131e8ef08f7a2c6f2a08af71ebee011a4cb88a367bd446b6c27 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetBalance.java -5bf41ab0be06328fa1e18bf9c98bfd89b7b16384fe185d96b3385595dabe61b3 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetCode.java -4c96b06073ff3830f4e0292fc732ec3fb9e0ed50ef741dceedf755b50f18d5ae ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetStorageAt.java -8e649d2a1c47cd912f0782408da3f3af5537e8c670c12cba72dbb502809c848d ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCount.java -41b5f7b0066ef6d8af25491814eeacf2bcbb29415f407020affde05b7d1916b7 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java +212e932427839cebf788cf90ffb7893f2905880219bbcd051ee0b9279810df35 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java +f76961f1c07629cd4d3061ba3b59167875cf517d7adc7ceb302894ef430baf52 datatypes/src/main/java/org/hyperledger/besu/datatypes/Address.java +cd815f46bd834f62f49551dd0dc44b9c0cf88d33311fc5928803949dc9964562 datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAlgorithm.java +d3658d8ccc923f1b890b80fa83673c15cc213615db0ac2a9e343cd18e931c471 datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAuthorization.java +17560a53ed57b6c22b43a6223eb0f52289eb1565dad2c1fe4a8a5565426a07cf datatypes/src/main/java/org/hyperledger/besu/datatypes/Transaction.java +9858c5116ce32ee4ebfa247a5eaa2befe1f5ec6507f61c7115f04a49ba0ece5f datatypes/src/main/java/org/hyperledger/besu/datatypes/TransactionType.java +9e370c961785ad6a2b16b3fc377a5341a5e930136bf9a47a8e1079ae501d1b62 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java +4f2fe5b68e55276d62181a8694fc8dbd44f4680ca5c0a28a555425135563cc43 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetBalance.java +b5394c8f60a8afc46c0e09b8198b9212e808c728050bc20808e4652d45019d39 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetCode.java +dfceadc9d220ef9b028ef30e09e12a2c44caffee1922b6aa98c95937862bc9f9 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetStorageAt.java +cd94a4c059dd5e79136544f9dc09163df722ecedd41b8552cd0cdc22ba6328d3 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCount.java +ce03332291e5a575cede22e2885e561ae5d27e815551d7d4af3a93045949eabc ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionBaseResult.java +6a54e53e6f1569b0bc00617188b8d82c6d69fa2de13db6285af0c5e959d1eae7 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java +2de0590a917378d7cd1cb2a4ab1a4f9d0ac27545c0e080a83417244718606ba5 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java 3ee087fc34b455e113ec9d2c53b5dbdb3574429fcf166f4c968a80120f70f0eb ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AereStateUnavailableIsAnErrorTest.java 6e99e9f399558abbd1d4fdb4b176ba413bf31cabb3c8aba1701c17ce5828ce8a ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/query/AereHeadStateRetryTest.java +bf6617807ee0dd03ed98e6f1a73d6d11366880c2f742f34e73ef4d74a4f01180 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +5d5032dbdad0f3dd15539b3ee6676cf3fd2bb0381d0db17d8abea2051b5c2c09 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionDecoder.java +95a30107a67e12d0f45df74365fd533bde15fab76036451590539a73714b0b84 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionEncoder.java +53f750f62dbf256a3d25fab595d651dd3c3872c207659aa839ab8403df4cf2ad ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionDecoder.java +2aa829c45cb5902f8467218244680ebc3f87162e6be8897b39d286813046dc23 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionEncoder.java +b3f5feddcae2e74f3b2517785a51e61998817fba4977d5f5a3fb9fbb48038af8 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AerePqTransactionSupport.java +10df69ba73a9847ea5b93d2fc2bc1c2dbf238b8b76d713568678403ea6edbd87 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java +73f66fae03826866c74e87bd262e7bac79ce3db814de782dc52b9263ce156059 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +c19d72f93d09f945a6c5a0144125f8501cc0cb0d0749cd08e8d8ca091ad40dcc ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java +f2257781bf894f96f0fd67bc8f5d97fb8df181a95029147a9e594b068ca04cf2 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsValidator.java +b3c6811983da5ce74f75a0fccdd87a2385881fa99210cdb0484d357ac0d844ab ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmEngine.java +20d842d8413034e506c6ab22288a9e6870f0fb8a83362ec5adecb70b49f6af07 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmPreprocessing.java +bd5d69c4ec63c212f364fa7fbb3ad658983654e772148bedb17c8028ce24540e ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmProcessor.java +fb4a98f4edf9e8c362ce241e44771f6c8db8c34f54b8e4efb026ba3747b3975e ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BaseReader.java +ae98ca304a11bfc7e051f5bfaf43b4270dca40dbae47f931e3d186358a87cb03 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BlockStmStats.java +85b13b2561f7541b6bb1187a2d6c07a8aafdd447bb04ad712fe540703f94f5a1 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/FlatBaseSnapshot.java +cdc1bd5ce7bdeecc5328885622f7b21028c2d3d5bf4b60d5da13d8e7819dc984 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvAccount.java +e74fcdcf9fa8874c0eaf7e4467be0a31351faf096bb4147fe0d8702068fa2bd6 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvBaseLayer.java +e3fa43a8ac48028631d63d6e1a13f32c27a14430b52d1a1d89d9c2a454d954a3 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvContext.java +6e83f1158908ac8262f5de81be72a3adbc7e35f5dce57db576437ba924fdd1e3 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvKey.java +1e6dbfb45124b5cad58ee760fd394ccd12f8d514508eb533d6fbb1ee7c4f94b7 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvMemory.java +a1422099123d6644e53129c9caa59bca7cb54669c5a07438f53bb8cd065c89b2 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/Scheduler.java +26a80db3a78556e874f6176e82c5e86475ba55fb574660a8a856806479ccd1c9 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarket.java +10c888f52c388ec4ce8767ff435a70ef14c341eaa632acb10f8e75622e9ca3ae ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java +cbd5d4e986818ad65259e2bf3c46b1fff1bbec96a5513212a1b2ef1448c18cb7 ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java +f3ba58dda5335659912c7bf9c336ee00b2b5645ed7f868ffa9955425cb2ced7d ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/AerePqTransactionTest.java +01ac1387d8a7d155a56274c2929b0f129053c78b2602137a7bdd00c8a99009f6 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD432GasCapUnsignedTest.java +aa4999574b158ed3a9d738d7439033d543ed0a748792e0e60d1eb8ca8af84427 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD433BftWithdrawalsTest.java +816c47aba04623a15e48ece16b440d6d9e16d23031841c22f9159830862d6dd6 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarketAereFloorTest.java +d34dd8c46ad4a394e8b15a2155c32f89883a0879c1cf7f1976f081938a3f56d3 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmBenchmarkTest.java +6f0119bcafb5247995b4399f766d4adc9275669fe3ca08fbe2df5b01800624e3 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmSmokeTest.java +f94acc1cd8aac6679c39c74f7d23375ba19781e9bfd1d14b8c8a9e4314384cc1 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +62abf412b67808f0648b1af363a376e567ba7796cb1c4aa93617adc3f2484b8e ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/BlockPropagationManager.java 3b690b72e80a0f9eaf868a3fa3f99ace42dad74dc30b2c98b9f3ca5a023917c0 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DownloadHeadersStep.java 7f611966d74f52ef2f6d55bb5882ffb72b10abf1985df2b01f03aad4a6e2baf1 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PropagatedBlockAdoptionGate.java +5a8187cc12bb1a16a9b77956c4e1c5d8c088a4e65164179c427cd7b2c2f24b7f ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncState.java +e068fc39b4f645d1ccdd778acdf77983869a73428513c11ce587547391e36ce3 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java +eb8d2305070adcf0424ab9884c8f09d9bc95a1662a4cf8775a136e3a314fa181 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java +76ce97047c7988c106969f69fbdcba6941657c482ea1e060b995e7ce0556c108 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncStateTest.java +cc003a07cc4cf27e20ebf249ae8d84b8f514d1c24e55dfaab00fea3c345e1a0c ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java +55e6c022442a970411bb677a1d0b27a4a7f8174b4fabd200de5b10393731b201 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java +e39a09c49cec66fa2766470ba29bbccced6f42ecb7f95df39ccf5086b666510c ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java +2190cd3df779c0b04ca1ca09098ea1a99df07dab6659a2d4cac058f34c27e495 evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java +eeeb63160aba1f889725d628b8623630baf227a82e367844b6d24619d8e83e7d evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java +72866e4744d34a120345a1ed3a0699292fd4a63d04d6256b3a7337738441e7bf evm/src/main/java/org/hyperledger/besu/evm/gascalculator/FrontierGasCalculator.java +417134787ef4f01af0674bb8e641a7eef3c701e1d9d4e8a1c807faab4bb97d99 evm/src/main/java/org/hyperledger/besu/evm/precompile/AereFalconSupport.java +c59d8c714155b347fe64bb378715e71670a8bd42180a97d08156e6c20caa359c evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon1024PrecompiledContract.java +6487810e9b167eb6c0a7046a5226e40936116771efb91013d6c2eb9559c60f97 evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon512PrecompiledContract.java +42e4d1442d3478eac8c3177e35318872c365a173d4a16f529356e52944901d50 evm/src/main/java/org/hyperledger/besu/evm/precompile/MLDSA44PrecompiledContract.java +d17d9fde238d5d6527380d9829d1b52cd3c7c76c9874c6172e4ddf1dcc26acba evm/src/main/java/org/hyperledger/besu/evm/precompile/MainnetPrecompiledContracts.java +c3b208adc9e81229eb9194fbedd40186aeccf43b8039827604d00169878d6690 evm/src/main/java/org/hyperledger/besu/evm/precompile/P256VerifyPrecompiledContract.java +c4ef4c11f13c7387b79c5d6532299f540d8c4cb82c98a865be2d01053bd5d35e evm/src/main/java/org/hyperledger/besu/evm/precompile/SHAKE256PrecompiledContract.java +976daf6471e817ca668942aa07adfc7ede7d59b4c941d0bbb91d02a25fa6fbb4 evm/src/main/java/org/hyperledger/besu/evm/precompile/SLHDSA128sPrecompiledContract.java diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftBlockHashing.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftBlockHashing.java new file mode 100644 index 0000000..06cb610 --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftBlockHashing.java @@ -0,0 +1,144 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.common.bft; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; + +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.tuweni.bytes.Bytes; + +/** The Bft block hashing. */ +public class BftBlockHashing { + private final BftExtraDataCodec bftExtraDataCodec; + + /** + * Instantiates a new Bft block hashing. + * + * @param bftExtraDataCodec the bft extra data codec + */ + public BftBlockHashing(final BftExtraDataCodec bftExtraDataCodec) { + this.bftExtraDataCodec = bftExtraDataCodec; + } + + /** + * Constructs a hash of the block header suitable for signing as a committed seal. The extra data + * in the hash uses an empty list for the committed seals. + * + * @param header The header for which a proposer seal is to be calculated (with or without extra + * data) + * @param bftExtraData The extra data block which is to be inserted to the header once seal is + * calculated + * @return the hash of the header including the validator and proposer seal in the extra data + */ + public Hash calculateDataHashForCommittedSeal( + final BlockHeader header, final BftExtraData bftExtraData) { + return Hash.hash( + serializeHeader( + header, + () -> bftExtraDataCodec.encodeWithoutCommitSeals(bftExtraData), + bftExtraDataCodec)); + } + + /** + * Calculate data hash for committed seal hash. + * + * @param header the header + * @return the hash + */ + public Hash calculateDataHashForCommittedSeal(final BlockHeader header) { + final BftExtraData bftExtraData = bftExtraDataCodec.decode(header); + return Hash.hash( + serializeHeader( + header, + () -> bftExtraDataCodec.encodeWithoutCommitSeals(bftExtraData), + bftExtraDataCodec)); + } + + /** + * Constructs a hash of the block header, but omits the committerSeals and sets round number to 0 + * (as these change on each of the potentially circulated blocks at the current chain height). + * + * @param header The header for which a block hash is to be calculated + * @return the hash of the header to be used when referencing the header on the blockchain + */ + public Hash calculateHashOfBftBlockOnchain(final BlockHeader header) { + final BftExtraData bftExtraData = bftExtraDataCodec.decode(header); + return Hash.hash( + serializeHeader( + header, + () -> bftExtraDataCodec.encodeWithoutCommitSealsAndRoundNumber(bftExtraData), + bftExtraDataCodec)); + } + + /** + * Recovers the {@link Address} for each validator that contributed a committed seal to the block. + * + * @param header the block header that was signed by the committed seals + * @param bftExtraData the parsed {@link BftExtraData} from the header + * @return the addresses of validators that provided a committed seal + */ + public List
recoverCommitterAddresses( + final BlockHeader header, final BftExtraData bftExtraData) { + final Hash committerHash = calculateDataHashForCommittedSeal(header, bftExtraData); + + return bftExtraData.getSeals().stream() + .map(p -> AerePqIdentity.author(Util.signatureToAddress(p, committerHash), header.getNumber())) + .collect(Collectors.toList()); + } + + /** + * Serialize header bytes. + * + * @param header the header + * @param extraDataSerializer the extra data serializer + * @param bftExtraDataCodec the bft extra data codec + * @return the bytes + */ + public static Bytes serializeHeader( + final BlockHeader header, + final Supplier extraDataSerializer, + final BftExtraDataCodec bftExtraDataCodec) { + + // create a block header which is a copy of the header supplied as parameter except of the + // extraData field + final BlockHeaderBuilder builder = BlockHeaderBuilder.fromHeader(header); + builder.blockHeaderFunctions(BftBlockHeaderFunctions.forOnchainBlock(bftExtraDataCodec)); + + // set the extraData field using the supplied extraDataSerializer if the block height is not 0 + if (header.getNumber() == BlockHeader.GENESIS_BLOCK_NUMBER) { + builder.extraData(header.getExtraData()); + } else { + builder.extraData(extraDataSerializer.get()); + } + + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + builder.buildBlockHeader().writeTo(out); + return out.encoded(); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java index dac0d63..716c67e 100644 --- a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java @@ -69,11 +69,39 @@ public final class PqSealCache { private static final Logger LOG = LoggerFactory.getLogger(PqSealCache.class); - /** How many distinct block hashes are retained. Several rounds at one height each add one. */ - private static final int MAX_ENTRIES = 1024; + /** + * AIP-21 (AERE, 2026-09-18): how far below the highest seen height entries are kept, in blocks. The + * consensus default stays 256 (what the producer needs is the parent); a node that serves + * aere_getPqFinality may keep more through the system property, at ~7 kB per block for ten seals. + * Read once, never below 256, never a consensus value: two nodes with different windows import + * the same chain. + */ + public static final String PROPERTY_FINALITY_WINDOW = "aere.pq.finality.window"; + + private static final long DEFAULT_HEIGHT_WINDOW = 256L; /** How far below the highest seen height entries are kept, in blocks. */ - private static final long HEIGHT_WINDOW = 256L; + private static final long HEIGHT_WINDOW = readWindow(); + + /** How many distinct block hashes are retained. Several rounds at one height each add one. */ + private static final int MAX_ENTRIES = (int) Math.max(1024L, Math.min(HEIGHT_WINDOW * 4L, 1 << 20)); + + private static long readWindow() { + final Long v = Long.getLong(PROPERTY_FINALITY_WINDOW); + if (v == null) { + return DEFAULT_HEIGHT_WINDOW; + } + return Math.max(DEFAULT_HEIGHT_WINDOW, Math.min(v, 1L << 18)); + } + + /** + * The retention window in blocks, as this node runs it. + * + * @return the window + */ + public long retentionBlocks() { + return HEIGHT_WINDOW; + } private static final PqSealCache INSTANCE = new PqSealCache(); @@ -90,6 +118,16 @@ public final class PqSealCache { private Hash persistedHash; private int persistedCount = -1; + // AIP-21, SECOND HALF (2026-09-18): the finality window survives a restart. Heard heights are + // appended to a segmented journal (PqSealWindowStore) when the NEXT height begins, and read back, + // re-verified seal by seal against the canonical chain, at startup. Null = journal off. + private volatile Path windowDirectory; + private volatile long windowPrunedSegment = -1L; + private long windowJournaledThrough = -1L; + + /** Heights restored at startup at most, newest first, whatever the configured window. */ + private static final long MAX_RESTORED_HEIGHTS = 4096L; + private PqSealCache() {} /** @@ -121,6 +159,9 @@ public final class PqSealCache { } this.persistenceChainId = chainId; this.persistenceFile = PqSealStore.fileIn(dataDirectory); + this.windowPrunedSegment = -1L; + this.windowDirectory = + PqSealWindowStore.enabled() ? PqSealWindowStore.directoryIn(dataDirectory) : null; LOG.info( "AERE PERSISTENTA-SIGILII: heard seals for the chain head are kept in {} (chainId={}). " + "Every seal read back is re-verified against the anchored registry before it is used.", @@ -133,6 +174,7 @@ public final class PqSealCache { */ public void disablePersistence() { this.persistenceFile = null; + this.windowDirectory = null; synchronized (writeLock) { persistedBlock = -1L; persistedHash = null; @@ -226,6 +268,7 @@ public final class PqSealCache { public void record( final long blockNumber, final Hash onchainBlockHash, final Collection seals) { final List toPersist; + List windowToJournal = List.of(); synchronized (this) { if (onchainBlockHash == null || seals == null || seals.isEmpty()) { return; @@ -238,6 +281,9 @@ public final class PqSealCache { } } if (blockNumber > highestSeen) { + // AIP-21 second half: a NEW height begins, so what was heard for the heights below it is as + // complete as this node will ever hold it. It goes to the window journal, outside this lock. + windowToJournal = windowSnapshotBelow(blockNumber); highestSeen = blockNumber; } prune(); @@ -251,6 +297,8 @@ public final class PqSealCache { // Deliberately OUTSIDE the map lock: a file write must never be able to stall a caller that only // wants to read the cache. persist(blockNumber, onchainBlockHash, toPersist); + // the head file first (liveness depends on it), the window journal after (best effort) + persistWindow(windowToJournal); } private void persist( @@ -396,6 +444,7 @@ public final class PqSealCache { public synchronized void clear() { byHash.clear(); highestSeen = -1L; + windowJournaledThrough = -1L; // what clear() forgets, the journal mark forgets too synchronized (writeLock) { persistedBlock = -1L; persistedHash = null; @@ -403,6 +452,157 @@ public final class PqSealCache { } } + /** + * AIP-21, second half. Everything heard BELOW a height that has just begun and not yet handed to + * the window journal. Caller holds the map lock. + * + * @param newHeight the height that has just begun + * @return the heard blocks to journal, possibly empty + */ + private List windowSnapshotBelow(final long newHeight) { + if (windowDirectory == null) { + return List.of(); + } + final List out = new ArrayList<>(); + for (final Map.Entry e : byHash.entrySet()) { + final long n = e.getValue().blockNumber; + if (n > windowJournaledThrough && n < newHeight && !e.getValue().seals.isEmpty()) { + out.add(PqSealWindowStore.heard(n, e.getKey().getBytes(), e.getValue().seals.values())); + } + } + windowJournaledThrough = Math.max(windowJournaledThrough, newHeight - 1L); + return out; + } + + private void persistWindow(final List heard) { + final Path directory = windowDirectory; + if (directory == null || heard.isEmpty()) { + return; + } + synchronized (writeLock) { + try { + long highest = -1L; + for (final PqSealWindowStore.Heard h : heard) { + PqSealWindowStore.append(directory, persistenceChainId, h); + highest = Math.max(highest, h.blockNumber()); + } + final long segment = highest / PqSealWindowStore.SEGMENT_BLOCKS; + if (segment != windowPrunedSegment) { + PqSealWindowStore.pruneSegments(directory, highest, HEIGHT_WINDOW); + windowPrunedSegment = segment; + } + } catch (final Exception e) { + // A disk fault must not be able to stop this node from taking part in consensus. It costs the + // answer aere_getPqFinality gives for these blocks after a restart, and nothing else. + LOG.warn( + "AERE FEREASTRA-SIGILII: could not append {} heard block(s) to the window journal {} ({}). " + + "This node keeps working; after a restart it answers 'unavailable' for them.", + heard.size(), + directory, + e.toString()); + } + } + } + + /** + * AIP-21, second half. Bring the finality window back after a restart. Nothing in the journal is + * believed: a frame counts only if its hash is the CANONICAL hash this node's own chain holds at + * that height, and each seal only if it verifies, against the registry epoch bound at that height, + * over the commit message rebuilt from that canonical hash. Never throws. + * + * @param headNumber this node's chain head + * @param canonicalHashAt the canonical on-chain hash at a height, empty when the chain holds none + * @param registry the anchored registry seals are verified against + * @return how many verified seals entered the window + */ + public int restoreWindowFromDisk( + final long headNumber, + final java.util.function.LongFunction> canonicalHashAt, + final PqSignerRegistry registry) { + final Path directory = windowDirectory; + if (directory == null || canonicalHashAt == null || registry == null) { + return 0; + } + int restored = 0; + try { + final long from = Math.max(0L, headNumber - Math.min(HEIGHT_WINDOW, MAX_RESTORED_HEIGHTS)); + final List frames = + PqSealWindowStore.readAll(directory, persistenceChainId, from, headNumber); + long highestInJournal = -1L; + int notCanonical = 0; + int rejected = 0; + final LinkedHashMap verified = new LinkedHashMap<>(); + for (final PqSealWindowStore.Heard frame : frames) { + final long n = frame.blockNumber(); + highestInJournal = Math.max(highestInJournal, n); + final java.util.Optional canonical = canonicalHashAt.apply(n); + if (canonical == null + || canonical.isEmpty() + || !canonical.get().getBytes().equals(frame.blockHash())) { + notCanonical++; + continue; + } + final org.apache.tuweni.bytes.Bytes32 message = + PqAnchor.commitMessage(persistenceChainId, n, frame.blockHash()); + for (final FalconSeal seal : frame.seals()) { + if (seal == null + || seal.getValidatorIndex() < 0 + || seal.getSignature() == null + || !registry.verifyAtHistoric( + n, seal.getValidatorIndex(), message, seal.getSignature())) { + rejected++; + continue; + } + final Entry entry = verified.computeIfAbsent(canonical.get(), unused -> new Entry(n)); + if (entry.seals.putIfAbsent(seal.getValidatorIndex(), seal) == null) { + restored++; + } + } + } + synchronized (this) { + // Oldest first, so the size cap still evicts the oldest: the restored heights go in BEFORE + // what this node already holds (its head, restored a moment ago from the head file). + final LinkedHashMap held = new LinkedHashMap<>(byHash); + byHash.clear(); + byHash.putAll(verified); + for (final Map.Entry e : held.entrySet()) { + final Entry target = byHash.get(e.getKey()); + if (target == null) { + byHash.put(e.getKey(), e.getValue()); + } else { + e.getValue().seals.forEach(target.seals::putIfAbsent); + e.getValue().extras.forEach(target.extras::putIfAbsent); + } + } + for (final Entry entry : verified.values()) { + if (entry.blockNumber > highestSeen) { + highestSeen = entry.blockNumber; + } + } + windowJournaledThrough = Math.max(windowJournaledThrough, highestInJournal); + prune(); + } + LOG.info( + "AERE FEREASTRA-SIGILII: {} verified seal(s) over {} block(s) restored into the finality " + + "window from {} ({} frame(s) read, {} not canonical on this chain, {} seal(s) FAILED " + + "verification and were dropped).", + restored, + verified.size(), + directory, + frames.size(), + notCanonical, + rejected); + } catch (final RuntimeException e) { + LOG.warn( + "AERE FEREASTRA-SIGILII: restoring the finality window from {} failed unexpectedly ({}); " + + "the window starts empty, exactly as before this journal existed.", + directory, + e.toString()); + return 0; + } + return restored; + } + private void prune() { final Iterator> it = byHash.entrySet().iterator(); while (it.hasNext()) { diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java new file mode 100644 index 0000000..b224d8f --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java @@ -0,0 +1,387 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.common.bft; + +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.CRC32; + +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * AIP-21, SECOND HALF (2026-09-18): the finality window on disk. + * + *

The first half gave every validator a window of heard Falcon seals ({@link PqSealCache}) and a + * method that reads it back ({@code aere_getPqFinality}). The window lived in memory only, so a node + * that restarted answered {@code unavailable} for every block it had heard before the restart. This + * class is the journal that carries the window across a restart. + * + *

SHAPE. One directory under the node's data directory, {@value #DIRECTORY_NAME}, holding + * SEGMENT files of {@value #SEGMENT_BLOCKS} heights each ({@code w-.rlp}). A segment is + * a sequence of FRAMES: {@code length(4, big endian) || payload || crc32(payload)(4)}, the payload + * being the RLP list {@code [DOMAIN, chainId, blockNumber, blockHash, certificate]} with the + * certificate encoded exactly as {@link PqSealStore} and the header encode it. Frames are only ever + * APPENDED, one per heard block, when the next height begins; whole segments below the window are + * deleted. The footprint is therefore bounded by the window, whatever the age of the node. + * + *

WHY THIS IS NOT A8 IN ANOTHER COAT, the same argument as {@link PqSealStore}: a seal is + * SELF-VERIFYING. Nothing read from this directory is believed. {@link PqSealCache} re-verifies + * every seal against the registry epoch bound at its height, over the message rebuilt from the + * CANONICAL hash this process reads from its own chain at that height; a frame for a hash that is + * not canonical is dropped before a single signature is looked at. A forged journal can obtain the + * empty window an absent journal already gives, and nothing else. + * + *

WHAT A FAULT COSTS. Every method here is written not to throw on bad input: a torn tail (the + * node died mid-append), a flipped byte, a directory where a file should be, a journal from another + * chain. A bad frame costs that frame and what FOLLOWS it in the same segment (a frame boundary + * cannot be trusted after a bad length), never the node. Writes are best effort and unsynced: the + * head file of {@link PqSealStore} is the one liveness depends on, and it keeps its fsync. + */ +public final class PqSealWindowStore { + + private static final Logger LOG = LoggerFactory.getLogger(PqSealWindowStore.class); + + /** Directory name under the node's data directory. */ + public static final String DIRECTORY_NAME = "aere-pq-seal-window"; + + /** Domain separator of a frame payload. */ + public static final String DOMAIN = "AERE-PQ-SEALWINDOW-1"; + + private static final Bytes DOMAIN_BYTES = Bytes.wrap(DOMAIN.getBytes(StandardCharsets.US_ASCII)); + + /** Heights per segment file. */ + public static final long SEGMENT_BLOCKS = 64L; + + /** A segment larger than this is ignored whole. */ + public static final long MAX_SEGMENT_BYTES = 64L * 1024L * 1024L; + + /** A frame payload larger than this ends the reading of its segment. */ + public static final int MAX_FRAME_BYTES = 4 * 1024 * 1024; + + /** More seals than this in one frame is not a certificate. */ + public static final int MAX_SEALS = 1024; + + /** System property: {@code false} turns the journal off (the window is then memory only). */ + public static final String PROPERTY_PERSIST = "aere.pq.finality.persist"; + + private static final Pattern SEGMENT_NAME = Pattern.compile("^w-(\\d{1,18})\\.rlp$"); + + private PqSealWindowStore() {} + + /** + * One heard block as the journal holds it. Unverified: the caller verifies. + * + * @param blockNumber the height + * @param blockHash the round-independent on-chain hash the seals were heard for + * @param seals the heard seals, index-sorted + */ + public record Heard(long blockNumber, Bytes blockHash, List seals) {} + + /** + * Whether the journal is enabled for this process. + * + * @return false only when {@value #PROPERTY_PERSIST} is {@code false} + */ + public static boolean enabled() { + return !"false".equalsIgnoreCase(System.getProperty(PROPERTY_PERSIST)); + } + + /** + * The journal directory of a node. + * + * @param dataDirectory the node's data directory + * @return the directory the segments live in + */ + public static Path directoryIn(final Path dataDirectory) { + return dataDirectory.resolve(DIRECTORY_NAME); + } + + /** + * The segment file a height belongs to. + * + * @param directory the journal directory + * @param blockNumber the height + * @return the segment path + */ + public static Path segmentFor(final Path directory, final long blockNumber) { + return directory.resolve("w-" + (blockNumber / SEGMENT_BLOCKS) + ".rlp"); + } + + /** + * Encode one frame payload. + * + * @param chainId the chain id + * @param heard the heard block + * @return the RLP payload + */ + public static byte[] encode(final long chainId, final Heard heard) { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + out.startList(); + out.writeBytes(DOMAIN_BYTES); + out.writeLongScalar(chainId); + out.writeLongScalar(heard.blockNumber()); + out.writeBytes(heard.blockHash()); + PqAnchor.writeCertificate(out, heard.seals()); + out.endList(); + return out.encoded().toArrayUnsafe(); + } + + /** + * Append one heard block to its segment. Best effort, unsynced. + * + * @param directory the journal directory + * @param chainId the chain id + * @param heard the heard block + * @throws IOException when the file system refuses; the caller logs and carries on + */ + public static void append(final Path directory, final long chainId, final Heard heard) + throws IOException { + Files.createDirectories(directory); + final byte[] payload = encode(chainId, heard); + final CRC32 crc = new CRC32(); + crc.update(payload, 0, payload.length); + final ByteBuffer frame = ByteBuffer.allocate(4 + payload.length + 4); + frame.putInt(payload.length); + frame.put(payload); + frame.putInt((int) crc.getValue()); + frame.flip(); + try (FileChannel channel = + FileChannel.open( + segmentFor(directory, heard.blockNumber()), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND)) { + while (frame.hasRemaining()) { + channel.write(frame); + } + } + } + + /** + * Delete every segment that lies WHOLLY below the window. + * + * @param directory the journal directory + * @param highestBlock the highest height heard + * @param window the retention window in blocks + * @return how many segment files were deleted + */ + public static int pruneSegments(final Path directory, final long highestBlock, final long window) { + if (directory == null || !Files.isDirectory(directory)) { + return 0; + } + final long oldestKept = Math.max(0L, highestBlock - window); + int deleted = 0; + try (DirectoryStream stream = Files.newDirectoryStream(directory)) { + for (final Path file : stream) { + final Matcher m = SEGMENT_NAME.matcher(file.getFileName().toString()); + if (!m.matches()) { + continue; + } + final long lastHeightOfSegment = (Long.parseLong(m.group(1)) + 1L) * SEGMENT_BLOCKS - 1L; + if (lastHeightOfSegment < oldestKept) { + try { + Files.deleteIfExists(file); + deleted++; + } catch (final IOException e) { + LOG.debug("AERE FEREASTRA-SIGILII: could not delete {} ({}).", file, e.toString()); + } + } + } + } catch (final IOException | RuntimeException e) { + LOG.debug("AERE FEREASTRA-SIGILII: could not list {} ({}).", directory, e.toString()); + } + return deleted; + } + + /** + * Read every frame for a height range, oldest first. Never throws; never verifies. + * + * @param directory the journal directory + * @param chainId the chain id this node runs + * @param fromBlock the lowest height wanted, inclusive + * @param toBlock the highest height wanted, inclusive + * @return the frames that decoded, for this chain, within the range + */ + public static List readAll( + final Path directory, final long chainId, final long fromBlock, final long toBlock) { + final List out = new ArrayList<>(); + if (directory == null || toBlock < fromBlock || !Files.isDirectory(directory)) { + return out; + } + final List segments = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(directory)) { + for (final Path file : stream) { + final Matcher m = SEGMENT_NAME.matcher(file.getFileName().toString()); + if (!m.matches()) { + continue; + } + final long first = Long.parseLong(m.group(1)) * SEGMENT_BLOCKS; + final long last = first + SEGMENT_BLOCKS - 1L; + if (last >= fromBlock && first <= toBlock) { + segments.add(file); + } + } + } catch (final IOException | RuntimeException e) { + LOG.warn( + "AERE FEREASTRA-SIGILII: the window journal {} could not be listed ({}); IGNORED, the " + + "window starts empty exactly as it did before this journal existed.", + directory, + e.toString()); + return out; + } + for (final Path segment : segments) { + readSegment(segment, chainId, fromBlock, toBlock, out); + } + out.sort(Comparator.comparingLong(Heard::blockNumber)); + return out; + } + + private static void readSegment( + final Path segment, + final long chainId, + final long fromBlock, + final long toBlock, + final List out) { + final byte[] raw; + try { + if (!Files.isRegularFile(segment)) { + return; + } + final long size = Files.size(segment); + if (size <= 0 || size > MAX_SEGMENT_BYTES) { + LOG.warn( + "AERE FEREASTRA-SIGILII: segment {} has an implausible size ({} bytes); IGNORED.", + segment, + size); + return; + } + raw = Files.readAllBytes(segment); + } catch (final IOException | RuntimeException e) { + LOG.warn( + "AERE FEREASTRA-SIGILII: segment {} could not be read ({}); IGNORED.", + segment, + e.toString()); + return; + } + final ByteBuffer buffer = ByteBuffer.wrap(raw); + int frames = 0; + while (buffer.remaining() >= 8) { + final int length = buffer.getInt(); + if (length <= 0 || length > MAX_FRAME_BYTES || buffer.remaining() < length + 4) { + LOG.info( + "AERE FEREASTRA-SIGILII: segment {} ends in a torn or implausible frame after {} good " + + "frame(s); what follows it is dropped.", + segment, + frames); + return; + } + final byte[] payload = new byte[length]; + buffer.get(payload); + final int storedCrc = buffer.getInt(); + final CRC32 crc = new CRC32(); + crc.update(payload, 0, payload.length); + if ((int) crc.getValue() != storedCrc) { + LOG.warn( + "AERE FEREASTRA-SIGILII: segment {} holds a frame whose checksum does not match after " + + "{} good frame(s); it and what follows it are dropped.", + segment, + frames); + return; + } + frames++; + final Heard heard = decode(payload, chainId); + if (heard != null && heard.blockNumber() >= fromBlock && heard.blockNumber() <= toBlock) { + out.add(heard); + } + } + } + + private static Heard decode(final byte[] payload, final long chainId) { + try { + final RLPInput in = new BytesValueRLPInput(Bytes.wrap(payload), false); + in.enterList(); + if (!DOMAIN_BYTES.equals(in.readBytes())) { + return null; + } + if (in.readLongScalar() != chainId) { + return null; + } + final long blockNumber = in.readLongScalar(); + final Bytes blockHash = in.readBytes(); + if (blockNumber < 0 || blockHash.size() != 32) { + return null; + } + final List seals = new ArrayList<>(); + in.enterList(); + while (!in.isEndOfCurrentList()) { + if (seals.size() >= MAX_SEALS) { + return null; + } + in.enterList(); + final int index = in.readIntScalar(); + final Bytes signature = in.readBytes(); + in.leaveList(); + seals.add(new FalconSeal(index, signature)); + } + in.leaveList(); + in.leaveList(); + return new Heard(blockNumber, blockHash, seals); + } catch (final RuntimeException e) { + return null; + } + } + + /** + * Count the frames of a journal, for tests and for the operator's curiosity. Never throws. + * + * @param directory the journal directory + * @param chainId the chain id + * @return how many frames decode for this chain, at any height + */ + public static int frameCount(final Path directory, final long chainId) { + return readAll(directory, chainId, 0L, Long.MAX_VALUE).size(); + } + + /** + * The seals of one frame collection, for callers that hold a plain collection. + * + * @param blockNumber the height + * @param blockHash the on-chain hash + * @param seals the heard seals + * @return the frame value, seals index-sorted + */ + public static Heard heard( + final long blockNumber, final Bytes blockHash, final Collection seals) { + return new Heard(blockNumber, blockHash, PqAnchor.sortedByIndex(new ArrayList<>(seals))); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java new file mode 100644 index 0000000..e43afac --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java @@ -0,0 +1,155 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-prevrandao-block-creation.patch. + */ +package org.hyperledger.besu.consensus.common.bft.blockcreation; + +import org.hyperledger.besu.config.BftConfigOptions; +import org.hyperledger.besu.consensus.common.ForksSchedule; +import org.hyperledger.besu.consensus.common.bft.BftBlockHeaderFunctions; +import org.hyperledger.besu.consensus.common.bft.BftExtraDataCodec; +import org.hyperledger.besu.consensus.common.bft.BftHelpers; +import org.hyperledger.besu.consensus.common.bft.BftProtocolSchedule; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; +import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.SealableBlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; + +import java.util.Collections; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes32; + +/** The Bft block creator. */ +// This class is responsible for creating a block without committer seals (basically it was just +// too hard to coordinate with the state machine). +public class BftBlockCreator extends AbstractBlockCreator { + + private final BftExtraDataCodec bftExtraDataCodec; + + /** + * Instantiates a new Bft block creator. + * + * @param miningConfiguration the mining parameters + * @param forksSchedule the forks schedule + * @param localAddress the local address + * @param extraDataCalculator the extra data calculator + * @param transactionPool the pending transactions + * @param protocolContext the protocol context + * @param protocolSchedule the protocol schedule + * @param bftExtraDataCodec the bft extra data codec + * @param ethScheduler the scheduler for asynchronous block creation tasks + */ + public BftBlockCreator( + final MiningConfiguration miningConfiguration, + final ForksSchedule forksSchedule, + final Address localAddress, + final ExtraDataCalculator extraDataCalculator, + final TransactionPool transactionPool, + final ProtocolContext protocolContext, + final ProtocolSchedule protocolSchedule, + final BftExtraDataCodec bftExtraDataCodec, + final EthScheduler ethScheduler) { + super( + miningConfiguration.setCoinbase(localAddress), + miningBeneficiaryCalculator(protocolSchedule), + extraDataCalculator, + transactionPool, + protocolContext, + protocolSchedule, + ethScheduler); + this.bftExtraDataCodec = bftExtraDataCodec; + } + + @Override + public BlockCreationResult createBlock(final long timestamp, final BlockHeader parentHeader) { + ProtocolSpec protocolSpec = + ((BftProtocolSchedule) protocolSchedule) + .getByBlockNumberOrTimestamp(parentHeader.getNumber() + 1, timestamp); + + if (protocolSpec.getWithdrawalsProcessor().isPresent()) { + return createEmptyWithdrawalsBlock(timestamp, parentHeader); + } else { + return createBlock( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(Bytes32.wrap(BftHelpers.EXPECTED_MIX_HASH.getBytes())), + Optional.empty(), + Optional.empty(), + timestamp, + true, + parentHeader); + } + } + + /** + * Construct a mining beneficiary calculator using the given protocolSchedule + * + * @param protocolSchedule protocol schedule + * @return mining beneficiary calculator + */ + public static MiningBeneficiaryCalculator miningBeneficiaryCalculator( + final ProtocolSchedule protocolSchedule) { + return (blockTimestamp, pendingHeader) -> { + BlockHeader newBlockHeader = + BlockHeaderBuilder.createDefault() + .coinbase(pendingHeader.getCoinbase()) + .buildBlockHeader(); + ProtocolSpec protocolSpec = + ((BftProtocolSchedule) protocolSchedule) + .getByBlockNumberOrTimestamp(pendingHeader.getNumber(), blockTimestamp); + return protocolSpec.getMiningBeneficiaryCalculator().calculateBeneficiary(newBlockHeader); + }; + } + + @Override + public BlockCreationResult createEmptyWithdrawalsBlock( + final long timestamp, final BlockHeader parentHeader) { + return createBlock( + Optional.empty(), + Optional.empty(), + Optional.of(Collections.emptyList()), + Optional.of(Bytes32.wrap(BftHelpers.EXPECTED_MIX_HASH.getBytes())), + Optional.empty(), + Optional.empty(), + timestamp, + true, + parentHeader); + } + + @Override + protected BlockHeader createFinalBlockHeader(final SealableBlockHeader sealableBlockHeader) { + final BlockHeaderBuilder builder = + BlockHeaderBuilder.create() + .populateFrom(sealableBlockHeader) + .mixHash(BftHelpers.EXPECTED_MIX_HASH) + .nonce(0L) + .blockHeaderFunctions(BftBlockHeaderFunctions.forCommittedSeal(bftExtraDataCodec)); + + return builder.buildBlockHeader(); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftMiningCoordinator.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftMiningCoordinator.java new file mode 100644 index 0000000..da4eeec --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftMiningCoordinator.java @@ -0,0 +1,279 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-coordinator-lifecycle-sync.patch. + */ +package org.hyperledger.besu.consensus.common.bft.blockcreation; + +import org.hyperledger.besu.consensus.common.bft.BftEventQueue; +import org.hyperledger.besu.consensus.common.bft.BftExecutors; +import org.hyperledger.besu.consensus.common.bft.BftProcessor; +import org.hyperledger.besu.consensus.common.bft.events.NewChainHead; +import org.hyperledger.besu.consensus.common.bft.statemachine.BftEventHandler; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; +import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; +import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.plugin.services.BesuEvents; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** The Bft mining coordinator. */ +public class BftMiningCoordinator implements MiningCoordinator, BlockAddedObserver { + + private enum State { + /** Idle state. */ + IDLE, + /** Running state. */ + RUNNING, + /** Stopped state. */ + STOPPED, + /** Paused state. */ + PAUSED, + } + + private static final Logger LOG = LoggerFactory.getLogger(BftMiningCoordinator.class); + + private final BftEventHandler eventHandler; + private final BftProcessor bftProcessor; + private final BftBlockCreatorFactory blockCreatorFactory; + + /** The Blockchain. */ + protected final Blockchain blockchain; + + private final BftEventQueue eventQueue; + private final BftExecutors bftExecutors; + + private long blockAddedObserverId; + private final AtomicReference state = new AtomicReference<>(State.PAUSED); + + private SyncState syncState; + + /** + * Instantiates a new Bft mining coordinator. + * + * @param bftExecutors the bft executors + * @param eventHandler the event handler + * @param bftProcessor the bft processor + * @param blockCreatorFactory the block creator factory + * @param blockchain the blockchain + * @param eventQueue the event queue + */ + public BftMiningCoordinator( + final BftExecutors bftExecutors, + final BftEventHandler eventHandler, + final BftProcessor bftProcessor, + final BftBlockCreatorFactory blockCreatorFactory, + final Blockchain blockchain, + final BftEventQueue eventQueue) { + this.bftExecutors = bftExecutors; + this.eventHandler = eventHandler; + this.bftProcessor = bftProcessor; + this.blockCreatorFactory = blockCreatorFactory; + this.eventQueue = eventQueue; + + this.blockchain = blockchain; + } + + /** + * Instantiates a new Bft mining coordinator. + * + * @param bftExecutors the bft executors + * @param eventHandler the event handler + * @param bftProcessor the bft processor + * @param blockCreatorFactory the block creator factory + * @param blockchain the blockchain + * @param eventQueue the event queue + * @param syncState the sync state + */ + public BftMiningCoordinator( + final BftExecutors bftExecutors, + final BftEventHandler eventHandler, + final BftProcessor bftProcessor, + final BftBlockCreatorFactory blockCreatorFactory, + final Blockchain blockchain, + final BftEventQueue eventQueue, + final SyncState syncState) { + this.bftExecutors = bftExecutors; + this.eventHandler = eventHandler; + this.bftProcessor = bftProcessor; + this.blockCreatorFactory = blockCreatorFactory; + this.eventQueue = eventQueue; + + this.blockchain = blockchain; + this.syncState = syncState; + } + + @Override + public void start() { + if (state.compareAndSet(State.IDLE, State.RUNNING) + || state.compareAndSet(State.STOPPED, State.RUNNING)) { + bftProcessor.start(); + bftExecutors.start(); + blockAddedObserverId = blockchain.observeBlockAdded(this); + eventHandler.start(); + bftExecutors.executeBftProcessor(bftProcessor); + } + } + + @Override + public void stop() { + if (state.compareAndSet(State.RUNNING, State.STOPPED)) { + blockchain.removeObserver(blockAddedObserverId); + bftProcessor.stop(); + // Make sure the processor has stopped before shutting down the executors + try { + bftProcessor.awaitStop(); + } catch (final InterruptedException e) { + LOG.debug("Interrupted while waiting for BftProcessor to stop.", e); + Thread.currentThread().interrupt(); + } + eventHandler.stop(); + bftExecutors.stop(); + } + } + + @Override + public void subscribe() { + if (syncState == null) { + return; + } + syncState.subscribeSyncStatus( + syncStatus -> { + if (syncState.syncTarget().isPresent() || !syncState.isInitialSyncPhaseDone()) { + // We're syncing so stop doing other stuff + LOG.info("Stopping BFT mining coordinator while we are syncing"); + stop(); + } else { + LOG.info("Starting BFT mining coordinator following sync"); + enable(); + start(); + } + }); + + syncState.subscribeCompletionReached( + new BesuEvents.InitialSyncCompletionListener() { + @Override + public void onInitialSyncCompleted() { + LOG.info("Starting BFT mining coordinator following initial sync"); + enable(); + start(); + } + + @Override + public void onInitialSyncRestart() { + // Nothing to do. The mining coordinator won't be started until + // sync has completed. + } + }); + } + + @Override + public void awaitStop() throws InterruptedException { + bftExecutors.awaitStop(); + } + + @Override + public synchronized boolean enable() { + // AERE D-434 (81a600236a, issue #10877): synchronized with start()/stop() so the multi-step + // state checks below cannot interleave with a concurrent lifecycle transition. + // Return true if we're already running or idle, or successfully switch to idle + if (state.get() == State.RUNNING + || state.get() == State.IDLE + || state.compareAndSet(State.PAUSED, State.IDLE)) { + return true; + } + return false; + } + + @Override + public synchronized boolean disable() { + // AERE D-434 (81a600236a, issue #10877): synchronized with start()/stop() so the multi-step + // state checks below cannot interleave with a concurrent lifecycle transition. + if (state.get() == State.PAUSED + || state.compareAndSet(State.IDLE, State.PAUSED) + || state.compareAndSet(State.RUNNING, State.PAUSED)) { + return true; + } + return false; + } + + @Override + public boolean isMining() { + return state.get() == State.RUNNING; + } + + @Override + public Wei getMinTransactionGasPrice() { + return blockCreatorFactory.getMinTransactionGasPrice(); + } + + @Override + public Wei getMinPriorityFeePerGas() { + return blockCreatorFactory.getMinPriorityFeePerGas(); + } + + @Override + public Optional

getCoinbase() { + return Optional.of(blockCreatorFactory.getLocalAddress()); + } + + @Override + public Optional createBlock( + final BlockHeader parentHeader, + final List transactions, + final List ommers) { + // One-off block creation has not been implemented + return Optional.empty(); + } + + @Override + public Optional createBlock(final BlockHeader parentHeader, final long timestamp) { + // One-off block creation has not been implemented + return Optional.empty(); + } + + @Override + public void changeTargetGasLimit(final Long targetGasLimit) { + blockCreatorFactory.changeTargetGasLimit(targetGasLimit); + } + + @Override + public void onBlockAdded(final BlockAddedEvent event) { + if (event.isNewCanonicalHead()) { + LOG.trace("New canonical head detected"); + eventQueue.add(new NewChainHead(event.getHeader())); + } + } + + @Override + public void removeObserver() { + blockchain.removeObserver(blockAddedObserverId); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/messagewrappers/BftMessage.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/messagewrappers/BftMessage.java new file mode 100644 index 0000000..e52c529 --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/messagewrappers/BftMessage.java @@ -0,0 +1,154 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-bounded-message-lists.patch. + */ +package org.hyperledger.besu.consensus.common.bft.messagewrappers; + +import org.hyperledger.besu.consensus.common.bft.ConsensusRoundIdentifier; +import org.hyperledger.besu.consensus.common.bft.payload.Authored; +import org.hyperledger.besu.consensus.common.bft.payload.Payload; +import org.hyperledger.besu.consensus.common.bft.payload.RoundSpecific; +import org.hyperledger.besu.consensus.common.bft.payload.SignedData; +import org.hyperledger.besu.crypto.SECPSignature; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.util.Objects; +import java.util.StringJoiner; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; + +/** + * The Bft message. + * + * @param

the type of Payload + */ +public class BftMessage

implements Authored, RoundSpecific { + + /** + * Maximum number of entries permitted in any embedded list (round-changes, prepares) when + * decoding a BFT message. AERE D-434: ported from upstream hyperledger/besu dd4101c9ec. + */ + public static final int MAX_LIST_ENTRIES = 512; + + private final SignedData

payload; + + /** + * Instantiates a new Bft message. + * + * @param payload the payload + */ + public BftMessage(final SignedData

payload) { + this.payload = payload; + } + + @Override + public Address getAuthor() { + return payload.getAuthor(); + } + + @Override + public ConsensusRoundIdentifier getRoundIdentifier() { + return payload.getPayload().getRoundIdentifier(); + } + + /** + * Encode. + * + * @return the bytes + */ + public Bytes encode() { + final BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + payload.writeTo(rlpOut); + return rlpOut.encoded(); + } + + /** + * Gets signed payload. + * + * @return the signed payload + */ + public SignedData

getSignedPayload() { + return payload; + } + + /** + * Gets message type. + * + * @return the message type + */ + public int getMessageType() { + return payload.getPayload().getMessageType(); + } + + /** + * Gets payload. + * + * @return the payload + */ + protected P getPayload() { + return payload.getPayload(); + } + + /** + * Read payload. + * + * @param the type parameter of Payload + * @param rlpInput the rlp input + * @param decoder the decoder + * @return the signed data + */ + protected static SignedData readPayload( + final RLPInput rlpInput, final Function decoder) { + rlpInput.enterList(); + final T unsignedMessageData = decoder.apply(rlpInput); + final SECPSignature signature = + rlpInput.readBytes((SignatureAlgorithmFactory.getInstance()::decodeSignature)); + rlpInput.leaveList(); + + return SignedData.create(unsignedMessageData, signature); + } + + @Override + public String toString() { + return new StringJoiner(", ", BftMessage.class.getSimpleName() + "[", "]") + .add("payload=" + payload) + .toString(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final BftMessage that = (BftMessage) o; + return Objects.equals(payload, that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(payload); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/network/ValidatorPeers.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/network/ValidatorPeers.java new file mode 100644 index 0000000..649fa75 --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/network/ValidatorPeers.java @@ -0,0 +1,128 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.common.bft.network; + +import org.hyperledger.besu.consensus.common.bft.AerePqIdentity; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection.PeerNotConnected; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Responsible for tracking the network peers which have a connection to this node, then + * multicasting packets to ONLY the peers which have been identified as being validators. + */ +public class ValidatorPeers implements ValidatorMulticaster, PeerConnectionTracker { + + private static final Logger LOG = LoggerFactory.getLogger(ValidatorPeers.class); + + // It's possible for multiple connections between peers to exist for brief periods, so map each + // address to a set of connections + private final Map> connectionsByAddress = new ConcurrentHashMap<>(); + private final ValidatorProvider validatorProvider; + private final String protocolName; + + /** + * Instantiates a new Validator peers. + * + * @param validatorProvider the validator provider + * @param protocolName the protocol name + */ + public ValidatorPeers(final ValidatorProvider validatorProvider, final String protocolName) { + this.validatorProvider = validatorProvider; + this.protocolName = protocolName; + } + + @Override + public void add(final PeerConnection newConnection) { + final Address peerAddress = newConnection.getPeerInfo().getAddress(); + final Set connections = + connectionsByAddress.computeIfAbsent( + peerAddress, (k) -> Collections.newSetFromMap(new ConcurrentHashMap<>())); + connections.add(newConnection); + } + + @Override + public void remove(final PeerConnection removedConnection) { + final Address peerAddress = removedConnection.getPeerInfo().getAddress(); + final Set connections = connectionsByAddress.get(peerAddress); + if (connections == null) { + return; + } + connections.remove(removedConnection); + } + + @Override + public void send(final MessageData message) { + sendMessageToSpecificAddresses(getLatestValidators(), message); + } + + @Override + public void send(final MessageData message, final Collection

denylist) { + final Collection
includedValidators = + getLatestValidators().stream() + .filter(a -> !denylist.contains(a)) + .collect(Collectors.toSet()); + sendMessageToSpecificAddresses(includedValidators, message); + } + + private void sendMessageToSpecificAddresses( + final Collection
recipients, final MessageData message) { + LOG.trace( + "Sending message to peers messageCode={} recipients={}", message.getCode(), recipients); + // AERE AIP-22 stage 3: from H_id the recipients are ids; they are reached through the node-key + // address their v3 row lists as transport, or through every peer when a row lists none + final java.util.Optional> targets = + AerePqIdentity.transportTargets(recipients); + (targets.isPresent() + ? targets.get().stream().map(connectionsByAddress::get).filter(Objects::nonNull) + : connectionsByAddress.values().stream()) + .flatMap(Set::stream) + .distinct() + .forEach( + connection -> { + try { + connection.sendForProtocol(protocolName, message); + } catch (final PeerNotConnected peerNotConnected) { + LOG.trace( + "Lost connection to a validator. remoteAddress={} peerInfo={}", + connection.getRemoteAddress(), + connection.getPeerInfo()); + } + }); + } + + private Collection
getLatestValidators() { + return validatorProvider.getValidatorsAtHead(); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/payload/SignedData.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/payload/SignedData.java new file mode 100644 index 0000000..2c59e69 --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/payload/SignedData.java @@ -0,0 +1,150 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.common.bft.payload; + +import org.hyperledger.besu.consensus.common.bft.AerePqFinality; +import org.hyperledger.besu.consensus.common.bft.AerePqIdentity; +import org.hyperledger.besu.crypto.SECPSignature; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.math.BigInteger; +import java.util.Objects; +import java.util.StringJoiner; + +import org.apache.tuweni.bytes.Bytes; + +/** + * The type Signed data. + * + * @param the type parameter of Payload + */ +public class SignedData implements Authored { + + private static final BigInteger HALF_CURVE_ORDER = + SignatureAlgorithmFactory.getInstance().getHalfCurveOrder(); + + private final Address sender; + private final SECPSignature signature; + private final M unsignedPayload; + + /** + * Create signed data. + * + * @param the type parameter + * @param payload the payload + * @param signature the signature + * @return the signed data + * @throws IllegalArgumentException if the signature is invalid + */ + public static SignedData create( + final T payload, final SECPSignature signature) { + final Hash msgHash = payload.hashForSignature(); + // AERE AIP-22 stage 3: from H_id the author is the post-quantum id of the signer's v3 row + // AERE AIP-22 stage 4: from H_fin the author is the id of the row the message's post-quantum seal index names, and the + // envelope's ECDSA signature is not read for authorship (a message without a post-quantum seal has no author then) + final long aereHeight = payload.getRoundIdentifier().getSequenceNumber(); + return new SignedData<>( + payload, + AerePqFinality.active(aereHeight) + ? AerePqFinality.authorFromSeal(payload, aereHeight) + : AerePqIdentity.author(Util.signatureToAddress(signature, msgHash), aereHeight), + signature); + } + + private SignedData(final M unsignedPayload, final Address sender, final SECPSignature signature) { + if (!(signature.getS().compareTo(HALF_CURVE_ORDER) <= 0)) { + throw new IllegalArgumentException("Signature is invalid"); + } + this.unsignedPayload = unsignedPayload; + this.sender = sender; + this.signature = signature; + } + + @Override + public Address getAuthor() { + return sender; + } + + /** + * Gets payload. + * + * @return the payload + */ + public M getPayload() { + return unsignedPayload; + } + + /** + * Write to. + * + * @param output the output + */ + public void writeTo(final RLPOutput output) { + output.startList(); + unsignedPayload.writeTo(output); + output.writeBytes(signature.encodedBytes()); + output.endList(); + } + + /** + * Encode. + * + * @return the bytes + */ + public Bytes encode() { + final BytesValueRLPOutput rlpEncode = new BytesValueRLPOutput(); + writeTo(rlpEncode); + return rlpEncode.encoded(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final SignedData that = (SignedData) o; + return Objects.equals(sender, that.sender) + && Objects.equals(signature, that.signature) + && Objects.equals(unsignedPayload, that.unsignedPayload); + } + + @Override + public int hashCode() { + return Objects.hash(sender, signature, unsignedPayload); + } + + @Override + public String toString() { + return new StringJoiner(", ", SignedData.class.getSimpleName() + "[", "]") + .add("sender=" + sender) + .add("signature=" + signature) + .add("unsignedPayload=" + unsignedPayload) + .toString(); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTally.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTally.java new file mode 100644 index 0000000..fed8a2a --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTally.java @@ -0,0 +1,160 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.common.validator.blockbased; + +import org.hyperledger.besu.consensus.common.bft.AerePqIdentity; +import org.hyperledger.besu.consensus.common.validator.ValidatorVote; +import org.hyperledger.besu.datatypes.Address; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +import com.google.common.collect.Maps; + +/** Tracks the current list of validators and votes to add or drop validators. */ +class VoteTally { + + private final NavigableSet
currentValidators; + + private final Map> addVotesBySubject; + private final Map> removeVotesBySubject; + + VoteTally(final Collection
initialValidators) { + this(new TreeSet<>(initialValidators), new HashMap<>(), new HashMap<>()); + } + + private VoteTally( + final Collection
initialValidators, + final Map> addVotesBySubject, + final Map> removeVotesBySubject) { + this.currentValidators = new TreeSet<>(initialValidators); + this.addVotesBySubject = addVotesBySubject; + this.removeVotesBySubject = removeVotesBySubject; + } + + /** + * Add a vote to the current tally. The current validator list will be updated if this vote takes + * the tally past the required votes to approve the change. + * + * @param validatorVote The vote which was cast in a block header. + */ + void addVote(final ValidatorVote validatorVote) { + final Set
addVotesForSubject = + addVotesBySubject.computeIfAbsent(validatorVote.getRecipient(), target -> new HashSet<>()); + final Set
removeVotesForSubject = + removeVotesBySubject.computeIfAbsent( + validatorVote.getRecipient(), target -> new HashSet<>()); + + if (validatorVote.isAuthVote()) { + addVotesForSubject.add(validatorVote.getProposer()); + removeVotesForSubject.remove(validatorVote.getProposer()); + } else { + removeVotesForSubject.add(validatorVote.getProposer()); + addVotesForSubject.remove(validatorVote.getProposer()); + } + + final int validatorLimit = validatorLimit(); + if (addVotesForSubject.size() >= validatorLimit) { + currentValidators.add(validatorVote.getRecipient()); + discardOutstandingVotesFor(validatorVote.getRecipient()); + } + if (removeVotesForSubject.size() >= validatorLimit) { + currentValidators.remove(validatorVote.getRecipient()); + discardOutstandingVotesFor(validatorVote.getRecipient()); + addVotesBySubject.values().forEach(votes -> votes.remove(validatorVote.getRecipient())); + removeVotesBySubject.values().forEach(votes -> votes.remove(validatorVote.getRecipient())); + } + } + + private void discardOutstandingVotesFor(final Address subject) { + addVotesBySubject.remove(subject); + removeVotesBySubject.remove(subject); + } + + Set
getOutstandingAddVotesFor(final Address subject) { + return Optional.ofNullable(addVotesBySubject.get(subject)).orElse(Collections.emptySet()); + } + + Set
getOutstandingRemoveVotesFor(final Address subject) { + return Optional.ofNullable(removeVotesBySubject.get(subject)).orElse(Collections.emptySet()); + } + + private int validatorLimit() { + return (currentValidators.size() / 2) + 1; + } + + /** + * Reset the outstanding vote tallies as required at each epoch. The current validator list is + * unaffected. + */ + void discardOutstandingVotes() { + addVotesBySubject.clear(); + removeVotesBySubject.clear(); + } + + /** + * AERE AIP-22 stage 3: after block H_id - 1 the validators become the sorted post-quantum ids of the + * same members, and votes still open on classical addresses are discarded (they name addresses the + * id space no longer contains). Anywhere else, and when the identity switch is not armed, nothing + * changes. + * + * @param blockNumber the block whose effects this tally now includes + */ + void translateToIds(final long blockNumber) { + final Collection
translated = + AerePqIdentity.translateSetAfter(currentValidators, blockNumber); + if (translated != currentValidators) { + currentValidators.clear(); + currentValidators.addAll(translated); + discardOutstandingVotes(); + } + } + + /** + * The validator addresses + * + * @return The collection of validators after the voting at the most recent block has been + * finalised. + */ + Collection
getValidators() { + return currentValidators; + } + + VoteTally copy() { + final Map> addVotesBySubject = Maps.newHashMap(); + final Map> removeVotesBySubject = Maps.newHashMap(); + + this.addVotesBySubject.forEach( + (key, value) -> addVotesBySubject.put(key, new TreeSet<>(value))); + this.removeVotesBySubject.forEach( + (key, value) -> removeVotesBySubject.put(key, new TreeSet<>(value))); + + return new VoteTally( + new TreeSet<>(this.currentValidators), addVotesBySubject, removeVotesBySubject); + } +} diff --git a/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTallyUpdater.java b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTallyUpdater.java new file mode 100644 index 0000000..0da2c63 --- /dev/null +++ b/anchor/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/validator/blockbased/VoteTallyUpdater.java @@ -0,0 +1,93 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.common.validator.blockbased; + +import org.hyperledger.besu.consensus.common.BlockInterface; +import org.hyperledger.besu.consensus.common.EpochManager; +import org.hyperledger.besu.consensus.common.bft.AerePqIdentity; +import org.hyperledger.besu.consensus.common.validator.ValidatorVote; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.Collection; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides the logic to extract vote tally state from the blockchain and update it as blocks are + * added. + */ +class VoteTallyUpdater { + + private static final Logger LOG = LoggerFactory.getLogger(VoteTallyUpdater.class); + + private final EpochManager epochManager; + private final BlockInterface blockInterface; + + VoteTallyUpdater(final EpochManager epochManager, final BlockInterface blockInterface) { + this.epochManager = epochManager; + this.blockInterface = blockInterface; + } + + /** + * Create a new VoteTally based on the current blockchain state. + * + * @param blockchain the blockchain to load the current state from + * @return a VoteTally reflecting the state of the blockchain head + */ + VoteTally buildVoteTallyFromBlockchain(final Blockchain blockchain) { + final long chainHeadBlockNumber = blockchain.getChainHeadBlockNumber(); + final long epochBlockNumber = epochManager.getLastEpochBlock(chainHeadBlockNumber); + LOG.info("Loading validator voting state starting from block {}", epochBlockNumber); + final BlockHeader epochBlock = blockchain.getBlockHeader(epochBlockNumber).get(); + final Collection
initialValidators = blockInterface.validatorsInBlock(epochBlock); + final VoteTally voteTally = new VoteTally(initialValidators); + for (long blockNumber = epochBlockNumber + 1; + blockNumber <= chainHeadBlockNumber; + blockNumber++) { + updateForBlock(blockchain.getBlockHeader(blockNumber).get(), voteTally); + } + return voteTally; + } + + /** + * Update the vote tally to reflect changes caused by appending a new block to the chain. + * + * @param header the header of the block being added + * @param voteTally the vote tally to update + */ + void updateForBlock(final BlockHeader header, final VoteTally voteTally) { + if (epochManager.isEpochBlock(header.getNumber())) { + voteTally.discardOutstandingVotes(); + } else { + final Optional vote = blockInterface.extractVoteFromHeader(header); + vote.ifPresent(voteTally::addVote); + } + // AERE AIP-22 stage 3: after block H_id - 1 the set becomes the post-quantum ids of its members; + // asked first, so that disarmed (or at any other block) the tally sees no extra call at all + if (AerePqIdentity.translatesAfter(header.getNumber())) { + voteTally.translateToIds(header.getNumber()); + } + } +} diff --git a/anchor/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java b/anchor/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java new file mode 100644 index 0000000..5c661d3 --- /dev/null +++ b/anchor/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java @@ -0,0 +1,478 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.common.bft; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import org.hyperledger.besu.consensus.common.bft.blockcreation.PqAnchorProducer; +import org.hyperledger.besu.crypto.SecureRandomProvider; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; + +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.LongFunction; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.crypto.digests.KeccakDigest; +import org.bouncycastle.pqc.crypto.falcon.FalconPrivateKeyParameters; +import org.bouncycastle.pqc.crypto.falcon.FalconSigner; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * AIP-21, SECOND HALF (2026-09-18): the finality window survives a restart. + * + *

The rule of the house: no proof without its negative pair. Real Falcon-512 keys, the real + * genesis-anchored registry, the real commit message, a real directory. What each test pins down: + * + *

    + *
  1. {@link #theWindowSurvivesARestartAndOnlyCanonicalBlocksComeBack()} - the point of the whole + * thing, with its negative: a journal frame for a hash that is NOT canonical at its height + * never comes back, and neither does a height this node's chain does not hold. + *
  2. {@link #aForgedSealInTheJournalNeverEntersTheWindow()} - THE load-bearing assertion, the one + * the build-time negative control plants against: three forgeries beside two genuine seals. + *
  3. {@link #aTornTailOrAFlippedByteCostsOnlyWhatFollowsIt()} - a node that died mid-append, a + * flipped byte, garbage, a directory where a file should be: never an exception, and the good + * frames BEFORE the damage still come back. + *
  4. {@link #aJournalFromAnotherChainOrOutsideTheWindowIsIgnored()}. + *
  5. {@link #oldSegmentsAreDeletedSoTheFootprintStaysBounded()}. + *
  6. {@link #theJournalIsOffWhenPersistenceIsOffOrThePropertySaysSo()}. + *
  7. {@link #aRestoredWindowIsNotAppendedASecondTime()} - the journal does not grow by what it + * already holds every time the node restarts. + *
  8. {@link #whatOneAppendCostsAgainstTheBlockInterval()} - the price on the consensus thread, + * as a number. + *
+ * + *

NOT MEASURED here: that a live validator answers {@code post-quantum} for a block heard before + * its restart. That is measured on testnet 28001 and recorded in the AIP. + */ +public class PqSealWindowPersistenceTest { + + private static final long H = 1_000L; + private static final long ATTACH = 900L; + private static final int N = 7; + private static final long CHAIN_ID = 2_800L; + private static final long BLOCK_INTERVAL_MS = 523L; + private static final long HEAD = 5_000L; + + @TempDir private Path tmp; + + private Path dataDirectory; + private final List privateKeys = new ArrayList<>(); + private final Map chain = new HashMap<>(); + + @BeforeEach + public void setUp() throws Exception { + dataDirectory = Files.createDirectories(tmp.resolve("besu-data")); + final KeccakDigest kd = new KeccakDigest(256); + final StringBuilder manifest = + new StringBuilder("{\"config\":{\"aereFalconRegistry\":{") + .append(PqV2Fixture.manifestHeader(N, CHAIN_ID, H)); + for (int i = 0; i < N; i++) { + privateKeys.add(PqV2Fixture.privateKey(i)); + final byte[] anchoredRow = PqV2Fixture.anchorPreimageRow(i); + kd.update(anchoredRow, 0, anchoredRow.length); + manifest.append(',').append(PqV2Fixture.manifestEntry(i, N, CHAIN_ID, H)); + } + final byte[] anchoredHash = new byte[32]; + kd.doFinal(anchoredHash, 0); + manifest + .append("}},\"alloc\":{\"0000000000000000000000000000000000000fa1\":{\"storage\":{\"0x") + .append("0".repeat(64)) + .append("\":\"0x") + .append(Bytes.wrap(anchoredHash).toUnprefixedHexString()) + .append("\"}}}}"); + final Path genesisPath = tmp.resolve("genesis-registry.json"); + Files.writeString(genesisPath, manifest.toString()); + System.setProperty("aere.falcon.genesis", genesisPath.toAbsolutePath().toString()); + System.setProperty("aere.falcon.attachBlock", Long.toString(ATTACH)); + System.clearProperty(PqSealWindowStore.PROPERTY_PERSIST); + + resetFalconSingleton(); + bindTheRegistryScheduleTheWayALiveNodeDoes(genesisPath); + PqSealCache.instance().disablePersistence(); + PqSealCache.instance().clear(); + PqAnchorProducer.useConfigForTesting( + new PqAnchorConfig(CHAIN_ID, H, Map.of(H, 0, H + 10L, 3), OptionalInt.empty(), false)); + chain.clear(); + } + + /** + * The HISTORY door ({@code verifyAtHistoric}) refuses to resolve a key above the arming height + * unless a registry schedule is loaded (D2, 2026-08-06): it will not fall back to the head + * registry. A live node holds that schedule from its genesis ({@code config.pqRegistryHash}); the + * window restore judges heights BELOW the head, so it goes through that door, and so this fixture + * binds the schedule exactly the way a node does. Measured 2026-09-18: without this every restore + * here returned zero, with the refusal in the log. + */ + private void bindTheRegistryScheduleTheWayALiveNodeDoes(final Path genesisPath) throws Exception { + final PqRegistryHash.Registry held = PqRegistryHash.loadAuto(genesisPath); + String hash = PqRegistryHash.hashFor(held, CHAIN_ID); + if (!hash.startsWith("0x")) { + hash = "0x" + hash; + } + final Path schedule = tmp.resolve("genesis-schedule.json"); + Files.writeString( + schedule, + "{\"config\":{\"pqRegistryHash\":[{\"block\":" + H + ",\"hash\":\"" + hash + "\"}]}}"); + FalconSealSupport.instance() + .verifyRegistryBindingOrAbort( + 0L, CHAIN_ID, PqRegistryHash.loadScheduleFromGenesis(schedule)); + } + + @AfterEach + public void tearDown() throws Exception { + System.clearProperty("aere.falcon.genesis"); + System.clearProperty("aere.falcon.attachBlock"); + System.clearProperty(PqSealWindowStore.PROPERTY_PERSIST); + PqSealCache.instance().disablePersistence(); + PqSealCache.instance().clear(); + PqAnchorProducer.useConfigForTesting(null); + resetFalconSingleton(); + } + + @Test + public void theWindowSurvivesARestartAndOnlyCanonicalBlocksComeBack() { + assertThat(FalconSealSupport.instance().genesisAnchored()).isTrue(); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + + // Ten heights heard live, three genuine seals each. A height reaches the journal when the NEXT + // one begins, so after hearing HEAD the journal holds HEAD-9 .. HEAD-1 and the head file HEAD. + for (long n = HEAD - 9L; n <= HEAD; n++) { + for (int i = 0; i < 3; i++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(i, header(n)))); + } + } + // and one losing proposal at HEAD-4: heard, journaled, NOT canonical + final BlockHeader loser = + new BlockHeaderTestFixture().number(HEAD - 4L).extraData(Bytes.of(7, 7, 7)).buildHeader(); + assertThat(loser.getHash()).isNotEqualTo(header(HEAD - 4L).getHash()); + + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + assertThat(journal).isDirectory(); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)).isEqualTo(9); + appendUnchecked(journal, loser, List.of(genuineSeal(0, loser))); + + // THE RESTART: the in-memory window is gone + PqSealCache.instance().clear(); + assertThat(PqSealCache.instance().sealCount(header(HEAD - 5L).getHash())).isZero(); + + // this node's chain holds every height except HEAD-7 (the negative: a height it cannot vouch for) + final LongFunction> canonical = + n -> n == HEAD - 7L ? Optional.empty() : Optional.of(header(n).getHash()); + final int restored = + PqSealCache.instance() + .restoreWindowFromDisk(HEAD, canonical, PqSignerRegistry.falconSealSupport()); + + assertThat(restored) + .describedAs("nine journaled heights, minus the one the chain does not hold, three seals each") + .isEqualTo(8 * 3); + for (long n = HEAD - 9L; n < HEAD; n++) { + assertThat(PqSealCache.instance().sealCount(header(n).getHash())) + .describedAs("height %d", n) + .isEqualTo(n == HEAD - 7L ? 0 : 3); + } + assertThat(PqSealCache.instance().sealCount(loser.getHash())) + .describedAs("a frame for a hash that is not canonical at its height never comes back") + .isZero(); + assertThat(PqSealCache.instance().sealsFor(HEAD - 2L, header(HEAD - 2L).getHash())) + .extracting(FalconSeal::getValidatorIndex) + .containsExactly(0, 1, 2); + } + + @Test + public void aForgedSealInTheJournalNeverEntersTheWindow() throws Exception { + final BlockHeader block = header(HEAD - 3L); + final FalconSeal genuine0 = genuineSeal(0, block); + final FalconSeal genuine1 = genuineSeal(1, block); + final Bytes32 anotherBlocksMessage = + PqAnchor.commitMessage(CHAIN_ID, block.getNumber() - 1L, Bytes32.leftPad(Bytes.of(9))); + final byte[] randomBytes = new byte[genuine0.getSignature().size()]; + SecureRandomProvider.createSecureRandom().nextBytes(randomBytes); + final List all = + List.of( + genuine0, + genuine1, + new FalconSeal(2, Bytes.wrap(falconSign(privateKeys.get(2), anotherBlocksMessage))), + new FalconSeal(3, Bytes.wrap(randomBytes)), + new FalconSeal(4, genuine0.getSignature())); + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + PqSealWindowStore.append( + journal, CHAIN_ID, PqSealWindowStore.heard(block.getNumber(), block.getHash().getBytes(), all)); + + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + final int restored = + PqSealCache.instance() + .restoreWindowFromDisk( + HEAD, n -> Optional.of(header(n).getHash()), PqSignerRegistry.falconSealSupport()); + + assertThat(restored) + .describedAs( + "THE LOAD-BEARING ASSERTION. Five seals were in the frame and only the two genuine ones " + + "may come out: a signature over another block's message, random bytes and a " + + "genuine seal re-labelled under another index all die at the verifier. Take the " + + "verifier out of restoreWindowFromDisk and this line goes red, which is what the " + + "build-time negative control proves.") + .isEqualTo(2); + assertThat(PqSealCache.instance().sealsFor(block.getNumber(), block.getHash())) + .extracting(FalconSeal::getValidatorIndex) + .containsExactly(0, 1); + } + + @Test + public void aTornTailOrAFlippedByteCostsOnlyWhatFollowsIt() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + // five heights in ONE segment (64 heights per segment; HEAD-5 .. HEAD-1 share 4992/64 = 78) + for (long n = HEAD - 5L; n < HEAD; n++) { + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard(n, header(n).getHash().getBytes(), List.of(genuineSeal(0, header(n))))); + } + final Path segment = PqSealWindowStore.segmentFor(journal, HEAD - 1L); + assertThat(PqSealWindowStore.segmentFor(journal, HEAD - 5L)).isEqualTo(segment); + final byte[] whole = Files.readAllBytes(segment); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + + // positive control first: the undamaged journal gives all five + assertThat(restoreAll()).isEqualTo(5); + + // (a) torn tail: the node died in the middle of the fifth append + Files.write(segment, java.util.Arrays.copyOf(whole, whole.length - 40)); + PqSealCache.instance().clear(); + assertThat(restoreAll()).describedAs("the four whole frames before the torn one").isEqualTo(4); + + // (b) one byte flipped inside the THIRD frame: the checksum catches it, frames one and two live + final int frame = whole.length / 5; + final byte[] flipped = whole.clone(); + flipped[2 * frame + frame / 2] ^= 0x20; + Files.write(segment, flipped); + PqSealCache.instance().clear(); + assertThat(restoreAll()).describedAs("the two frames before the damaged one").isEqualTo(2); + + // (c) garbage, empty, a directory where the segment should be: nothing, and no exception + Files.write(segment, new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9}); + PqSealCache.instance().clear(); + assertThat(restoreAll()).isZero(); + Files.write(segment, new byte[0]); + assertThat(restoreAll()).isZero(); + Files.delete(segment); + Files.createDirectory(segment); + assertThatCode(this::restoreAll).doesNotThrowAnyException(); + assertThat(restoreAll()).isZero(); + } + + @Test + public void aJournalFromAnotherChainOrOutsideTheWindowIsIgnored() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + final BlockHeader inWindow = header(HEAD - 10L); + final BlockHeader tooOld = header(HEAD - PqSealCache.instance().retentionBlocks() - 50L); + PqSealWindowStore.append( + journal, + CHAIN_ID + 1L, + PqSealWindowStore.heard( + inWindow.getNumber(), inWindow.getHash().getBytes(), List.of(genuineSeal(0, inWindow)))); + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard( + tooOld.getNumber(), tooOld.getHash().getBytes(), List.of(genuineSeal(0, tooOld)))); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + assertThat(restoreAll()) + .describedAs("another chain's frame and a height below the window: neither comes back") + .isZero(); + // positive pair: the same height, this chain + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard( + inWindow.getNumber(), inWindow.getHash().getBytes(), List.of(genuineSeal(0, inWindow)))); + assertThat(restoreAll()).isEqualTo(1); + } + + @Test + public void oldSegmentsAreDeletedSoTheFootprintStaysBounded() throws Exception { + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + final long window = PqSealCache.instance().retentionBlocks(); + final long heights = window + 5L * PqSealWindowStore.SEGMENT_BLOCKS; + final FalconSeal one = new FalconSeal(0, Bytes.wrap(new byte[650])); + for (long n = HEAD; n < HEAD + heights; n++) { + PqSealCache.instance().record(n, Hash.hash(Bytes.ofUnsignedLong(n)), List.of(one)); + } + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + final long highest = HEAD + heights - 1L; + try (Stream files = Files.list(journal)) { + final List segments = files.toList(); + assertThat(segments.size()) + .describedAs("segments kept: the window plus the one being written plus one of slack") + .isLessThanOrEqualTo((int) (window / PqSealWindowStore.SEGMENT_BLOCKS) + 2); + assertThat(segments) + .describedAs("no segment wholly below the window is left behind") + .doesNotContain(PqSealWindowStore.segmentFor(journal, highest - window - 2L * PqSealWindowStore.SEGMENT_BLOCKS)); + assertThat(segments).contains(PqSealWindowStore.segmentFor(journal, highest - 1L)); + } + } + + @Test + public void theJournalIsOffWhenPersistenceIsOffOrThePropertySaysSo() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + // persistence never enabled: memory only, exactly as before this class existed + for (long n = HEAD; n < HEAD + 3L; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + assertThat(journal).doesNotExist(); + + // the operator's switch + System.setProperty(PqSealWindowStore.PROPERTY_PERSIST, "false"); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + for (long n = HEAD + 3L; n < HEAD + 6L; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + assertThat(journal).doesNotExist(); + assertThat(PqSealStore.fileIn(dataDirectory)) + .describedAs("the head file, which liveness depends on, is NOT governed by this switch") + .exists(); + + // positive pair: the switch back on + System.clearProperty(PqSealWindowStore.PROPERTY_PERSIST); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + for (long n = HEAD + 6L; n < HEAD + 9L; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + assertThat(journal).isDirectory(); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)).isGreaterThanOrEqualTo(2); + } + + @Test + public void aRestoredWindowIsNotAppendedASecondTime() { + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + for (long n = HEAD - 4L; n <= HEAD; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)).isEqualTo(4); + + PqSealCache.instance().clear(); + PqSealCache.instance() + .restoreFromDisk(HEAD, header(HEAD).getHash(), PqSignerRegistry.falconSealSupport()); + assertThat(restoreAll()).isEqualTo(4); + + // the chain moves on: exactly ONE new frame (the head, which was only in the head file) + PqSealCache.instance() + .record(HEAD + 1L, header(HEAD + 1L).getHash(), List.of(genuineSeal(0, header(HEAD + 1L)))); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)) + .describedAs("four restored heights are not written again; the head is written once") + .isEqualTo(5); + } + + @Test + public void whatOneAppendCostsAgainstTheBlockInterval() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + final List ten = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ten.add(new FalconSeal(i, Bytes.wrap(new byte[660]))); + } + final int rounds = 200; + final long t0 = System.nanoTime(); + for (int r = 0; r < rounds; r++) { + PqSealWindowStore.append( + journal, CHAIN_ID, PqSealWindowStore.heard(HEAD + r, Bytes32.leftPad(Bytes.of(r)), ten)); + } + final double meanMs = (System.nanoTime() - t0) / 1_000_000.0 / rounds; + System.out.printf( + "AERE FEREASTRA-SIGILII: one append of 10 seals costs %.3f ms on average, %.2f%% of a %d ms block%n", + meanMs, 100.0 * meanMs / BLOCK_INTERVAL_MS, BLOCK_INTERVAL_MS); + assertThat(meanMs) + .describedAs("an append on the consensus thread must stay far below the block interval") + .isLessThan(BLOCK_INTERVAL_MS * 0.05); + } + + /** + * THE SHARED VECTOR, literal for literal in the Nethermind-derived client's + * AereAip21SealWindowJournalProofTests: the two clients write the SAME frame payload. Chain 28001, + * block 5, hash 32 x 0xaa, one seal (index 1, signature 0x0102). The frame around it is + * length(4, big endian) || payload || crc32(payload)(4, big endian), java.util.zip.CRC32. + */ + @Test + public void theBytesOnDiskAreTheOnesTheOtherClientWrites() { + final Bytes hash = Bytes.fromHexString("0x" + "aa".repeat(32)); + final byte[] payload = + PqSealWindowStore.encode( + 28_001L, PqSealWindowStore.heard(5L, hash, List.of(new FalconSeal(1, Bytes.of(1, 2))))); + assertThat(Bytes.wrap(payload).toUnprefixedHexString()) + .isEqualTo( + "f84094414552452d50512d5345414c57494e444f572d31826d6105a0" + + "aa".repeat(32) + + "c5c401820102"); + final java.util.zip.CRC32 crc = new java.util.zip.CRC32(); + crc.update("123456789".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + assertThat(crc.getValue()).isEqualTo(0xCBF43926L); + } + + private int restoreAll() { + return PqSealCache.instance() + .restoreWindowFromDisk( + HEAD, n -> Optional.of(header(n).getHash()), PqSignerRegistry.falconSealSupport()); + } + + private BlockHeader header(final long number) { + return chain.computeIfAbsent( + number, n -> new BlockHeaderTestFixture().number(n).buildHeader()); + } + + private FalconSeal genuineSeal(final int index, final BlockHeader block) { + final Bytes32 m = + PqAnchor.commitMessage(CHAIN_ID, block.getNumber(), block.getHash().getBytes()); + return new FalconSeal(index, Bytes.wrap(falconSign(privateKeys.get(index), m))); + } + + private static void appendUnchecked( + final Path journal, final BlockHeader block, final List seals) { + try { + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard(block.getNumber(), block.getHash().getBytes(), seals)); + } catch (final java.io.IOException e) { + throw new java.io.UncheckedIOException(e); + } + } + + private static byte[] falconSign(final FalconPrivateKeyParameters key, final Bytes32 m) { + final FalconSigner signer = new FalconSigner(); + signer.init(true, key); + return signer.generateSignature(m.toArray()); + } + + private static void resetFalconSingleton() throws Exception { + final Field f = FalconSealSupport.class.getDeclaredField("instance"); + f.setAccessible(true); + f.set(null, null); + } +} diff --git a/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/Proposal.java b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/Proposal.java new file mode 100644 index 0000000..64ece7d --- /dev/null +++ b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/Proposal.java @@ -0,0 +1,136 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-bounded-message-lists.patch. + */ +package org.hyperledger.besu.consensus.qbft.core.messagewrappers; + +import org.hyperledger.besu.consensus.common.bft.messagewrappers.BftMessage; +import org.hyperledger.besu.consensus.common.bft.payload.SignedData; +import org.hyperledger.besu.consensus.qbft.core.payload.PreparePayload; +import org.hyperledger.besu.consensus.qbft.core.payload.ProposalPayload; +import org.hyperledger.besu.consensus.qbft.core.payload.RoundChangePayload; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlock; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockCodec; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +/** The Proposal. */ +public class Proposal extends BftMessage { + + private final List> roundChanges; + private final List> prepares; + + /** + * Instantiates a new Proposal. + * + * @param payload the payload + * @param roundChanges the round changes + * @param prepares the prepares + */ + public Proposal( + final SignedData payload, + final List> roundChanges, + final List> prepares) { + super(payload); + this.roundChanges = roundChanges; + this.prepares = prepares; + } + + /** + * Gets round changes. + * + * @return the round changes + */ + public List> getRoundChanges() { + return roundChanges; + } + + /** + * Gets list of Prepare payload. + * + * @return the list of Prepare payload + */ + public List> getPrepares() { + return prepares; + } + + /** + * Gets block. + * + * @return the block + */ + public QbftBlock getBlock() { + return getPayload().getProposedBlock(); + } + + /** + * Gets block access list. + * + * @return the block access list + */ + public Optional getBlockAccessList() { + return getPayload().getBlockAccessList(); + } + + @Override + public Bytes encode() { + final BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + rlpOut.startList(); + getSignedPayload().writeTo(rlpOut); + + rlpOut.startList(); + rlpOut.writeList(roundChanges, SignedData::writeTo); + rlpOut.writeList(prepares, SignedData::writeTo); + rlpOut.endList(); + + rlpOut.endList(); + return rlpOut.encoded(); + } + + /** + * Decode. + * + * @param data the data + * @param blockEncoder the qbft block encoder + * @return the proposal + */ + public static Proposal decode(final Bytes data, final QbftBlockCodec blockEncoder) { + final RLPInput rlpIn = RLP.input(data); + rlpIn.enterList(); + final SignedData payload = + readPayload(rlpIn, rlpInput -> ProposalPayload.readFrom(rlpInput, blockEncoder)); + + rlpIn.enterList(); + final List> roundChanges = + rlpIn.readList(r -> readPayload(r, RoundChangePayload::readFrom), MAX_LIST_ENTRIES); + final List> prepares = + rlpIn.readList(r -> readPayload(r, PreparePayload::readFrom), MAX_LIST_ENTRIES); + rlpIn.leaveList(); + + rlpIn.leaveList(); + return new Proposal(payload, roundChanges, prepares); + } +} diff --git a/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/RoundChange.java b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/RoundChange.java new file mode 100644 index 0000000..5d7cfce --- /dev/null +++ b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/RoundChange.java @@ -0,0 +1,168 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-bounded-message-lists.patch. + */ +package org.hyperledger.besu.consensus.qbft.core.messagewrappers; + +import org.hyperledger.besu.consensus.common.bft.messagewrappers.BftMessage; +import org.hyperledger.besu.consensus.common.bft.payload.SignedData; +import org.hyperledger.besu.consensus.qbft.core.payload.PreparePayload; +import org.hyperledger.besu.consensus.qbft.core.payload.PreparedRoundMetadata; +import org.hyperledger.besu.consensus.qbft.core.payload.RoundChangePayload; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlock; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockCodec; +import org.hyperledger.besu.ethereum.core.encoding.BlockAccessListDecoder; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +/** The Round change payload message. */ +public class RoundChange extends BftMessage { + + private final Optional proposedBlock; + private final Optional blockAccessList; + private final QbftBlockCodec blockEncoder; + private final List> prepares; + + /** + * Instantiates a new Round change. + * + * @param payload the payload + * @param proposedBlock the proposed block + * @param blockAccessList the block access list + * @param blockEncoder the qbft block encoder + * @param prepares the prepares + */ + public RoundChange( + final SignedData payload, + final Optional proposedBlock, + final Optional blockAccessList, + final QbftBlockCodec blockEncoder, + final List> prepares) { + super(payload); + this.proposedBlock = proposedBlock; + this.blockAccessList = blockAccessList; + this.blockEncoder = blockEncoder; + this.prepares = prepares; + } + + /** + * Gets proposed block. + * + * @return the proposed block + */ + public Optional getProposedBlock() { + return proposedBlock; + } + + /** + * Gets block access list. + * + * @return the block access list + */ + public Optional getBlockAccessList() { + return blockAccessList; + } + + /** + * Gets list of Prepare payload signed data. + * + * @return the prepares + */ + public List> getPrepares() { + return prepares; + } + + /** + * Gets prepared round metadata. + * + * @return the prepared round metadata + */ + public Optional getPreparedRoundMetadata() { + return getPayload().getPreparedRoundMetadata(); + } + + /** + * Gets prepared round. + * + * @return the prepared round + */ + public Optional getPreparedRound() { + return getPayload().getPreparedRoundMetadata().map(PreparedRoundMetadata::getPreparedRound); + } + + @Override + public Bytes encode() { + final BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); + rlpOut.startList(); + getSignedPayload().writeTo(rlpOut); + proposedBlock.ifPresentOrElse(pb -> blockEncoder.writeTo(pb, rlpOut), rlpOut::writeEmptyList); + blockAccessList.ifPresentOrElse((bal) -> bal.writeTo(rlpOut), rlpOut::writeNull); + rlpOut.writeList(prepares, SignedData::writeTo); + rlpOut.endList(); + return rlpOut.encoded(); + } + + /** + * Decode. + * + * @param data the data + * @param blockEncoder the qbft block encoder + * @return the round change + */ + public static RoundChange decode(final Bytes data, final QbftBlockCodec blockEncoder) { + + final RLPInput rlpIn = RLP.input(data); + rlpIn.enterList(); + final SignedData payload = readPayload(rlpIn, RoundChangePayload::readFrom); + + final Optional block; + if (rlpIn.nextIsList() && rlpIn.nextSize() == 0) { + rlpIn.skipNext(); + block = Optional.empty(); + } else { + block = Optional.of(blockEncoder.readFrom(rlpIn)); + } + + final Optional blockAccessList = readBlockAccessList(rlpIn); + + final List> prepares = + rlpIn.readList(r -> readPayload(r, PreparePayload::readFrom), MAX_LIST_ENTRIES); + rlpIn.leaveList(); + + return new RoundChange(payload, block, blockAccessList, blockEncoder, prepares); + } + + private static Optional readBlockAccessList(final RLPInput rlpIn) { + if (rlpIn.isEndOfCurrentList()) { + return Optional.empty(); + } + if (!rlpIn.nextIsNull()) { + return Optional.of(BlockAccessListDecoder.decode(rlpIn)); + } + rlpIn.skipNext(); + return Optional.empty(); + } +} diff --git a/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRound.java b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRound.java index 189760f..8dc438b 100644 --- a/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRound.java +++ b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRound.java @@ -112,7 +112,7 @@ public class QbftRound { // usual case). Each used to sign on its own, so at the parent of an anchor every Besu validator // made two SLH-DSA signatures back to back on the consensus thread, the first before its Prepare // and the second between "prepared" and its Commit. Measured on 2800 (2026-09-28, import times in - // ms on rpc2, scripts/masuratori/ritm-pe-propunator.mjs): the anchor-parent slot 1,036 ms against + // ms measured at a reading node of the network): the anchor-parent slot 1,036 ms against // 599 for the other slots. The message is fixed by (block, commit hash), both fixed for the round // once the proposal is accepted, so the second signing could only produce another valid seal over // the same bytes. It now reuses the first, and the local and wire Commits carry the same seals diff --git a/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRoundFactory.java b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRoundFactory.java new file mode 100644 index 0000000..a797951 --- /dev/null +++ b/anchor/consensus/qbft-core/src/main/java/org/hyperledger/besu/consensus/qbft/core/statemachine/QbftRoundFactory.java @@ -0,0 +1,124 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.qbft.core.statemachine; + +import org.hyperledger.besu.consensus.common.bft.AerePqIdentity; +import org.hyperledger.besu.consensus.common.bft.ConsensusRoundIdentifier; +import org.hyperledger.besu.consensus.qbft.core.network.QbftMessageTransmitter; +import org.hyperledger.besu.consensus.qbft.core.payload.MessageFactory; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockCreator; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockCreatorFactory; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockHeader; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockInterface; +import org.hyperledger.besu.consensus.qbft.core.types.QbftFinalState; +import org.hyperledger.besu.consensus.qbft.core.types.QbftMinedBlockObserver; +import org.hyperledger.besu.consensus.qbft.core.types.QbftProtocolSchedule; +import org.hyperledger.besu.consensus.qbft.core.validation.MessageValidatorFactory; +import org.hyperledger.besu.util.Subscribers; + +/** The Qbft round factory. */ +public class QbftRoundFactory { + + private final QbftFinalState finalState; + private final QbftBlockCreatorFactory blockCreatorFactory; + private final QbftBlockInterface blockInterface; + private final QbftProtocolSchedule protocolSchedule; + private final Subscribers minedBlockObservers; + private final MessageValidatorFactory messageValidatorFactory; + private final MessageFactory messageFactory; + + /** + * Instantiates a new Qbft round factory. + * + * @param finalState the final state + * @param blockInterface the block interface + * @param protocolSchedule the protocol schedule + * @param minedBlockObservers the mined block observers + * @param messageValidatorFactory the message validator factory + * @param messageFactory the message factory + */ + public QbftRoundFactory( + final QbftFinalState finalState, + final QbftBlockInterface blockInterface, + final QbftProtocolSchedule protocolSchedule, + final Subscribers minedBlockObservers, + final MessageValidatorFactory messageValidatorFactory, + final MessageFactory messageFactory) { + this.finalState = finalState; + this.blockCreatorFactory = finalState.getBlockCreatorFactory(); + this.blockInterface = blockInterface; + this.protocolSchedule = protocolSchedule; + this.minedBlockObservers = minedBlockObservers; + this.messageValidatorFactory = messageValidatorFactory; + this.messageFactory = messageFactory; + } + + /** + * Create new round qbft round. + * + * @param parentHeader the parent header + * @param round the round + * @return the qbft round + */ + public QbftRound createNewRound(final QbftBlockHeader parentHeader, final int round) { + long nextBlockHeight = parentHeader.getNumber() + 1; + final ConsensusRoundIdentifier roundIdentifier = + new ConsensusRoundIdentifier(nextBlockHeight, round); + + final RoundState roundState = + new RoundState( + roundIdentifier, + finalState.getQuorum(), + messageValidatorFactory.createMessageValidator(roundIdentifier, parentHeader)); + + return createNewRoundWithState(parentHeader, roundState); + } + + /** + * Create new Qbft round with state. + * + * @param parentHeader the parent header + * @param roundState the round state + * @return the qbft round + */ + public QbftRound createNewRoundWithState( + final QbftBlockHeader parentHeader, final RoundState roundState) { + final QbftBlockCreator blockCreator = + blockCreatorFactory.create(roundState.getRoundIdentifier().getRoundNumber()); + + // TODO(tmm): Why is this created everytime?! + final QbftMessageTransmitter messageTransmitter = + new QbftMessageTransmitter(messageFactory, finalState.getValidatorMulticaster()); + + return new QbftRound( + roundState, + blockCreator, + blockInterface, + protocolSchedule, + minedBlockObservers, + finalState.getNodeKey(), + AerePqIdentity.local(finalState.getLocalAddress(), parentHeader.getNumber() + 1L), + messageFactory, + messageTransmitter, + finalState.getRoundTimer(), + parentHeader); + } +} diff --git a/anchor/consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/AereD434ListBoundsTest.java b/anchor/consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/AereD434ListBoundsTest.java new file mode 100644 index 0000000..889e0e7 --- /dev/null +++ b/anchor/consensus/qbft-core/src/test/java/org/hyperledger/besu/consensus/qbft/core/messagewrappers/AereD434ListBoundsTest.java @@ -0,0 +1,73 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Aere Network notice (Apache-2.0 section 4(b)): this file was added by Aere Network, 2026-09-28 (D-434), adapted from the tests of the + * ported upstream fix hyperledger/besu dd4101c9ec ("bound prepares/round-change lists at decode"). + */ +package org.hyperledger.besu.consensus.qbft.core.messagewrappers; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.hyperledger.besu.consensus.common.bft.ConsensusRoundIdentifier; +import org.hyperledger.besu.consensus.common.bft.messagewrappers.BftMessage; +import org.hyperledger.besu.consensus.common.bft.payload.SignedData; +import org.hyperledger.besu.consensus.qbft.core.payload.PreparePayload; +import org.hyperledger.besu.consensus.qbft.core.payload.RoundChangePayload; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockCodec; +import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.cryptoservices.NodeKeyUtils; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.rlp.RLPException; + +import java.util.Collections; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** + * AERE D-434: a RoundChange whose prepares list holds more than BftMessage.MAX_LIST_ENTRIES (512) entries is refused at DECODE, before + * any signature or author check; before the port any peer could make a node allocate and verify an arbitrarily long list. With ten + * validators an honest list never exceeds ten, so the bound never binds an honest network. + */ +class AereD434ListBoundsTest { + + private static final QbftBlockCodec CODEC = Mockito.mock(QbftBlockCodec.class); + + private static Bytes roundChangeWith(final int prepares) { + final NodeKey key = NodeKeyUtils.generate(); + final RoundChangePayload rc = new RoundChangePayload(new ConsensusRoundIdentifier(1, 0), Optional.empty()); + final SignedData signedRc = SignedData.create(rc, key.sign(Bytes32.wrap(rc.hashForSignature().getBytes()))); + final PreparePayload p = new PreparePayload(new ConsensusRoundIdentifier(1, 0), Hash.ZERO); + final SignedData one = SignedData.create(p, key.sign(Bytes32.wrap(p.hashForSignature().getBytes()))); + return new RoundChange(signedRc, Optional.empty(), Optional.empty(), CODEC, Collections.nCopies(prepares, one)).encode(); + } + + @Test + void aPreparesListAboveTheBoundIsRefusedAtDecode() { + final Bytes oversized = roundChangeWith(BftMessage.MAX_LIST_ENTRIES + 1); + assertThatThrownBy(() -> RoundChange.decode(oversized, CODEC)) + .isInstanceOf(RLPException.class) + .hasMessageContaining("exceeds the maximum permitted size"); + } + + @Test + void aPreparesListAtTheBoundStillDecodes() { + // positive control: the fixture decodes, and the bound is inclusive + assertThat(RoundChange.decode(roundChangeWith(BftMessage.MAX_LIST_ENTRIES), CODEC).getPrepares()).hasSize(BftMessage.MAX_LIST_ENTRIES); + } +} diff --git a/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/adaptor/QbftFinalStateImpl.java b/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/adaptor/QbftFinalStateImpl.java new file mode 100644 index 0000000..03d2c64 --- /dev/null +++ b/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/adaptor/QbftFinalStateImpl.java @@ -0,0 +1,196 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.consensus.qbft.adaptor; + +import org.hyperledger.besu.consensus.common.bft.AerePqIdentity; +import org.hyperledger.besu.consensus.common.bft.BftHelpers; +import org.hyperledger.besu.consensus.common.bft.BlockTimer; +import org.hyperledger.besu.consensus.common.bft.ConsensusRoundIdentifier; +import org.hyperledger.besu.consensus.common.bft.RoundTimer; +import org.hyperledger.besu.consensus.common.bft.blockcreation.ProposerSelector; +import org.hyperledger.besu.consensus.common.bft.network.ValidatorMulticaster; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.consensus.qbft.core.types.QbftBlockCreatorFactory; +import org.hyperledger.besu.consensus.qbft.core.types.QbftFinalState; +import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.datatypes.Address; + +import java.time.Clock; +import java.util.Collection; + +/** Besu implementation of QbftFinalState for maintaining the state of a QBFT network. */ +public class QbftFinalStateImpl implements QbftFinalState { + private final ValidatorProvider validatorProvider; + private final NodeKey nodeKey; + private final Address localAddress; + private final ProposerSelector proposerSelector; + private final ValidatorMulticaster validatorMulticaster; + private final RoundTimer roundTimer; + private final BlockTimer blockTimer; + private final QbftBlockCreatorFactory blockCreatorFactory; + private final Clock clock; + + /** + * Constructs a new QBFT final state. + * + * @param validatorProvider the validator provider + * @param nodeKey the node key + * @param localAddress the local address + * @param proposerSelector the proposer selector + * @param validatorMulticaster the validator multicaster + * @param roundTimer the round timer + * @param blockTimer the block timer + * @param blockCreatorFactory the block creator factory + * @param clock the clock + */ + public QbftFinalStateImpl( + final ValidatorProvider validatorProvider, + final NodeKey nodeKey, + final Address localAddress, + final ProposerSelector proposerSelector, + final ValidatorMulticaster validatorMulticaster, + final RoundTimer roundTimer, + final BlockTimer blockTimer, + final QbftBlockCreatorFactory blockCreatorFactory, + final Clock clock) { + this.validatorProvider = validatorProvider; + this.nodeKey = nodeKey; + this.localAddress = localAddress; + this.proposerSelector = proposerSelector; + this.validatorMulticaster = validatorMulticaster; + this.roundTimer = roundTimer; + this.blockTimer = blockTimer; + this.blockCreatorFactory = blockCreatorFactory; + this.clock = clock; + } + + /** + * Gets validators. + * + * @return the validators + */ + @Override + public Collection

getValidators() { + return validatorProvider.getValidatorsAtHead(); + } + + /** + * Gets the validator multicaster. + * + * @return the validator multicaster + */ + @Override + public ValidatorMulticaster getValidatorMulticaster() { + return validatorMulticaster; + } + + /** + * Gets node key. + * + * @return the node key + */ + @Override + public NodeKey getNodeKey() { + return nodeKey; + } + + /** + * Gets local address. + * + * @return the local address + */ + @Override + public Address getLocalAddress() { + return localAddress; + } + + /** + * Is local node validator. + * + * @return the boolean + */ + @Override + public boolean isLocalNodeValidator() { + return getValidators().stream().anyMatch(a -> AerePqIdentity.isLocal(a, localAddress)); + } + + /** + * Gets round timer. + * + * @return the round timer + */ + @Override + public RoundTimer getRoundTimer() { + return roundTimer; + } + + /** + * Gets block creator factory. + * + * @return the block creator factory + */ + @Override + public QbftBlockCreatorFactory getBlockCreatorFactory() { + return blockCreatorFactory; + } + + @Override + public int getQuorum() { + return BftHelpers.calculateRequiredValidatorQuorum(getValidators().size()); + } + + @Override + public BlockTimer getBlockTimer() { + return blockTimer; + } + + /** + * Is local node proposer for round. + * + * @param roundIdentifier the round identifier + * @return the boolean + */ + @Override + public boolean isLocalNodeProposerForRound(final ConsensusRoundIdentifier roundIdentifier) { + return getProposerForRound(roundIdentifier) + .equals(AerePqIdentity.local(localAddress, roundIdentifier.getSequenceNumber())); + } + + /** + * Gets proposer for round. + * + * @param roundIdentifier the round identifier + * @return the proposer for round + */ + public Address getProposerForRound(final ConsensusRoundIdentifier roundIdentifier) { + return proposerSelector.selectProposerForRound(roundIdentifier); + } + + /** + * Gets clock. + * + * @return the clock + */ + @Override + public Clock getClock() { + return clock; + } +} diff --git a/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java b/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java new file mode 100644 index 0000000..674b39e --- /dev/null +++ b/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java @@ -0,0 +1,107 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0007-aere-pq-finality.patch. + */ +package org.hyperledger.besu.consensus.qbft.jsonrpc; + +import org.hyperledger.besu.config.BftConfigOptions; +import org.hyperledger.besu.consensus.common.BlockInterface; +import org.hyperledger.besu.consensus.common.bft.BftContext; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftDiscardValidatorVote; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetPendingVotes; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.AereGetPqFinality; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetRequestTimeoutSeconds; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetSignerMetrics; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetValidatorsByBlockHash; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetValidatorsByBlockNumber; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftProposeValidatorVote; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.methods.ApiGroupJsonRpcMethods; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; + +import java.util.Map; + +/** The Qbft json rpc methods. */ +public class QbftJsonRpcMethods extends ApiGroupJsonRpcMethods { + + private final ProtocolContext context; + private final ValidatorProvider readOnlyValidatorProvider; + private final ProtocolSchedule protocolSchedule; + private final MiningConfiguration miningConfiguration; + private final BftConfigOptions bftConfig; + + /** + * Instantiates a new Qbft json rpc methods. + * + * @param context the protocol context + * @param protocolSchedule the protocol schedule + * @param miningConfiguration the mining parameters + * @param readOnlyValidatorProvider the read only validator provider + * @param bftConfig the BFT config options, containing QBFT-specific settings + */ + public QbftJsonRpcMethods( + final ProtocolContext context, + final ProtocolSchedule protocolSchedule, + final MiningConfiguration miningConfiguration, + final ValidatorProvider readOnlyValidatorProvider, + final BftConfigOptions bftConfig) { + this.context = context; + this.readOnlyValidatorProvider = readOnlyValidatorProvider; + this.protocolSchedule = protocolSchedule; + this.miningConfiguration = miningConfiguration; + this.bftConfig = bftConfig; + } + + @Override + protected String getApiGroup() { + return RpcApis.QBFT.name(); + } + + @Override + protected Map create() { + final BlockchainQueries blockchainQueries = + new BlockchainQueries( + protocolSchedule, + context.getBlockchain(), + context.getWorldStateArchive(), + miningConfiguration); + final BftContext bftContext = context.getConsensusContext(BftContext.class); + final BlockInterface blockInterface = bftContext.getBlockInterface(); + final ValidatorProvider validatorProvider = bftContext.getValidatorProvider(); + + Map methods = + mapOf( + new QbftProposeValidatorVote(validatorProvider), + new QbftGetValidatorsByBlockNumber(blockchainQueries, readOnlyValidatorProvider), + new QbftDiscardValidatorVote(validatorProvider), + new QbftGetValidatorsByBlockHash(context.getBlockchain(), readOnlyValidatorProvider), + new QbftGetSignerMetrics(readOnlyValidatorProvider, blockInterface, blockchainQueries), + new QbftGetPendingVotes(validatorProvider), + new QbftGetRequestTimeoutSeconds(bftConfig), + // AIP-21 (AERE): per-block post-quantum finality, non-consensus, verified against the anchored registry + new AereGetPqFinality(blockchainQueries, readOnlyValidatorProvider)); + + return methods; + } +} diff --git a/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java b/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java new file mode 100644 index 0000000..7fc5d24 --- /dev/null +++ b/anchor/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java @@ -0,0 +1,254 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.qbft.jsonrpc.methods; + +import org.hyperledger.besu.consensus.common.bft.BftHelpers; +import org.hyperledger.besu.consensus.common.bft.FalconSeal; +import org.hyperledger.besu.consensus.common.bft.PqSealCache; +import org.hyperledger.besu.consensus.common.bft.PqSignerRegistry; +import org.hyperledger.besu.consensus.common.bft.SchemeSeal; +import org.hyperledger.besu.consensus.common.bft.SealSchemes; +import org.hyperledger.besu.consensus.common.bft.blockcreation.PqAnchorProducer; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.AbstractBlockParameterMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeSet; + +import org.apache.tuweni.bytes.Bytes; + +/** + * AIP-21 (AERE, 2026-09-18), first half: the post-quantum finality of ONE block, verifiable per block, + * without growing the header. + * + *

WHAT IT ANSWERS. Every block of chain 2800 is committed with a Falcon-512 seal on each Commit + * message (enforced since H1, 2026-09-05), and every validator keeps the seals it heard for recent + * blocks in {@link PqSealCache}. Those seals never reach the header (only every 128th block carries a + * certificate, over its parent), so until now the immediate post-quantum finality of a block was true + * and NOT observable. This method exposes it: for a block it returns the seals heard, re-verifies each + * one right now against the anchored registry over the exact message the validators signed, and gives + * the verdict "verified distinct signers >= the chain's commit quorum". + * + *

WHAT IT DOES NOT CLAIM. Seals travel only on Commit messages, which validators send to + * validators: a reader node hears none and answers {@code finality: "unavailable"} honestly. The + * retention window is the cache's (system property {@code aere.pq.finality.window}, blocks; default + * 256), so an older block answers "unavailable" too, and the permanent record for it is the anchor + * certificate. Hybrid extras (SLH-DSA seals on anchor parents) are listed as heard but NOT verified + * here ({@code verified: null}); their verification is the anchor certificate's job. Below the anchor + * form (never on a live chain today) the signed message is the round-specific commit digest, which + * cannot be rebuilt from the header, so the method says so instead of guessing. + * + *

NON-CONSENSUS. Nothing here changes what a block is; a node with or without this method imports + * the same chain. That is why it needs no activation height. + */ +public class AereGetPqFinality extends AbstractBlockParameterMethod { + + /** Verifies one Falcon seal over the message signed for a block; injectable for tests. */ + @FunctionalInterface + public interface SealVerifier { + /** + * Verify. + * + * @param blockNumber the height + * @param validatorIndex the signer index in the anchored registry + * @param message the message that was signed + * @param signature the Falcon signature + * @return true iff it verifies against the key bound at that height + */ + boolean verify(long blockNumber, int validatorIndex, Bytes message, Bytes signature); + } + + /** Rebuilds the message the seals of a block signed, when that is possible from the header. */ + @FunctionalInterface + public interface MessageBuilder { + /** + * The signed message. + * + * @param blockNumber the height + * @param onchainHash the round-independent on-chain hash + * @return the message, or empty when the form at that height cannot be rebuilt from the header + */ + Optional messageFor(long blockNumber, Hash onchainHash); + } + + private final ValidatorProvider validatorProvider; + private final PqSealCache cache; + private final SealVerifier verifier; + private final MessageBuilder messageBuilder; + + /** + * Production wiring: the process-wide seal cache, the anchored Falcon registry at the block's + * height, and the anchor-form commit message. + * + * @param blockchainQueries the blockchain queries + * @param validatorProvider the read-only validator provider + */ + public AereGetPqFinality( + final BlockchainQueries blockchainQueries, final ValidatorProvider validatorProvider) { + this( + blockchainQueries, + validatorProvider, + PqSealCache.instance(), + (n, i, m, s) -> PqSignerRegistry.falconSealSupport().verifyAtHistoric(n, i, m, s), + (n, h) -> + PqAnchorProducer.sealMessageIsAnchorForm(n) + ? Optional.of(PqAnchorProducer.commitSealMessage(n, h.getBytes(), Hash.ZERO)) + : Optional.empty()); + } + + /** + * Full wiring, for tests. + * + * @param blockchainQueries the blockchain queries + * @param validatorProvider the validator provider + * @param cache the seal cache to read + * @param verifier the seal verifier + * @param messageBuilder the signed-message builder + */ + public AereGetPqFinality( + final BlockchainQueries blockchainQueries, + final ValidatorProvider validatorProvider, + final PqSealCache cache, + final SealVerifier verifier, + final MessageBuilder messageBuilder) { + super(blockchainQueries); + this.validatorProvider = validatorProvider; + this.cache = cache; + this.verifier = verifier; + this.messageBuilder = messageBuilder; + } + + @Override + public String getName() { + return RpcMethod.AERE_GET_PQ_FINALITY.getMethodName(); + } + + @Override + protected BlockParameter blockParameter(final JsonRpcRequestContext request) { + try { + return request.getOptionalParameter(0, BlockParameter.class).orElse(BlockParameter.LATEST); + } catch (final JsonRpcParameterException e) { + throw new InvalidJsonRpcParameters( + "Invalid block parameter (index 0)", RpcErrorType.INVALID_BLOCK_PARAMS, e); + } + } + + @Override + protected Object pendingResult(final JsonRpcRequestContext request) { + return describe(getBlockchainQueries().headBlockHeader()); + } + + @Override + protected Object resultByBlockNumber(final JsonRpcRequestContext request, final long blockNumber) { + return getBlockchainQueries().getBlockHeaderByNumber(blockNumber).map(this::describe).orElse(null); + } + + /** + * The finality record of one block. + * + * @param header the block's header + * @return the JSON-shaped record + */ + public Map describe(final BlockHeader header) { + final long number = header.getNumber(); + final Hash hash = header.getHash(); + final int validators = validatorProvider.getValidatorsForBlock(header).size(); + final int quorum = validators == 0 ? 0 : BftHelpers.calculateRequiredValidatorQuorum(validators); + final Optional message = messageBuilder.messageFor(number, hash); + final List heard = cache.sealsFor(number, hash); + final List extras = cache.extrasFor(number, hash); + + final List> seals = new ArrayList<>(); + final TreeSet verifiedSigners = new TreeSet<>(); + for (final FalconSeal seal : heard) { + final boolean ok = + message.isPresent() + && verifier.verify(number, seal.getValidatorIndex(), message.get(), seal.getSignature()); + if (ok) { + verifiedSigners.add(seal.getValidatorIndex()); + } + final Map s = new LinkedHashMap<>(); + s.put("scheme", "falcon-512"); + s.put("index", seal.getValidatorIndex()); + s.put("verified", ok); + s.put("signature", seal.getSignature().toHexString()); + seals.add(s); + } + final List> hybrid = new ArrayList<>(); + for (final SchemeSeal seal : extras) { + final Map s = new LinkedHashMap<>(); + s.put( + "scheme", + SealSchemes.byWireId(seal.getSchemeWireId()).map(x -> x.id()).orElse("wire-" + (seal.getSchemeWireId() & 0xff))); + s.put("index", seal.getValidatorIndex()); + s.put("verified", null); + s.put("signature", seal.getSignature().toHexString()); + hybrid.add(s); + } + + final String finality; + final String note; + if (message.isEmpty()) { + finality = "unverifiable-form"; + note = + "below the anchor form the signed message is the round-specific commit digest, which this node cannot rebuild from the header; nothing is guessed"; + } else if (heard.isEmpty()) { + finality = "unavailable"; + note = + "no seal heard for this block: seals travel on Commit messages between validators, so a reader node hears none, and a validator keeps only the last " + + cache.retentionBlocks() + + " blocks; the permanent record for older blocks is the anchor certificate"; + } else if (verifiedSigners.size() >= quorum && quorum > 0) { + finality = "post-quantum"; + note = "at least the commit quorum of distinct validators sealed this block hash with Falcon-512 and every counted seal verifies now against the anchored registry"; + } else if (!verifiedSigners.isEmpty()) { + finality = "partial"; + note = "fewer verified distinct signers than the commit quorum: this node did not hear enough seals, or some it heard no longer verify"; + } else { + finality = "none"; + note = "seals were heard but none verifies against the anchored registry at this height"; + } + + final Map out = new LinkedHashMap<>(); + out.put("blockNumber", number); + out.put("blockHash", hash.getBytes().toHexString()); + out.put("validators", validators); + out.put("quorum", quorum); + out.put("sealForm", message.isPresent() ? "anchor" : "commit-digest"); + out.put("signedMessage", message.map(Bytes::toHexString).orElse(null)); + out.put("falconSealsHeard", heard.size()); + out.put("falconSealsVerified", verifiedSigners.size()); + out.put("verifiedIndexes", new ArrayList<>(verifiedSigners)); + out.put("falconSeals", seals); + out.put("hybridSeals", hybrid); + out.put("finality", finality); + out.put("retentionBlocks", cache.retentionBlocks()); + out.put("note", note); + return out; + } +} diff --git a/anchor/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java b/anchor/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java new file mode 100644 index 0000000..b3b6d95 --- /dev/null +++ b/anchor/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java @@ -0,0 +1,205 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.qbft.jsonrpc.methods; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.consensus.common.bft.FalconSeal; +import org.hyperledger.besu.consensus.common.bft.PqSealCache; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * AIP-21: the finality record is a MEASUREMENT of the seals heard, re-verified, against the chain's + * commit quorum; every claim it can make has its negative pair here (too few signers, none verifying, + * nothing heard, a form that cannot be rebuilt), and the verifier is consulted with the exact message + * and signature, so a stub that verifies nothing turns the verdict from post-quantum to none. + */ +@ExtendWith(MockitoExtension.class) +public class AereGetPqFinalityTest { + + private static final long NUMBER = 19_018_096L; + private static final Hash HASH = Hash.fromHexString("0xf776f80e1710dc0e0f1927958dd589b98fc35ce8b9854e4fdb12320459fbb3f1"); + private static final Bytes MESSAGE = Bytes32.fromHexStringLenient("0x1122"); + + @Mock private BlockchainQueries blockchainQueries; + @Mock private BlockHeader header; + @Mock private ValidatorProvider validatorProvider; + + private final PqSealCache cache = PqSealCache.instance(); + private final List verifierCalls = new ArrayList<>(); + + @BeforeEach + public void setUp() { + cache.clear(); + // lenient: the name and unknown-block cases never touch the header, and strict stubs would call that a defect + lenient().when(header.getNumber()).thenReturn(NUMBER); + lenient().when(header.getHash()).thenReturn(HASH); + } + + @AfterEach + public void tearDown() { + cache.clear(); + } + + private static List

validators(final int n) { + final List
out = new ArrayList<>(); + for (int i = 0; i < n; i++) { + out.add(Address.fromHexString(String.format("0x%040x", 0xA000 + i))); + } + return out; + } + + private void heard(final int count) { + final List seals = new ArrayList<>(); + for (int i = 0; i < count; i++) { + seals.add(new FalconSeal(i, Bytes.of(0x29, i, 7, 7, 7))); + } + cache.record(NUMBER, HASH, seals); + } + + private AereGetPqFinality method(final int verifyBelowIndex, final Optional message) { + return new AereGetPqFinality( + blockchainQueries, + validatorProvider, + cache, + (n, i, m, s) -> { + verifierCalls.add(new Object[] {n, i, m, s}); + return i < verifyBelowIndex; + }, + (n, h) -> message); + } + + @SuppressWarnings("unchecked") + private Map call(final AereGetPqFinality method, final Object... params) { + final JsonRpcRequestContext request = + new JsonRpcRequestContext(new JsonRpcRequest("2.0", method.getName(), params)); + final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) method.response(request); + return (Map) response.getResult(); + } + + @Test + public void nameIsTheAipMethod() { + assertThat(method(7, Optional.of(MESSAGE)).getName()).isEqualTo("aere_getPqFinality"); + } + + @Test + public void quorumOfVerifiedDistinctSignersIsPostQuantumFinality() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(5, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("post-quantum"); + assertThat(r.get("quorum")).isEqualTo(5); + assertThat(r.get("falconSealsHeard")).isEqualTo(7); + assertThat(r.get("falconSealsVerified")).isEqualTo(5); + assertThat(r.get("sealForm")).isEqualTo("anchor"); + assertThat(r.get("signedMessage")).isEqualTo(MESSAGE.toHexString()); + // the verifier saw exactly the message and each seal's own signature + assertThat(verifierCalls).hasSize(7); + assertThat(verifierCalls.get(3)[1]).isEqualTo(3); + assertThat(verifierCalls.get(3)[2]).isEqualTo(MESSAGE); + assertThat(verifierCalls.get(3)[3]).isEqualTo(Bytes.of(0x29, 3, 7, 7, 7)); + } + + @Test + public void oneSignerShortOfQuorumIsPartial() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(4, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("partial"); + assertThat(r.get("falconSealsVerified")).isEqualTo(4); + } + + @Test + public void negativeControlNothingVerifiesIsNone() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(0, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("none"); + assertThat(r.get("falconSealsVerified")).isEqualTo(0); + assertThat(r.get("falconSealsHeard")).isEqualTo(7); + } + + @Test + public void nothingHeardIsUnavailableNotFalse() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + final Map r = call(method(7, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("unavailable"); + assertThat(r.get("falconSealsHeard")).isEqualTo(0); + assertThat(verifierCalls).isEmpty(); + } + + @Test + public void aFormThatCannotBeRebuiltIsSaidNotGuessed() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(7, Optional.empty()), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("unverifiable-form"); + assertThat(r.get("sealForm")).isEqualTo("commit-digest"); + assertThat(r.get("falconSealsVerified")).isEqualTo(0); + assertThat(verifierCalls).isEmpty(); + } + + @Test + public void latestIsTheDefaultBlock() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(10)); + // "latest" resolves through the head NUMBER, then the header by number (AbstractBlockParameterMethod) + when(blockchainQueries.headBlockNumber()).thenReturn(NUMBER); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(10); + final Map r = call(method(10, Optional.of(MESSAGE))); + assertThat(r.get("quorum")).isEqualTo(7); + assertThat(r.get("finality")).isEqualTo("post-quantum"); + assertThat(r.get("blockNumber")).isEqualTo(NUMBER); + } + + @Test + public void unknownBlockIsNull() { + when(blockchainQueries.getBlockHeaderByNumber(5L)).thenReturn(Optional.empty()); + final JsonRpcRequestContext request = + new JsonRpcRequestContext(new JsonRpcRequest("2.0", "aere_getPqFinality", new Object[] {"0x5"})); + final JsonRpcSuccessResponse response = + (JsonRpcSuccessResponse) method(7, Optional.of(MESSAGE)).response(request); + assertThat(response.getResult()).isNull(); + } +} diff --git a/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/Address.java b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/Address.java new file mode 100644 index 0000000..3a63362 --- /dev/null +++ b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/Address.java @@ -0,0 +1,274 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0001-aere-pqc-precompiles-mainnet.patch. + */ +package org.hyperledger.besu.datatypes; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.hyperledger.besu.crypto.Hash.keccak256; + +import org.hyperledger.besu.crypto.SECPPublicKey; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPException; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.util.concurrent.ExecutionException; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +/** A 160-bits account address. */ +public class Address extends BytesHolder { + + /** The constant SIZE. */ + public static final int SIZE = 20; + + /** Specific addresses of the "precompiled" contracts. */ + public static final Address ECREC = Address.precompiled(0x01); + + /** The constant SHA256. */ + public static final Address SHA256 = Address.precompiled(0x02); + + /** The constant RIPEMD160. */ + public static final Address RIPEMD160 = Address.precompiled(0x03); + + /** The constant ID. */ + public static final Address ID = Address.precompiled(0x04); + + /** The constant MODEXP. */ + public static final Address MODEXP = Address.precompiled(0x05); + + /** The constant ALTBN128_ADD. */ + public static final Address ALTBN128_ADD = Address.precompiled(0x06); + + /** The constant ALTBN128_MUL. */ + public static final Address ALTBN128_MUL = Address.precompiled(0x07); + + /** The constant ALTBN128_PAIRING. */ + public static final Address ALTBN128_PAIRING = Address.precompiled(0x08); + + /** The constant BLAKE2B_F_COMPRESSION. */ + public static final Address BLAKE2B_F_COMPRESSION = Address.precompiled(0x09); + + /** The constant KZG_POINT_EVAL aka POINT_EVALUATION_PRECOMPILE_ADDRESS. */ + public static final Address KZG_POINT_EVAL = Address.precompiled(0xA); + + /** The constant BLS12_G1ADD. */ + public static final Address BLS12_G1ADD = Address.precompiled(0xB); + + /** The constant BLS12_G1MULTIEXP. */ + public static final Address BLS12_G1MULTIEXP = Address.precompiled(0xC); + + /** The constant BLS12_G2ADD. */ + public static final Address BLS12_G2ADD = Address.precompiled(0xD); + + /** The constant BLS12_G2MULTIEXP. */ + public static final Address BLS12_G2MULTIEXP = Address.precompiled(0xE); + + /** The constant BLS12_PAIRING. */ + public static final Address BLS12_PAIRING = Address.precompiled(0xF); + + /** The constant BLS12_MAP_FP_TO_G1. */ + public static final Address BLS12_MAP_FP_TO_G1 = Address.precompiled(0x10); + + /** The constant BLS12_MAP_FP2_TO_G2. */ + public static final Address BLS12_MAP_FP2_TO_G2 = Address.precompiled(0x11); + + /** Precompile address for P256_VERIFY. */ + public static final Address P256_VERIFY = Address.precompiled(0x0100); + + /** AERE PQC precompile: Falcon-512 verify. */ + public static final Address AERE_FALCON512 = Address.fromHexString("0x0000000000000000000000000000000000000ae1"); + + /** AERE PQC precompile: Falcon-1024 verify. */ + public static final Address AERE_FALCON1024 = Address.fromHexString("0x0000000000000000000000000000000000000ae2"); + + /** AERE PQC precompile: ML-DSA-44 (FIPS 204) verify. */ + public static final Address AERE_MLDSA44 = Address.fromHexString("0x0000000000000000000000000000000000000ae3"); + + /** AERE PQC precompile: SLH-DSA-SHA2-128s (FIPS 205) verify. */ + public static final Address AERE_SLHDSA128S = Address.fromHexString("0x0000000000000000000000000000000000000ae4"); + + /** AERE PQC precompile: SHAKE256 XOF (FIPS 202). */ + public static final Address AERE_SHAKE256 = Address.fromHexString("0x0000000000000000000000000000000000000ae5"); + + /** The constant ZERO. */ + public static final Address ZERO = Address.fromHexString("0x0"); + + static LoadingCache hashCache = + CacheBuilder.newBuilder() + .maximumSize(4000) + // .weakKeys() // unless we "intern" all addresses we cannot use weak or soft keys. + .build( + new CacheLoader<>() { + @Override + public Hash load(final Address key) { + return Hash.hash(key.getBytes()); + } + }); + + /** + * Instantiates a new Address. + * + * @param bytes the bytes + */ + protected Address(final Bytes bytes) { + super(bytes); + } + + /** + * Wrap address. + * + * @param value the value + * @return the address + */ + public static Address wrap(final Bytes value) { + checkArgument( + value.size() == SIZE, + "An account address must be %s bytes long, got %s", + SIZE, + value.size()); + return new Address(value); + } + + /** + * Creates an address from the given RLP-encoded input. + * + * @param input The input to read from + * @return the input's corresponding address + */ + public static Address readFrom(final RLPInput input) { + final Bytes bytes = input.readBytes(); + if (bytes.size() != SIZE) { + throw new RLPException( + String.format("Address unexpected size of %s (needs %s)", bytes.size(), SIZE)); + } + return Address.wrap(bytes); + } + + /** + * Extracts an address from a ECDSARECOVER result hash. + * + * @param hash A hash that has been obtained through hashing the return of the + * ECDSARECOVER function from Appendix F (Signing Transactions) of the Ethereum + * Yellow Paper. + * @return The ethereum address from the provided hash. + */ + public static Address extract(final Bytes32 hash) { + return wrap(hash.slice(12, 20)); + } + + /** + * Extract address. + * + * @param publicKey the public key + * @return the address + */ + public static Address extract(final SECPPublicKey publicKey) { + return Address.extract(keccak256(publicKey.getEncodedBytes())); + } + + /** + * Parse a hexadecimal string representing an account address. + * + * @param str A hexadecimal string (with or without the leading '0x') representing a valid account + * address. + * @return The parsed address: {@code null} if the provided string is {@code null}. + * @throws IllegalArgumentException if the string is either not hexadecimal, or not the valid + * representation of an address. + */ + @JsonCreator + public static Address fromHexString(final String str) { + if (str == null) return null; + return wrap(Bytes.fromHexStringLenient(str, SIZE)); + } + + /** + * Parse a hexadecimal string representing an account address. + * + * @param str A hexadecimal string representing a valid account address (strictly 20 bytes). + * @return The parsed address. + * @throws IllegalArgumentException if the provided string is {@code null}. + * @throws IllegalArgumentException if the string is either not hexadecimal, or not the valid + * representation of a 20 byte address. + */ + public static Address fromHexStringStrict(final String str) { + checkArgument(str != null); + final Bytes value = Bytes.fromHexString(str); + checkArgument( + value.size() == SIZE, + "An account address must be %s bytes long, got %s", + SIZE, + value.size()); + return new Address(value); + } + + /** + * Precompiled address. + * + * @param value the value + * @return the address + */ + public static Address precompiled(final int value) { + // Allow values up to 0x01FF (511) to encompass layer2 precompile address space + checkArgument(value < 0x01FF, "Precompiled value must be <= 0x01FF"); + final byte[] address = new byte[SIZE]; + address[SIZE - 2] = (byte) (value >>> 8); // High byte + address[SIZE - 1] = (byte) (value & 0xFF); // Low byte + return new Address(Bytes.wrap(address)); + } + + /** + * Address of the created contract. + * + *

This implement equation (86) in Section 7 of the Yellow Paper (rev. a91c29c). + * + * @param senderAddress the address of the transaction sender. + * @param nonce the nonce of this transaction. + * @return The generated address of the created contract. + */ + public static Address contractAddress(final Address senderAddress, final long nonce) { + return Address.extract( + keccak256( + RLP.encode( + out -> { + out.startList(); + out.writeBytes(senderAddress.getBytes()); + out.writeLongScalar(nonce); + out.endList(); + }))); + } + + /** + * Returns the hash of the address. Backed by a cache for performance reasons. + * + * @return the hash of the address. + */ + public Hash addressHash() { + try { + return hashCache.get(this); + } catch (ExecutionException e) { + return Hash.hash(getBytes()); + } + } +} diff --git a/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAlgorithm.java b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAlgorithm.java new file mode 100644 index 0000000..ef2d7f6 --- /dev/null +++ b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAlgorithm.java @@ -0,0 +1,98 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.datatypes; + +import java.util.Optional; + +/** + * The post-quantum signature algorithms a type-0x50 transaction (AIP-20) may be authorized with. + * + *

The identifier, the exact key and signature lengths and the verification gas are the table of + * AIP-20. Only the three FIPS 204 ML-DSA parameter sets are valid in v1; the reserved identifiers + * (SLH-DSA 0x10, Falcon-512 0x20, hybrid flag 0x80) are deliberately absent so that a transaction + * naming them is invalid, not silently accepted. + */ +public enum AerePqAlgorithm { + /** ML-DSA-44, FIPS 204, NIST category 2. */ + ML_DSA_44(0x01, 1312, 2420, 55_000L), + /** ML-DSA-65, FIPS 204, NIST category 3, the recommended default. */ + ML_DSA_65(0x02, 1952, 3309, 75_000L), + /** ML-DSA-87, FIPS 204, NIST category 5. */ + ML_DSA_87(0x03, 2592, 4627, 100_000L); + + private final int id; + private final int publicKeyLength; + private final int signatureLength; + private final long verifyGas; + + AerePqAlgorithm( + final int id, final int publicKeyLength, final int signatureLength, final long verifyGas) { + this.id = id; + this.publicKeyLength = publicKeyLength; + this.signatureLength = signatureLength; + this.verifyGas = verifyGas; + } + + /** + * Looks an algorithm up by its AIP-20 identifier. + * + * @param id the identifier as carried in the transaction + * @return the algorithm, or empty for any identifier this implementation does not accept + */ + public static Optional byId(final int id) { + for (final AerePqAlgorithm a : values()) { + if (a.id == id) { + return Optional.of(a); + } + } + return Optional.empty(); + } + + /** + * The AIP-20 identifier. + * + * @return the identifier byte value + */ + public int id() { + return id; + } + + /** + * The exact public key length in bytes. + * + * @return the length + */ + public int publicKeyLength() { + return publicKeyLength; + } + + /** + * The exact signature length in bytes. + * + * @return the length + */ + public int signatureLength() { + return signatureLength; + } + + /** + * The gas charged for one signature verification (AIP-20 table, Draft values). + * + * @return the gas + */ + public long verifyGas() { + return verifyGas; + } +} diff --git a/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAuthorization.java b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAuthorization.java new file mode 100644 index 0000000..37cda70 --- /dev/null +++ b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/AerePqAuthorization.java @@ -0,0 +1,110 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.datatypes; + +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +/** + * The post-quantum authorization of a type-0x50 transaction (AIP-20): the algorithm, the raw public + * key and the raw signature. No secp256k1 material exists for such a transaction. + * + * @param algorithm the ML-DSA parameter set + * @param publicKey the raw public key, exactly {@code algorithm.publicKeyLength()} bytes + * @param signature the raw signature, exactly {@code algorithm.signatureLength()} bytes + */ +public record AerePqAuthorization(AerePqAlgorithm algorithm, Bytes publicKey, Bytes signature) { + + /** The EIP-2718 type byte of the transaction this authorization belongs to. */ + public static final byte TYPE_BYTE = (byte) 0x50; + + /** + * Validates the record's invariants at construction. + * + * @param algorithm the algorithm + * @param publicKey the public key + * @param signature the signature + */ + public AerePqAuthorization( + final AerePqAlgorithm algorithm, final Bytes publicKey, final Bytes signature) { + this.algorithm = Objects.requireNonNull(algorithm, "algorithm"); + this.publicKey = Objects.requireNonNull(publicKey, "publicKey"); + this.signature = Objects.requireNonNull(signature, "signature"); + } + + /** + * Whether the key and signature have exactly the lengths of the algorithm's table. A wrong + * length is an invalid transaction, never a truncated or padded verification. + * + * @return true when both lengths match + */ + public boolean lengthsValid() { + return publicKey.size() == algorithm.publicKeyLength() + && signature.size() == algorithm.signatureLength(); + } + + /** + * The bytes charged as authorization data (AIP-20 gas rule). + * + * @return key length plus signature length + */ + public int authorizationByteCount() { + return publicKey.size() + signature.size(); + } + + /** + * The sender address of AIP-20: {@code keccak256(0x50 || alg_id || public_key)[12:32]}. + * + * @return the derived sender + */ + public Address senderAddress() { + return deriveSender(algorithm, publicKey); + } + + /** + * Derives the AIP-20 sender address for a public key under an algorithm. + * + * @param algorithm the algorithm + * @param publicKey the raw public key + * @return the address + */ + public static Address deriveSender(final AerePqAlgorithm algorithm, final Bytes publicKey) { + final Bytes preimage = + Bytes.concatenate(Bytes.of(TYPE_BYTE), Bytes.of((byte) algorithm.id()), publicKey); + return Address.extract(Bytes32.wrap(Hash.hash(preimage).getBytes())); + } + + /** + * A copy that shares no byte array with the original (for detached transaction copies). + * + * @return the copy + */ + public AerePqAuthorization detachedCopy() { + return new AerePqAuthorization(algorithm, publicKey.copy(), signature.copy()); + } + + @Override + public String toString() { + return "AerePqAuthorization{" + + algorithm + + ", pk=" + + publicKey.size() + + "B, sig=" + + signature.size() + + "B}"; + } +} diff --git a/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/Transaction.java b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/Transaction.java new file mode 100644 index 0000000..8175f0b --- /dev/null +++ b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/Transaction.java @@ -0,0 +1,300 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.datatypes; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +/** + * A transaction is a single cryptographically-signed instruction constructed by an actor externally + * to the scope of Ethereum. While it is assumed that the ultimate external actor will be human in + * nature, software tools will be used in its construction and dissemination. + * + *

There are two types of transactions: those which result in message calls and those which + * result in the creation of new accounts with associated code (known informally as ‘contract + * creation’). Message call transactions will have an address present in the {@link #getTo} method + * whereas contract creation transactions will not. + */ +public interface Transaction { + + /** + * The Keccak 256-bit hash of this transaction. + * + * @return The Keccak 256-bit hash of this transaction. + */ + Hash getHash(); + + /** + * A scalar value equal to the number of transactions sent by the sender. + * + * @return the number of transactions sent by the sender. + */ + long getNonce(); + + /** + * A scalar value equal to the number of Wei to be paid per unit of gas for all computation costs + * incurred as a result of the execution of this transaction. + * + * @return the quantity of Wei per gas unit paid. + */ + Optional getGasPrice(); + + /** + * A scalar value equal to the number of Wei to be paid on top of base fee, as specified in + * EIP-1559. + * + * @return the quantity of Wei for max fee per gas + */ + default Optional getMaxPriorityFeePerGas() { + return Optional.empty(); + } + + /** + * A scalar value equal to the number of Wei to be paid in total, as specified in EIP-1559. + * + * @return the quantity of Wei for fee cap. + */ + default Optional getMaxFeePerGas() { + return Optional.empty(); + } + + /** + * A scalar value equal to the max number of Wei to be paid for blob gas, as specified in + * EIP-4844. + * + * @return the quantity of Wei for fee per blob gas. + */ + default Optional getMaxFeePerBlobGas() { + return Optional.empty(); + } + + /** + * A scalar value equal to the maximum amount of gas that should be used in executing this + * transaction. This is paid up-front, before any computation is done and may not be increased + * later. + * + * @return the maximum amount of gas that should be used in executing this * transaction. + */ + long getGasLimit(); + + /** + * The 160-bit address of the message call’s recipient. For a contract creation transaction this + * address will not be present. + * + * @return address of the recipient + */ + Optional getTo(); + + /** + * A scalar value equal to the number of Wei to be transferred to the message call’s recipient or, + * in the case of contract creation, as an endowment to the newly created account. + * + * @return value equal to the number of Wei to be transferred + */ + Quantity getValue(); + + /** + * Value corresponding to the 'yParity' component of non-legacy signatures. + * + * @return the 'yParity' component of the signature + */ + BigInteger getYParity(); + + /** + * Value corresponding to the 'v' component of legacy signatures, which encodes chainId and + * yParity. + * + * @return the 'V' component of the signature + */ + BigInteger getV(); + + /** + * Value corresponding to the 'R' component of the signature of the transaction. + * + * @return the 'R' component of the signature + */ + BigInteger getR(); + + /** + * Value corresponding to the 'S' component of the signature of the transaction. + * + * @return the 'S' component of the signature + */ + BigInteger getS(); + + /** + * The 160-bit address of the account sending the transaction, extracted from the v, r, s + * parameters. + * + * @return The address of the account that sent this transaction. + */ + Address getSender(); + + /** + * The chainId, computed from the 'V' portion of the signature. Used for replay protection. If + * replay protection is not enabled, this value will not be present. + * + * @return The chainId for transaction. + */ + Optional getChainId(); + + /** + * An unlimited size byte array specifying the EVM-code for the account // initialisation + * procedure. + * + *

Only present if this is a contract creation transaction, which is only true if {@link + * #getTo} is empty. + * + * @return if present, the contract init code. + */ + Optional getInit(); + + /** + * An unlimited size byte array specifying the input data of the message call. + * + *

Only present if this is a message call transaction, which is only true if {@link #getTo} is + * present. + * + * @return if present, the message call data + */ + Optional getData(); + + /** + * The data payload of this transaction. + * + *

If this transaction is a message-call to an account (the {@link #getTo} field is present), + * this same value will be exposed by {@link #getData}. If instead this is a contract-creation + * transaction (the {@link #getTo} field is absent), the payload is also exposed by {@link + * #getInit}. + * + * @return the transaction payload + */ + Bytes getPayload(); + + /** + * The number of zero bytes in the payload. + * + * @return the number of zero bytes in the payload + */ + long getPayloadZeroBytes(); + + /** + * Returns the type of the transaction. + * + * @return the type of the transaction + */ + TransactionType getType(); + + /** + * Return the versioned hashes for this transaction. + * + * @return optional list of versioned hashes + */ + Optional> getVersionedHashes(); + + /** + * Return the blobs with commitments for this transaction. + * + * @return optional blobs with commitments + */ + Optional getBlobsWithCommitments(); + + /** + * Return the address of the contract, if the transaction creates one + * + * @return address of new contract or empty otherwise + */ + Optional

contractAddress(); + + /** + * Return the access list in case of EIP-2930 transaction + * + * @return optional access list + */ + Optional> getAccessList(); + + /** + * Returns the transaction with the proper encoding + * + * @return the encoded transaction as Bytes + */ + Bytes encoded(); + + /** + * Returns the preimage of the transaction with the proper encoding + * + * @return the encoded preimage of the transaction as Bytes + */ + Bytes encodedPreimage(); + + /** + * Returns the size in bytes of the encoded transaction. This is the size of the transaction when + * it is announced to other peers. The difference between this and {@link + * #getSizeForBlockInclusion()} is that transactions that are included in a block cannot contain + * blobs, whereas transactions that are announced and sent to other peers have to contain blobs. + * + * @return the size in bytes of the encoded transaction. + */ + int getSizeForAnnouncement(); + + /** + * Returns the size in bytes of the encoded transaction for block inclusion. This is the size of + * the transaction when it is included in a block. The difference between this and {@link + * #getSizeForAnnouncement()} is that transactions that are included in a block cannot contain + * blobs, whereas transactions that are announced and sent to other peers have to contain blobs. + * + * @return the size in bytes of the encoded transaction for block inclusion. + */ + int getSizeForBlockInclusion(); + + /** + * Returns whether the transaction is a contract creation + * + * @return {@code true} if this is a contract-creation transaction; otherwise {@code false} + */ + boolean isContractCreation(); + + /** + * Returns the code delegations if this transaction is a 7702 transaction. + * + * @return the code delegations + */ + Optional> getCodeDelegationList(); + + /** + * Returns the size of the code delegation list. + * + * @return the size of the code delegation list + */ + int codeDelegationListSize(); + + /** + * The AIP-20 post-quantum authorization, present only for type 0x50 transactions. + * + * @return the authorization, or empty for every other type + */ + default Optional getPqAuthorization() { + return Optional.empty(); + } +} diff --git a/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/TransactionType.java b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/TransactionType.java new file mode 100644 index 0000000..cf6e077 --- /dev/null +++ b/anchor/datatypes/src/main/java/org/hyperledger/besu/datatypes/TransactionType.java @@ -0,0 +1,209 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.datatypes; + +import java.util.EnumSet; +import java.util.Optional; +import java.util.Set; + +/** The enum Transaction type. */ +public enum TransactionType { + /** The Frontier. */ + FRONTIER(0xf8, 0x00), + /** Access list transaction type. */ + ACCESS_LIST(0x01), + /** Eip1559 transaction type. */ + EIP1559(0x02), + /** Blob transaction type. */ + BLOB(0x03), + /** Eip7702 transaction type. */ + DELEGATE_CODE(0x04), + /** AIP-20: Aere post-quantum authorized transaction (ML-DSA), no secp256k1 signature. */ + AERE_PQ(0x50); + + private static final Set ACCESS_LIST_SUPPORTED_TRANSACTION_TYPES = + EnumSet.of(ACCESS_LIST, EIP1559, BLOB, DELEGATE_CODE, AERE_PQ); + + private static final Set LEGACY_FEE_MARKET_TRANSACTION_TYPES = + EnumSet.of(FRONTIER, ACCESS_LIST); + + // The theoretical boundaries of the first byte of the RLP of a Frontier transaction. + // In practice Frontier transactions almost always start with 0xf8 or 0xf9. + private static final byte MIN_LEGACY_TX_OPAQUE_BYTE = (byte) 0xc0; + private static final byte MAX_LEGACY_TX_OPAQUE_BYTE = (byte) 0xfe; + + // This array helps to translate the first opaque byte of an RLP encoded transaction + // to its type. + // Note that Frontier type occupies more slots, since there are different first byte values, + // that represent such type. + // Holes in the array represents invalid first byte values, for which there is not (yet) + // a defined transaction type + private static final TransactionType[] transactionTypeByOpaqueByte = + new TransactionType[Byte.toUnsignedInt(MAX_LEGACY_TX_OPAQUE_BYTE) + 1]; + + private static final TransactionType[] transactionTypeByEthSerializedType = + // sized by the byte space, not by the enum: AERE_PQ is 0x50 (AIP-20), far past values().length + new TransactionType[256]; + + static { + EnumSet.allOf(TransactionType.class).stream() + .forEach( + tt -> { + tt.requireChainId = tt != FRONTIER; + tt.supportAccessList = ACCESS_LIST_SUPPORTED_TRANSACTION_TYPES.contains(tt); + tt.supportBaseFeeMarket = !LEGACY_FEE_MARKET_TRANSACTION_TYPES.contains(tt); + tt.supportBlob = tt == BLOB; + tt.supportDelegatedCode = tt == DELEGATE_CODE; + tt.supportPqAuthorization = tt == AERE_PQ; + if (tt == FRONTIER) { + for (int i = Byte.toUnsignedInt(MIN_LEGACY_TX_OPAQUE_BYTE); + i < Byte.toUnsignedInt(MAX_LEGACY_TX_OPAQUE_BYTE); + i++) { + transactionTypeByOpaqueByte[i] = FRONTIER; + } + } else { + transactionTypeByOpaqueByte[tt.getSerializedType()] = tt; + } + transactionTypeByEthSerializedType[tt.getEthSerializedType()] = tt; + }); + } + + private final byte typeValue; + private final byte serializedType; + boolean requireChainId; + boolean supportAccessList; + boolean supportBaseFeeMarket; + boolean supportBlob; + boolean supportDelegatedCode; + boolean supportPqAuthorization; + + TransactionType(final int typeValue, final int serializedType) { + this.typeValue = (byte) typeValue; + this.serializedType = (byte) serializedType; + } + + TransactionType(final int typeValue) { + this(typeValue, typeValue); + } + + /** + * Gets serialized type. + * + * @return the serialized type + */ + public byte getSerializedType() { + return typeValue; + } + + /** + * Gets serialized type for returning in an eth transaction result, factoring in the special case + * for FRONTIER transactions which have enum type 0xf8 but are represented as 0x00 in transaction + * results. + * + * @return the serialized type + */ + public byte getEthSerializedType() { + return serializedType; + } + + /** + * Convert from opaque byte to type value. + * + * @param opaqueByte the opaque byte from serialized bytes + * @return the transaction type + */ + public static Optional fromOpaque(final byte opaqueByte) { + try { + return Optional.ofNullable(transactionTypeByOpaqueByte[Byte.toUnsignedInt(opaqueByte)]); + } catch (final ArrayIndexOutOfBoundsException e) { + return Optional.empty(); + } + } + + /** + * Convert from ETH serialized byte to type value. + * + * @param ethSerializedType the opaque byte from serialized bytes + * @return the transaction type + */ + public static Optional fromEthSerializedType(final byte ethSerializedType) { + try { + return Optional.ofNullable(transactionTypeByEthSerializedType[ethSerializedType]); + } catch (final ArrayIndexOutOfBoundsException e) { + return Optional.empty(); + } + } + + /** + * Does transaction type support access list. + * + * @return the boolean + */ + public boolean supportsAccessList() { + return supportAccessList; + } + + /** + * Does transaction type support EIP-1559 fee market. + * + * @return the boolean + */ + public boolean supports1559FeeMarket() { + return supportBaseFeeMarket; + } + + /** + * Does transaction type require chain id. + * + * @return the boolean + */ + public boolean requiresChainId() { + return requireChainId; + } + + /** + * Does transaction type support data blobs. + * + * @return the boolean + */ + public boolean supportsBlob() { + return supportBlob; + } + + /** + * Does transaction type support delegate code. + * + * @return the boolean + */ + public boolean supportsDelegateCode() { + return supportDelegatedCode; + } + + /** + * Does transaction type carry an AIP-20 post-quantum authorization instead of a secp256k1 + * signature. + * + * @return the boolean + */ + public boolean supportsPqAuthorization() { + return supportPqAuthorization; + } +} diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java new file mode 100644 index 0000000..cdcaafc --- /dev/null +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java @@ -0,0 +1,210 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0007-aere-pq-finality.patch. + */ +package org.hyperledger.besu.ethereum.api.jsonrpc; + +import java.util.Collection; +import java.util.HashSet; + +public enum RpcMethod { + ADMIN_ADD_PEER("admin_addPeer"), + ADMIN_NODE_INFO("admin_nodeInfo"), + ADMIN_PEERS("admin_peers"), + ADMIN_REMOVE_PEER("admin_removePeer"), + ADMIN_CHANGE_LOG_LEVEL("admin_changeLogLevel"), + ADMIN_GENERATE_LOG_BLOOM_CACHE("admin_generateLogBloomCache"), + ADMIN_LOGS_REPAIR_CACHE("admin_logsRepairCache"), + ADMIN_LOGS_REMOVE_CACHE("admin_logsRemoveCache"), + DEBUG_ACCOUNT_AT("debug_accountAt"), + DEBUG_ACCOUNT_RANGE("debug_accountRange"), + DEBUG_METRICS("debug_metrics"), + DEBUG_RESYNC_WORLDSTATE("debug_resyncWorldState"), + DEBUG_SET_HEAD("debug_setHead"), + DEBUG_REPLAY_BLOCK("debug_replayBlock"), + DEBUG_STORAGE_RANGE_AT("debug_storageRangeAt"), + DEBUG_TRACE_BLOCK("debug_traceBlock"), + DEBUG_TRACE_BLOCK_BY_HASH("debug_traceBlockByHash"), + DEBUG_TRACE_BLOCK_BY_NUMBER("debug_traceBlockByNumber"), + DEBUG_STANDARD_TRACE_BLOCK_TO_FILE("debug_standardTraceBlockToFile"), + DEBUG_STANDARD_TRACE_BAD_BLOCK_TO_FILE("debug_standardTraceBadBlockToFile"), + DEBUG_TRACE_TRANSACTION("debug_traceTransaction"), + DEBUG_TRACE_CALL("debug_traceCall"), + DEBUG_BATCH_RAW_TRANSACTION("debug_batchSendRawTransaction"), + DEBUG_GET_BAD_BLOCKS("debug_getBadBlocks"), + DEBUG_GET_RAW_HEADER("debug_getRawHeader"), + DEBUG_GET_RAW_BLOCK("debug_getRawBlock"), + DEBUG_GET_RAW_RECEIPTS("debug_getRawReceipts"), + DEBUG_GET_RAW_TRANSACTION("debug_getRawTransaction"), + ENGINE_GET_BLOBS_V1("engine_getBlobsV1"), + ENGINE_GET_BLOBS_V2("engine_getBlobsV2"), + ENGINE_GET_BLOBS_V3("engine_getBlobsV3"), + ENGINE_GET_PAYLOAD_V1("engine_getPayloadV1"), + ENGINE_GET_PAYLOAD_V2("engine_getPayloadV2"), + ENGINE_GET_PAYLOAD_V3("engine_getPayloadV3"), + ENGINE_GET_PAYLOAD_V4("engine_getPayloadV4"), + ENGINE_GET_PAYLOAD_V5("engine_getPayloadV5"), + ENGINE_GET_PAYLOAD_V6("engine_getPayloadV6"), + ENGINE_NEW_PAYLOAD_V1("engine_newPayloadV1"), + ENGINE_NEW_PAYLOAD_V2("engine_newPayloadV2"), + ENGINE_NEW_PAYLOAD_V3("engine_newPayloadV3"), + ENGINE_NEW_PAYLOAD_V4("engine_newPayloadV4"), + ENGINE_NEW_PAYLOAD_V5("engine_newPayloadV5"), + ENGINE_FORKCHOICE_UPDATED_V1("engine_forkchoiceUpdatedV1"), + ENGINE_FORKCHOICE_UPDATED_V2("engine_forkchoiceUpdatedV2"), + ENGINE_FORKCHOICE_UPDATED_V3("engine_forkchoiceUpdatedV3"), + ENGINE_FORKCHOICE_UPDATED_V4("engine_forkchoiceUpdatedV4"), + ENGINE_EXCHANGE_TRANSITION_CONFIGURATION("engine_exchangeTransitionConfigurationV1"), + ENGINE_GET_CLIENT_VERSION_V1("engine_getClientVersionV1"), + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1("engine_getPayloadBodiesByHashV1"), + ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V2("engine_getPayloadBodiesByHashV2"), + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1("engine_getPayloadBodiesByRangeV1"), + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V2("engine_getPayloadBodiesByRangeV2"), + ENGINE_EXCHANGE_CAPABILITIES("engine_exchangeCapabilities"), + ENGINE_PREPARE_PAYLOAD_DEBUG("engine_preparePayload_debug"), + ETH_ACCOUNTS("eth_accounts"), + ETH_BLOCK_NUMBER("eth_blockNumber"), + ETH_CALL("eth_call"), + ETH_CHAIN_ID("eth_chainId"), + ETH_COINBASE("eth_coinbase"), + ETH_CONFIG("eth_config"), + ETH_ESTIMATE_GAS("eth_estimateGas"), + ETH_CREATE_ACCESS_LIST("eth_createAccessList"), + ETH_FEE_HISTORY("eth_feeHistory"), + ETH_GAS_PRICE("eth_gasPrice"), + ETH_BLOB_BASE_FEE("eth_blobBaseFee"), + ETH_GET_BALANCE("eth_getBalance"), + ETH_GET_BLOCK_ACCESS_LIST("eth_getBlockAccessList"), + ETH_GET_BLOCK_BY_HASH("eth_getBlockByHash"), + ETH_GET_BLOCK_BY_NUMBER("eth_getBlockByNumber"), + ETH_GET_BLOCK_RECEIPTS("eth_getBlockReceipts"), + ETH_GET_BLOCK_TRANSACTION_COUNT_BY_HASH("eth_getBlockTransactionCountByHash"), + ETH_GET_BLOCK_TRANSACTION_COUNT_BY_NUMBER("eth_getBlockTransactionCountByNumber"), + ETH_GET_CODE("eth_getCode"), + ETH_GET_FILTER_CHANGES("eth_getFilterChanges"), + ETH_GET_FILTER_LOGS("eth_getFilterLogs"), + ETH_GET_LOGS("eth_getLogs"), + ETH_GET_MAX_PRIORITY_FEE_PER_GAS("eth_maxPriorityFeePerGas"), + ETH_GET_PROOF("eth_getProof"), + ETH_GET_STORAGE_AT("eth_getStorageAt"), + ETH_GET_TRANSACTION_BY_BLOCK_HASH_AND_INDEX("eth_getTransactionByBlockHashAndIndex"), + ETH_GET_TRANSACTION_BY_BLOCK_NUMBER_AND_INDEX("eth_getTransactionByBlockNumberAndIndex"), + ETH_GET_TRANSACTION_BY_HASH("eth_getTransactionByHash"), + ETH_GET_TRANSACTION_COUNT("eth_getTransactionCount"), + ETH_GET_TRANSACTION_RECEIPT("eth_getTransactionReceipt"), + ETH_GET_UNCLE_BY_BLOCK_HASH_AND_INDEX("eth_getUncleByBlockHashAndIndex"), + ETH_GET_UNCLE_BY_BLOCK_NUMBER_AND_INDEX("eth_getUncleByBlockNumberAndIndex"), + ETH_GET_UNCLE_COUNT_BY_BLOCK_HASH("eth_getUncleCountByBlockHash"), + ETH_GET_UNCLE_COUNT_BY_BLOCK_NUMBER("eth_getUncleCountByBlockNumber"), + ETH_MINING("eth_mining"), + ETH_NEW_BLOCK_FILTER("eth_newBlockFilter"), + ETH_NEW_FILTER("eth_newFilter"), + ETH_NEW_PENDING_TRANSACTION_FILTER("eth_newPendingTransactionFilter"), + ETH_PROTOCOL_VERSION("eth_protocolVersion"), + ETH_SEND_RAW_TRANSACTION("eth_sendRawTransaction"), + ETH_SEND_TRANSACTION("eth_sendTransaction"), + ETH_SIMULATE_V1("eth_simulateV1"), + ETH_SUBSCRIBE("eth_subscribe"), + ETH_SYNCING("eth_syncing"), + ETH_UNINSTALL_FILTER("eth_uninstallFilter"), + ETH_UNSUBSCRIBE("eth_unsubscribe"), + IBFT_DISCARD_VALIDATOR_VOTE("ibft_discardValidatorVote"), + IBFT_GET_PENDING_VOTES("ibft_getPendingVotes"), + IBFT_GET_VALIDATORS_BY_BLOCK_HASH("ibft_getValidatorsByBlockHash"), + IBFT_GET_VALIDATORS_BY_BLOCK_NUMBER("ibft_getValidatorsByBlockNumber"), + IBFT_PROPOSE_VALIDATOR_VOTE("ibft_proposeValidatorVote"), + IBFT_GET_SIGNER_METRICS("ibft_getSignerMetrics"), + QBFT_DISCARD_VALIDATOR_VOTE("qbft_discardValidatorVote"), + QBFT_GET_PENDING_VOTES("qbft_getPendingVotes"), + QBFT_GET_VALIDATORS_BY_BLOCK_HASH("qbft_getValidatorsByBlockHash"), + QBFT_GET_VALIDATORS_BY_BLOCK_NUMBER("qbft_getValidatorsByBlockNumber"), + QBFT_PROPOSE_VALIDATOR_VOTE("qbft_proposeValidatorVote"), + QBFT_GET_SIGNER_METRICS("qbft_getSignerMetrics"), + QBFT_GET_REQUEST_TIMEOUT_SECONDS("qbft_getRequestTimeoutSeconds"), + // AIP-21 (AERE, 2026-09-18): the per-block post-quantum finality record, read from the seals heard on Commit messages + AERE_GET_PQ_FINALITY("aere_getPqFinality"), + MINER_CHANGE_TARGET_GAS_LIMIT("miner_changeTargetGasLimit"), + MINER_START("miner_start"), + MINER_STOP("miner_stop"), + MINER_GET_MIN_PRIORITY_FEE("miner_getMinPriorityFee"), + MINER_SET_MIN_PRIORITY_FEE("miner_setMinPriorityFee"), + MINER_GET_MIN_GAS_PRICE("miner_getMinGasPrice"), + MINER_SET_MIN_GAS_PRICE("miner_setMinGasPrice"), + MINER_GET_EXTRA_DATA("miner_getExtraData"), + MINER_SET_EXTRA_DATA("miner_setExtraData"), + NET_ENODE("net_enode"), + NET_LISTENING("net_listening"), + NET_PEER_COUNT("net_peerCount"), + NET_SERVICES("net_services"), + NET_VERSION("net_version"), + PERM_ADD_ACCOUNTS_TO_WHITELIST("perm_addAccountsToWhitelist"), + PERM_ADD_ACCOUNTS_TO_ALLOWLIST("perm_addAccountsToAllowlist"), + PERM_ADD_NODES_TO_WHITELIST("perm_addNodesToWhitelist"), + PERM_ADD_NODES_TO_ALLOWLIST("perm_addNodesToAllowlist"), + PERM_GET_ACCOUNTS_WHITELIST("perm_getAccountsWhitelist"), + PERM_GET_ACCOUNTS_ALLOWLIST("perm_getAccountsAllowlist"), + PERM_GET_NODES_WHITELIST("perm_getNodesWhitelist"), + PERM_GET_NODES_ALLOWLIST("perm_getNodesAllowlist"), + PERM_RELOAD_PERMISSIONS_FROM_FILE("perm_reloadPermissionsFromFile"), + PERM_REMOVE_ACCOUNTS_FROM_ALLOWLIST("perm_removeAccountsFromAllowlist"), + PERM_REMOVE_NODES_FROM_ALLOWLIST("perm_removeNodesFromAllowlist"), + RPC_MODULES("rpc_modules"), + TRACE_BLOCK("trace_block"), + TRACE_CALL("trace_call"), + TRACE_CALL_MANY("trace_callMany"), + TRACE_GET("trace_get"), + TRACE_FILTER("trace_filter"), + TRACE_RAW_TRANSACTION("trace_rawTransaction"), + TRACE_REPLAY_BLOCK_TRANSACTIONS("trace_replayBlockTransactions"), + TRACE_TRANSACTION("trace_transaction"), + TX_POOL_BESU_STATISTICS("txpool_besuStatistics"), + TX_POOL_BESU_TRANSACTIONS("txpool_besuTransactions"), + TX_POOL_BESU_PENDING_TRANSACTIONS("txpool_besuPendingTransactions"), + TX_POOL_STATUS("txpool_status"), + TX_POOL_CONTENT_FROM("txpool_contentFrom"), + TX_POOL_CONTENT("txpool_content"), + TX_POOL_INSPECT("txpool_inspect"), + WEB3_CLIENT_VERSION("web3_clientVersion"), + WEB3_SHA3("web3_sha3"), + PLUGINS_RELOAD_CONFIG("plugins_reloadPluginConfig"), + TESTING_BUILD_BLOCK_V1("testing_buildBlockV1"); + + private final String methodName; + + private static final Collection allMethodNames; + + public String getMethodName() { + return methodName; + } + + static { + allMethodNames = new HashSet<>(); + for (RpcMethod m : RpcMethod.values()) { + allMethodNames.add(m.getMethodName()); + } + } + + RpcMethod(final String methodName) { + this.methodName = methodName; + } + + public static boolean rpcMethodExists(final String rpcMethodName) { + return allMethodNames.contains(rpcMethodName); + } +} diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetBalance.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetBalance.java index 0f43576..5c74d44 100644 --- a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetBalance.java +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetBalance.java @@ -11,6 +11,12 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetCode.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetCode.java index b4be077..52bc65e 100644 --- a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetCode.java +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetCode.java @@ -11,6 +11,12 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetStorageAt.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetStorageAt.java index fcd9909..b6a3e95 100644 --- a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetStorageAt.java +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetStorageAt.java @@ -11,6 +11,12 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCount.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCount.java index d8ba86c..291056a 100644 --- a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCount.java +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetTransactionCount.java @@ -11,6 +11,12 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionBaseResult.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionBaseResult.java new file mode 100644 index 0000000..b24ba10 --- /dev/null +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionBaseResult.java @@ -0,0 +1,322 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.results; + +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.BytesHolder; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; + +import java.util.List; +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.apache.tuweni.bytes.Bytes; + +@JsonPropertyOrder({ + "accessList", + "authorizationList", + "blockHash", + "blockNumber", + "blockTimestamp", + "chainId", + "from", + "gas", + "gasPrice", + "maxPriorityFeePerGas", + "maxFeePerGas", + "maxFeePerBlobGas", + "hash", + "input", + "nonce", + "to", + "transactionIndex", + "type", + "value", + "yParity", + "v", + "r", + "s", + "algId", + "pqPublicKey", + "pqSignature", + "blobVersionedHashes" +}) +public class TransactionBaseResult implements TransactionResult { + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final List accessList; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String chainId; + + private final String from; + private final String gas; + private final String gasPrice; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String maxPriorityFeePerGas; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String maxFeePerGas; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String maxFeePerBlobGas; + + private final String hash; + private final String input; + private final String nonce; + private final String to; + private final String type; + private final String value; + private final String yParity; + private final String v; + private final String r; + private final String s; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String algId; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String pqPublicKey; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String pqSignature; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final List versionedHashes; + + @JsonInclude(JsonInclude.Include.NON_NULL) + private final List authorizationList; + + public TransactionBaseResult(final Transaction transaction) { + this(transaction, Optional.empty()); + } + + public TransactionBaseResult(final Transaction transaction, final Optional maybeBaseFee) { + final TransactionType transactionType = transaction.getType(); + this.accessList = + transaction.getAccessList().orElse(transactionType.supportsAccessList() ? List.of() : null); + this.chainId = transaction.getChainId().map(Quantity::create).orElse(null); + this.from = transaction.getSender().toString(); + this.gas = Quantity.create(transaction.getGasLimit()); + this.maxPriorityFeePerGas = + transaction.getMaxPriorityFeePerGas().map(Wei::toShortHexString).orElse(null); + this.maxFeePerGas = transaction.getMaxFeePerGas().map(Wei::toShortHexString).orElse(null); + this.maxFeePerBlobGas = + transaction.getMaxFeePerBlobGas().map(Wei::toShortHexString).orElse(null); + this.gasPrice = + Quantity.create( + transaction + .getGasPrice() + .orElseGet( + () -> + maybeBaseFee.isPresent() + ? transaction.getEffectiveGasPrice(maybeBaseFee) + : transaction.getMaxFeePerGas().get())); + this.hash = transaction.getHash().toString(); + this.input = transaction.getPayload().toString(); + this.nonce = Quantity.create(transaction.getNonce()); + this.to = transaction.getTo().map(a -> a.getBytes().toHexString()).orElse(null); + if (transactionType == TransactionType.FRONTIER) { + this.type = Quantity.create(0); + this.yParity = null; + this.v = Quantity.create(transaction.getV()); + } else { + this.type = Quantity.create(transactionType.getSerializedType()); + this.yParity = + transaction.getYParity() == null ? null : Quantity.create(transaction.getYParity()); + this.v = + (transactionType == TransactionType.ACCESS_LIST + || transactionType == TransactionType.EIP1559 + || transactionType == TransactionType.DELEGATE_CODE + || transactionType == TransactionType.BLOB) + ? Quantity.create(transaction.getYParity()) + : null; + } + this.value = Quantity.create(transaction.getValue()); + this.r = transaction.getR() == null ? null : Quantity.create(transaction.getR()); + this.s = transaction.getS() == null ? null : Quantity.create(transaction.getS()); + this.algId = + transaction.getPqAuthorization().map(a -> Quantity.create(a.algorithm().id())).orElse(null); + this.pqPublicKey = + transaction.getPqAuthorization().map(a -> a.publicKey().toHexString()).orElse(null); + this.pqSignature = + transaction.getPqAuthorization().map(a -> a.signature().toHexString()).orElse(null); + this.versionedHashes = + transaction + .getVersionedHashes() + .map( + hashes -> + hashes.stream().map(BytesHolder::getBytes).map(Bytes::toHexString).toList()) + .orElse(null); + this.authorizationList = + transaction + .getCodeDelegationList() + .map(cds -> cds.stream().map(CodeDelegationResult::new).toList()) + .orElse(null); + } + + @JsonGetter(value = "accessList") + public List getAccessList() { + return accessList; + } + + @JsonGetter(value = "chainId") + public String getChainId() { + return chainId; + } + + @JsonGetter(value = "from") + public String getFrom() { + return from; + } + + @JsonGetter(value = "gas") + public String getGas() { + return gas; + } + + @JsonGetter(value = "gasPrice") + public String getGasPrice() { + return gasPrice; + } + + @JsonGetter(value = "maxPriorityFeePerGas") + public String getMaxPriorityFeePerGas() { + return maxPriorityFeePerGas; + } + + @JsonGetter(value = "maxFeePerGas") + public String getMaxFeePerGas() { + return maxFeePerGas; + } + + @JsonGetter(value = "maxFeePerBlobGas") + public String getMaxFeePerBlobGas() { + return maxFeePerBlobGas; + } + + @JsonGetter(value = "hash") + public String getHash() { + return hash; + } + + @JsonGetter(value = "input") + public String getInput() { + return input; + } + + @JsonGetter(value = "nonce") + public String getNonce() { + return nonce; + } + + @JsonGetter(value = "to") + public String getTo() { + return to; + } + + @JsonGetter(value = "type") + public String getType() { + return type; + } + + @JsonGetter(value = "value") + public String getValue() { + return value; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "yParity") + public String getYParity() { + return yParity; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "v") + public String getV() { + return v; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "r") + public String getR() { + return r; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "s") + public String getS() { + return s; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "algId") + public String getAlgId() { + return algId; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "pqPublicKey") + public String getPqPublicKey() { + return pqPublicKey; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "pqSignature") + public String getPqSignature() { + return pqSignature; + } + + @JsonGetter(value = "blobVersionedHashes") + public List getVersionedHashes() { + return versionedHashes; + } + + @JsonGetter(value = "authorizationList") + public List getAuthorizationList() { + return authorizationList; + } + + @JsonGetter(value = "blockHash") + public String getBlockHash() { + return null; + } + + @JsonGetter(value = "blockNumber") + public String getBlockNumber() { + return null; + } + + @JsonGetter(value = "blockTimestamp") + public String getBlockTimestamp() { + return null; + } + + @JsonGetter(value = "transactionIndex") + public String getTransactionIndex() { + return null; + } +} diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java new file mode 100644 index 0000000..d03b782 --- /dev/null +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java @@ -0,0 +1,155 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.pending; + +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.JsonRpcResult; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.Quantity; +import org.hyperledger.besu.ethereum.core.Transaction; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({ + "from", + "gas", + "gasPrice", + "hash", + "input", + "nonce", + "to", + "type", + "value", + "yParity", + "v", + "r", + "s" +}) +public class PendingTransactionDetailResult implements JsonRpcResult { + private final String from; + private final String gas; + private final String gasPrice; + private final String hash; + private final String input; + private final String nonce; + private final String to; + private final String type; + private final String value; + private final String yParity; + private final String v; + private final String r; + private final String s; + + public PendingTransactionDetailResult(final Transaction tx) { + TransactionType transactionType = tx.getType(); + this.from = tx.getSender().toString(); + this.gas = Quantity.create(tx.getGasLimit()); + this.gasPrice = tx.getGasPrice().map(Quantity::create).orElse(null); + this.hash = tx.getHash().toString(); + this.input = tx.getPayload().toString(); + this.nonce = Quantity.create(tx.getNonce()); + this.to = tx.getTo().map(a -> a.getBytes().toHexString()).orElse(null); + if (transactionType == TransactionType.FRONTIER) { + this.type = Quantity.create(0); + this.yParity = null; + this.v = Quantity.create(tx.getV()); + } else { + this.type = Quantity.create(transactionType.getSerializedType()); + this.yParity = tx.getYParity() == null ? null : Quantity.create(tx.getYParity()); + this.v = + (transactionType == TransactionType.ACCESS_LIST + || transactionType == TransactionType.EIP1559) + ? Quantity.create(tx.getYParity()) + : null; + } + this.value = Quantity.create(tx.getValue()); + this.r = tx.getR() == null ? null : Quantity.create(tx.getR()); + this.s = tx.getS() == null ? null : Quantity.create(tx.getS()); + } + + @JsonGetter(value = "from") + public String getFrom() { + return from; + } + + @JsonGetter(value = "gas") + public String getGas() { + return gas; + } + + @JsonGetter(value = "gasPrice") + public String getGasPrice() { + return gasPrice; + } + + @JsonGetter(value = "hash") + public String getHash() { + return hash; + } + + @JsonGetter(value = "input") + public String getInput() { + return input; + } + + @JsonGetter(value = "nonce") + public String getNonce() { + return nonce; + } + + @JsonGetter(value = "to") + public String getTo() { + return to; + } + + @JsonGetter(value = "type") + public String getType() { + return type; + } + + @JsonGetter(value = "value") + public String getValue() { + return value; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "yParity") + public String getyParity() { + return yParity; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(value = "v") + public String getV() { + return v; + } + + @JsonGetter(value = "r") + public String getR() { + return r; + } + + @JsonGetter(value = "s") + public String getS() { + return s; + } +} diff --git a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java index caed487..b7e4c45 100644 --- a/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java +++ b/anchor/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java @@ -11,6 +11,12 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-drop-per-request-console-writes.patch. */ package org.hyperledger.besu.ethereum.api.query; @@ -935,7 +941,7 @@ public class BlockchainQueries { } } } catch (final IOException e) { - e.printStackTrace(System.out); + // AERE D-434 (fbfb5f9ecd): removed printStackTrace(System.out) on the cached-log-blooms error path; a cheap malformed log query could otherwise spam the process console, bypassing log4j. LOG.error below records the exception. LOG.error("Error reading cached log blooms", e); } return results; diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java new file mode 100644 index 0000000..ec42cfd --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -0,0 +1,1647 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.core; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static org.hyperledger.besu.crypto.Hash.keccak256; +import static org.hyperledger.besu.datatypes.VersionedHash.SHA256_VERSION_ID; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SECPPublicKey; +import org.hyperledger.besu.crypto.SECPSignature; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.AerePqAuthorization; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.BlobType; +import org.hyperledger.besu.datatypes.BytesHolder; +import org.hyperledger.besu.datatypes.CodeDelegation; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Sha256Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.VersionedHash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.encoding.AccessListTransactionEncoder; +import org.hyperledger.besu.ethereum.core.encoding.AerePqTransactionEncoder; +import org.hyperledger.besu.ethereum.core.encoding.BlobTransactionEncoder; +import org.hyperledger.besu.ethereum.core.encoding.CodeDelegationTransactionEncoder; +import org.hyperledger.besu.ethereum.core.encoding.EncodingContext; +import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder; +import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder; +import org.hyperledger.besu.ethereum.core.kzg.Blob; +import org.hyperledger.besu.ethereum.core.kzg.BlobsWithCommitments; +import org.hyperledger.besu.ethereum.core.kzg.KZGCommitment; +import org.hyperledger.besu.ethereum.core.kzg.KZGProof; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.math.BigInteger; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.primitives.Longs; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.apache.tuweni.units.bigints.UInt256s; + +/** An operation submitted by an external actor to be applied to the system. */ +public class Transaction + implements org.hyperledger.besu.datatypes.Transaction, + org.hyperledger.besu.plugin.data.UnsignedPrivateMarkerTransaction { + + // Used for transactions that are not tied to a specific chain + // (e.g. does not have a chain id associated with it). + public static final BigInteger REPLAY_UNPROTECTED_V_BASE = BigInteger.valueOf(27); + public static final BigInteger REPLAY_UNPROTECTED_V_BASE_PLUS_1 = BigInteger.valueOf(28); + + public static final BigInteger REPLAY_PROTECTED_V_BASE = BigInteger.valueOf(35); + + // The v signature parameter starts at 36 because 1 is the first valid chainId so: + // chainId > 1 implies that 2 * chainId + V_BASE > 36. + public static final BigInteger REPLAY_PROTECTED_V_MIN = BigInteger.valueOf(36); + + public static final BigInteger TWO = BigInteger.valueOf(2); + + private static final Cache senderCache = + CacheBuilder.newBuilder().recordStats().maximumSize(100_000L).build(); + + private final long nonce; + + private final Optional gasPrice; + + private final Optional maxPriorityFeePerGas; + + private final Optional maxFeePerGas; + private final Optional maxFeePerBlobGas; + + private final long gasLimit; + + private final Optional
to; + + private final Wei value; + + private final SECPSignature signature; + + private final Payload payload; + + private final Optional> maybeAccessList; + + private final Optional chainId; + + // Caches a "hash" of a portion of the transaction used for sender recovery. + // Note that this hash does not include the transaction signature, so it does not + // fully identify the transaction (use the result of the {@code hash()} for that). + // It is only used to compute said signature and recover the sender from it. + private volatile Bytes32 hashNoSignature; + + // Caches the transaction sender. + private volatile Address sender; + + // Caches the hash used to uniquely identify the transaction. + private volatile Hash hash; + // Caches the size in bytes of the encoded transaction. + private volatile int sizeForAnnouncement = -1; + private volatile int sizeForBlockInclusion = -1; + private final TransactionType transactionType; + + private final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance(); + private final Optional> versionedHashes; + + private final Optional blobsWithCommitments; + private final Optional> maybeCodeDelegationList; + + private final Optional rawRlp; + + // AIP-20: present only for type 0x50; then `signature` is null and the sender comes from the key. + private final Optional pqAuthorization; + + public static Builder builder() { + return new Builder(); + } + + public static Transaction readFrom(final Bytes rlpBytes) { + return readFrom(RLP.input(rlpBytes)); + } + + /** + * Recreate a transaction from RLP serialized in the block body format + * + * @param rlpInput the RLP input in block body format + * @return the transaction + */ + public static Transaction readFrom(final RLPInput rlpInput) { + return readFrom(rlpInput, EncodingContext.BLOCK_BODY); + } + + /** + * Recreate a transaction from RLP serialized according to the specified format + * + * @param rlpInput the RLP input in block body format + * @param context specify in which format the RLP was serialized + * @return the transaction + */ + public static Transaction readFrom(final RLPInput rlpInput, final EncodingContext context) { + return TransactionDecoder.decodeRLP(rlpInput, context); + } + + /** + * Instantiates a transaction instance. + * + * @param forCopy true when using to create a copy of an already validated transaction avoid to + * redo the validation + * @param transactionType the transaction type + * @param nonce the nonce + * @param gasPrice the gas price + * @param maxPriorityFeePerGas the max priority fee per gas + * @param maxFeePerGas the max fee per gas + * @param maxFeePerBlobGas the max fee per blob gas + * @param gasLimit the gas limit + * @param to the transaction recipient + * @param value the value being transferred to the recipient + * @param signature the signature + * @param payload the payload + * @param maybeAccessList the optional list of addresses/storage slots this transaction intends to + * preload + * @param sender the transaction sender + * @param chainId the chain id to apply the transaction to + *

The {@code to} will be an {@code Optional.empty()} for a contract creation transaction; + * otherwise it should contain an address. + *

The {@code chainId} must be greater than 0 to be applied to a specific chain; otherwise + * it will default to any chain. + * @param hash the transaction hash + * @param sizeForAnnouncement the size of the transaction, used for announcement + * @param sizeForBlockInclusion the size of the transaction, used for block inclusion + */ + private Transaction( + final boolean forCopy, + final TransactionType transactionType, + final long nonce, + final Optional gasPrice, + final Optional maxPriorityFeePerGas, + final Optional maxFeePerGas, + final Optional maxFeePerBlobGas, + final long gasLimit, + final Optional

to, + final Wei value, + final SECPSignature signature, + final Payload payload, + final Optional> maybeAccessList, + final Address sender, + final Optional chainId, + final Optional> versionedHashes, + final Optional blobsWithCommitments, + final Optional> maybeCodeDelegationList, + final Optional rawRlp, + final Optional pqAuthorization, + final Optional hash, + final Optional sizeForAnnouncement, + final Optional sizeForBlockInclusion) { + + if (!forCopy) { + if (transactionType.requiresChainId()) { + checkArgument( + chainId.isPresent(), + "Chain id must be present for transaction type %s", + transactionType); + } + + if (maybeAccessList.isPresent()) { + checkArgument( + transactionType.supportsAccessList(), + "Must not specify access list for transaction not supporting it"); + } + + if (Objects.equals(transactionType, TransactionType.ACCESS_LIST)) { + checkArgument( + maybeAccessList.isPresent(), "Must specify access list for access list transaction"); + } + + if (versionedHashes.isPresent() || maxFeePerBlobGas.isPresent()) { + checkArgument( + transactionType.supportsBlob(), + "Must not specify blob versioned hashes or max fee per blob gas for transaction not supporting it"); + } + + if (transactionType.supportsBlob()) { + checkArgument( + versionedHashes.isPresent(), "Must specify blob versioned hashes for blob transaction"); + checkArgument( + !versionedHashes.get().isEmpty(), + "Blob transaction must have at least one versioned hash"); + checkArgument( + maxFeePerBlobGas.isPresent(), "Must specify max fee per blob gas for blob transaction"); + } + + if (transactionType.supportsDelegateCode()) { + checkArgument( + maybeCodeDelegationList.isPresent(), + "Must specify code delegation authorizations for code delegation transaction"); + } + + if (transactionType.supportsPqAuthorization()) { + checkArgument( + pqAuthorization.isPresent(), + "Must specify a post-quantum authorization for an AIP-20 transaction"); + checkArgument(signature == null, "An AIP-20 transaction carries no secp256k1 signature"); + } else { + checkArgument( + pqAuthorization.isEmpty(), + "Must not specify a post-quantum authorization for transaction type %s", + transactionType); + } + } + + this.transactionType = transactionType; + this.nonce = nonce; + this.gasPrice = gasPrice; + this.maxPriorityFeePerGas = maxPriorityFeePerGas; + this.maxFeePerGas = maxFeePerGas; + this.maxFeePerBlobGas = maxFeePerBlobGas; + this.gasLimit = gasLimit; + this.to = to; + this.value = value; + this.signature = signature; + this.payload = payload; + this.maybeAccessList = maybeAccessList; + this.sender = sender; + this.chainId = chainId; + this.versionedHashes = versionedHashes; + this.blobsWithCommitments = blobsWithCommitments; + this.maybeCodeDelegationList = maybeCodeDelegationList; + this.rawRlp = rawRlp; + this.pqAuthorization = pqAuthorization; + hash.ifPresent(h -> this.hash = h); + sizeForAnnouncement.ifPresent(i -> this.sizeForAnnouncement = i); + sizeForBlockInclusion.ifPresent(i -> this.sizeForBlockInclusion = i); + } + + /** + * Returns the transaction nonce. + * + * @return the transaction nonce + */ + @Override + public long getNonce() { + return nonce; + } + + /** + * Return the transaction gas price. + * + * @return the transaction gas price + */ + @Override + public Optional getGasPrice() { + return gasPrice; + } + + /** + * Return the transaction max priority per gas. + * + * @return the transaction max priority per gas + */ + @Override + public Optional getMaxPriorityFeePerGas() { + return maxPriorityFeePerGas; + } + + /** + * Return the transaction max fee per gas. + * + * @return the transaction max fee per gas + */ + @Override + public Optional getMaxFeePerGas() { + return maxFeePerGas; + } + + /** + * Return the transaction max fee per blob gas. + * + * @return the transaction max fee per blob gas + */ + @Override + public Optional getMaxFeePerBlobGas() { + return maxFeePerBlobGas; + } + + /** + * Return the effective priority fee per gas for this transaction. + * + * @param maybeBaseFee base fee in case of EIP-1559 transaction + * @return priority fee per gas in wei + */ + public Wei getEffectivePriorityFeePerGas(final Optional maybeBaseFee) { + return maybeBaseFee + .map( + baseFee -> { + if (getType().supports1559FeeMarket()) { + if (baseFee.greaterOrEqualThan(getMaxFeePerGas().get())) { + return Wei.ZERO; + } + return UInt256s.min( + getMaxPriorityFeePerGas().get(), getMaxFeePerGas().get().subtract(baseFee)); + } else { + if (baseFee.greaterOrEqualThan(getGasPrice().get())) { + return Wei.ZERO; + } + return getGasPrice().get().subtract(baseFee); + } + }) + .orElseGet(() -> getGasPrice().orElse(Wei.ZERO)); + } + + /** + * Returns the transaction gas limit. + * + * @return the transaction gas limit + */ + @Override + public long getGasLimit() { + return gasLimit; + } + + /** + * Returns the number of blobs this transaction has, or 0 if not a blob transaction type + * + * @return return the count + */ + public int getBlobCount() { + return versionedHashes.map(List::size).orElse(0); + } + + /** + * Returns the transaction recipient. + * + *

The {@code Optional

} will be {@code Optional.empty()} if the transaction is a + * contract creation; otherwise it will contain the message call transaction recipient. + * + * @return the transaction recipient if a message call; otherwise {@code Optional.empty()} + */ + @Override + public Optional
getTo() { + return to; + } + + /** + * Returns the value transferred in the transaction. + * + * @return the value transferred in the transaction + */ + @Override + public Wei getValue() { + return value; + } + + /** + * Returns the signature used to sign the transaction. + * + * @return the signature used to sign the transaction + */ + public SECPSignature getSignature() { + return signature; + } + + /** + * Returns the transaction payload. + * + * @return the transaction payload + */ + @Override + public Bytes getPayload() { + return payload.getPayloadBytes(); + } + + @Override + public long getPayloadZeroBytes() { + return payload.getZeroBytesCount(); + } + + /** + * Returns the payload if this is a contract creation transaction. + * + * @return if present the init code + */ + @Override + public Optional getInit() { + return getTo().isPresent() ? Optional.empty() : Optional.of(payload.getPayloadBytes()); + } + + /** + * Returns the payload if this is a message call transaction. + * + * @return if present the init code + */ + @Override + public Optional getData() { + return getTo().isPresent() ? Optional.of(payload.getPayloadBytes()) : Optional.empty(); + } + + @Override + public Optional> getAccessList() { + return maybeAccessList; + } + + /** + * Return the transaction chain id (if it exists) + * + *

The {@code OptionalInt} will be {@code OptionalInt.empty()} if the transaction is not tied + * to a specific chain. + * + * @return the transaction chain id if it exists; otherwise {@code OptionalInt.empty()} + */ + @Override + public Optional getChainId() { + return chainId; + } + + /** + * Returns the transaction sender. + * + * @return the transaction sender + */ + @Override + public Address getSender() { + if (sender == null) { + Optional

cachedSender = Optional.ofNullable(senderCache.getIfPresent(getHash())); + sender = cachedSender.orElseGet(this::computeSender); + } + return sender; + } + + private Address computeSender() { + if (pqAuthorization.isPresent()) { + // AIP-20: no recovery; the sender is derived from the post-quantum public key. + final Address pqSender = pqAuthorization.get().senderAddress(); + senderCache.put(getHash(), pqSender); + return pqSender; + } + final SECPPublicKey publicKey = + signatureAlgorithm + .recoverPublicKeyFromSignature(getOrComputeSenderRecoveryHash(), signature) + .orElseThrow( + () -> + new IllegalStateException( + "Cannot recover public key from signature for " + this)); + final Address calculatedSender = + Address.extract(Bytes32.wrap(Hash.hash(publicKey.getEncodedBytes()).getBytes())); + senderCache.put(getHash(), calculatedSender); + return calculatedSender; + } + + /** + * Returns the public key extracted from the signature. + * + * @return the public key + */ + public Optional getPublicKey() { + if (pqAuthorization.isPresent()) { + return Optional.of(pqAuthorization.get().publicKey().toHexString()); + } + return signatureAlgorithm + .recoverPublicKeyFromSignature(getOrComputeSenderRecoveryHash(), signature) + .map(SECPPublicKey::toString); + } + + private Bytes32 getOrComputeSenderRecoveryHash() { + if (hashNoSignature == null) { + hashNoSignature = + computeSenderRecoveryHash( + transactionType, + nonce, + gasPrice.orElse(null), + maxPriorityFeePerGas.orElse(null), + maxFeePerGas.orElse(null), + maxFeePerBlobGas.orElse(null), + gasLimit, + to, + value, + payload, + maybeAccessList, + versionedHashes.orElse(null), + maybeCodeDelegationList, + chainId, + pqAuthorization); + } + return hashNoSignature; + } + + /** + * Writes the transaction to RLP using the block body format + * + * @param out the output to write the transaction to + */ + public void writeTo(final RLPOutput out) { + writeTo(out, EncodingContext.BLOCK_BODY); + } + + /** + * Writes the transaction to RLP using the specified format + * + * @param out the output to write the transaction to + * @param context the serialization format to use + */ + public void writeTo(final RLPOutput out, final EncodingContext context) { + TransactionEncoder.encodeRLP(this, out, context); + } + + @Override + public Bytes encoded() { + final BytesValueRLPOutput rplOutput = new BytesValueRLPOutput(); + writeTo(rplOutput); + return rplOutput.encoded(); + } + + @Override + public BigInteger getR() { + return signature == null ? null : signature.getR(); + } + + @Override + public BigInteger getS() { + return signature == null ? null : signature.getS(); + } + + @Override + public BigInteger getV() { + if (transactionType != null && transactionType != TransactionType.FRONTIER) { + // EIP-2718 typed transaction, use yParity: + return null; + } else { + final BigInteger recId = BigInteger.valueOf(signature.getRecId()); + return chainId + .map(bigInteger -> recId.add(REPLAY_PROTECTED_V_BASE).add(TWO.multiply(bigInteger))) + .orElseGet(() -> recId.add(REPLAY_UNPROTECTED_V_BASE)); + } + } + + @Override + public BigInteger getYParity() { + if (transactionType != null && transactionType != TransactionType.FRONTIER) { + // EIP-2718 typed transaction, return yParity (none for an AIP-20 transaction): + return signature == null ? null : BigInteger.valueOf(signature.getRecId()); + } else { + // legacy types never return yParity + return null; + } + } + + /** + * Returns the transaction hash. + * + * @return the transaction hash + */ + @Override + public Hash getHash() { + if (hash == null) { + memoizeHashAndSizeForBlockInclusion(); + } + return hash; + } + + /** + * Returns the size in bytes of the encoded transaction. + * + * @return the size in bytes of the encoded transaction. + */ + @Override + public int getSizeForAnnouncement() { + if (sizeForAnnouncement == -1) { + memoizeSizeForAnnouncement(); + } + return sizeForAnnouncement; + } + + /** + * Returns the size in bytes of the encoded transaction for block inclusion. + * + * @return the size in bytes of the encoded transaction for block inclusion. + */ + @Override + public int getSizeForBlockInclusion() { + if (sizeForBlockInclusion == -1) { + memoizeHashAndSizeForBlockInclusion(); + } + return sizeForBlockInclusion; + } + + private void memoizeHashAndSizeForBlockInclusion() { + final Bytes bytes = TransactionEncoder.encodeOpaqueBytes(this, EncodingContext.BLOCK_BODY); + hash = Hash.hash(bytes); + sizeForBlockInclusion = bytes.size(); + if (!transactionType.supportsBlob() || this.getBlobsWithCommitments().isEmpty()) { + // for transactions not containing blobs the encoding is the same, so we can set this as well: + sizeForAnnouncement = sizeForBlockInclusion; + } + } + + private void memoizeSizeForAnnouncement() { + final Bytes pooledBytes = + TransactionEncoder.encodeOpaqueBytes(this, EncodingContext.POOLED_TRANSACTION); + sizeForAnnouncement = pooledBytes.size(); + if (!transactionType.supportsBlob() || this.getBlobsWithCommitments().isEmpty()) { + // for transactions not containing blobs the encoding is the same, so we can set these as + // well: + sizeForBlockInclusion = sizeForAnnouncement; + if (hash == null) { + hash = Hash.hash(pooledBytes); + } + } + } + + @Override + public boolean isContractCreation() { + return getTo().isEmpty(); + } + + /** + * Calculates the max up-front cost for the gas the transaction can use. + * + * @return the max up-front cost for the gas the transaction can use. + */ + private Wei getMaxUpfrontGasCost(final long blobGasPerBlock) { + return getUpfrontGasCost( + getMaxGasPrice(), getMaxFeePerBlobGas().orElse(Wei.ZERO), blobGasPerBlock); + } + + /** + * Calculates the up-front cost for the gas and blob gas the transaction can use. + * + * @param gasPrice the gas price to use + * @param blobGasPrice the blob gas price to use + * @return the up-front cost for the gas the transaction can use. + */ + public Wei getUpfrontGasCost( + final Wei gasPrice, final Wei blobGasPrice, final long totalBlobGas) { + if (gasPrice == null) { + return Wei.ZERO; + } + + final var cost = calculateUpfrontGasCost(gasPrice, blobGasPrice, totalBlobGas); + + if (cost.bitLength() > 256) { + return Wei.MAX_WEI; + } else { + return Wei.of(cost); + } + } + + public BigInteger calculateUpfrontGasCost( + final Wei gasPrice, final Wei blobGasPrice, final long totalBlobGas) { + var cost = + new BigInteger(1, Longs.toByteArray(getGasLimit())).multiply(gasPrice.getAsBigInteger()); + + if (transactionType.supportsBlob()) { + cost = cost.add(blobGasPrice.getAsBigInteger().multiply(BigInteger.valueOf(totalBlobGas))); + } + + return cost; + } + + /** + * Calculates the up-front cost for the transaction. + * + *

The up-front cost is paid by the sender account before the transaction is executed. The + * sender must have the amount in its account balance to execute and some of this amount may be + * refunded after the transaction has executed. + * + * @return the up-front gas cost for the transaction + */ + public Wei getUpfrontCost(final long totalBlobGas) { + Wei maxUpfrontGasCost = getMaxUpfrontGasCost(totalBlobGas); + Wei result = maxUpfrontGasCost.add(getValue()); + return (maxUpfrontGasCost.compareTo(result) > 0) ? Wei.MAX_WEI : result; + } + + /** + * Return the maximum fee per gas the sender is willing to pay for this transaction. + * + * @return max fee per gas in wei + */ + public Wei getMaxGasPrice() { + return maxFeePerGas.orElseGet( + () -> + gasPrice.orElseThrow( + () -> + new IllegalStateException( + "Transaction requires either gasPrice or maxFeePerGas"))); + } + + /** + * Calculates the effectiveGasPrice of a transaction on the basis of an {@code Optional} + * baseFee and handles unwrapping Optional fee parameters. If baseFee is present, effective gas is + * calculated as: + * + *

min((baseFeePerGas + maxPriorityFeePerGas), maxFeePerGas) + * + *

Otherwise, return gasPrice for legacy transactions. + * + * @param baseFeePerGas optional baseFee from the block header, if we are post-london + * @return the effective gas price. + */ + public final Wei getEffectiveGasPrice(final Optional baseFeePerGas) { + return getEffectivePriorityFeePerGas(baseFeePerGas).addExact(baseFeePerGas.orElse(Wei.ZERO)); + } + + public Optional getRawRlp() { + return rawRlp; + } + + @Override + public TransactionType getType() { + return this.transactionType; + } + + @Override + public Optional> getVersionedHashes() { + return versionedHashes; + } + + @Override + public Optional getBlobsWithCommitments() { + return blobsWithCommitments; + } + + @Override + public Optional> getCodeDelegationList() { + return maybeCodeDelegationList; + } + + @Override + public Optional getPqAuthorization() { + return pqAuthorization; + } + + /** + * The AIP-20 signing hash of a type-0x50 transaction: keccak256 of the signing pre-image. + * + * @return the 32-byte hash + */ + public Bytes getPqSigningHash() { + return getOrComputeSenderRecoveryHash(); + } + + @Override + public int codeDelegationListSize() { + return maybeCodeDelegationList.map(List::size).orElse(0); + } + + /** + * Return the list of transaction hashes extracted from the collection of Transaction passed as + * argument + * + * @param transactions a collection of transactions + * @return the list of transaction hashes + */ + public static List toHashList(final Collection transactions) { + return transactions.stream().map(Transaction::getHash).toList(); + } + + private static Bytes32 computeSenderRecoveryHash( + final TransactionType transactionType, + final long nonce, + final Wei gasPrice, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final Wei maxFeePerBlobGas, + final long gasLimit, + final Optional

to, + final Wei value, + final Payload payload, + final Optional> accessList, + final List versionedHashes, + final Optional> codeDelegationList, + final Optional chainId, + final Optional pqAuthorization) { + if (transactionType.requiresChainId()) { + checkArgument(chainId.isPresent(), "Transaction type %s requires chainId", transactionType); + } + final Bytes preimage = + getPreimage( + transactionType, + nonce, + gasPrice, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + gasLimit, + to, + value, + payload.getPayloadBytes(), + accessList, + versionedHashes, + codeDelegationList, + chainId, + pqAuthorization); + return keccak256(preimage); + } + + @Override + public Bytes encodedPreimage() { + return getPreimage( + transactionType, + nonce, + gasPrice.orElse(null), + maxPriorityFeePerGas.orElse(null), + maxFeePerGas.orElse(null), + maxFeePerBlobGas.orElse(null), + gasLimit, + to, + value, + payload.getPayloadBytes(), + maybeAccessList, + versionedHashes.orElse(null), + maybeCodeDelegationList, + chainId, + pqAuthorization); + } + + private static Bytes getPreimage( + final TransactionType transactionType, + final long nonce, + final Wei gasPrice, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final Wei maxFeePerBlobGas, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional> accessList, + final List versionedHashes, + final Optional> codeDelegationList, + final Optional chainId, + final Optional pqAuthorization) { + final Bytes preimage = + switch (transactionType) { + case AERE_PQ -> + AerePqTransactionEncoder.signingPreimage( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + chainId, + accessList, + pqAuthorization.orElseThrow( + () -> + new IllegalStateException( + "Developer error: an AIP-20 transaction must carry its authorization here"))); + case FRONTIER -> frontierPreimage(nonce, gasPrice, gasLimit, to, value, payload, chainId); + case EIP1559 -> + eip1559Preimage( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + chainId, + accessList); + case BLOB -> + blobPreimage( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + gasLimit, + to, + value, + payload, + chainId, + accessList, + versionedHashes); + case ACCESS_LIST -> + accessListPreimage( + nonce, + gasPrice, + gasLimit, + to, + value, + payload, + accessList.orElseThrow( + () -> + new IllegalStateException( + "Developer error: the transaction should be guaranteed to have an access list here")), + chainId); + case DELEGATE_CODE -> + codeDelegationPreimage( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + chainId, + accessList, + codeDelegationList.orElseThrow( + () -> + new IllegalStateException( + "Developer error: the transaction should be guaranteed to have a code delegations here"))); + }; + return preimage; + } + + private static Bytes frontierPreimage( + final long nonce, + final Wei gasPrice, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional chainId) { + return RLP.encode( + rlpOutput -> { + rlpOutput.startList(); + rlpOutput.writeLongScalar(nonce); + rlpOutput.writeUInt256Scalar(gasPrice); + rlpOutput.writeLongScalar(gasLimit); + rlpOutput.writeBytes(to.map(BytesHolder::getBytes).map(Bytes::copy).orElse(Bytes.EMPTY)); + rlpOutput.writeUInt256Scalar(value); + rlpOutput.writeBytes(payload); + if (chainId.isPresent()) { + rlpOutput.writeBigIntegerScalar(chainId.get()); + rlpOutput.writeUInt256Scalar(UInt256.ZERO); + rlpOutput.writeUInt256Scalar(UInt256.ZERO); + } + rlpOutput.endList(); + }); + } + + private static Bytes eip1559Preimage( + final long nonce, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional chainId, + final Optional> accessList) { + final Bytes encoded = + RLP.encode( + rlpOutput -> { + rlpOutput.startList(); + eip1559PreimageFields( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + chainId, + accessList, + rlpOutput); + rlpOutput.endList(); + }); + return Bytes.concatenate(Bytes.of(TransactionType.EIP1559.getSerializedType()), encoded); + } + + private static void eip1559PreimageFields( + final long nonce, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional chainId, + final Optional> accessList, + final RLPOutput rlpOutput) { + rlpOutput.writeBigIntegerScalar(chainId.orElseThrow()); + rlpOutput.writeLongScalar(nonce); + rlpOutput.writeUInt256Scalar(maxPriorityFeePerGas); + rlpOutput.writeUInt256Scalar(maxFeePerGas); + rlpOutput.writeLongScalar(gasLimit); + rlpOutput.writeBytes(to.map(BytesHolder::getBytes).map(Bytes::copy).orElse(Bytes.EMPTY)); + rlpOutput.writeUInt256Scalar(value); + rlpOutput.writeBytes(payload); + AccessListTransactionEncoder.writeAccessList(rlpOutput, accessList); + } + + private static Bytes blobPreimage( + final long nonce, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final Wei maxFeePerBlobGas, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional chainId, + final Optional> accessList, + final List versionedHashes) { + + final Bytes encoded = + RLP.encode( + rlpOutput -> { + rlpOutput.startList(); + eip1559PreimageFields( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + chainId, + accessList, + rlpOutput); + rlpOutput.writeUInt256Scalar(maxFeePerBlobGas); + BlobTransactionEncoder.writeBlobVersionedHashes(rlpOutput, versionedHashes); + rlpOutput.endList(); + }); + return Bytes.concatenate(Bytes.of(TransactionType.BLOB.getSerializedType()), encoded); + } + + private static Bytes accessListPreimage( + final long nonce, + final Wei gasPrice, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final List accessList, + final Optional chainId) { + final Bytes encode = + RLP.encode( + rlpOutput -> { + rlpOutput.startList(); + AccessListTransactionEncoder.encodeAccessListInner( + chainId, nonce, gasPrice, gasLimit, to, value, payload, accessList, rlpOutput); + rlpOutput.endList(); + }); + return Bytes.concatenate(Bytes.of(TransactionType.ACCESS_LIST.getSerializedType()), encode); + } + + private static Bytes codeDelegationPreimage( + final long nonce, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional chainId, + final Optional> accessList, + final List authorizationList) { + final Bytes encoded = + RLP.encode( + rlpOutput -> { + rlpOutput.startList(); + eip1559PreimageFields( + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + chainId, + accessList, + rlpOutput); + CodeDelegationTransactionEncoder.encodeCodeDelegationInner( + authorizationList, rlpOutput); + rlpOutput.endList(); + }); + return Bytes.concatenate(Bytes.of(TransactionType.DELEGATE_CODE.getSerializedType()), encoded); + } + + @Override + public boolean equals(final Object other) { + if (!(other instanceof Transaction that)) { + return false; + } + return Objects.equals(this.chainId, that.chainId) + && this.gasLimit == that.gasLimit + && Objects.equals(this.gasPrice, that.gasPrice) + && Objects.equals(this.maxPriorityFeePerGas, that.maxPriorityFeePerGas) + && Objects.equals(this.maxFeePerGas, that.maxFeePerGas) + && Objects.equals(this.maxFeePerBlobGas, that.maxFeePerBlobGas) + && this.nonce == that.nonce + && Objects.equals(this.payload, that.payload) + && Objects.equals(this.signature, that.signature) + && Objects.equals(this.pqAuthorization, that.pqAuthorization) + && Objects.equals(this.to, that.to) + && Objects.equals(this.value, that.value) + && Objects.equals(this.getV(), that.getV()); + } + + @Override + public int hashCode() { + return Objects.hash( + nonce, + gasPrice, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + gasLimit, + to, + value, + payload.getPayloadBytes(), + signature, + pqAuthorization, + chainId); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append( + transactionType.supportsBlob() + ? "Blob" + : isContractCreation() ? "ContractCreation" : "MessageCall") + .append("{"); + sb.append("type=").append(getType()).append(", "); + sb.append("nonce=").append(getNonce()).append(", "); + getGasPrice() + .ifPresent(gp -> sb.append("gasPrice=").append(gp.toHumanReadableString()).append(", ")); + if (getMaxPriorityFeePerGas().isPresent() && getMaxFeePerGas().isPresent()) { + sb.append("maxPriorityFeePerGas=") + .append(getMaxPriorityFeePerGas().map(Wei::toHumanReadableString).get()) + .append(", "); + sb.append("maxFeePerGas=") + .append(getMaxFeePerGas().map(Wei::toHumanReadableString).get()) + .append(", "); + getMaxFeePerBlobGas() + .ifPresent( + wei -> + sb.append("maxFeePerBlobGas=").append(wei.toHumanReadableString()).append(", ")); + } + sb.append("gasLimit=").append(getGasLimit()).append(", "); + if (getTo().isPresent()) sb.append("to=").append(getTo().get()).append(", "); + sb.append("value=").append(getValue()).append(", "); + sb.append("sig=").append(getSignature()).append(", "); + pqAuthorization.ifPresent(a -> sb.append("pq=").append(a).append(", ")); + if (chainId.isPresent()) sb.append("chainId=").append(getChainId().get()).append(", "); + if (transactionType.equals(TransactionType.ACCESS_LIST)) { + sb.append("accessList=").append(maybeAccessList).append(", "); + } + if (versionedHashes.isPresent()) { + final List vhs = versionedHashes.get(); + if (!vhs.isEmpty()) { + sb.append("versionedHashes=["); + sb.append( + vhs.get(0) + .toString()); // can't be empty if present, as this is checked in the constructor + for (int i = 1; i < vhs.size(); i++) { + sb.append(", ").append(vhs.get(i).toString()); + } + sb.append("], "); + } + } + if (transactionType.supportsBlob() && this.blobsWithCommitments.isPresent()) { + sb.append("numberOfBlobs=").append(blobsWithCommitments.get().getBlobs().size()).append(", "); + } + sb.append("payload=").append(getPayload()); + return sb.append("}").toString(); + } + + public String toTraceLog() { + final StringBuilder sb = new StringBuilder(); + sb.append(getHash()).append("={"); + sb.append( + transactionType.supportsBlob() + ? "Blob" + : isContractCreation() ? "ContractCreation" : "MessageCall") + .append(", "); + sb.append(getNonce()).append(", "); + sb.append(getSender()).append(", "); + sb.append(getType()).append(", "); + getGasPrice() + .ifPresent(gp -> sb.append("gp: ").append(gp.toHumanReadableString()).append(", ")); + if (getMaxPriorityFeePerGas().isPresent() && getMaxFeePerGas().isPresent()) { + sb.append("mf: ") + .append(getMaxFeePerGas().map(Wei::toHumanReadableString).get()) + .append(", "); + sb.append("pf: ") + .append(getMaxPriorityFeePerGas().map(Wei::toHumanReadableString).get()) + .append(", "); + getMaxFeePerBlobGas() + .ifPresent(wei -> sb.append("df: ").append(wei.toHumanReadableString()).append(", ")); + } + sb.append("gl: ").append(getGasLimit()).append(", "); + sb.append("v: ").append(getValue().toHumanReadableString()).append(", "); + getTo().ifPresent(t -> sb.append("to: ").append(t)); + return sb.append("}").toString(); + } + + @Override + public Optional
contractAddress() { + if (isContractCreation()) { + return Optional.of(Address.contractAddress(getSender(), getNonce())); + } + return Optional.empty(); + } + + /** + * Creates a copy of this transaction that does not share any underlying byte array. + * + *

This is useful in case the transaction is built from a block body and fields, like to or + * payload, are wrapping (and so keeping references) sections of the large RLP encoded block body, + * and we plan to keep the transaction around for some time, like in the txpool in case of a + * reorg, and do not want to keep all the block body in memory for a long time, but only the + * actual transaction. + * + * @return a copy of the transaction + */ + public Transaction detachedCopy() { + final Optional

detachedTo = to.map(address -> Address.wrap(address.getBytes().copy())); + final Optional> detachedAccessList = + maybeAccessList.map( + accessListEntries -> + accessListEntries.stream().map(this::accessListDetachedCopy).toList()); + final Optional> detachedVersionedHashes = + versionedHashes.map( + hashes -> + hashes.stream() + .map(vh -> new VersionedHash(Bytes32.wrap(vh.getBytes().copy()))) + .toList()); + final Optional detachedBlobsWithCommitments = + blobsWithCommitments.map( + withCommitments -> + blobsWithCommitmentsDetachedCopy(withCommitments, detachedVersionedHashes.get())); + final Optional> detachedCodeDelegationList = + maybeCodeDelegationList.map( + codeDelegations -> + codeDelegations.stream().map(this::codeDelegationDetachedCopy).toList()); + + final var copiedTx = + new Transaction( + true, + transactionType, + nonce, + gasPrice, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + gasLimit, + detachedTo, + value, + signature, + payload, + detachedAccessList, + sender, + chainId, + detachedVersionedHashes, + detachedBlobsWithCommitments, + detachedCodeDelegationList, + Optional.empty(), + pqAuthorization.map(AerePqAuthorization::detachedCopy), + Optional.ofNullable(hash), + Optional.of(sizeForAnnouncement), + Optional.of(sizeForBlockInclusion)); + + // copy also the computed fields, to avoid to recompute them + copiedTx.sender = this.sender; + copiedTx.hashNoSignature = this.hashNoSignature; + + return copiedTx; + } + + private AccessListEntry accessListDetachedCopy(final AccessListEntry accessListEntry) { + final Address detachedAddress = Address.wrap(accessListEntry.address().getBytes().copy()); + final var detachedStorage = accessListEntry.storageKeys().stream().map(Bytes32::copy).toList(); + return new AccessListEntry(detachedAddress, detachedStorage); + } + + private CodeDelegation codeDelegationDetachedCopy(final CodeDelegation codeDelegation) { + final Address detachedAddress = Address.wrap(codeDelegation.address().getBytes().copy()); + return new org.hyperledger.besu.ethereum.core.CodeDelegation( + codeDelegation.chainId(), + detachedAddress, + codeDelegation.nonce(), + codeDelegation.signature()); + } + + private BlobsWithCommitments blobsWithCommitmentsDetachedCopy( + final BlobsWithCommitments blobsWithCommitments, final List versionedHashes) { + final var detachedCommitments = + blobsWithCommitments.getKzgCommitments().stream() + .map(kc -> new KZGCommitment(kc.getData().copy())) + .toList(); + final var detachedBlobs = + blobsWithCommitments.getBlobs().stream() + .map(blob -> new Blob(blob.getData().copy())) + .toList(); + final var detachedProofs = + blobsWithCommitments.getKzgProofs().stream() + .map(proof -> new KZGProof(proof.getData().copy())) + .toList(); + return new BlobsWithCommitments( + blobsWithCommitments.getBlobType(), + detachedCommitments, + detachedBlobs, + detachedProofs, + versionedHashes); + } + + public static class Builder { + private static final Optional> EMPTY_ACCESS_LIST = Optional.of(List.of()); + + protected TransactionType transactionType; + + protected long nonce = -1L; + + protected Wei gasPrice; + + protected Wei maxPriorityFeePerGas; + + protected Wei maxFeePerGas; + protected Wei maxFeePerBlobGas; + + protected long gasLimit = -1L; + + protected Optional
to = Optional.empty(); + + protected Wei value; + + protected SECPSignature signature; + + protected Payload payload; + + protected Optional> accessList = Optional.empty(); + + protected Address sender; + + protected Optional chainId = Optional.empty(); + protected Optional v = Optional.empty(); + protected List versionedHashes = null; + private BlobsWithCommitments blobsWithCommitments; + protected Optional> codeDelegationAuthorizations = Optional.empty(); + protected Bytes rawRlp = null; + protected Optional pqAuthorization = Optional.empty(); + private Optional hash = Optional.empty(); + private Optional sizeForAnnouncement = Optional.empty(); + private Optional sizeForBlockInclusion = Optional.empty(); + + public Builder copiedFrom(final Transaction toCopy) { + this.transactionType = toCopy.transactionType; + this.nonce = toCopy.nonce; + this.gasPrice = toCopy.gasPrice.orElse(null); + this.maxPriorityFeePerGas = toCopy.maxPriorityFeePerGas.orElse(null); + this.maxFeePerGas = toCopy.maxFeePerGas.orElse(null); + this.maxFeePerBlobGas = toCopy.maxFeePerBlobGas.orElse(null); + this.gasLimit = toCopy.gasLimit; + this.to = toCopy.to; + this.value = toCopy.value; + this.signature = toCopy.signature; + this.payload = toCopy.payload; + this.accessList = toCopy.maybeAccessList; + this.sender = toCopy.sender; + this.chainId = toCopy.chainId; + this.versionedHashes = toCopy.versionedHashes.orElse(null); + this.blobsWithCommitments = toCopy.blobsWithCommitments.orElse(null); + this.codeDelegationAuthorizations = toCopy.maybeCodeDelegationList; + this.pqAuthorization = toCopy.pqAuthorization; + return this; + } + + public Builder type(final TransactionType transactionType) { + this.transactionType = transactionType; + return this; + } + + public Builder chainId(final BigInteger chainId) { + this.chainId = Optional.of(chainId); + return this; + } + + public Builder v(final BigInteger v) { + this.v = Optional.of(v); + return this; + } + + public Builder gasPrice(final Wei gasPrice) { + this.gasPrice = gasPrice; + return this; + } + + public Builder maxPriorityFeePerGas(final Wei maxPriorityFeePerGas) { + this.maxPriorityFeePerGas = maxPriorityFeePerGas; + return this; + } + + public Builder maxFeePerGas(final Wei maxFeePerGas) { + this.maxFeePerGas = maxFeePerGas; + return this; + } + + public Builder maxFeePerBlobGas(final Wei maxFeePerBlobGas) { + this.maxFeePerBlobGas = maxFeePerBlobGas; + return this; + } + + public Builder gasLimit(final long gasLimit) { + this.gasLimit = gasLimit; + return this; + } + + public Builder nonce(final long nonce) { + this.nonce = nonce; + return this; + } + + public Builder value(final Wei value) { + this.value = value; + return this; + } + + public Builder to(final Address to) { + this.to = Optional.ofNullable(to); + return this; + } + + public Builder payload(final Bytes payload) { + this.payload = new Payload(payload); + return this; + } + + public Builder accessList(final List accessList) { + this.accessList = + accessList == null + ? Optional.empty() + : accessList.isEmpty() ? EMPTY_ACCESS_LIST : Optional.of(accessList); + return this; + } + + public Builder sender(final Address sender) { + this.sender = sender; + return this; + } + + public Builder signature(final SECPSignature signature) { + this.signature = signature; + return this; + } + + public Builder versionedHashes(final List versionedHashes) { + this.versionedHashes = versionedHashes; + return this; + } + + public Builder rawRlp(final Bytes rawRlp) { + this.rawRlp = rawRlp; + return this; + } + + public Builder pqAuthorization(final AerePqAuthorization pqAuthorization) { + this.pqAuthorization = Optional.ofNullable(pqAuthorization); + return this; + } + + public Builder hash(final Hash hash) { + this.hash = Optional.ofNullable(hash); + return this; + } + + public Builder sizeForAnnouncement(final int sizeForAnnouncement) { + this.sizeForAnnouncement = Optional.of(sizeForAnnouncement); + return this; + } + + public Builder sizeForBlockInclusion(final int sizeForBlockInclusion) { + this.sizeForBlockInclusion = Optional.of(sizeForBlockInclusion); + return this; + } + + public Builder guessType() { + if (pqAuthorization.isPresent()) { + transactionType = TransactionType.AERE_PQ; + } else if (codeDelegationAuthorizations.isPresent()) { + transactionType = TransactionType.DELEGATE_CODE; + } else if (versionedHashes != null && !versionedHashes.isEmpty()) { + transactionType = TransactionType.BLOB; + } else if (maxPriorityFeePerGas != null || maxFeePerGas != null) { + transactionType = TransactionType.EIP1559; + } else if (accessList.isPresent()) { + transactionType = TransactionType.ACCESS_LIST; + } else { + transactionType = TransactionType.FRONTIER; + } + return this; + } + + public TransactionType getTransactionType() { + return transactionType; + } + + public Transaction build() { + if (transactionType == null) guessType(); + return new Transaction( + false, + transactionType, + nonce, + Optional.ofNullable(gasPrice), + Optional.ofNullable(maxPriorityFeePerGas), + Optional.ofNullable(maxFeePerGas), + Optional.ofNullable(maxFeePerBlobGas), + gasLimit, + to, + value, + signature, + payload, + accessList, + sender, + chainId, + Optional.ofNullable(versionedHashes), + Optional.ofNullable(blobsWithCommitments), + codeDelegationAuthorizations, + Optional.ofNullable(rawRlp), + pqAuthorization, + hash, + sizeForAnnouncement, + sizeForBlockInclusion); + } + + public Transaction signAndBuild(final KeyPair keys) { + checkState( + signature == null, "The transaction signature has already been provided to this builder"); + signature(computeSignature(keys)); + sender( + Address.extract( + Bytes32.wrap(Hash.hash(keys.getPublicKey().getEncodedBytes()).getBytes()))); + return build(); + } + + SECPSignature computeSignature(final KeyPair keys) { + return SignatureAlgorithmFactory.getInstance() + .sign( + computeSenderRecoveryHash( + transactionType, + nonce, + gasPrice, + maxPriorityFeePerGas, + maxFeePerGas, + maxFeePerBlobGas, + gasLimit, + to, + value, + payload, + accessList, + versionedHashes, + codeDelegationAuthorizations, + chainId, + Optional.empty()), + keys); + } + + public Builder kzgBlobs( + final BlobType blobType, + final List kzgCommitments, + final List blobs, + final List kzgProofs) { + if (this.versionedHashes == null || this.versionedHashes.isEmpty()) { + this.versionedHashes = + kzgCommitments.stream() + .map(c -> new VersionedHash(SHA256_VERSION_ID, Sha256Hash.sha256(c.getData()))) + .toList(); + } + this.blobsWithCommitments = + new BlobsWithCommitments(blobType, kzgCommitments, blobs, kzgProofs, versionedHashes); + return this; + } + + public Builder blobsWithCommitments(final BlobsWithCommitments blobsWithCommitments) { + this.blobsWithCommitments = blobsWithCommitments; + return this; + } + + public Builder codeDelegations(final List codeDelegations) { + this.codeDelegationAuthorizations = Optional.ofNullable(codeDelegations); + return this; + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionDecoder.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionDecoder.java new file mode 100644 index 0000000..29eaa60 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionDecoder.java @@ -0,0 +1,111 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core.encoding; + +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.AerePqAlgorithm; +import org.hyperledger.besu.datatypes.AerePqAuthorization; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.math.BigInteger; + +import org.apache.tuweni.bytes.Bytes; + +/** + * Decodes an AIP-20 type-0x50 transaction: {@code 0x50 || rlp([chain_id, nonce, + * max_priority_fee_per_gas, max_fee_per_gas, gas_limit, to, value, data, access_list, alg_id, + * public_key, signature])}. + * + *

The decoder is strict on shape: an unknown {@code alg_id} or a key/signature of the wrong + * length is rejected here, before any cryptography runs, with an {@link IllegalArgumentException} + * that the callers of {@link TransactionDecoder} already treat as "malformed transaction". + */ +public class AerePqTransactionDecoder { + + private AerePqTransactionDecoder() { + // static + } + + /** + * Decodes the opaque bytes of a type-0x50 transaction. + * + * @param input the opaque bytes, type byte included + * @return the transaction + */ + public static Transaction decode(final Bytes input) { + final RLPInput txRlp = RLP.input(input.slice(1)); // skip the transaction type byte + txRlp.enterList(); + final BigInteger chainId = txRlp.readBigIntegerScalar(); + final Transaction.Builder builder = + Transaction.builder() + .type(TransactionType.AERE_PQ) + .chainId(chainId) + .nonce(txRlp.readLongScalar()) + .maxPriorityFeePerGas(Wei.of(txRlp.readUInt256Scalar())) + .maxFeePerGas(Wei.of(txRlp.readUInt256Scalar())) + .gasLimit(txRlp.readLongScalar()) + .to(txRlp.readBytes(v -> v.isEmpty() ? null : Address.wrap(v))) + .value(Wei.of(txRlp.readUInt256Scalar())) + .payload(txRlp.readBytes()) + .rawRlp(txRlp.raw()) + .accessList( + txRlp.readList( + accessListEntryRLPInput -> { + accessListEntryRLPInput.enterList(); + final AccessListEntry accessListEntry = + new AccessListEntry( + Address.wrap(accessListEntryRLPInput.readBytes()), + accessListEntryRLPInput.readList(RLPInput::readBytes32)); + accessListEntryRLPInput.leaveList(); + return accessListEntry; + })) + .sizeForAnnouncement(input.size()) + .sizeForBlockInclusion(input.size()) + .hash(Hash.hash(input)); + final int algId = txRlp.readIntScalar(); + final AerePqAlgorithm algorithm = + AerePqAlgorithm.byId(algId) + .orElseThrow( + () -> + new IllegalArgumentException( + "AIP-20: unknown post-quantum algorithm identifier " + algId)); + final Bytes publicKey = txRlp.readBytes(); + final Bytes signature = txRlp.readBytes(); + final AerePqAuthorization authorization = + new AerePqAuthorization(algorithm, publicKey, signature); + if (!authorization.lengthsValid()) { + throw new IllegalArgumentException( + "AIP-20: " + + algorithm + + " expects a " + + algorithm.publicKeyLength() + + "-byte public key and a " + + algorithm.signatureLength() + + "-byte signature, got " + + publicKey.size() + + " and " + + signature.size()); + } + final Transaction transaction = builder.pqAuthorization(authorization).build(); + txRlp.leaveList(); + return transaction; + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionEncoder.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionEncoder.java new file mode 100644 index 0000000..f2de33f --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/AerePqTransactionEncoder.java @@ -0,0 +1,152 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core.encoding; + +import static org.hyperledger.besu.ethereum.core.encoding.AccessListTransactionEncoder.writeAccessList; + +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.AerePqAuthorization; +import org.hyperledger.besu.datatypes.BytesHolder; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +/** + * Encodes an AIP-20 type-0x50 transaction and computes its signing pre-image. + * + *

Signing pre-image (AIP-20): {@code 0x50 || rlp([chain_id, nonce, max_priority_fee_per_gas, + * max_fee_per_gas, gas_limit, to, value, data, access_list, alg_id, public_key])}; the signing hash + * is its keccak256 and is what ML-DSA signs, in the pure (non pre-hash) interface with an empty + * context string. + */ +public class AerePqTransactionEncoder { + + private AerePqTransactionEncoder() { + // static + } + + /** + * Writes the payload list of the transaction (everything after the type byte). + * + * @param transaction the transaction, type 0x50 + * @param out the RLP output + */ + public static void encode(final Transaction transaction, final RLPOutput out) { + final AerePqAuthorization auth = + transaction + .getPqAuthorization() + .orElseThrow( + () -> + new IllegalStateException( + "type 0x50 transaction without a post-quantum authorization")); + out.startList(); + writeUnsignedFields( + transaction.getChainId().orElseThrow(), + transaction.getNonce(), + transaction.getMaxPriorityFeePerGas().orElseThrow(), + transaction.getMaxFeePerGas().orElseThrow(), + transaction.getGasLimit(), + transaction.getTo(), + transaction.getValue(), + transaction.getPayload(), + transaction.getAccessList(), + auth, + out); + out.writeBytes(auth.signature()); + out.endList(); + } + + /** + * The signing pre-image of AIP-20 for the given fields. + * + * @param nonce the nonce + * @param maxPriorityFeePerGas the max priority fee per gas + * @param maxFeePerGas the max fee per gas + * @param gasLimit the gas limit + * @param to the recipient, empty for contract creation + * @param value the value + * @param payload the calldata + * @param chainId the chain id (mandatory for this type) + * @param accessList the access list + * @param authorization the authorization whose algorithm and public key are covered + * @return {@code 0x50 || rlp([...])} + */ + public static Bytes signingPreimage( + final long nonce, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final long gasLimit, + final Optional

to, + final Wei value, + final Bytes payload, + final Optional chainId, + final Optional> accessList, + final AerePqAuthorization authorization) { + final Bytes encoded = + RLP.encode( + rlpOutput -> { + rlpOutput.startList(); + writeUnsignedFields( + chainId.orElseThrow( + () -> new IllegalStateException("type 0x50 requires a chain id")), + nonce, + maxPriorityFeePerGas, + maxFeePerGas, + gasLimit, + to, + value, + payload, + accessList, + authorization, + rlpOutput); + rlpOutput.endList(); + }); + return Bytes.concatenate(Bytes.of(TransactionType.AERE_PQ.getSerializedType()), encoded); + } + + private static void writeUnsignedFields( + final BigInteger chainId, + final long nonce, + final Wei maxPriorityFeePerGas, + final Wei maxFeePerGas, + final long gasLimit, + final Optional
to, + final Wei value, + final Bytes payload, + final Optional> accessList, + final AerePqAuthorization authorization, + final RLPOutput out) { + out.writeBigIntegerScalar(chainId); + out.writeLongScalar(nonce); + out.writeUInt256Scalar(maxPriorityFeePerGas); + out.writeUInt256Scalar(maxFeePerGas); + out.writeLongScalar(gasLimit); + out.writeBytes(to.map(BytesHolder::getBytes).map(Bytes::copy).orElse(Bytes.EMPTY)); + out.writeUInt256Scalar(value); + out.writeBytes(payload); + writeAccessList(out, accessList); + out.writeIntScalar(authorization.algorithm().id()); + out.writeBytes(authorization.publicKey()); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionDecoder.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionDecoder.java new file mode 100644 index 0000000..65aff2e --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionDecoder.java @@ -0,0 +1,184 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.core.encoding; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +public class TransactionDecoder { + + @FunctionalInterface + interface Decoder { + Transaction decode(Bytes input); + } + + private static final EnumMap TRANSACTION_DECODERS = + new EnumMap<>( + Map.of( + TransactionType.FRONTIER, FrontierTransactionDecoder::decode, + TransactionType.ACCESS_LIST, AccessListTransactionDecoder::decode, + TransactionType.EIP1559, EIP1559TransactionDecoder::decode, + TransactionType.BLOB, BlobTransactionDecoder::decode, + TransactionType.DELEGATE_CODE, CodeDelegationTransactionDecoder::decode, + TransactionType.AERE_PQ, AerePqTransactionDecoder::decode)); + + private static final EnumMap POOLED_TRANSACTION_DECODERS = + new EnumMap<>(Map.of(TransactionType.BLOB, BlobPooledTransactionDecoder::decode)); + + /** + * Decodes an RLP input into a transaction. If the input represents a typed transaction, it uses + * the appropriate decoder for that type. Otherwise, it uses the frontier decoder. + * + * @param rlpInput the RLP input + * @return the decoded transaction + */ + public static Transaction decodeRLP( + final RLPInput rlpInput, final EncodingContext encodingContext) { + if (isTypedTransaction(rlpInput)) { + return decodeTransaction(rlpInput, encodingContext); + } else { + return FrontierTransactionDecoder.decode(rlpInput.readAsRlp()); + } + } + + /** + * Decodes a typed transaction from an RLP input. It first reads the transaction type from the + * input, then uses the appropriate decoder for that type. + * + * @param rlpInput the RLP input + * @return the decoded transaction + */ + private static Transaction decodeTransaction( + final RLPInput rlpInput, final EncodingContext context) { + // Read the typed transaction bytes from the RLP input + final Bytes typedTransactionBytes = rlpInput.readBytes(); + + // Determine the transaction type from the typed transaction bytes + TransactionType transactionType = + getTransactionType(typedTransactionBytes) + .orElseThrow((() -> new IllegalArgumentException("Unsupported transaction type"))); + return decodeTransaction(typedTransactionBytes, transactionType, context); + } + + /** + * Decodes a typed transaction. The method first slices the transaction bytes to exclude the + * transaction type, then uses the appropriate decoder for the transaction type to decode the + * remaining bytes. + * + * @param transactionBytes the transaction bytes + * @param transactionType the type of the transaction + * @param context the encoding context + * @return the decoded transaction + */ + private static Transaction decodeTransaction( + final Bytes transactionBytes, + final TransactionType transactionType, + final EncodingContext context) { + // Slice the transaction bytes to exclude the transaction type and prepare for decoding + // Use the appropriate decoder for the transaction type to decode the remaining bytes + return getDecoder(transactionType, context).decode(transactionBytes); + } + + /** + * Decodes a transaction from opaque bytes. The method first determines the transaction type from + * the bytes. If the type is present, it delegates the decoding process to the appropriate decoder + * for that type. If the type is not present, it decodes the bytes as an RLP input. + * + * @param opaqueBytes the opaque bytes + * @param context the encoding context + * @return the decoded transaction + */ + public static Transaction decodeOpaqueBytes( + final Bytes opaqueBytes, final EncodingContext context) { + if (opaqueBytes.isEmpty()) { + throw new IllegalArgumentException("Empty opaque bytes"); + } + var transactionType = getTransactionType(opaqueBytes); + if (transactionType.isPresent()) { + return decodeTransaction(opaqueBytes, transactionType.get(), context); + } else { + // If the transaction type is not present, decode the opaque bytes as RLP + return decodeRLP(RLP.input(opaqueBytes), context); + } + } + + /** + * Retrieves the transaction type from the provided bytes. The method attempts to extract the + * first byte from the input bytes and interpret it as a transaction type. If the byte does not + * correspond to a valid transaction type, the method returns an empty Optional. + * + * @param opaqueBytes the bytes from which to extract the transaction type + * @return an Optional containing the TransactionType if the first byte of the input corresponds + * to a valid transaction type, or an empty Optional if it does not + */ + private static Optional getTransactionType(final Bytes opaqueBytes) { + byte transactionTypeByte = opaqueBytes.get(0); + return TransactionType.fromOpaque(transactionTypeByte); + } + + /** + * Checks if the given RLP input is a typed transaction. + * + *

See EIP-2718 + * + *

If it starts with a value in the range [0, 0x7f] then it is a new transaction type + * + *

if it starts with a value in the range [0xc0, 0xfe] then it is a legacy transaction type + * + * @param rlpInput the RLP input + * @return true if the RLP input is a typed transaction, false otherwise + */ + private static boolean isTypedTransaction(final RLPInput rlpInput) { + return !rlpInput.nextIsList(); + } + + /** + * Gets the decoder for a given transaction type and encoding context. If the context is + * POOLED_TRANSACTION, it uses the network decoder for the type. Otherwise, it uses the typed + * decoder. + * + * @param transactionType the transaction type + * @param encodingContext the encoding context + * @return the decoder + */ + private static Decoder getDecoder( + final TransactionType transactionType, final EncodingContext encodingContext) { + if (encodingContext.equals(EncodingContext.POOLED_TRANSACTION)) { + if (POOLED_TRANSACTION_DECODERS.containsKey(transactionType)) { + return POOLED_TRANSACTION_DECODERS.get(transactionType); + } + } + return checkNotNull( + TRANSACTION_DECODERS.get(transactionType), + "Developer Error. A supported transaction type %s has no associated decoding logic", + transactionType); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionEncoder.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionEncoder.java new file mode 100644 index 0000000..45da4a4 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionEncoder.java @@ -0,0 +1,153 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.core.encoding; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import com.google.common.collect.ImmutableMap; +import org.apache.tuweni.bytes.Bytes; + +public class TransactionEncoder { + + @FunctionalInterface + interface Encoder { + void encode(Transaction transaction, RLPOutput output); + } + + private static final ImmutableMap TYPED_TRANSACTION_ENCODERS = + ImmutableMap.of( + TransactionType.ACCESS_LIST, + AccessListTransactionEncoder::encode, + TransactionType.EIP1559, + EIP1559TransactionEncoder::encode, + TransactionType.BLOB, + BlobTransactionEncoder::encode, + TransactionType.DELEGATE_CODE, + CodeDelegationTransactionEncoder::encode, + TransactionType.AERE_PQ, + AerePqTransactionEncoder::encode); + + private static final ImmutableMap POOLED_TRANSACTION_ENCODERS = + ImmutableMap.of(TransactionType.BLOB, BlobPooledTransactionEncoder::encode); + + /** + * Encodes a transaction into RLP format. + * + * @param transaction the transaction to encode + * @param rlpOutput the RLP output stream + * @param encodingContext the encoding context + */ + public static void encodeRLP( + final Transaction transaction, + final RLPOutput rlpOutput, + final EncodingContext encodingContext) { + final TransactionType transactionType = getTransactionType(transaction); + + Bytes opaqueBytes = encodeOpaqueBytes(transaction, encodingContext); + encodeRLP(transactionType, opaqueBytes, rlpOutput); + } + + /** + * Encodes a transaction into RLP format. + * + * @param transactionType the type of the transaction + * @param opaqueBytes the bytes of the transaction + * @param rlpOutput the RLP output stream + */ + public static void encodeRLP( + final TransactionType transactionType, final Bytes opaqueBytes, final RLPOutput rlpOutput) { + checkNotNull(transactionType, "Transaction type was not specified."); + if (TransactionType.FRONTIER.equals(transactionType)) { + rlpOutput.writeRaw(opaqueBytes); + } else { + rlpOutput.writeBytes(opaqueBytes); + } + } + + /** + * Encodes a transaction into opaque bytes. + * + * @param transaction the transaction to encode + * @param encodingContext the encoding context + * @return the encoded transaction as bytes + */ + public static Bytes encodeOpaqueBytes( + final Transaction transaction, final EncodingContext encodingContext) { + final TransactionType transactionType = getTransactionType(transaction); + if (TransactionType.FRONTIER.equals(transactionType)) { + return RLP.encode(rlpOutput -> FrontierTransactionEncoder.encode(transaction, rlpOutput)); + } else { + final Encoder encoder = getEncoder(transactionType, encodingContext); + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + transaction + .getRawRlp() + .ifPresentOrElse( + (rawRlp) -> + out.writeRLPBytes( + Bytes.concatenate(Bytes.of(transactionType.getSerializedType()), rawRlp)), + () -> { + out.writeByte(transaction.getType().getSerializedType()); + encoder.encode(transaction, out); + }); + return out.encoded(); + } + } + + static void writeSignatureAndV(final Transaction transaction, final RLPOutput out) { + out.writeBigIntegerScalar(transaction.getV()); + writeSignature(transaction, out); + } + + static void writeSignatureAndRecoveryId(final Transaction transaction, final RLPOutput out) { + out.writeIntScalar(transaction.getSignature().getRecId()); + writeSignature(transaction, out); + } + + static void writeSignature(final Transaction transaction, final RLPOutput out) { + out.writeBigIntegerScalar(transaction.getSignature().getR()); + out.writeBigIntegerScalar(transaction.getSignature().getS()); + } + + private static TransactionType getTransactionType(final Transaction transaction) { + return checkNotNull( + transaction.getType(), "Transaction type for %s was not specified.", transaction); + } + + private static Encoder getEncoder( + final TransactionType transactionType, final EncodingContext encodingContext) { + + if (encodingContext.equals(EncodingContext.POOLED_TRANSACTION)) { + if (POOLED_TRANSACTION_ENCODERS.containsKey(transactionType)) { + return POOLED_TRANSACTION_ENCODERS.get(transactionType); + } + } + return checkNotNull( + TYPED_TRANSACTION_ENCODERS.get(transactionType), + "Developer Error. A supported transaction type %s has no associated encoding logic", + transactionType); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AerePqTransactionSupport.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AerePqTransactionSupport.java new file mode 100644 index 0000000..1eaa9ae --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AerePqTransactionSupport.java @@ -0,0 +1,196 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet; + +import org.hyperledger.besu.datatypes.AerePqAlgorithm; +import org.hyperledger.besu.datatypes.AerePqAuthorization; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.pqc.crypto.mldsa.MLDSAParameters; +import org.bouncycastle.pqc.crypto.mldsa.MLDSAPublicKeyParameters; +import org.bouncycastle.pqc.crypto.mldsa.MLDSASigner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * AIP-20 support on the validation path: the activation height and the ML-DSA verification of a + * type-0x50 transaction. + * + *

Activation follows every other Aere rule: a per-node system property, {@code + * aere.pqtx.forkBlock}, set uniformly on every node that validates or reads blocks. Absent means + * NEVER: a binary that has this code and no property behaves exactly like one that does not have + * it, which is what makes staging the binary ahead of the height safe. + * + *

Verification is FIPS 204 ML-DSA in the pure interface with the empty context string, over the + * 32-byte signing hash of AIP-20. Bouncy Castle's {@link MLDSASigner} implements exactly that + * interface when initialised without a context parameter. + */ +public final class AerePqTransactionSupport { + + private static final Logger LOG = LoggerFactory.getLogger(AerePqTransactionSupport.class); + + /** The system property that names the activation height; absent means never. */ + public static final String FORK_BLOCK_PROPERTY = "aere.pqtx.forkBlock"; + + private static final long NEVER = Long.MAX_VALUE; + private static final long FORK_BLOCK = readForkBlock(); + + private AerePqTransactionSupport() { + // static + } + + private static long readForkBlock() { + final String raw = System.getProperty(FORK_BLOCK_PROPERTY); + if (raw == null || raw.isBlank()) { + LOG.info("AERE PQ-TX: {} absent, type 0x50 transactions are never accepted", FORK_BLOCK_PROPERTY); + return NEVER; + } + try { + final long value = Long.parseLong(raw.trim()); + if (value < 0) { + throw new IllegalArgumentException("negative"); + } + LOG.info("AERE PQ-TX: type 0x50 transactions accepted from block {}", value); + return value; + } catch (final IllegalArgumentException e) { + throw new IllegalStateException( + "AERE PQ-TX: " + + FORK_BLOCK_PROPERTY + + " must be a non-negative block number, got '" + + raw + + "'. Refusing to start with an unreadable consensus parameter.", + e); + } + } + + /** + * The configured activation height. + * + * @return the height, or {@link Long#MAX_VALUE} when the type is never active + */ + public static long forkBlock() { + return FORK_BLOCK; + } + + /** + * Whether type-0x50 transactions are valid in a block at this height. + * + * @param blockNumber the block number being validated or built + * @return true at or above the activation height + */ + public static boolean isActiveAt(final long blockNumber) { + return blockNumber >= FORK_BLOCK; + } + + /** + * The message for a type-0x50 transaction seen below the activation height. + * + * @param blockNumber the block number + * @return a human-readable reason + */ + public static String inactiveMessage(final long blockNumber) { + return FORK_BLOCK == NEVER + ? "AIP-20 type 0x50 transactions are not enabled on this node (" + + FORK_BLOCK_PROPERTY + + " absent)" + : "AIP-20 type 0x50 transactions are valid from block " + + FORK_BLOCK + + ", this is block " + + blockNumber; + } + + /** + * Validates the post-quantum authorization of a type-0x50 transaction: presence, exact lengths, + * chain id presence, and the ML-DSA signature over the AIP-20 signing hash. On success the sender + * is derivable from the public key (no recovery is involved). + * + * @param transaction the transaction, type 0x50 + * @return valid, or INVALID_SIGNATURE / INVALID_TRANSACTION_FORMAT with the reason + */ + public static ValidationResult validateAuthorization( + final Transaction transaction) { + final Optional maybeAuth = transaction.getPqAuthorization(); + if (maybeAuth.isEmpty()) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + "type 0x50 transaction carries no post-quantum authorization"); + } + final AerePqAuthorization auth = maybeAuth.get(); + if (!auth.lengthsValid()) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_SIGNATURE, + "post-quantum key or signature length does not match " + auth.algorithm()); + } + if (transaction.getChainId().isEmpty()) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + "type 0x50 transaction must carry a chain id"); + } + final Bytes32 signingHash; + try { + signingHash = Bytes32.wrap(transaction.getPqSigningHash()); + } catch (final RuntimeException e) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + "type 0x50 signing pre-image could not be computed: " + e.getMessage()); + } + final boolean ok; + try { + ok = verify(auth.algorithm(), auth.publicKey().toArrayUnsafe(), signingHash.toArrayUnsafe(), auth.signature().toArrayUnsafe()); + } catch (final RuntimeException e) { + // A malformed key that the length check let through: the verifier throws, the tx is invalid. + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_SIGNATURE, + "post-quantum verification failed to run: " + e.getClass().getSimpleName()); + } + if (!ok) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_SIGNATURE, + "post-quantum signature does not verify over the AIP-20 signing hash"); + } + return ValidationResult.valid(); + } + + /** + * FIPS 204 ML-DSA verification, pure interface, empty context. + * + * @param algorithm the parameter set + * @param publicKey the raw public key + * @param message the message (here the 32-byte signing hash) + * @param signature the raw signature + * @return true when the signature verifies + */ + public static boolean verify( + final AerePqAlgorithm algorithm, + final byte[] publicKey, + final byte[] message, + final byte[] signature) { + final MLDSAParameters parameters = + switch (algorithm) { + case ML_DSA_44 -> MLDSAParameters.ml_dsa_44; + case ML_DSA_65 -> MLDSAParameters.ml_dsa_65; + case ML_DSA_87 -> MLDSAParameters.ml_dsa_87; + }; + final MLDSAPublicKeyParameters pub = new MLDSAPublicKeyParameters(parameters, publicKey); + final MLDSASigner verifier = new MLDSASigner(); + verifier.init(false, pub); + verifier.update(message, 0, message.length); + return verifier.verifySignature(signature); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java new file mode 100644 index 0000000..3acaea6 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java @@ -0,0 +1,1592 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0005-aere-eip2935-futureeips.patch. + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.AMSTERDAM; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.ARROW_GLACIER; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BERLIN; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BPO1; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BPO2; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BPO3; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BPO4; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BPO5; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.BYZANTIUM; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.CANCUN; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.CONSTANTINOPLE; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.DAO_RECOVERY_INIT; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.DAO_RECOVERY_TRANSITION; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.EXPERIMENTAL_EIPS; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.FRONTIER; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.FUTURE_EIPS; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.GRAY_GLACIER; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.HOMESTEAD; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.ISTANBUL; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.LONDON; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.MUIR_GLACIER; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.OSAKA; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.PARIS; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.PETERSBURG; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.PRAGUE; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.SHANGHAI; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.SPURIOUS_DRAGON; +import static org.hyperledger.besu.datatypes.HardforkId.MainnetHardforkId.TANGERINE_WHISTLE; +import static org.hyperledger.besu.ethereum.mainnet.requests.MainnetRequestsProcessor.pragueRequestsProcessors; + +import org.hyperledger.besu.config.BlobSchedule; +import org.hyperledger.besu.config.BlobScheduleOptions; +import org.hyperledger.besu.config.GenesisConfigOptions; +import org.hyperledger.besu.config.PowAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.BlobType; +import org.hyperledger.besu.datatypes.HardforkId; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.BlockProcessingResult; +import org.hyperledger.besu.ethereum.MainnetBlockValidatorBuilder; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.core.feemarket.CoinbaseFeePriceCalculator; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor.TransactionReceiptFactory; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessListFactory; +import org.hyperledger.besu.ethereum.mainnet.blockhash.CancunPreExecutionProcessor; +import org.hyperledger.besu.ethereum.mainnet.blockhash.FrontierPreExecutionProcessor; +import org.hyperledger.besu.ethereum.mainnet.blockhash.PraguePreExecutionProcessor; +import org.hyperledger.besu.ethereum.mainnet.blockhash.PreExecutionProcessor; +import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.mainnet.parallelization.MainnetParallelBlockProcessor; +import org.hyperledger.besu.ethereum.mainnet.requests.MainnetRequestsValidator; +import org.hyperledger.besu.ethereum.mainnet.requests.RequestContractAddresses; +import org.hyperledger.besu.ethereum.mainnet.requests.RequestProcessorCoordinator; +import org.hyperledger.besu.ethereum.mainnet.staterootcommitter.BalStateRootCommitterFactory; +import org.hyperledger.besu.ethereum.mainnet.transactionpool.OsakaTransactionPoolPreProcessor; +import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.evm.MainnetEVMs; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.contractvalidation.MaxCodeSizeRule; +import org.hyperledger.besu.evm.contractvalidation.PrefixCodeRule; +import org.hyperledger.besu.evm.gascalculator.AmsterdamGasCalculator; +import org.hyperledger.besu.evm.gascalculator.BerlinGasCalculator; +import org.hyperledger.besu.evm.gascalculator.ByzantiumGasCalculator; +import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator; +import org.hyperledger.besu.evm.gascalculator.ConstantinopleGasCalculator; +import org.hyperledger.besu.evm.gascalculator.FrontierGasCalculator; +import org.hyperledger.besu.evm.gascalculator.HomesteadGasCalculator; +import org.hyperledger.besu.evm.gascalculator.IstanbulGasCalculator; +import org.hyperledger.besu.evm.gascalculator.LondonGasCalculator; +import org.hyperledger.besu.evm.gascalculator.OsakaGasCalculator; +import org.hyperledger.besu.evm.gascalculator.PetersburgGasCalculator; +import org.hyperledger.besu.evm.gascalculator.PragueGasCalculator; +import org.hyperledger.besu.evm.gascalculator.ShanghaiGasCalculator; +import org.hyperledger.besu.evm.gascalculator.SpuriousDragonGasCalculator; +import org.hyperledger.besu.evm.gascalculator.TangerineWhistleGasCalculator; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.log.EIP7708TransferLogEmitter; +import org.hyperledger.besu.evm.processor.ContractCreationProcessor; +import org.hyperledger.besu.evm.processor.MessageCallProcessor; +import org.hyperledger.besu.evm.worldstate.CodeDelegationService; +import org.hyperledger.besu.evm.worldstate.WorldState; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.io.IOException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.IntStream; + +import com.google.common.io.Resources; +import io.vertx.core.json.JsonArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Provides the various {@link ProtocolSpec}s on mainnet hard forks. */ +public abstract class MainnetProtocolSpecs { + + private static final Address RIPEMD160_PRECOMPILE = + Address.fromHexString("0x0000000000000000000000000000000000000003"); + + private static final SignatureAlgorithm SIGNATURE_ALGORITHM = + SignatureAlgorithmFactory.getInstance(); + + // A consensus bug at Ethereum mainnet transaction 0xcf416c53 + // deleted an empty account even when the message execution scope + // failed, but the transaction itself succeeded. + private static final HashSet

SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES; + + private static final Wei FRONTIER_BLOCK_REWARD = Wei.fromEth(5); + + private static final Wei BYZANTIUM_BLOCK_REWARD = Wei.fromEth(3); + + private static final Wei CONSTANTINOPLE_BLOCK_REWARD = Wei.fromEth(2); + + private static final Logger LOG = LoggerFactory.getLogger(MainnetProtocolSpecs.class); + private static final int POW_SLOT_TIME_ESTIMATION = 13; + + static { + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES = new HashSet<>(); + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES.add(RIPEMD160_PRECOMPILE); + } + + private MainnetProtocolSpecs() {} + + public static ProtocolSpecBuilder frontierDefinition( + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return new ProtocolSpecBuilder() + .gasCalculator(FrontierGasCalculator::new) + .gasLimitCalculatorBuilder( + (feeMarket, gasCalculator, blobSchedule) -> new FrontierTargetingGasLimitCalculator()) + .evmBuilder(MainnetEVMs::frontier) + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::frontier) + .messageCallProcessorBuilder(MessageCallProcessor::new) + .contractCreationProcessorBuilder( + evm -> + new ContractCreationProcessor( + evm, false, Collections.singletonList(MaxCodeSizeRule.from(evm)), 0)) + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), gasLimitCalculator, false, Optional.empty())) + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidatorFactory, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidatorFactory) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(false) + .warmCoinbase(false) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(FeeMarket.legacy()) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.frontier()) + .build()) + .difficultyCalculator(MainnetDifficultyCalculators.FRONTIER) + .blockHeaderValidatorBuilder( + (feeMarket, gasCalculator, gasLimitCalculator) -> MainnetBlockHeaderValidator.create()) + .ommerHeaderValidatorBuilder( + (feeMarket, gasCalculator, gasLimitCalculator) -> + MainnetBlockHeaderValidator.createLegacyFeeMarketOmmerValidator()) + .blockBodyValidatorBuilder(MainnetBlockBodyValidator::new) + .blockAccessListValidatorBuilder(__ -> BlockAccessListValidator.ALWAYS_REJECT_BAL) + .transactionReceiptFactory(new FrontierTransactionReceiptFactory()) + .blockReward(FRONTIER_BLOCK_REWARD) + .skipZeroBlockRewards(false) + .balConfiguration(balConfiguration) + .blockProcessorBuilder( + isParallelTxProcessingEnabled + ? new MainnetParallelBlockProcessor.ParallelBlockProcessorBuilder(metricsSystem) + : new MainnetBlockProcessor.MainnetBlockProcessorBuilder(metricsSystem)) + .blockValidatorBuilder(MainnetBlockValidatorBuilder::frontier) + .blockImporterBuilder(MainnetBlockImporter::new) + .blockHeaderFunctions(new MainnetBlockHeaderFunctions()) + .miningBeneficiaryCalculator(BlockHeader::getCoinbase) + .evmConfiguration(evmConfiguration) + .preExecutionProcessor(new FrontierPreExecutionProcessor()) + .slotDuration(getSlotDurationFromGenesis(genesisConfigOptions)) + .hardforkId(FRONTIER); + } + + private static Duration getSlotDurationFromGenesis( + final GenesisConfigOptions genesisConfigOptions) { + + if (genesisConfigOptions.isIbft2()) { + return Duration.ofSeconds(genesisConfigOptions.getBftConfigOptions().getBlockPeriodSeconds()); + } + if (genesisConfigOptions.isQbft()) { + return Duration.ofSeconds( + genesisConfigOptions.getQbftConfigOptions().getBlockPeriodSeconds()); + } + if (genesisConfigOptions.isClique()) { + return Duration.ofSeconds( + genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds()); + } + // if no hints are present in the genesis file, then we are in PoW and there is not predefined + // block period, so just return an estimation. + // We also get here if we are in PoS mode, but the right value for PoS slot duration will + // override this value. + return Duration.ofSeconds(POW_SLOT_TIME_ESTIMATION); + } + + public static PoWHasher powHasher(final PowAlgorithm powAlgorithm) { + if (powAlgorithm == null) { + return PoWHasher.UNSUPPORTED; + } + return powAlgorithm == PowAlgorithm.ETHASH ? PoWHasher.ETHASH_LIGHT : PoWHasher.UNSUPPORTED; + } + + public static ProtocolSpecBuilder homesteadDefinition( + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return frontierDefinition( + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(HomesteadGasCalculator::new) + .evmBuilder(MainnetEVMs::homestead) + .contractCreationProcessorBuilder( + evm -> + new ContractCreationProcessor( + evm, true, Collections.singletonList(MaxCodeSizeRule.from(evm)), 0)) + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), gasLimitCalculator, true, Optional.empty())) + .difficultyCalculator(MainnetDifficultyCalculators.HOMESTEAD) + .hardforkId(HOMESTEAD); + } + + public static ProtocolSpecBuilder daoRecoveryInitDefinition( + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return homesteadDefinition( + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .blockHeaderValidatorBuilder( + (feeMarket, gasCalculator, gasLimitCalculator) -> + MainnetBlockHeaderValidator.createDaoValidator()) + .blockProcessorBuilder( + (transactionProcessor, + transactionReceiptFactory, + blockReward, + miningBeneficiaryCalculator, + skipZeroBlockRewards, + protocolSchedule, + balConfig) -> + new DaoBlockProcessor( + isParallelTxProcessingEnabled + ? new MainnetParallelBlockProcessor( + transactionProcessor, + transactionReceiptFactory, + blockReward, + miningBeneficiaryCalculator, + skipZeroBlockRewards, + protocolSchedule, + balConfig, + metricsSystem) + : new MainnetBlockProcessor( + transactionProcessor, + transactionReceiptFactory, + blockReward, + miningBeneficiaryCalculator, + skipZeroBlockRewards, + protocolSchedule, + balConfig, + metricsSystem))) + .hardforkId(DAO_RECOVERY_INIT); + } + + public static ProtocolSpecBuilder daoRecoveryTransitionDefinition( + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return daoRecoveryInitDefinition( + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .blockProcessorBuilder( + isParallelTxProcessingEnabled + ? new MainnetParallelBlockProcessor.ParallelBlockProcessorBuilder(metricsSystem) + : new MainnetBlockProcessor.MainnetBlockProcessorBuilder(metricsSystem)) + .hardforkId(DAO_RECOVERY_TRANSITION); + } + + public static ProtocolSpecBuilder tangerineWhistleDefinition( + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return homesteadDefinition( + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(TangerineWhistleGasCalculator::new) + .hardforkId(TANGERINE_WHISTLE); + } + + public static ProtocolSpecBuilder spuriousDragonDefinition( + final Optional chainId, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return tangerineWhistleDefinition( + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .isReplayProtectionSupported(true) + .gasCalculator(SpuriousDragonGasCalculator::new) + .skipZeroBlockRewards(true) + .messageCallProcessorBuilder( + (evm, precompileContractRegistry) -> + new MessageCallProcessor( + evm, + precompileContractRegistry, + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES)) + .contractCreationProcessorBuilder( + evm -> + new ContractCreationProcessor( + evm, + true, + Collections.singletonList(MaxCodeSizeRule.from(evm)), + 1, + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES)) + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), gasLimitCalculator, true, chainId)) + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidator, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidator) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(true) + .warmCoinbase(false) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(feeMarket) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.frontier()) + .build()) + .hardforkId(SPURIOUS_DRAGON); + } + + public static ProtocolSpecBuilder byzantiumDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return spuriousDragonDefinition( + chainId, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(ByzantiumGasCalculator::new) + .evmBuilder(MainnetEVMs::byzantium) + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::byzantium) + .difficultyCalculator(MainnetDifficultyCalculators.BYZANTIUM) + .transactionReceiptFactory(new ByzantiumTransactionReceiptFactory(enableRevertReason)) + .blockReward(BYZANTIUM_BLOCK_REWARD) + .hardforkId(BYZANTIUM); + } + + public static ProtocolSpecBuilder constantinopleDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return byzantiumDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .difficultyCalculator(MainnetDifficultyCalculators.CONSTANTINOPLE) + .gasCalculator(ConstantinopleGasCalculator::new) + .evmBuilder(MainnetEVMs::constantinople) + .blockReward(CONSTANTINOPLE_BLOCK_REWARD) + .hardforkId(CONSTANTINOPLE); + } + + public static ProtocolSpecBuilder petersburgDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return constantinopleDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(PetersburgGasCalculator::new) + .hardforkId(PETERSBURG); + } + + public static ProtocolSpecBuilder istanbulDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return petersburgDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(IstanbulGasCalculator::new) + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.istanbul( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::istanbul) + .contractCreationProcessorBuilder( + evm -> + new ContractCreationProcessor( + evm, + true, + Collections.singletonList(MaxCodeSizeRule.from(evm)), + 1, + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES)) + .hardforkId(ISTANBUL); + } + + static ProtocolSpecBuilder muirGlacierDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return istanbulDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .difficultyCalculator(MainnetDifficultyCalculators.MUIR_GLACIER) + .hardforkId(MUIR_GLACIER); + } + + static ProtocolSpecBuilder berlinDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return muirGlacierDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(BerlinGasCalculator::new) + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), + gasLimitCalculator, + true, + chainId, + Set.of(TransactionType.FRONTIER, TransactionType.ACCESS_LIST))) + .transactionReceiptFactory(new BerlinTransactionReceiptFactory(enableRevertReason)) + .hardforkId(BERLIN); + } + + static ProtocolSpecBuilder londonDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + final long londonForkBlockNumber = + genesisConfigOptions.getLondonBlockNumber().orElse(Long.MAX_VALUE); + return berlinDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .feeMarketBuilder( + createFeeMarket( + londonForkBlockNumber, + genesisConfigOptions.isZeroBaseFee(), + genesisConfigOptions.isFixedBaseFee(), + false, + miningConfiguration.getMinTransactionGasPrice(), + (blobSchedule) -> + FeeMarket.london( + londonForkBlockNumber, genesisConfigOptions.getBaseFeePerGas()))) + .gasCalculator(LondonGasCalculator::new) + .gasLimitCalculatorBuilder( + (feeMarket, gasCalculator, blobSchedule) -> + new LondonTargetingGasLimitCalculator( + londonForkBlockNumber, (BaseFeeMarket) feeMarket)) + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), + gasLimitCalculator, + feeMarket, + true, + chainId, + Set.of( + TransactionType.FRONTIER, + TransactionType.ACCESS_LIST, + TransactionType.EIP1559), + Integer.MAX_VALUE)) + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidatorFactory, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidatorFactory) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(true) + .warmCoinbase(false) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(feeMarket) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.eip1559()) + .build()) + .contractCreationProcessorBuilder( + evm -> + new ContractCreationProcessor( + evm, + true, + List.of(MaxCodeSizeRule.from(evm), PrefixCodeRule.of()), + 1, + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES)) + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.london( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + .difficultyCalculator(MainnetDifficultyCalculators.LONDON) + .blockHeaderValidatorBuilder( + (feeMarket, gasCalculator, gasLimitCalculator) -> + MainnetBlockHeaderValidator.createBaseFeeMarketValidator((BaseFeeMarket) feeMarket)) + .ommerHeaderValidatorBuilder( + (feeMarket, gasCalculator, gasLimitCalculator) -> + MainnetBlockHeaderValidator.createBaseFeeMarketOmmerValidator( + (BaseFeeMarket) feeMarket)) + .blockBodyValidatorBuilder(BaseFeeBlockBodyValidator::new) + .hardforkId(LONDON); + } + + static ProtocolSpecBuilder arrowGlacierDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return londonDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .difficultyCalculator(MainnetDifficultyCalculators.ARROW_GLACIER) + .hardforkId(ARROW_GLACIER); + } + + static ProtocolSpecBuilder grayGlacierDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return arrowGlacierDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .difficultyCalculator(MainnetDifficultyCalculators.GRAY_GLACIER) + .hardforkId(GRAY_GLACIER); + } + + static ProtocolSpecBuilder parisDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + + return grayGlacierDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.paris(gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + .difficultyCalculator(MainnetDifficultyCalculators.PROOF_OF_STAKE_DIFFICULTY) + .blockHeaderValidatorBuilder(MainnetBlockHeaderValidator::mergeBlockHeaderValidator) + .blockReward(Wei.ZERO) + .skipZeroBlockRewards(true) + .isPoS(true) + .slotDuration(Duration.ofSeconds(miningConfiguration.getUnstable().getPosSlotDuration())) + .hardforkId(PARIS); + } + + static ProtocolSpecBuilder shanghaiDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return parisDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + // gas calculator has new code to support EIP-3860 limit and meter initcode + .gasCalculator(ShanghaiGasCalculator::new) + // EVM has a new operation for EIP-3855 PUSH0 instruction + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.shanghai( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + // we need to flip the Warm Coinbase flag for EIP-3651 warm coinbase + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidatorFactory, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidatorFactory) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(true) + .warmCoinbase(true) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(feeMarket) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.eip1559()) + .build()) + // Contract creation rules for EIP-3860 Limit and meter intitcode + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), + gasLimitCalculator, + feeMarket, + true, + chainId, + Set.of( + TransactionType.FRONTIER, + TransactionType.ACCESS_LIST, + TransactionType.EIP1559), + evm.getMaxInitcodeSize())) + .withdrawalsProcessor(new WithdrawalsProcessor()) + .withdrawalsValidator(new WithdrawalsValidator.AllowedWithdrawals()) + .blockHeaderValidatorBuilder(MainnetBlockHeaderValidator::noBlobBlockHeaderValidator) + .hardforkId(SHANGHAI); + } + + static ProtocolSpecBuilder cancunDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + final long londonForkBlockNumber = genesisConfigOptions.getLondonBlockNumber().orElse(0L); + + return shanghaiDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .feeMarketBuilder( + createFeeMarket( + londonForkBlockNumber, + genesisConfigOptions.isZeroBaseFee(), + genesisConfigOptions.isFixedBaseFee(), + true, + miningConfiguration.getMinTransactionGasPrice(), + (blobSchedule) -> + FeeMarket.cancun( + londonForkBlockNumber, + genesisConfigOptions.getBaseFeePerGas(), + blobSchedule))) + .blobSchedule( + genesisConfigOptions + .getBlobScheduleOptions() + .flatMap(BlobScheduleOptions::getCancun) + .orElse(BlobSchedule.CANCUN_DEFAULT)) + // gas calculator for EIP-4844 blob gas + .gasCalculator(CancunGasCalculator::new) + // gas limit with EIP-4844 max blob gas per block + .gasLimitCalculatorBuilder( + (feeMarket, gasCalculator, blobSchedule) -> + new CancunTargetingGasLimitCalculator( + londonForkBlockNumber, + (BaseFeeMarket) feeMarket, + gasCalculator, + blobSchedule.getMax(), + blobSchedule.getTarget())) + // EVM changes to support EIP-1153: TSTORE and EIP-5656: MCOPY + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.cancun( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + // use Cancun fee market + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidator, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidator) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(true) + .warmCoinbase(true) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(feeMarket) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.eip1559()) + .build()) + // change to check for max blob gas per block for EIP-4844 + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), + gasLimitCalculator, + feeMarket, + true, + chainId, + Set.of( + TransactionType.FRONTIER, + TransactionType.ACCESS_LIST, + TransactionType.EIP1559, + TransactionType.BLOB), + Set.of(BlobType.KZG_PROOF), + evm.getMaxInitcodeSize())) + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::cancun) + .blockHeaderValidatorBuilder(MainnetBlockHeaderValidator::blobAwareBlockHeaderValidator) + .preExecutionProcessor(getPreExecutionProcessor(genesisConfigOptions)) + .hardforkId(CANCUN); + } + + private static PreExecutionProcessor getPreExecutionProcessor( + final GenesisConfigOptions genesisConfigOptions) { + if (isPoAConsensus(genesisConfigOptions) && !hasSystemContractAddresses(genesisConfigOptions)) { + return new FrontierPreExecutionProcessor(); + } + + return new CancunPreExecutionProcessor(); + } + + private static PreExecutionProcessor getPraguePreExecutionProcessor( + final GenesisConfigOptions genesisConfigOptions) { + if (isPoAConsensus(genesisConfigOptions) && !hasSystemContractAddresses(genesisConfigOptions)) { + return new FrontierPreExecutionProcessor(); + } + + return new PraguePreExecutionProcessor(); + } + + static ProtocolSpecBuilder pragueDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + ProtocolSpecBuilder pragueSpecBuilder = + cancunDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .blobSchedule( + genesisConfigOptions + .getBlobScheduleOptions() + .flatMap(BlobScheduleOptions::getPrague) + .orElse(BlobSchedule.PRAGUE_DEFAULT)) + .gasCalculator(PragueGasCalculator::new) + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.prague( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + + // EIP-2537 BLS12-381 precompiles + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::prague) + + // EIP-7002 Withdrawals / EIP-6610 Deposits / EIP-7685 Requests + .requestsValidator(new MainnetRequestsValidator()) + + // change to accept EIP-7702 transactions + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), + gasLimitCalculator, + feeMarket, + true, + chainId, + Set.of( + TransactionType.FRONTIER, + TransactionType.ACCESS_LIST, + TransactionType.EIP1559, + TransactionType.BLOB, + TransactionType.DELEGATE_CODE), + Set.of(BlobType.KZG_PROOF), + evm.getMaxInitcodeSize())) + // CodeDelegationProcessor + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidator, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidator) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(true) + .warmCoinbase(true) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(feeMarket) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.eip1559()) + .codeDelegationProcessor( + new CodeDelegationProcessor( + chainId, + SIGNATURE_ALGORITHM.getHalfCurveOrder(), + new CodeDelegationService())) + .build()) + // EIP-2935 Blockhash processor + .preExecutionProcessor(getPraguePreExecutionProcessor(genesisConfigOptions)) + .hardforkId(PRAGUE); + if (isPoAConsensus(genesisConfigOptions) && !hasSystemContractAddresses(genesisConfigOptions)) { + LOG.warn( + "Skipping system contract request processors for PoA consensus (clique/ibft/qbft) without system contract addresses."); + pragueSpecBuilder.requestProcessorCoordinator(RequestProcessorCoordinator.noOp()); + } else { + try { + RequestContractAddresses requestContractAddresses = + RequestContractAddresses.fromGenesis(genesisConfigOptions); + + pragueSpecBuilder.requestProcessorCoordinator( + pragueRequestsProcessors(requestContractAddresses)); + } catch (NoSuchElementException nsee) { + LOG.warn("Prague definitions require system contract addresses in genesis"); + throw nsee; + } + } + + return pragueSpecBuilder; + } + + private static boolean isPoAConsensus(final GenesisConfigOptions genesisConfigOptions) { + return genesisConfigOptions.isClique() + || genesisConfigOptions.isIbft2() + || genesisConfigOptions.isQbft(); + } + + private static boolean hasSystemContractAddresses( + final GenesisConfigOptions genesisConfigOptions) { + return genesisConfigOptions.getDepositContractAddress().isPresent() + && genesisConfigOptions.getWithdrawalRequestContractAddress().isPresent() + && genesisConfigOptions.getConsolidationRequestContractAddress().isPresent(); + } + + static ProtocolSpecBuilder osakaDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + final long londonForkBlockNumber = genesisConfigOptions.getLondonBlockNumber().orElse(0L); + + return pragueDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(OsakaGasCalculator::new) + // tx gas limit cap EIP-7825 + .gasLimitCalculatorBuilder( + (feeMarket, gasCalculator, blobSchedule) -> + new OsakaTargetingGasLimitCalculator( + londonForkBlockNumber, + (BaseFeeMarket) feeMarket, + gasCalculator, + blobSchedule.getMax(), + blobSchedule.getTarget(), + miningConfiguration.getMaxBlobsPerTransaction(), + miningConfiguration.getMaxBlobsPerBlock())) + .evmBuilder( + (gasCalculator, __) -> + MainnetEVMs.osaka(gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + .transactionValidatorFactoryBuilder( + (evm, gasLimitCalculator, feeMarket) -> + new TransactionValidatorFactory( + evm.getGasCalculator(), + gasLimitCalculator, + feeMarket, + true, + chainId, + Set.of( + TransactionType.FRONTIER, + TransactionType.ACCESS_LIST, + TransactionType.EIP1559, + TransactionType.BLOB, + TransactionType.DELEGATE_CODE, + TransactionType.AERE_PQ), + Set.of(BlobType.KZG_CELL_PROOFS), + evm.getMaxInitcodeSize())) + .transactionPoolPreProcessor(new OsakaTransactionPoolPreProcessor()) + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::osaka) + .blockValidatorBuilder(MainnetBlockValidatorBuilder::osaka) + .hardforkId(OSAKA); + } + + static ProtocolSpecBuilder bpo1Definition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + ProtocolSpecBuilder builder = + osakaDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem); + return applyBlobSchedule( + builder, + genesisConfigOptions, + BlobScheduleOptions::getBpo1, + GenesisConfigOptions::getBpo1Time, + BPO1); + } + + static ProtocolSpecBuilder bpo2Definition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + ProtocolSpecBuilder builder = + bpo1Definition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem); + return applyBlobSchedule( + builder, + genesisConfigOptions, + BlobScheduleOptions::getBpo2, + GenesisConfigOptions::getBpo2Time, + BPO2); + } + + static ProtocolSpecBuilder bpo3Definition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + ProtocolSpecBuilder builder = + bpo2Definition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem); + return applyBlobSchedule( + builder, + genesisConfigOptions, + BlobScheduleOptions::getBpo3, + GenesisConfigOptions::getBpo3Time, + BPO3); + } + + static ProtocolSpecBuilder bpo4Definition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + ProtocolSpecBuilder builder = + bpo3Definition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem); + return applyBlobSchedule( + builder, + genesisConfigOptions, + BlobScheduleOptions::getBpo4, + GenesisConfigOptions::getBpo4Time, + BPO4); + } + + static ProtocolSpecBuilder bpo5Definition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + ProtocolSpecBuilder builder = + bpo4Definition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem); + return applyBlobSchedule( + builder, + genesisConfigOptions, + BlobScheduleOptions::getBpo5, + GenesisConfigOptions::getBpo5Time, + BPO5); + } + + static ProtocolSpecBuilder amsterdamDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + return bpo5Definition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .gasCalculator(AmsterdamGasCalculator::new) + // EIP-7708: Override evmBuilder to use Amsterdam EVM with transfer logging + .evmBuilder( + (gasCalculator, __) -> + MainnetEVMs.amsterdam( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + // EIP-7708: ContractCreationProcessor with transfer log emission enabled + .contractCreationProcessorBuilder( + evm -> + new ContractCreationProcessor( + evm, + true, + List.of(MaxCodeSizeRule.from(evm), PrefixCodeRule.of()), + 1, + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES, + EIP7708TransferLogEmitter.INSTANCE)) + // EIP-7708: MessageCallProcessor with transfer log emission enabled + .messageCallProcessorBuilder( + (evm, precompileContractRegistry) -> + new MessageCallProcessor( + evm, + precompileContractRegistry, + SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES, + EIP7708TransferLogEmitter.INSTANCE)) + // EIP-7708: TransactionProcessor configured for Amsterdam with transfer log emission + .transactionProcessorBuilder( + (gasCalculator, + feeMarket, + transactionValidator, + contractCreationProcessor, + messageCallProcessor) -> + MainnetTransactionProcessor.builder() + .gasCalculator(gasCalculator) + .transactionValidatorFactory(transactionValidator) + .contractCreationProcessor(contractCreationProcessor) + .messageCallProcessor(messageCallProcessor) + .clearEmptyAccounts(true) + .warmCoinbase(true) + .maxStackSize(evmConfiguration.evmStackSize()) + .feeMarket(feeMarket) + .coinbaseFeePriceCalculator(CoinbaseFeePriceCalculator.eip1559()) + .codeDelegationProcessor( + new CodeDelegationProcessor( + chainId, + SIGNATURE_ALGORITHM.getHalfCurveOrder(), + new CodeDelegationService())) + .transferLogEmitter(EIP7708TransferLogEmitter.INSTANCE) + .build()) + .blockAccessListFactory(new BlockAccessListFactory()) + .blockAccessListValidatorBuilder(MainnetBlockAccessListValidator::create) + .stateRootCommitterFactory(new BalStateRootCommitterFactory(balConfiguration)) + // EIP-8037: Disable validation-time TX_MAX_GAS_LIMIT cap (enforced at runtime on regular + // gas) + .gasLimitCalculatorBuilder( + (feeMarket, gasCalculator, blobSchedule) -> { + final long londonForkBlock = genesisConfigOptions.getLondonBlockNumber().orElse(0L); + return new AmsterdamTargetingGasLimitCalculator( + londonForkBlock, + (BaseFeeMarket) feeMarket, + gasCalculator, + blobSchedule.getMax(), + blobSchedule.getTarget(), + miningConfiguration.getMaxBlobsPerTransaction(), + miningConfiguration.getMaxBlobsPerBlock()); + }) + // EIP-8037: Amsterdam gas calculator with state gas cost support + .gasCalculator(AmsterdamGasCalculator::new) + // Amsterdam (EIP-7778 + EIP-8037): Pre-refund 2D gas accounting + .blockGasAccountingStrategy(BlockGasAccountingStrategy.AMSTERDAM) + // Amsterdam: Validator uses pre-refund gas_metered = max(regular, state) from processing + .blockGasUsedValidator(BlockGasUsedValidator.AMSTERDAM) + .hardforkId(AMSTERDAM); + } + + private static ProtocolSpecBuilder applyBlobSchedule( + final ProtocolSpecBuilder builder, + final GenesisConfigOptions genesisConfigOptions, + final Function> blobGetter, + final Function blobScheduleTimestampGetter, + final HardforkId hardforkId) { + // Only apply a fork's blob schedule if the fork is actually activated (has a timestamp). + // This prevents inactive BPO forks from overriding the blob schedule with stale values + // from the genesis config. + if (blobScheduleTimestampGetter.apply(genesisConfigOptions).isPresent()) { + genesisConfigOptions + .getBlobScheduleOptions() + .flatMap(blobGetter) + .ifPresent(builder::blobSchedule); + } + return builder.hardforkId(hardforkId); + } + + static ProtocolSpecBuilder futureEipsDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + // AERE MILESTONE ISOLATION: base the AerePQC (futureEips) activation on the + // OSAKA spec, NOT amsterdam. Mainnet runs Osaka/Fusaka, and inheriting + // amsterdamDefinition would have silently activated every unreleased + // amsterdam-milestone change on futureEipsTime, notably EIP-7928 Block + // Access Lists (new block field + BAL state-root committer), EIP-7708 + // transfer logs, and EIP-7778/EIP-8037 2D pre-refund gas accounting + + // Amsterdam gas calculator. Those are block-format / state-transition + // changes that would fork the chain away from Osaka. Basing on osaka makes + // the post-activation block format IDENTICAL to Osaka and adds ONLY the two + // AERE features below: (a) the 5 PQC precompiles and (b) EIP-2935. + return osakaDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + // AERE EIP-2935: historical block hashes (ring buffer in the system + // contract at 0x0000f9...2935), wired into the SAME futureEips (AerePQC) + // hard fork as the PQC precompiles so a single futureEipsTime activation + // delivers both. Forced on unconditionally here: the inherited Prague + // definition falls back to FrontierPreExecutionProcessor for QBFT/PoA + // chains without system-contract addresses, which would silently skip + // EIP-2935 history storage. This override restores it, protocol-native. + .preExecutionProcessor(new PraguePreExecutionProcessor()) + .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::futureEips) + // AERE AerePQC fork also activates Block-STM optimistic parallel execution when + // AERE_BLOCKSTM=1, so one hard fork delivers PQC + EIP-2935 + parallel exec together. + // When unset this reproduces the inherited default builder exactly (live-safe). + .blockProcessorBuilder( + "1".equals(System.getenv("AERE_BLOCKSTM")) + ? new org.hyperledger.besu.ethereum.mainnet.blockstm.AereBlockStmProcessor + .AereBlockStmProcessorBuilder(metricsSystem) + : (isParallelTxProcessingEnabled + ? new org.hyperledger.besu.ethereum.mainnet.parallelization + .MainnetParallelBlockProcessor.ParallelBlockProcessorBuilder(metricsSystem) + : new org.hyperledger.besu.ethereum.mainnet.MainnetBlockProcessor + .MainnetBlockProcessorBuilder(metricsSystem))) + .hardforkId(FUTURE_EIPS); + } + + static ProtocolSpecBuilder experimentalEipsDefinition( + final Optional chainId, + final boolean enableRevertReason, + final GenesisConfigOptions genesisConfigOptions, + final EvmConfiguration evmConfiguration, + final MiningConfiguration miningConfiguration, + final boolean isParallelTxProcessingEnabled, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + + return futureEipsDefinition( + chainId, + enableRevertReason, + genesisConfigOptions, + evmConfiguration, + miningConfiguration, + isParallelTxProcessingEnabled, + balConfiguration, + metricsSystem) + .evmBuilder( + (gasCalculator, jdCacheConfig) -> + MainnetEVMs.experimentalEips( + gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration)) + .hardforkId(EXPERIMENTAL_EIPS); + } + + private static class FrontierTransactionReceiptFactory implements TransactionReceiptFactory { + + @Override + public TransactionReceipt create( + final TransactionType transactionType, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { + return new TransactionReceipt( + worldState.frontierRootHash(), + gasUsed, + result.getLogs(), + Optional.empty()); // No revert reason in Frontier + } + + @Override + public TransactionReceipt create( + final TransactionType transactionType, + final TransactionProcessingResult result, + final long gasUsed) { + throw new UnsupportedOperationException("No stateless transaction receipt in Frontier"); + } + } + + private abstract static class PostFrontierTransactionReceiptFactory + implements TransactionReceiptFactory { + protected final boolean revertReasonEnabled; + + public PostFrontierTransactionReceiptFactory(final boolean revertReasonEnabled) { + this.revertReasonEnabled = revertReasonEnabled; + } + + @Override + public TransactionReceipt create( + final TransactionType transactionType, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { + return create(transactionType, result, gasUsed); + } + } + + static class ByzantiumTransactionReceiptFactory extends PostFrontierTransactionReceiptFactory { + public ByzantiumTransactionReceiptFactory(final boolean revertReasonEnabled) { + super(revertReasonEnabled); + } + + @Override + public TransactionReceipt create( + final TransactionType transactionType, + final TransactionProcessingResult result, + final long gasUsed) { + return new TransactionReceipt( + result.isSuccessful() ? 1 : 0, + gasUsed, + result.getLogs(), + revertReasonEnabled ? result.getRevertReason() : Optional.empty()); + } + } + + static class BerlinTransactionReceiptFactory extends PostFrontierTransactionReceiptFactory { + + public BerlinTransactionReceiptFactory(final boolean revertReasonEnabled) { + super(revertReasonEnabled); + } + + @Override + public TransactionReceipt create( + final TransactionType transactionType, + final TransactionProcessingResult result, + final long gasUsed) { + return new TransactionReceipt( + transactionType, + result.isSuccessful() ? 1 : 0, + gasUsed, + result.getLogs(), + revertReasonEnabled ? result.getRevertReason() : Optional.empty()); + } + } + + private record DaoBlockProcessor(BlockProcessor wrapped) implements BlockProcessor { + + @Override + public BlockProcessingResult processBlock( + final ProtocolContext protocolContext, + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block) { + updateWorldStateForDao(worldState); + return wrapped.processBlock( + protocolContext, + blockchain, + worldState, + block, + new AbstractBlockProcessor.PreprocessingFunction.NoPreprocessing()); + } + + @Override + public BlockProcessingResult processBlock( + final ProtocolContext protocolContext, + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block, + final Optional blockAccessList) { + updateWorldStateForDao(worldState); + return wrapped.processBlock(protocolContext, blockchain, worldState, block, blockAccessList); + } + + @Override + public BlockProcessingResult processBlock( + final ProtocolContext protocolContext, + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block, + final AbstractBlockProcessor.PreprocessingFunction preprocessingBlockFunction) { + return processBlock( + protocolContext, + blockchain, + worldState, + block, + Optional.empty(), + preprocessingBlockFunction); + } + + @Override + public BlockProcessingResult processBlock( + final ProtocolContext protocolContext, + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block, + final Optional blockAccessList, + final AbstractBlockProcessor.PreprocessingFunction preprocessingBlockFunction) { + updateWorldStateForDao(worldState); + return wrapped.processBlock( + protocolContext, + blockchain, + worldState, + block, + blockAccessList, + preprocessingBlockFunction); + } + + private static final Address DAO_REFUND_CONTRACT_ADDRESS = + Address.fromHexString("0xbf4ed7b27f1d666546e30d74d50d173d20bca754"); + + private void updateWorldStateForDao(final MutableWorldState worldState) { + try { + final JsonArray json = + new JsonArray( + Resources.toString( + Objects.requireNonNull(this.getClass().getResource("/daoAddresses.json")), + StandardCharsets.UTF_8)); + final List
addresses = + IntStream.range(0, json.size()) + .mapToObj(json::getString) + .map(Address::fromHexString) + .toList(); + final WorldUpdater worldUpdater = worldState.updater(); + final MutableAccount daoRefundContract = + worldUpdater.getOrCreate(DAO_REFUND_CONTRACT_ADDRESS); + for (final Address address : addresses) { + final MutableAccount account = worldUpdater.getOrCreate(address); + final Wei balance = account.getBalance(); + account.decrementBalance(balance); + daoRefundContract.incrementBalance(balance); + } + worldUpdater.commit(); + } catch (final IOException e) { + throw new IllegalStateException(e); + } + } + } + + static ProtocolSpecBuilder.FeeMarketBuilder createFeeMarket( + final long londonForkBlockNumber, + final boolean isZeroBaseFee, + final boolean isFixedBaseFee, + final boolean supportsBlobs, + final Wei minTransactionGasPrice, + final ProtocolSpecBuilder.FeeMarketBuilder feeMarketBuilder) { + if (isZeroBaseFee) { + var baseFeeMarket = + supportsBlobs + ? FeeMarket.zeroBlobFee(londonForkBlockNumber) + : FeeMarket.zeroBaseFee(londonForkBlockNumber); + return blobSchedule -> baseFeeMarket; + } + if (isFixedBaseFee) { + return blobSchedule -> FeeMarket.fixedBaseFee(londonForkBlockNumber, minTransactionGasPrice); + } + return feeMarketBuilder; + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java new file mode 100644 index 0000000..cb8ddfa --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -0,0 +1,877 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static org.hyperledger.besu.evm.internal.Words.clampedAdd; +import static org.hyperledger.besu.evm.worldstate.CodeDelegationHelper.getTarget; +import static org.hyperledger.besu.evm.worldstate.CodeDelegationHelper.hasCodeDelegation; + +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.feemarket.CoinbaseFeePriceCalculator; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.AccessLocationTracker; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.PartialBlockAccessView; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; +import org.hyperledger.besu.ethereum.trie.MerkleTrieException; +import org.hyperledger.besu.evm.Code; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; +import org.hyperledger.besu.evm.log.TransferLogEmitter; +import org.hyperledger.besu.evm.processor.AbstractMessageProcessor; +import org.hyperledger.besu.evm.processor.ContractCreationProcessor; +import org.hyperledger.besu.evm.processor.MessageCallProcessor; +import org.hyperledger.besu.evm.tracing.OperationTracer; +import org.hyperledger.besu.evm.worldstate.CodeDelegationHelper; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; + +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MainnetTransactionProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(MainnetTransactionProcessor.class); + + private static final Set
EMPTY_ADDRESS_SET = Set.of(); + + protected final GasCalculator gasCalculator; + + protected final TransactionValidatorFactory transactionValidatorFactory; + + private final ContractCreationProcessor contractCreationProcessor; + + private final MessageCallProcessor messageCallProcessor; + + private final int maxStackSize; + + private final boolean clearEmptyAccounts; + + protected final boolean warmCoinbase; + + protected final FeeMarket feeMarket; + private final CoinbaseFeePriceCalculator coinbaseFeePriceCalculator; + + private final Optional maybeCodeDelegationProcessor; + + private final TransferLogEmitter transferLogEmitter; + + private MainnetTransactionProcessor( + final GasCalculator gasCalculator, + final TransactionValidatorFactory transactionValidatorFactory, + final ContractCreationProcessor contractCreationProcessor, + final MessageCallProcessor messageCallProcessor, + final boolean clearEmptyAccounts, + final boolean warmCoinbase, + final int maxStackSize, + final FeeMarket feeMarket, + final CoinbaseFeePriceCalculator coinbaseFeePriceCalculator, + final CodeDelegationProcessor maybeCodeDelegationProcessor, + final TransferLogEmitter transferLogEmitter) { + this.gasCalculator = gasCalculator; + this.transactionValidatorFactory = transactionValidatorFactory; + this.contractCreationProcessor = contractCreationProcessor; + this.messageCallProcessor = messageCallProcessor; + this.clearEmptyAccounts = clearEmptyAccounts; + this.warmCoinbase = warmCoinbase; + this.maxStackSize = maxStackSize; + this.feeMarket = feeMarket; + this.coinbaseFeePriceCalculator = coinbaseFeePriceCalculator; + this.maybeCodeDelegationProcessor = Optional.ofNullable(maybeCodeDelegationProcessor); + this.transferLogEmitter = transferLogEmitter; + } + + /** + * Applies a transaction to the current system state. + * + * @param worldState The current world state + * @param blockHeader The current block header + * @param transaction The transaction to process + * @param miningBeneficiary The address which is to receive the transaction fee + * @param blockHashLookup The {@link BlockHashLookup} to use for BLOCKHASH operations + * @param transactionValidationParams Validation parameters that will be used by the {@link + * MainnetTransactionValidator} + * @return the transaction result + * @see MainnetTransactionValidator + * @see TransactionValidationParams + */ + public TransactionProcessingResult processTransaction( + final WorldUpdater worldState, + final ProcessableBlockHeader blockHeader, + final Transaction transaction, + final Address miningBeneficiary, + final BlockHashLookup blockHashLookup, + final TransactionValidationParams transactionValidationParams, + final Wei blobGasPrice) { + return processTransaction( + worldState, + blockHeader, + transaction, + miningBeneficiary, + OperationTracer.NO_TRACING, + blockHashLookup, + transactionValidationParams, + blobGasPrice); + } + + /** + * Applies a transaction to the current system state. + * + * @param worldState The current world state + * @param blockHeader The current block header + * @param transaction The transaction to process + * @param miningBeneficiary The address which is to receive the transaction fee + * @param operationTracer The tracer to record results of each EVM operation + * @param blockHashLookup The {@link BlockHashLookup} to use for BLOCKHASH operations + * @return the transaction result + */ + public TransactionProcessingResult processTransaction( + final WorldUpdater worldState, + final ProcessableBlockHeader blockHeader, + final Transaction transaction, + final Address miningBeneficiary, + final OperationTracer operationTracer, + final BlockHashLookup blockHashLookup, + final Wei blobGasPrice) { + return processTransaction( + worldState, + blockHeader, + transaction, + miningBeneficiary, + operationTracer, + blockHashLookup, + ImmutableTransactionValidationParams.builder().build(), + blobGasPrice); + } + + public TransactionProcessingResult processTransaction( + final WorldUpdater worldState, + final ProcessableBlockHeader blockHeader, + final Transaction transaction, + final Address miningBeneficiary, + final OperationTracer operationTracer, + final BlockHashLookup blockHashLookup, + final TransactionValidationParams transactionValidationParams, + final Wei blobGasPrice) { + return processTransaction( + worldState, + blockHeader, + transaction, + miningBeneficiary, + operationTracer, + blockHashLookup, + transactionValidationParams, + blobGasPrice, + Optional.empty()); + } + + public TransactionProcessingResult processTransaction( + final WorldUpdater worldState, + final ProcessableBlockHeader blockHeader, + final Transaction transaction, + final Address miningBeneficiary, + final OperationTracer operationTracer, + final BlockHashLookup blockHashLookup, + final TransactionValidationParams transactionValidationParams, + final Wei blobGasPrice, + final Optional accessLocationTracker) { + try { + final var transactionValidator = transactionValidatorFactory.get(); + LOG.trace("Starting execution of {}", transaction); + // AIP-20: below the activation height the type is invalid in blocks, on every node alike. + if (transaction.getType().supportsPqAuthorization() + && !AerePqTransactionSupport.isActiveAt(blockHeader.getNumber())) { + return TransactionProcessingResult.invalid( + ValidationResult.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + AerePqTransactionSupport.inactiveMessage(blockHeader.getNumber()))); + } + ValidationResult validationResult = + transactionValidator.validate( + transaction, + blockHeader.getBaseFee(), + Optional.ofNullable(blobGasPrice), + transactionValidationParams); + // Make sure the transaction is intrinsically valid before trying to + // compare against a sender account (because the transaction may not + // be signed correctly to extract the sender). + if (!validationResult.isValid()) { + LOG.debug("Invalid transaction: {}", validationResult.getErrorMessage()); + return TransactionProcessingResult.invalid(validationResult); + } + + final Address senderAddress = transaction.getSender(); + final MutableAccount sender = worldState.getOrCreateSenderAccount(senderAddress); + accessLocationTracker.ifPresent(t -> t.addTouchedAccount(senderAddress)); + + validationResult = + transactionValidator.validateForSender(transaction, sender, transactionValidationParams); + if (!validationResult.isValid()) { + LOG.debug("Invalid transaction: {}", validationResult.getErrorMessage()); + return TransactionProcessingResult.invalid(validationResult); + } + + operationTracer.tracePrepareTransaction(worldState, transaction); + + final Set
eip2930WarmAddressList = new HashSet<>(Address.SIZE); + + final long previousNonce = sender.incrementNonce(); + LOG.trace( + "Incremented sender {} nonce ({} -> {})", + senderAddress, + previousNonce, + sender.getNonce()); + + final Wei transactionGasPrice = + feeMarket.getTransactionPriceCalculator().price(transaction, blockHeader.getBaseFee()); + + final long blobGas = gasCalculator.blobGasCost(transaction.getBlobCount()); + + final Wei upfrontGasCost = + transaction.getUpfrontGasCost(transactionGasPrice, blobGasPrice, blobGas); + try { + final Wei previousBalance = sender.decrementBalance(upfrontGasCost); + LOG.trace( + "Deducted sender {} upfront gas cost {} ({} -> {})", + senderAddress, + upfrontGasCost, + previousBalance, + sender.getBalance()); + } catch (final IllegalStateException ise) { + if (transactionValidationParams.allowUnderpriced()) { + LOG.trace("Allowing account balance underflow as requested"); + } else { + throw ise; + } + } + + long codeDelegationRefund = 0L; + long alreadyExistingDelegators = 0L; + if (transaction.getType().equals(TransactionType.DELEGATE_CODE)) { + if (maybeCodeDelegationProcessor.isEmpty()) { + throw new RuntimeException("Code delegation processor is required for 7702 transactions"); + } + + final WorldUpdater delegationUpdater = worldState.updater(); + final CodeDelegationResult codeDelegationResult = + maybeCodeDelegationProcessor + .get() + .process(delegationUpdater, transaction, accessLocationTracker); + eip2930WarmAddressList.addAll(codeDelegationResult.accessedDelegatorAddresses()); + alreadyExistingDelegators = codeDelegationResult.alreadyExistingDelegators(); + codeDelegationRefund = + gasCalculator.calculateDelegateCodeGasRefund(alreadyExistingDelegators); + delegationUpdater.commit(); + } + + final List eip2930AccessListEntries = + transaction.getAccessList().orElse(List.of()); + // we need to keep a separate hash set of addresses in case they specify no storage. + // No-storage is a common pattern, especially for Externally Owned Accounts + final Multimap eip2930StorageList = HashMultimap.create(); + int accessListStorageCount = 0; + for (final var entry : eip2930AccessListEntries) { + final Address address = entry.address(); + eip2930WarmAddressList.add(address); + final List storageKeys = entry.storageKeys(); + eip2930StorageList.putAll(address, storageKeys); + accessListStorageCount += storageKeys.size(); + } + if (warmCoinbase) { + eip2930WarmAddressList.add(miningBeneficiary); + } + + final long accessListGas = + gasCalculator.accessListGasCost(eip2930AccessListEntries.size(), accessListStorageCount); + final long codeDelegationGas = + gasCalculator.delegateCodeGasCost(transaction.codeDelegationListSize()); + final long intrinsicRegularGas = + gasCalculator.transactionIntrinsicGasCost( + transaction, clampedAdd(accessListGas, codeDelegationGas)); + + // EIP-8037: Validate that gas limit covers both regular AND state intrinsic gas. + // This must be checked before frame construction to reject the tx at the intrinsic level. + final var stateGasCalc = gasCalculator.stateGasCostCalculator(); + final long intrinsicStateGas = + stateGasCalc.transactionIntrinsicStateGas( + blockHeader.getGasLimit(), + transaction.isContractCreation(), + transaction.codeDelegationListSize()); + if (transaction.getGasLimit() < intrinsicRegularGas + intrinsicStateGas) { + LOG.trace( + "Insufficient gas for intrinsic cost: gasLimit={}, regularIntrinsic={}, stateIntrinsic={}", + transaction.getGasLimit(), + intrinsicRegularGas, + intrinsicStateGas); + return TransactionProcessingResult.invalid( + ValidationResult.invalid( + TransactionInvalidReason.INTRINSIC_GAS_EXCEEDS_GAS_LIMIT, + String.format( + "intrinsic gas cost %d (regular %d + state %d) exceeds gas limit %d", + intrinsicRegularGas + intrinsicStateGas, + intrinsicRegularGas, + intrinsicStateGas, + transaction.getGasLimit()))); + } + + final long gasAvailable = transaction.getGasLimit() - intrinsicRegularGas; + LOG.trace( + "Gas available for execution {} = {} - {} (limit - intrinsic)", + gasAvailable, + transaction.getGasLimit(), + intrinsicRegularGas); + + final WorldUpdater worldUpdater = worldState.updater(); + + operationTracer.traceStartTransaction(worldUpdater, transaction); + + final MessageFrame.Builder commonMessageFrameBuilder = + MessageFrame.builder() + .maxStackSize(maxStackSize) + .worldUpdater(worldUpdater.updater()) + .initialGas(gasAvailable) + .originator(senderAddress) + .gasPrice(transactionGasPrice) + .blobGasPrice(blobGasPrice) + .sender(senderAddress) + .value(transaction.getValue()) + .apparentValue(transaction.getValue()) + .blockValues(blockHeader) + .completer(__ -> {}) + .miningBeneficiary(miningBeneficiary) + .blockHashLookup(blockHashLookup) + .eip2930AccessListWarmStorage(eip2930StorageList); + + accessLocationTracker.ifPresent(commonMessageFrameBuilder::eip7928AccessList); + + if (transaction.getVersionedHashes().isPresent()) { + commonMessageFrameBuilder.versionedHashes( + Optional.of(transaction.getVersionedHashes().get().stream().toList())); + } else { + commonMessageFrameBuilder.versionedHashes(Optional.empty()); + } + + final MessageFrame initialFrame; + if (transaction.isContractCreation()) { + final Address contractAddress = + Address.contractAddress(senderAddress, sender.getNonce() - 1L); + accessLocationTracker.ifPresent(t -> t.addTouchedAccount(contractAddress)); + + final Bytes initCodeBytes = transaction.getPayload(); + Code code = new Code(initCodeBytes); + initialFrame = + commonMessageFrameBuilder + .type(MessageFrame.Type.CONTRACT_CREATION) + .address(contractAddress) + .contract(contractAddress) + .inputData(initCodeBytes.slice(code.getSize())) + .code(code) + .eip2930AccessListWarmAddresses(eip2930WarmAddressList) + .build(); + } else { + @SuppressWarnings("OptionalGetWithoutIsPresent") // isContractCall tests isPresent + final Address to = transaction.getTo().get(); + accessLocationTracker.ifPresent(t -> t.addTouchedAccount(to)); + final Code code = + processCodeFromAccount( + worldState, eip2930WarmAddressList, worldState.get(to), accessLocationTracker); + + initialFrame = + commonMessageFrameBuilder + .type(MessageFrame.Type.MESSAGE_CALL) + .address(to) + .contract(to) + .inputData(transaction.getPayload()) + .code(code) + .eip2930AccessListWarmAddresses(eip2930WarmAddressList) + .build(); + } + // EIP-8037: Initialize the state gas reservoir for Amsterdam+ forks. + // When multidimensional gas is active, regular gas is capped at TX_MAX_GAS_LIMIT - intrinsic. + // Gas beyond that cap goes into the state gas reservoir. + if (stateGasCalc.isActive()) { + final long regularBudget = + Math.max(0L, stateGasCalc.transactionRegularGasLimit() - intrinsicRegularGas); + final long gasLeft = Math.min(regularBudget, gasAvailable); + final long reservoir = gasAvailable - gasLeft; + initialFrame.setGasRemaining(gasLeft); + initialFrame.setStateGasReservoir(reservoir); + } + // EIP-8037: Charge state gas for intrinsic costs + if (transaction.isContractCreation()) { + stateGasCalc.chargeCreateStateGas(initialFrame); + } + if (transaction.getType().equals(TransactionType.DELEGATE_CODE)) { + stateGasCalc.chargeCodeDelegationStateGas( + initialFrame, transaction.codeDelegationListSize(), alreadyExistingDelegators); + } + // EIP-8037: Advance the undo mark so intrinsic state gas charges (auth delegation, + // contract creation) are not rolled back if the initial frame's execution reverts. + // These are transaction-level costs that persist regardless of execution outcome. + initialFrame.advanceUndoMark(); + + Deque messageFrameStack = initialFrame.getMessageFrameStack(); + + // EIP-8037: Track spillBurned before the initial frame's final processing step. + // When the initial frame reverts/halts, its state gas spill should count as state gas + // for block accounting. Child frame spills (tracked earlier) should not. + long spillBurnedBeforeInitialFinal = 0; + while (!messageFrameStack.isEmpty()) { + if (messageFrameStack.size() == 1) { + spillBurnedBeforeInitialFinal = initialFrame.getStateGasSpillBurned(); + } + process(messageFrameStack.peekFirst(), operationTracer); + } + final long initialFrameStateGasSpill = + initialFrame.getStateGasSpillBurned() - spillBurnedBeforeInitialFinal; + + // EIP-8037: Runtime TX_MAX_GAS_LIMIT enforcement on regular gas only. + // With multidimensional gas, tx.gasLimit can exceed TX_MAX_GAS_LIMIT to accommodate + // state gas, but regular gas consumption is still bounded at runtime. + // For pre-Amsterdam forks, transactionRegularGasLimit() returns Long.MAX_VALUE (always + // passes). + // We also need to include leftover reservoir in remaining gas for correct consumption + // calculation + final long totalRemaining = + initialFrame.getRemainingGas() + initialFrame.getStateGasReservoir(); + final long totalConsumed = transaction.getGasLimit() - totalRemaining; + final long regularConsumed = totalConsumed - initialFrame.getStateGasUsed(); + final boolean regularGasLimitExceeded = + regularConsumed > stateGasCalc.transactionRegularGasLimit(); + if (regularGasLimitExceeded) { + LOG.debug( + "Transaction {} regular gas {} exceeds TX_MAX_GAS_LIMIT {}, reverting", + transaction.getHash(), + regularConsumed, + stateGasCalc.transactionRegularGasLimit()); + } + + final boolean txSucceeded = + initialFrame.getState() == MessageFrame.State.COMPLETED_SUCCESS + && !regularGasLimitExceeded; + + if (txSucceeded) { + worldUpdater.commit(); + } else { + if (initialFrame.getExceptionalHaltReason().isPresent()) { + validationResult = + ValidationResult.invalid( + TransactionInvalidReason.EXECUTION_HALTED, + initialFrame.getExceptionalHaltReason().get().getDescription()); + } + if (regularGasLimitExceeded) { + validationResult = + ValidationResult.invalid( + TransactionInvalidReason.EXECUTION_HALTED, + "Regular gas consumption exceeds TX_MAX_GAS_LIMIT"); + } + } + + // TODO SLD are the log correct following EIP-7623? + if (LOG.isTraceEnabled()) { + LOG.trace( + "Gas used by transaction: {}, by message call/contract creation: {}", + transaction.getGasLimit() - initialFrame.getRemainingGas(), + gasAvailable - initialFrame.getRemainingGas()); + } + + // Refund the sender by what we should and pay the miner fee (note that we're doing them one + // after the other so that if it is the same account somehow, we end up with the right result) + // EIP-8037: No refund when regular gas limit is exceeded (all gas consumed) + final long refundedGas = + regularGasLimitExceeded + ? 0L + : gasCalculator.calculateGasRefund(transaction, initialFrame, codeDelegationRefund); + final Wei refundedWei = transactionGasPrice.multiply(refundedGas); + final Wei balancePriorToRefund = sender.getBalance(); + sender.incrementBalance(refundedWei); + LOG.atTrace() + .setMessage("refunded sender {} {} wei ({} -> {})") + .addArgument(senderAddress) + .addArgument(refundedWei) + .addArgument(balancePriorToRefund) + .addArgument(sender.getBalance()) + .log(); + // Calculate gas used: max of execution gas and transaction floor cost (EIP-7623) + // For pre-Prague forks, floor cost is 0, so this returns just execution gas + // For Prague+ forks with EIP-7778, this ensures block gas accounts for data floor + // EIP-8037: Gas accounting with multidimensional gas support + final long floorCost = + gasCalculator.transactionFloorCost( + transaction.getPayload(), transaction.getPayloadZeroBytes()); + final TransactionGasAccounting.GasResult gasResult = + TransactionGasAccounting.builder() + .txGasLimit(transaction.getGasLimit()) + .remainingGas(initialFrame.getRemainingGas()) + .stateGasReservoir(initialFrame.getStateGasReservoir()) + .stateGasUsed(initialFrame.getStateGasUsed()) + .initialFrameStateGasSpill(initialFrameStateGasSpill) + .stateGasSpillBurned(initialFrame.getStateGasSpillBurned()) + .initialFrameRegularHaltBurn(initialFrame.getInitialFrameRegularHaltBurn()) + .refundedGas(refundedGas) + .floorCost(floorCost) + .regularGasLimitExceeded(regularGasLimitExceeded) + .build() + .calculate(); + final long effectiveStateGas = gasResult.effectiveStateGas(); + final long gasUsedByTransaction = gasResult.gasUsedByTransaction(); + final long usedGas = gasResult.usedGas(); + final CoinbaseFeePriceCalculator coinbaseCalculator; + if (blockHeader.getBaseFee().isPresent()) { + final Wei baseFee = blockHeader.getBaseFee().get(); + final boolean gasPriceBelowBaseFee = transactionGasPrice.compareTo(baseFee) < 0; + if (transactionValidationParams.allowUnderpriced() + || transactionValidationParams.isPreserveCallerGasPricing()) { + coinbaseCalculator = + gasPriceBelowBaseFee ? (a, b, c) -> Wei.ZERO : coinbaseFeePriceCalculator; + } else { + if (gasPriceBelowBaseFee) { + final Optional partialBlockAccessView = + accessLocationTracker.map( + tracker -> tracker.createPartialBlockAccessView(worldState)); + return TransactionProcessingResult.failed( + gasUsedByTransaction, + refundedGas, + usedGas, + effectiveStateGas, + ValidationResult.invalid( + TransactionInvalidReason.TRANSACTION_PRICE_TOO_LOW, + "transaction price must be greater than base fee"), + Optional.empty(), + Optional.empty(), + partialBlockAccessView); + } + coinbaseCalculator = coinbaseFeePriceCalculator; + } + } else { + coinbaseCalculator = CoinbaseFeePriceCalculator.frontier(); + } + + final Wei coinbaseWeiDelta = + coinbaseCalculator.price(usedGas, transactionGasPrice, blockHeader.getBaseFee()); + + operationTracer.traceBeforeRewardTransaction(worldUpdater, transaction, coinbaseWeiDelta); + + // EIP-158 & EIP-7928: coinbase is considered "touched" even when fees are zero. + // Touching ensures an *empty* coinbase can be deleted during state clearing. + final MutableAccount coinbase = worldState.getOrCreate(miningBeneficiary); + accessLocationTracker.ifPresent(t -> t.addTouchedAccount(miningBeneficiary)); + if (!coinbaseWeiDelta.isZero()) { + coinbase.incrementBalance(coinbaseWeiDelta); + } + + // EIP-7708: Emit closure logs for accounts with remaining balance before deletion + // Noop before Amsterdam + transferLogEmitter.emitClosureLogs( + worldState, initialFrame.getSelfDestructs(), initialFrame::addLog); + + operationTracer.traceEndTransaction( + worldState.updater(), + transaction, + txSucceeded, + initialFrame.getOutputData(), + initialFrame.getLogs(), + gasUsedByTransaction, + initialFrame.getSelfDestructs(), + 0L); + + initialFrame.getSelfDestructs().forEach(worldState::deleteAccount); + + if (clearEmptyAccounts) { + worldState.clearAccountsThatAreEmpty(); + } + + final Optional partialBlockAccessView = + accessLocationTracker.map(tracker -> tracker.createPartialBlockAccessView(worldState)); + + if (txSucceeded) { + return TransactionProcessingResult.successful( + initialFrame.getLogs(), + gasUsedByTransaction, + refundedGas, + usedGas, + effectiveStateGas, + initialFrame.getOutputData(), + partialBlockAccessView, + validationResult); + } else { + if (initialFrame.getExceptionalHaltReason().isPresent()) { + LOG.debug( + "Transaction {} processing halted: {}", + transaction.getHash(), + initialFrame.getExceptionalHaltReason().get()); + } + if (initialFrame.getRevertReason().isPresent()) { + LOG.debug( + "Transaction {} reverted: {}", + transaction.getHash(), + initialFrame.getRevertReason().get()); + } + return TransactionProcessingResult.failed( + gasUsedByTransaction, + refundedGas, + usedGas, + effectiveStateGas, + validationResult, + initialFrame.getRevertReason(), + initialFrame.getExceptionalHaltReason(), + partialBlockAccessView); + } + } catch (final MerkleTrieException re) { + operationTracer.traceEndTransaction( + worldState.updater(), + transaction, + false, + Bytes.EMPTY, + List.of(), + 0, + EMPTY_ADDRESS_SET, + 0L); + + // need to throw to trigger the heal + throw re; + } catch (final RuntimeException re) { + final var cause = re.getCause(); + // in case of an interruption then just return without calling any other tracing method + if (cause != null && cause instanceof InterruptedException) { + LOG.atDebug() + .setMessage("Interrupted while processing the transaction with hash {}") + .addArgument(transaction::getHash) + .log(); + return TransactionProcessingResult.invalid( + ValidationResult.invalid(TransactionInvalidReason.EXECUTION_INTERRUPTED)); + } + + operationTracer.traceEndTransaction( + worldState.updater(), + transaction, + false, + Bytes.EMPTY, + List.of(), + 0, + EMPTY_ADDRESS_SET, + 0L); + + LOG.error("Critical Exception Processing Transaction", re); + return TransactionProcessingResult.invalid( + ValidationResult.invalid( + TransactionInvalidReason.INTERNAL_ERROR, + "Internal Error in Besu - " + re + "\n" + printableStackTraceFromThrowable(re))); + } + } + + public void process(final MessageFrame frame, final OperationTracer operationTracer) { + final AbstractMessageProcessor executor = getMessageProcessor(frame.getType()); + + executor.process(frame, operationTracer); + } + + public AbstractMessageProcessor getMessageProcessor(final MessageFrame.Type type) { + return switch (type) { + case MESSAGE_CALL -> messageCallProcessor; + case CONTRACT_CREATION -> contractCreationProcessor; + }; + } + + public MessageCallProcessor getMessageCallProcessor() { + return messageCallProcessor; + } + + public boolean getClearEmptyAccounts() { + return clearEmptyAccounts; + } + + private String printableStackTraceFromThrowable(final RuntimeException re) { + final StringBuilder builder = new StringBuilder(); + + for (final StackTraceElement stackTraceElement : re.getStackTrace()) { + builder.append("\tat ").append(stackTraceElement.toString()).append("\n"); + } + + return builder.toString(); + } + + private Code processCodeFromAccount( + final WorldUpdater worldUpdater, + final Set
warmAddressList, + final Account contract, + final Optional accessLocationTracker) { + if (contract == null) { + return Code.EMPTY_CODE; + } + + final Hash codeHash = contract.getCodeHash(); + if (codeHash == null || codeHash.equals(Hash.EMPTY)) { + return Code.EMPTY_CODE; + } + + if (hasCodeDelegation(contract.getCode())) { + return delegationTargetCode(worldUpdater, warmAddressList, contract, accessLocationTracker); + } + + // Bonsai accounts may have a fully cached code, so we use that one + if (contract.getCodeCache() != null) { + return contract.getOrCreateCachedCode(); + } + + // Any other account can only use the cached jump dest analysis if available + return messageCallProcessor.getOrCreateCachedJumpDest( + contract.getCodeHash(), contract.getCode()); + } + + private Code delegationTargetCode( + final WorldUpdater worldUpdater, + final Set
warmAddressList, + final Account contract, + final Optional accessLocationTracker) { + // we need to look up the target account and its code, but do NOT charge gas for it + final CodeDelegationHelper.Target target = + getTarget(worldUpdater, gasCalculator::isPrecompile, contract, accessLocationTracker); + warmAddressList.add(target.address()); + + return target.code(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private GasCalculator gasCalculator; + private TransactionValidatorFactory transactionValidatorFactory; + private ContractCreationProcessor contractCreationProcessor; + private MessageCallProcessor messageCallProcessor; + private boolean clearEmptyAccounts; + private boolean warmCoinbase; + private int maxStackSize; + private FeeMarket feeMarket; + private CoinbaseFeePriceCalculator coinbaseFeePriceCalculator; + private CodeDelegationProcessor codeDelegationProcessor; + private TransferLogEmitter transferLogEmitter = TransferLogEmitter.NOOP; + + public Builder gasCalculator(final GasCalculator gasCalculator) { + this.gasCalculator = gasCalculator; + return this; + } + + public Builder transactionValidatorFactory( + final TransactionValidatorFactory transactionValidatorFactory) { + this.transactionValidatorFactory = transactionValidatorFactory; + return this; + } + + public Builder contractCreationProcessor( + final ContractCreationProcessor contractCreationProcessor) { + this.contractCreationProcessor = contractCreationProcessor; + return this; + } + + public Builder messageCallProcessor(final MessageCallProcessor messageCallProcessor) { + this.messageCallProcessor = messageCallProcessor; + return this; + } + + public Builder clearEmptyAccounts(final boolean clearEmptyAccounts) { + this.clearEmptyAccounts = clearEmptyAccounts; + return this; + } + + public Builder warmCoinbase(final boolean warmCoinbase) { + this.warmCoinbase = warmCoinbase; + return this; + } + + public Builder maxStackSize(final int maxStackSize) { + this.maxStackSize = maxStackSize; + return this; + } + + public Builder feeMarket(final FeeMarket feeMarket) { + this.feeMarket = feeMarket; + return this; + } + + public Builder coinbaseFeePriceCalculator( + final CoinbaseFeePriceCalculator coinbaseFeePriceCalculator) { + this.coinbaseFeePriceCalculator = coinbaseFeePriceCalculator; + return this; + } + + public Builder codeDelegationProcessor( + final CodeDelegationProcessor maybeCodeDelegationProcessor) { + this.codeDelegationProcessor = maybeCodeDelegationProcessor; + return this; + } + + public Builder transferLogEmitter(final TransferLogEmitter transferLogEmitter) { + this.transferLogEmitter = transferLogEmitter; + return this; + } + + public Builder populateFrom(final MainnetTransactionProcessor processor) { + this.gasCalculator = processor.gasCalculator; + this.transactionValidatorFactory = processor.transactionValidatorFactory; + this.contractCreationProcessor = processor.contractCreationProcessor; + this.messageCallProcessor = processor.messageCallProcessor; + this.clearEmptyAccounts = processor.clearEmptyAccounts; + this.warmCoinbase = processor.warmCoinbase; + this.maxStackSize = processor.maxStackSize; + this.feeMarket = processor.feeMarket; + this.coinbaseFeePriceCalculator = processor.coinbaseFeePriceCalculator; + this.codeDelegationProcessor = processor.maybeCodeDelegationProcessor.orElse(null); + this.transferLogEmitter = processor.transferLogEmitter; + return this; + } + + public MainnetTransactionProcessor build() { + return new MainnetTransactionProcessor( + gasCalculator, + transactionValidatorFactory, + contractCreationProcessor, + messageCallProcessor, + clearEmptyAccounts, + warmCoinbase, + maxStackSize, + feeMarket, + coinbaseFeePriceCalculator, + codeDelegationProcessor, + transferLogEmitter); + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java new file mode 100644 index 0000000..804f4be --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java @@ -0,0 +1,395 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-tx-gas-cap-unsigned.patch. + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static org.hyperledger.besu.evm.account.Account.MAX_NONCE; +import static org.hyperledger.besu.evm.internal.Words.clampedAdd; +import static org.hyperledger.besu.evm.worldstate.CodeDelegationHelper.hasCodeDelegation; + +import org.hyperledger.besu.crypto.SECPSignature; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.BlobType; +import org.hyperledger.besu.datatypes.CodeDelegation; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +import java.math.BigInteger; +import java.util.Optional; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Validates a transaction based on Frontier protocol runtime requirements. + * + *

The {@link MainnetTransactionValidator} performs the intrinsic gas cost check on the given + * {@link Transaction}. + */ +public class MainnetTransactionValidator implements TransactionValidator { + private static final Logger LOG = LoggerFactory.getLogger(MainnetTransactionValidator.class); + + public static final BigInteger TWO_POW_256 = BigInteger.TWO.pow(256); + + private final GasCalculator gasCalculator; + private final GasLimitCalculator gasLimitCalculator; + private final FeeMarket feeMarket; + + private final boolean disallowSignatureMalleability; + + private final Optional chainId; + + private final Set acceptedTransactionTypes; + + private final int maxInitcodeSize; + private final MainnetBlobsValidator blobsValidator; + + public MainnetTransactionValidator( + final GasCalculator gasCalculator, + final GasLimitCalculator gasLimitCalculator, + final FeeMarket feeMarket, + final boolean checkSignatureMalleability, + final Optional chainId, + final Set acceptedTransactionTypes, + final Set acceptedBlobVersions, + final int maxInitcodeSize) { + this.gasCalculator = gasCalculator; + this.gasLimitCalculator = gasLimitCalculator; + this.feeMarket = feeMarket; + this.disallowSignatureMalleability = checkSignatureMalleability; + this.chainId = chainId; + this.acceptedTransactionTypes = acceptedTransactionTypes; + this.maxInitcodeSize = maxInitcodeSize; + this.blobsValidator = + new MainnetBlobsValidator(acceptedBlobVersions, gasLimitCalculator, gasCalculator); + } + + @Override + public ValidationResult validate( + final Transaction transaction, + final Optional baseFee, + final Optional blobFee, + final TransactionValidationParams transactionValidationParams) { + final ValidationResult signatureResult = + validateTransactionSignature(transaction); + if (!signatureResult.isValid()) { + return signatureResult; + } + + final TransactionType transactionType = transaction.getType(); + if (!acceptedTransactionTypes.contains(transactionType)) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + String.format( + "Transaction type %s is invalid, accepted transaction types are %s", + transactionType, acceptedTransactionTypes)); + } + + if (!transactionValidationParams.isAllowFutureNonce() && transaction.getNonce() == MAX_NONCE) { + return ValidationResult.invalid( + TransactionInvalidReason.NONCE_OVERFLOW, "Nonce must be less than 2^64-1"); + } + + if (!transactionValidationParams.isAllowExceedingGasLimit() + // AERE D-432: unsigned, like the intrinsic-gas check below. The decoder reads gasLimit with readLongScalar, so a limit of 2^63 + // or more arrives as a negative long, and a signed compare let it through the EIP-7825 cap. + && Long.compareUnsigned(transaction.getGasLimit(), gasLimitCalculator.transactionGasLimitCap()) > 0) { + return ValidationResult.invalid( + TransactionInvalidReason.EXCEEDS_TRANSACTION_GAS_LIMIT, + "Transaction gas limit must be at most " + gasLimitCalculator.transactionGasLimitCap()); + } + + if (transactionType.supportsBlob()) { + final ValidationResult blobTransactionResult = + blobsValidator.validate(transaction); + if (!blobTransactionResult.isValid()) { + LOG.debug( + "Blob transaction {} validation failed: {}", + transaction.getHash().getBytes().toHexString(), + blobTransactionResult.getErrorMessage()); + return blobTransactionResult; + } + } + + if (transaction.isContractCreation() && transaction.getPayload().size() > maxInitcodeSize) { + return ValidationResult.invalid( + TransactionInvalidReason.INITCODE_TOO_LARGE, + String.format( + "Initcode size of %d exceeds maximum size of %s", + transaction.getPayload().size(), maxInitcodeSize)); + } + + if (transactionType == TransactionType.DELEGATE_CODE) { + ValidationResult codeDelegationValidation = + validateCodeDelegation(transaction); + if (!codeDelegationValidation.isValid()) { + return codeDelegationValidation; + } + } + + return validateCostAndFee(transaction, baseFee, blobFee, transactionValidationParams); + } + + private static ValidationResult validateCodeDelegation( + final Transaction transaction) { + if (isDelegateCodeEmpty(transaction)) { + return ValidationResult.invalid( + TransactionInvalidReason.EMPTY_CODE_DELEGATION, + "transaction code delegation transactions must have a non-empty code delegation list"); + } + + if (transaction.getTo().isEmpty()) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + "transaction code delegation transactions must have a to address"); + } + + final Optional> validationResult = + transaction + .getCodeDelegationList() + .map( + codeDelegations -> { + for (CodeDelegation codeDelegation : codeDelegations) { + if (codeDelegation.chainId().compareTo(TWO_POW_256) >= 0) { + throw new IllegalArgumentException( + "Invalid 'chainId' value, should be < 2^256 but got " + + codeDelegation.chainId()); + } + + if (codeDelegation.r().compareTo(TWO_POW_256) >= 0) { + throw new IllegalArgumentException( + "Invalid 'r' value, should be < 2^256 but got " + codeDelegation.r()); + } + + if (codeDelegation.s().compareTo(TWO_POW_256) >= 0) { + throw new IllegalArgumentException( + "Invalid 's' value, should be < 2^256 but got " + codeDelegation.s()); + } + } + + return ValidationResult.valid(); + }); + + if (validationResult.isPresent() && !validationResult.get().isValid()) { + return validationResult.get(); + } + + return ValidationResult.valid(); + } + + private static boolean isDelegateCodeEmpty(final Transaction transaction) { + return transaction.getCodeDelegationList().isEmpty() + || transaction.getCodeDelegationList().get().isEmpty(); + } + + private ValidationResult validateCostAndFee( + final Transaction transaction, + final Optional maybeBaseFee, + final Optional maybeBlobFee, + final TransactionValidationParams transactionValidationParams) { + + if (maybeBaseFee.isPresent()) { + final Wei price = feeMarket.getTransactionPriceCalculator().price(transaction, maybeBaseFee); + if (!transactionValidationParams.allowUnderpriced() + && !transactionValidationParams.isPreserveCallerGasPricing() + && price.compareTo(maybeBaseFee.orElseThrow()) < 0) { + return ValidationResult.invalid( + TransactionInvalidReason.GAS_PRICE_BELOW_CURRENT_BASE_FEE, + "gasPrice is less than the current BaseFee"); + } + + // assert transaction.max_fee_per_gas >= transaction.max_priority_fee_per_gas + if (transaction.getType().supports1559FeeMarket() + && transaction + .getMaxPriorityFeePerGas() + .get() + .getAsBigInteger() + .compareTo(transaction.getMaxFeePerGas().get().getAsBigInteger()) + > 0) { + return ValidationResult.invalid( + TransactionInvalidReason.MAX_PRIORITY_FEE_PER_GAS_EXCEEDS_MAX_FEE_PER_GAS, + "max priority fee per gas cannot be greater than max fee per gas"); + } + } + + if (transaction.getType().supportsBlob()) { + final long txTotalBlobGas = gasCalculator.blobGasCost(transaction.getBlobCount()); + if (txTotalBlobGas > gasLimitCalculator.currentBlobGasLimit()) { + return ValidationResult.invalid( + TransactionInvalidReason.TOTAL_BLOB_GAS_TOO_HIGH, + String.format( + "total blob gas %d exceeds max blob gas per block %d", + txTotalBlobGas, gasLimitCalculator.currentBlobGasLimit())); + } + if (maybeBlobFee.isEmpty()) { + throw new IllegalArgumentException( + "blob fee must be provided from blocks containing blobs"); + // tx.getMaxFeePerBlobGas can be empty for eth_call + } else if (!transactionValidationParams.allowUnderpriced() + && maybeBlobFee.get().compareTo(transaction.getMaxFeePerBlobGas().get()) > 0) { + return ValidationResult.invalid( + TransactionInvalidReason.BLOB_GAS_PRICE_BELOW_CURRENT_BLOB_BASE_FEE, + String.format( + "tx max fee per blob gas less than block blob gas fee: address %s blobGasFeeCap: %s, blobBaseFee: %s", + transaction.getSender().getBytes().toHexString(), + transaction.getMaxFeePerBlobGas().get().toHumanReadableString(), + maybeBlobFee.get().toHumanReadableString())); + } + } + + final long baselineGas = + clampedAdd( + transaction.getAccessList().map(gasCalculator::accessListGasCost).orElse(0L), + gasCalculator.delegateCodeGasCost(transaction.codeDelegationListSize())); + final long intrinsicGasCostOrFloor = + Math.max( + gasCalculator.transactionIntrinsicGasCost(transaction, baselineGas), + gasCalculator.transactionFloorCost( + transaction.getPayload(), transaction.getPayloadZeroBytes())); + + if (Long.compareUnsigned(intrinsicGasCostOrFloor, transaction.getGasLimit()) > 0) { + return ValidationResult.invalid( + TransactionInvalidReason.INTRINSIC_GAS_EXCEEDS_GAS_LIMIT, + String.format( + "intrinsic gas cost %s exceeds gas limit %s", + intrinsicGasCostOrFloor, transaction.getGasLimit())); + } + + if (transaction.calculateUpfrontGasCost(transaction.getMaxGasPrice(), Wei.ZERO, 0).bitLength() + > 256) { + return ValidationResult.invalid( + TransactionInvalidReason.UPFRONT_COST_EXCEEDS_UINT256, + "Upfront gas cost cannot exceed 2^256 Wei"); + } + + return ValidationResult.valid(); + } + + @Override + public ValidationResult validateForSender( + final Transaction transaction, + final Account sender, + final TransactionValidationParams validationParams) { + Wei senderBalance = Account.DEFAULT_BALANCE; + long senderNonce = Account.DEFAULT_NONCE; + Hash codeHash = Hash.EMPTY; + + if (sender != null) { + senderBalance = sender.getBalance(); + senderNonce = sender.getNonce(); + if (sender.getCodeHash() != null) codeHash = sender.getCodeHash(); + } + + final Wei upfrontCost = + transaction.getUpfrontCost(gasCalculator.blobGasCost(transaction.getBlobCount())); + if (!validationParams.allowUnderpriced() && upfrontCost.compareTo(senderBalance) > 0) { + return ValidationResult.invalid( + TransactionInvalidReason.UPFRONT_COST_EXCEEDS_BALANCE, + String.format( + "transaction up-front cost %s exceeds transaction sender account balance %s for sender %s", + upfrontCost.toQuantityHexString(), + senderBalance.toQuantityHexString(), + transaction.getSender())); + } + + if (Long.compareUnsigned(transaction.getNonce(), senderNonce) < 0) { + return ValidationResult.invalid( + TransactionInvalidReason.NONCE_TOO_LOW, + String.format( + "transaction nonce %s below sender account nonce %s for sender %s", + transaction.getNonce(), senderNonce, transaction.getSender())); + } + + if (!validationParams.isAllowFutureNonce() && senderNonce != transaction.getNonce()) { + return ValidationResult.invalid( + TransactionInvalidReason.NONCE_TOO_HIGH, + String.format( + "transaction nonce %s does not match sender account nonce %s for sender %s", + transaction.getNonce(), senderNonce, transaction.getSender())); + } + + if (!validationParams.isAllowContractAddressAsSender() + && !canSendTransaction(sender, codeHash)) { + return ValidationResult.invalid( + TransactionInvalidReason.TX_SENDER_NOT_AUTHORIZED, + String.format( + "Sender %s has deployed code and so is not authorized to send transactions", + transaction.getSender())); + } + + return ValidationResult.valid(); + } + + private static boolean canSendTransaction(final Account sender, final Hash codeHash) { + return codeHash.equals(Hash.EMPTY) || hasCodeDelegation(sender.getCode()); + } + + private ValidationResult validateTransactionSignature( + final Transaction transaction) { + if (chainId.isPresent() + && (transaction.getChainId().isPresent() && !transaction.getChainId().equals(chainId))) { + return ValidationResult.invalid( + TransactionInvalidReason.WRONG_CHAIN_ID, + String.format( + "transaction was meant for chain id %s and not this chain id %s", + transaction.getChainId().get(), chainId.get())); + } + + if (chainId.isEmpty() && transaction.getChainId().isPresent()) { + return ValidationResult.invalid( + TransactionInvalidReason.REPLAY_PROTECTED_SIGNATURES_NOT_SUPPORTED, + "replay protected signatures is not supported"); + } + + if (transaction.getType().supportsPqAuthorization()) { + // AIP-20: no secp256k1 material exists; the ML-DSA authorization is what is verified. + return AerePqTransactionSupport.validateAuthorization(transaction); + } + + final SECPSignature signature = transaction.getSignature(); + final BigInteger halfCurveOrder = SignatureAlgorithmFactory.getInstance().getHalfCurveOrder(); + if (disallowSignatureMalleability && signature.getS().compareTo(halfCurveOrder) > 0) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_SIGNATURE, + String.format( + "Signature s value should be less than %s, but got %s", + halfCurveOrder, signature.getS())); + } + + // org.bouncycastle.math.ec.ECCurve.AbstractFp.decompressPoint throws an + // IllegalArgumentException for "Invalid point compression" for bad signatures. + try { + transaction.getSender(); + } catch (final IllegalArgumentException e) { + return ValidationResult.invalid( + TransactionInvalidReason.INVALID_SIGNATURE, + "sender could not be extracted from transaction signature"); + } + return ValidationResult.valid(); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsValidator.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsValidator.java new file mode 100644 index 0000000..fc04a7c --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsValidator.java @@ -0,0 +1,149 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-withdrawals-not-applicable.patch. + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.Withdrawal; + +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface WithdrawalsValidator { + + boolean validateWithdrawals(Optional> withdrawals); + + boolean validateWithdrawalsRoot(Block block); + + class ProhibitedWithdrawals implements WithdrawalsValidator { + + private static final Logger LOG = LoggerFactory.getLogger(ProhibitedWithdrawals.class); + + @Override + public boolean validateWithdrawals(final Optional> withdrawals) { + final boolean isValid = withdrawals.isEmpty(); + if (!isValid) { + LOG.warn( + "withdrawals must be null when Withdrawals are prohibited but were: {}", withdrawals); + } + return isValid; + } + + @Override + public boolean validateWithdrawalsRoot(final Block block) { + final Optional withdrawalsRoot = block.getHeader().getWithdrawalsRoot(); + if (withdrawalsRoot.isPresent()) { + LOG.warn( + "withdrawalsRoot must be null when Withdrawals are prohibited but was: {}", + withdrawalsRoot.get()); + return false; + } + + return true; + } + } + + class AllowedWithdrawals implements WithdrawalsValidator { + + private static final Logger LOG = LoggerFactory.getLogger(AllowedWithdrawals.class); + + @Override + public boolean validateWithdrawals(final Optional> withdrawals) { + final boolean isValid = withdrawals.isPresent(); + if (!isValid) { + LOG.warn("withdrawals must not be null when Withdrawals are activated"); + } + return isValid; + } + + @Override + public boolean validateWithdrawalsRoot(final Block block) { + checkArgument( + block.getBody().getWithdrawals().isPresent(), "Block body must contain withdrawals"); + final Optional withdrawalsRoot = block.getHeader().getWithdrawalsRoot(); + if (withdrawalsRoot.isEmpty()) { + LOG.warn("withdrawalsRoot must not be null when Withdrawals are activated"); + return false; + } + + final List withdrawals = block.getBody().getWithdrawals().get(); + final Hash expectedWithdrawalsRoot = BodyValidation.withdrawalsRoot(withdrawals); + if (!expectedWithdrawalsRoot.equals(withdrawalsRoot.get())) { + LOG.info( + "Invalid block: transaction root mismatch (expected={}, actual={})", + expectedWithdrawalsRoot, + withdrawalsRoot.get()); + return false; + } + + return true; + } + } + + class NotApplicableWithdrawals implements WithdrawalsValidator { + + private static final Logger LOG = LoggerFactory.getLogger(NotApplicableWithdrawals.class); + + // AERE D-433: ported from upstream hyperledger/besu ffe8104ee2 (GHSA-p4h2-gvh4-pv6j) and b7f1387cb4. On a BFT chain a non-empty list is a + // Byzantine proposer's injection that WithdrawalsProcessor would credit as native balance on every honest node. + @Override + public boolean validateWithdrawals(final Optional> withdrawals) { + // Pre-Shanghai blocks have no withdrawals field (Optional.empty()) - valid. + // Shanghai+ blocks must carry an empty list; a non-empty list means a Byzantine + // proposer injected unauthorized withdrawals that would be executed as balance credits. + if (withdrawals.isPresent() && !withdrawals.get().isEmpty()) { + LOG.warn( + "Withdrawals not applicable but block contained {} withdrawal(s)", + withdrawals.get().size()); + return false; + } + return true; + } + + @Override + public boolean validateWithdrawalsRoot(final Block block) { + final Optional withdrawalsRoot = block.getHeader().getWithdrawalsRoot(); + if (withdrawalsRoot.isEmpty()) { + return true; // pre-Shanghai block, no withdrawals root field + } + // A block with a withdrawalsRoot in the header but no withdrawals field in the body is + // malformed. + if (block.getBody().getWithdrawals().isEmpty()) { + LOG.warn( + "Invalid block: withdrawalsRoot present in header but withdrawals absent from body"); + return false; + } + if (!Hash.EMPTY_TRIE_HASH.equals(withdrawalsRoot.get())) { + LOG.warn( + "Invalid block: withdrawals root mismatch (expected={}, actual={})", + Hash.EMPTY_TRIE_HASH, + withdrawalsRoot.get()); + return false; + } + return true; + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmEngine.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmEngine.java new file mode 100644 index 0000000..1ef6489 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmEngine.java @@ -0,0 +1,528 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList.BlockAccessListBuilder; +import org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockTransactionProcessor; +import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; +import org.hyperledger.besu.evm.tracing.OperationTracer; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.metrics.Counter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * AERE Block-STM optimistic parallel execution engine, wired into Besu's block-processing + * preprocessing hook. Faithful realisation of the AERE Rust Block-STM (Gelashvili et al. 2022): + * {@link Scheduler} + {@link MvMemory} drive a worker pool that runs the block's transactions + * concurrently against a multi-version world view ({@link MvBaseLayer}/{@link MvAccount}), captures + * read/write sets, re-executes conflicting transactions, and yields per-transaction outcomes that + * the block processor commits IN BLOCK ORDER — producing a state root bit-identical to sequential + * execution. + * + *

Safety: any transaction whose parallel outcome is missing, unsuccessful, or produced by a run + * that did not reach {@code scheduler.done()} falls back to Besu's sequential execution in the block + * loop (see {@link #getProcessingResult}). The engine can never commit an unverified result. + */ +@SuppressWarnings("MissingSummary") +public class AereBlockStmEngine extends ParallelBlockTransactionProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(AereBlockStmEngine.class); + private static final long PARALLEL_TIMEOUT_SECONDS = 60; + + private final MainnetTransactionProcessor transactionProcessor; + private final int numThreads; + + // Populated by runAsyncBlock, consumed in block order by getProcessingResult. + private volatile MvOutcome[] outcomes; + private volatile boolean success = false; + private final AtomicInteger totalExecutions = new AtomicInteger(); + private final AtomicInteger totalValidations = new AtomicInteger(); + private final AtomicInteger totalAborts = new AtomicInteger(); + private final AtomicInteger committedParallel = new AtomicInteger(); + + // Execution-phase instrumentation (see BlockStmStats): snapshotNanos = time to mint the single + // parent Bonsai view + flat pre-snapshot wrapper; execPhaseNanos = the PARALLEL EXECUTION PHASE + // (worker pool start -> termination) in isolation. Neither includes the downstream serial Bonsai + // trie commit, which happens in the block processor after this preprocessing returns. + private volatile long snapshotNanos = 0L; + private volatile long execPhaseNanos = 0L; + + /** + * @param transactionProcessor the mainnet transaction processor (EVM) + * @param numThreads worker-pool size (typically NCPU) + */ + public AereBlockStmEngine( + final MainnetTransactionProcessor transactionProcessor, final int numThreads) { + this.transactionProcessor = transactionProcessor; + this.numThreads = Math.max(1, numThreads); + } + + /** Per-transaction net outcome captured by the parallel phase. */ + private static final class MvOutcome { + final Map writes; + final Set writeKeys; + final List>> storageByAddr; + final Map nonceByAddr; // addr -> [nonce] if written + final Map balanceByAddr; // addr -> balance if written + final Map codeByAddr; // addr -> code if written + final Set

deleted; + Wei beneficiaryDelta = Wei.ZERO; + TransactionProcessingResult result; + + MvOutcome() { + this.writes = new HashMap<>(); + this.writeKeys = new HashSet<>(); + this.storageByAddr = new ArrayList<>(); + this.nonceByAddr = new HashMap<>(); + this.balanceByAddr = new HashMap<>(); + this.codeByAddr = new HashMap<>(); + this.deleted = new HashSet<>(); + } + } + + // ---- shared per-block execution state ---- + private MvMemory mv; + private Scheduler scheduler; + private List txns; + private BlockHeader blockHeader; + private Address beneficiary; + private BlockHashLookup blockHashLookup; + private Wei blobGasPrice; + private Object[] slotLocks; + private Map[] lastReads; + private Set[] lastWriteKeys; + private final AtomicBoolean hardFail = new AtomicBoolean(false); + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public void runAsyncBlock( + final ProtocolContext protocolContext, + final BlockHeader header, + final List transactions, + final Address miningBeneficiary, + final BlockHashLookup blockHashLookupParam, + final Wei blobGasPriceParam, + final java.util.concurrent.Executor executorUnused, + final Optional blockAccessListBuilder, + final Optional maybeParentHeader) { + + final int n = transactions.size(); + outcomes = new MvOutcome[n]; + success = false; + if (n == 0 || maybeParentHeader.isEmpty()) { + return; + } + + // AERE Block-STM Track B: FLAT immutable base pre-snapshot shared lock-free by all workers. + // Iteration 1 gave each worker its OWN frozen BonsaiWorldState and re-walked the parent trie on + // every base read (task bottleneck #2: per-worker Bonsai-view minting + layered-trie traversal + // per read). Here we mint the parent Bonsai view exactly ONCE, wrap it in a FlatBaseSnapshot + // that materialises each base value ONCE into an in-memory ConcurrentHashMap, and hand the SAME + // snapshot to every worker. Hot-path base reads become lock-free O(1) hashmap lookups with no + // trie traversal and no per-worker view. The base state is immutable for the round, so all + // workers observe identical parent values and serial equivalence holds (parallelRoot == + // sequentialRoot). snapshotNanos captures this one-time mint separately from execPhaseNanos. + final long snapT0 = System.nanoTime(); + final List openStates = new ArrayList<>(1); + try { + final BonsaiWorldState ws = + getWorldState(protocolContext, maybeParentHeader.get()).orElse(null); + if (ws == null) { + return; // success stays false -> sequential fallback + } + ws.disableCacheMerkleTrieLoader(); + openStates.add(ws); + final BaseReader sharedBase = new FlatBaseSnapshot(ws); + this.snapshotNanos = System.nanoTime() - snapT0; + + this.mv = new MvMemory(); + this.scheduler = new Scheduler(n); + this.txns = transactions; + this.blockHeader = header; + this.beneficiary = miningBeneficiary; + this.blockHashLookup = blockHashLookupParam; + this.blobGasPrice = blobGasPriceParam; + this.slotLocks = new Object[n]; + this.lastReads = new Map[n]; + this.lastWriteKeys = new Set[n]; + for (int i = 0; i < n; i++) { + slotLocks[i] = new Object(); + lastWriteKeys[i] = new HashSet<>(); + } + hardFail.set(false); + + // EXECUTION PHASE (timed in isolation from the snapshot mint above and from the downstream + // serial Bonsai trie commit that runs after this method returns). All workers share the one + // FlatBaseSnapshot; its hot-path reads are lock-free, so distinct-key base reads never contend. + final long execT0 = System.nanoTime(); + final ExecutorService pool = Executors.newFixedThreadPool(numThreads); + try { + for (int t = 0; t < numThreads; t++) { + pool.execute(() -> worker(sharedBase)); + } + pool.shutdown(); + final boolean finished = pool.awaitTermination(PARALLEL_TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (!finished) { + pool.shutdownNow(); + } + this.execPhaseNanos = System.nanoTime() - execT0; + success = finished && scheduler.done() && !hardFail.get(); + } catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + pool.shutdownNow(); + this.execPhaseNanos = System.nanoTime() - execT0; + success = false; + } + if (!success) { + LOG.info( + "AERE Block-STM: parallel phase did not verify (done={}, hardFail={}); block falls back" + + " to sequential per-transaction execution", + scheduler.done(), + hardFail.get()); + } + } catch (final RuntimeException ex) { + LOG.info("AERE Block-STM: parallel preprocessing error; sequential fallback", ex); + success = false; + } finally { + for (final BonsaiWorldState ws : openStates) { + try { + ws.close(); + } catch (final Exception ignored) { + // best-effort close + } + } + } + } + + private void worker(final BaseReader base) { + try { + Scheduler.Task task = scheduler.nextTask(); + while (!hardFail.get()) { + switch (task.kind()) { + case DONE: + return; + case NO_TASK: + task = scheduler.nextTask(); + break; + case EXECUTION: + task = tryExecute(base, task.idx(), task.inc()); + break; + case VALIDATION: + task = tryValidate(task.idx(), task.inc()); + break; + default: + return; + } + } + } catch (final Throwable th) { + hardFail.set(true); + LOG.debug("AERE Block-STM worker aborted", th); + } + } + + private Scheduler.Task tryExecute(final BaseReader base, final int idx, final int inc) { + while (true) { + if (hardFail.get()) { + return Scheduler.Task.NO_TASK; + } + final MvContext ctx = new MvContext(idx, mv, base, beneficiary); + final MvBaseLayer baseLayer = new MvBaseLayer(ctx); + final WorldUpdater txUpdater = baseLayer.updater(); + totalExecutions.incrementAndGet(); + + TransactionProcessingResult res = null; + try { + res = + transactionProcessor.processTransaction( + txUpdater, + blockHeader, + txns.get(idx).detachedCopy(), + beneficiary, + OperationTracer.NO_TRACING, + blockHashLookup, + TransactionValidationParams.processingBlock(), + blobGasPrice, + Optional.empty()); + } catch (final RuntimeException ex) { + if (ctx.blockedOn < 0) { + throw ex; // genuine failure -> worker aborts -> sequential fallback + } + // else: a routed read parked; fall through to dependency handling + } + + if (ctx.blockedOn >= 0) { + final int blocking = ctx.blockedOn; + if (scheduler.addDependency(idx, blocking)) { + return Scheduler.Task.NO_TASK; // parked + } + continue; // dependency already resolved; re-run same incarnation + } + + final MvOutcome outcome = harvest(ctx, baseLayer, txUpdater, res); + final Set prevKeys; + synchronized (slotLocks[idx]) { + prevKeys = lastWriteKeys[idx]; + } + final boolean wroteNewPath = mv.applyWriteSet(idx, inc, outcome.writes, prevKeys); + synchronized (slotLocks[idx]) { + lastReads[idx] = ctx.reads; + lastWriteKeys[idx] = outcome.writeKeys; + outcomes[idx] = outcome; + } + return scheduler.finishExecution(idx, inc, wroteNewPath); + } + } + + private Scheduler.Task tryValidate(final int idx, final int inc) { + totalValidations.incrementAndGet(); + final Map reads; + synchronized (slotLocks[idx]) { + reads = lastReads[idx]; + } + boolean valid = true; + if (reads != null) { + for (final Map.Entry e : reads.entrySet()) { + final MvMemory.ReadResult now = mv.read(e.getKey(), idx); + final boolean matches = !now.isBlocked() && e.getValue().sameProvenance(now.descriptor()); + if (!matches) { + valid = false; + break; + } + } + } + final boolean aborted = !valid && scheduler.tryValidationAbort(idx, inc); + if (aborted) { + totalAborts.incrementAndGet(); + final Set keys; + synchronized (slotLocks[idx]) { + keys = lastWriteKeys[idx]; + } + if (keys != null) { + mv.markEstimates(idx, keys); + } + } + return scheduler.finishValidation(idx, aborted); + } + + private MvOutcome harvest( + final MvContext ctx, + final MvBaseLayer baseLayer, + final WorldUpdater txUpdater, + final TransactionProcessingResult res) { + final MvOutcome o = new MvOutcome(); + o.result = res; + + for (final Account acct : txUpdater.getTouchedAccounts()) { + final Address a = acct.getAddress(); + final MvAccount seed = baseLayer.materializedAccount(a); + final long seedNonce = seed == null ? 0L : seed.seededNonce(); + final Wei seedBalance = seed == null ? Wei.ZERO : seed.seededBalance(); + final Hash seedCodeHash = seed == null ? Hash.EMPTY : seed.seededCodeHash(); + + final long nonce = acct.getNonce(); + if (nonce != seedNonce) { + o.nonceByAddr.put(a, new long[] {nonce}); + final MvKey k = MvKey.nonce(a); + o.writes.put(k, Long.valueOf(nonce)); + o.writeKeys.add(k); + } + + if (ctx.isBeneficiary(a)) { + final Wei bal = acct.getBalance(); + o.beneficiaryDelta = bal.subtract(seedBalance); + } else { + final Wei bal = acct.getBalance(); + if (!bal.equals(seedBalance)) { + o.balanceByAddr.put(a, bal); + final MvKey k = MvKey.balance(a); + o.writes.put(k, bal); + o.writeKeys.add(k); + } + } + + final Hash codeHash = acct.getCodeHash(); + if (!codeHash.equals(seedCodeHash)) { + final Bytes code = acct.getCode(); + o.codeByAddr.put(a, code); + final MvKey k = MvKey.code(a); + o.writes.put(k, code); + o.writeKeys.add(k); + } + + final Map updated = + (acct instanceof MutableAccount) ? ((MutableAccount) acct).getUpdatedStorage() : null; + if (updated != null && !updated.isEmpty()) { + final Map copy = new HashMap<>(updated); + o.storageByAddr.add(Map.entry(a, copy)); + for (final Map.Entry se : copy.entrySet()) { + final MvKey k = MvKey.storage(a, se.getKey()); + o.writes.put(k, se.getValue()); + o.writeKeys.add(k); + } + } + } + + for (final Address d : txUpdater.getDeletedAccountAddresses()) { + o.deleted.add(d); + // reflect clearing in MVMemory so higher transactions read cleared state + final MvKey bk = MvKey.balance(d); + final MvKey nk = MvKey.nonce(d); + final MvKey ck = MvKey.code(d); + o.writes.put(bk, Wei.ZERO); + o.writes.put(nk, Long.valueOf(0L)); + o.writes.put(ck, Bytes.EMPTY); + o.writeKeys.add(bk); + o.writeKeys.add(nk); + o.writeKeys.add(ck); + } + return o; + } + + /** + * Commit phase: called by the block processor in strict block order (location 0..n-1) on the real + * world state. Applies transaction {@code location}'s parallel outcome and returns its result, or + * empty to request sequential re-execution for this transaction. + */ + @Override + public Optional getProcessingResult( + final MutableWorldState worldState, + final Address miningBeneficiary, + final Transaction transaction, + final int location, + final Optional confirmedParallelizedTransactionCounter, + final Optional conflictingButCachedTransactionCounter) { + + if (!success || outcomes == null || location >= outcomes.length) { + return Optional.empty(); + } + final MvOutcome o = outcomes[location]; + if (o == null || o.result == null || !o.result.isSuccessful()) { + return Optional.empty(); + } + try { + final WorldUpdater updater = worldState.updater(); + // deletions win over writes for the same address + for (final Map.Entry e : o.balanceByAddr.entrySet()) { + if (!o.deleted.contains(e.getKey())) { + updater.getOrCreate(e.getKey()).setBalance(e.getValue()); + } + } + for (final Map.Entry e : o.nonceByAddr.entrySet()) { + if (!o.deleted.contains(e.getKey())) { + updater.getOrCreate(e.getKey()).setNonce(e.getValue()[0]); + } + } + for (final Map.Entry e : o.codeByAddr.entrySet()) { + if (!o.deleted.contains(e.getKey())) { + updater.getOrCreate(e.getKey()).setCode(e.getValue()); + } + } + for (final Map.Entry> e : o.storageByAddr) { + if (o.deleted.contains(e.getKey())) { + continue; + } + final MutableAccount ra = updater.getOrCreate(e.getKey()); + for (final Map.Entry se : e.getValue().entrySet()) { + ra.setStorageValue(se.getKey(), se.getValue()); + } + } + if (!o.beneficiaryDelta.isZero()) { + updater.getOrCreate(miningBeneficiary).incrementBalance(o.beneficiaryDelta); + } + for (final Address d : o.deleted) { + updater.deleteAccount(d); + } + updater.commit(); + committedParallel.incrementAndGet(); + confirmedParallelizedTransactionCounter.ifPresent(Counter::inc); + o.result.setIsProcessedInParallel(Optional.of(Boolean.TRUE)); + return Optional.of(o.result); + } catch (final RuntimeException ex) { + LOG.info("AERE Block-STM: commit of tx {} failed; sequential fallback", location, ex); + return Optional.empty(); + } + } + + @Override + protected org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelizedTransactionContext + runTransaction( + final ProtocolContext protocolContext, + final BlockHeader header, + final int transactionLocation, + final Transaction transaction, + final Address miningBeneficiary, + final BlockHashLookup blockHashLookupParam, + final Wei blobGasPriceParam, + final Optional blockAccessListBuilder, + final Optional maybeParentHeader) { + // Unused: AERE Block-STM overrides runAsyncBlock wholesale. + return null; + } + + /** @return whether the last parallel run verified (reached done with no hard failure). */ + public boolean lastRunSucceeded() { + return success; + } + + /** @return Block-STM statistics for the last run. */ + public BlockStmStats stats() { + return new BlockStmStats( + outcomes == null ? 0 : outcomes.length, + numThreads, + totalExecutions.get(), + totalValidations.get(), + totalAborts.get(), + committedParallel.get(), + success, + snapshotNanos, + execPhaseNanos); + } + + /** @return wall-clock nanos of the last parallel EXECUTION PHASE (worker pool only). */ + public long execPhaseNanos() { + return execPhaseNanos; + } + + /** @return wall-clock nanos spent building the last flat base pre-snapshot. */ + public long snapshotNanos() { + return snapshotNanos; + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmPreprocessing.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmPreprocessing.java new file mode 100644 index 0000000..db6562e --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmPreprocessing.java @@ -0,0 +1,119 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor.PreprocessingFunction; +import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList.BlockAccessListBuilder; +import org.hyperledger.besu.ethereum.mainnet.parallelization.PreprocessingContext; +import org.hyperledger.besu.ethereum.trie.pathbased.common.provider.PathBasedWorldStateProvider; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; + +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Preprocessing hook that runs AERE Block-STM optimistic parallel execution over the block's + * transactions before the sequential commit loop. Occupies the same {@code PreprocessingFunction} + * slot Besu uses for its built-in single-round parallelizer, but drives the AERE multi-version + * Block-STM engine instead. A fresh {@link AereBlockStmEngine} is created per block, so no state + * leaks between blocks. + */ +@SuppressWarnings("MissingSummary") +public class AereBlockStmPreprocessing implements PreprocessingFunction { + + private static final Logger LOG = LoggerFactory.getLogger(AereBlockStmPreprocessing.class); + + private final MainnetTransactionProcessor transactionProcessor; + private final int numThreads; + + /** + * @param transactionProcessor the EVM transaction processor + * @param numThreads worker-pool size (typically NCPU) + */ + public AereBlockStmPreprocessing( + final MainnetTransactionProcessor transactionProcessor, final int numThreads) { + this.transactionProcessor = transactionProcessor; + this.numThreads = numThreads; + } + + @Override + public Optional run( + final ProtocolContext protocolContext, + final BlockHeader blockHeader, + final List transactions, + final Address miningBeneficiary, + final BlockHashLookup blockHashLookup, + final Wei blobGasPrice, + final Optional blockAccessListBuilder, + final Optional maybeBlockBal, + final Optional maybeParentHeader) { + + if (!(protocolContext.getWorldStateArchive() instanceof PathBasedWorldStateProvider)) { + // Block-STM multi-version view requires a Bonsai/path-based world state. + return Optional.empty(); + } + + final AereBlockStmEngine engine = new AereBlockStmEngine(transactionProcessor, numThreads); + engine.runAsyncBlock( + protocolContext, + blockHeader, + transactions, + miningBeneficiary, + blockHashLookup, + blobGasPrice, + Runnable::run, + blockAccessListBuilder, + maybeParentHeader); + + final BlockStmStats stats = engine.stats(); + LOG.debug( + "AERE Block-STM block #{}: txns={} threads={} exec={} valid={} aborts={} committedParallel={}" + + " verified={}", + blockHeader.getNumber(), + stats.numTxns(), + stats.numThreads(), + stats.totalExecutions(), + stats.totalValidations(), + stats.totalAborts(), + stats.committedParallel(), + stats.verified()); + // Parseable single-line marker for the benchmark harness: the isolated EXECUTION-PHASE nanos + // (worker pool only, excludes the flat-snapshot mint AND the downstream serial Bonsai trie + // commit) plus the snapshot-mint nanos, per block. Gated on AERE_BLOCKSTM_STATS=1 so it is + // silent in normal operation and only emitted during benchmarking. + if ("1".equals(System.getenv("AERE_BLOCKSTM_STATS")) && stats.numTxns() > 0) { + LOG.info( + "AERE-BLOCKSTM-STATS blk={} n={} thr={} execNs={} snapNs={} exec={} aborts={}" + + " committedParallel={} verified={}", + blockHeader.getNumber(), + stats.numTxns(), + stats.numThreads(), + stats.execPhaseNanos(), + stats.snapshotNanos(), + stats.totalExecutions(), + stats.totalAborts(), + stats.committedParallel(), + stats.verified()); + } + + return Optional.of(new PreprocessingContext(engine)); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmProcessor.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmProcessor.java new file mode 100644 index 0000000..db2065a --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/AereBlockStmProcessor.java @@ -0,0 +1,160 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.BlockProcessingResult; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor.PreprocessingFunction.NoPreprocessing; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor.TransactionReceiptFactory; +import org.hyperledger.besu.ethereum.mainnet.BalConfiguration; +import org.hyperledger.besu.ethereum.mainnet.BlockProcessor; +import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.MiningBeneficiaryCalculator; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecBuilder; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.mainnet.parallelization.MainnetParallelBlockProcessor; +import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * L1 block processor that executes a block's transactions with AERE Block-STM optimistic parallel + * execution, committing in block order to produce a state root bit-identical to sequential + * execution. It reuses Besu's parallel consume path ({@link MainnetParallelBlockProcessor}: the + * overridden {@code getTransactionProcessingResult} pulls per-transaction results from the + * preprocessing context) but swaps Besu's single-round parallelizer for the AERE multi-version + * Block-STM engine via {@link AereBlockStmPreprocessing}. + * + *

Fork gate: this processor is selected only for the AERE {@code AerePQC} (FUTURE_EIPS) hard fork + * and only when {@code AERE_BLOCKSTM=1}, so a single activation delivers PQC precompiles + EIP-2935 + * + parallel execution together. On any parallel-phase failure the whole block falls back to + * sequential execution, so the processor can never persist an unverified state root. + */ +@SuppressWarnings("MissingSummary") +public class AereBlockStmProcessor extends MainnetParallelBlockProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(AereBlockStmProcessor.class); + private static final int NCPU = Runtime.getRuntime().availableProcessors(); + + /** + * @param transactionProcessor the EVM transaction processor + * @param transactionReceiptFactory receipt factory + * @param blockReward block reward + * @param miningBeneficiaryCalculator beneficiary calculator + * @param skipZeroBlockRewards whether to skip zero rewards + * @param protocolSchedule the protocol schedule + * @param balConfiguration BAL configuration + * @param metricsSystem metrics + */ + public AereBlockStmProcessor( + final MainnetTransactionProcessor transactionProcessor, + final TransactionReceiptFactory transactionReceiptFactory, + final Wei blockReward, + final MiningBeneficiaryCalculator miningBeneficiaryCalculator, + final boolean skipZeroBlockRewards, + final ProtocolSchedule protocolSchedule, + final BalConfiguration balConfiguration, + final MetricsSystem metricsSystem) { + super( + transactionProcessor, + transactionReceiptFactory, + blockReward, + miningBeneficiaryCalculator, + skipZeroBlockRewards, + protocolSchedule, + balConfiguration, + metricsSystem); + } + + @Override + public BlockProcessingResult processBlock( + final ProtocolContext protocolContext, + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block) { + return processBlock(protocolContext, blockchain, worldState, block, Optional.empty()); + } + + @Override + public BlockProcessingResult processBlock( + final ProtocolContext protocolContext, + final Blockchain blockchain, + final MutableWorldState worldState, + final Block block, + final Optional blockAccessList) { + + final BlockProcessingResult result = + super.processBlock( + protocolContext, + blockchain, + worldState, + block, + blockAccessList, + new AereBlockStmPreprocessing(transactionProcessor, NCPU)); + + if (result.isFailed()) { + LOG.info( + "AERE Block-STM processing failure for block #{} ({}); falling back to sequential", + block.getHeader().getNumber(), + block.getHash()); + if (worldState instanceof BonsaiWorldState) { + ((BonsaiWorldStateUpdateAccumulator) worldState.updater()).reset(); + } + return super.processBlock( + protocolContext, blockchain, worldState, block, blockAccessList, new NoPreprocessing()); + } + return result; + } + + /** Builds {@link AereBlockStmProcessor}, mirroring Besu's parallel builder. */ + public static class AereBlockStmProcessorBuilder + implements ProtocolSpecBuilder.BlockProcessorBuilder { + + private final MetricsSystem metricsSystem; + + /** + * @param metricsSystem metrics + */ + public AereBlockStmProcessorBuilder(final MetricsSystem metricsSystem) { + this.metricsSystem = metricsSystem; + } + + @Override + public BlockProcessor apply( + final MainnetTransactionProcessor transactionProcessor, + final TransactionReceiptFactory transactionReceiptFactory, + final Wei blockReward, + final MiningBeneficiaryCalculator miningBeneficiaryCalculator, + final boolean skipZeroBlockRewards, + final ProtocolSchedule protocolSchedule, + final BalConfiguration balConfiguration) { + return new AereBlockStmProcessor( + transactionProcessor, + transactionReceiptFactory, + blockReward, + miningBeneficiaryCalculator, + skipZeroBlockRewards, + protocolSchedule, + balConfiguration, + metricsSystem); + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BaseReader.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BaseReader.java new file mode 100644 index 0000000..2020655 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BaseReader.java @@ -0,0 +1,67 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * Read-only access to the immutable base (parent-block) world state. A {@code Base} read in + * MVMemory falls through to here. Implementations must be safe for concurrent reads (the base state + * does not change during block execution). + */ +@SuppressWarnings("MissingSummary") +public interface BaseReader { + + /** + * Whether the account exists in base state. + * + * @param address the account + * @return true if present + */ + boolean accountExists(Address address); + + /** + * Base nonce (0 if absent). + * + * @param address the account + * @return nonce + */ + long getNonce(Address address); + + /** + * Base balance (ZERO if absent). + * + * @param address the account + * @return balance + */ + Wei getBalance(Address address); + + /** + * Base code (EMPTY if absent). + * + * @param address the account + * @return code + */ + Bytes getCode(Address address); + + /** + * Base storage slot value (ZERO if absent). + * + * @param address the account + * @param slot the slot + * @return value + */ + UInt256 getStorageValue(Address address, UInt256 slot); +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BlockStmStats.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BlockStmStats.java new file mode 100644 index 0000000..a6a05f0 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/BlockStmStats.java @@ -0,0 +1,38 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +/** + * Aggregate statistics for one Block-STM run. + * + * @param numTxns transactions in the block + * @param numThreads worker-pool size + * @param totalExecutions total incarnations executed (excess over numTxns = re-executions) + * @param totalValidations total validation tasks run + * @param totalAborts validation failures that triggered re-execution + * @param committedParallel transactions committed from a parallel outcome (not sequential fallback) + * @param verified whether the run reached scheduler.done() with no hard failure + * @param snapshotNanos wall-clock nanos spent building the flat base pre-snapshot (single Bonsai + * view mint + wrapper), excluded from the parallel execution phase + * @param execPhaseNanos wall-clock nanos of the PARALLEL EXECUTION PHASE only (worker pool start to + * termination); excludes both the snapshot build and the downstream serial Bonsai trie commit + */ +@SuppressWarnings("MissingSummary") +public record BlockStmStats( + int numTxns, + int numThreads, + int totalExecutions, + int totalValidations, + int totalAborts, + int committedParallel, + boolean verified, + long snapshotNanos, + long execPhaseNanos) {} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/FlatBaseSnapshot.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/FlatBaseSnapshot.java new file mode 100644 index 0000000..20c3441 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/FlatBaseSnapshot.java @@ -0,0 +1,165 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.evm.account.Account; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * A FLAT, immutable, in-memory pre-snapshot of the parent-block base world state, shared lock-free + * by every Block-STM worker. + * + *

Iteration-1 gave each worker its OWN frozen {@link BonsaiWorldState} view and routed every base + * read through a layered-trie traversal. Two costs dominated cheap-tx blocks and capped scaling on + * execution-bound blocks (task bottleneck #2): (1) minting {@code numThreads} BonsaiWorldState + * snapshots per block, and (2) re-walking the parent trie on every read, on every worker, on every + * re-execution incarnation, even for keys already seen. + * + *

This snapshot mints the parent Bonsai view exactly ONCE. Each distinct base value (account + * facet triple, or one storage slot) is materialised the first time any worker touches it and cached + * in a {@link ConcurrentHashMap}; every subsequent read by any worker/incarnation is a lock-free + * O(1) hashmap lookup with no trie traversal. Cold loads (cache misses) route through the single + * shared BonsaiWorldState and are serialised on {@link #loadLock} because a BonsaiWorldState is not + * safe for concurrent {@code get}; this cold path is one-time-per-key and negligible for + * execution-bound workloads, and the hot path is fully lock-free. + * + *

Serial-equivalence is preserved: the base state is immutable for the whole parallel round, so + * every worker observes identical parent values regardless of load order (parallelRoot == + * sequentialRoot). An account is materialised as a whole {@link Shell} (existence + nonce + balance + * + code) in one trie hit, so the common pattern of reading several facets of the same account costs + * a single traversal. + */ +@SuppressWarnings("MissingSummary") +final class FlatBaseSnapshot implements BaseReader { + + /** Immutable materialised account facet triple. */ + private static final class Shell { + final boolean exists; + final long nonce; + final Wei balance; + final Bytes code; + + Shell(final boolean exists, final long nonce, final Wei balance, final Bytes code) { + this.exists = exists; + this.nonce = nonce; + this.balance = balance; + this.code = code; + } + } + + private static final Shell ABSENT = new Shell(false, 0L, Wei.ZERO, Bytes.EMPTY); + + /** Composite key for a materialised storage slot. */ + private static final class SlotKey { + private final Address address; + private final UInt256 slot; + private final int hash; + + SlotKey(final Address address, final UInt256 slot) { + this.address = address; + this.slot = slot; + this.hash = Objects.hash(address, slot); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SlotKey)) { + return false; + } + final SlotKey k = (SlotKey) o; + return address.equals(k.address) && slot.equals(k.slot); + } + + @Override + public int hashCode() { + return hash; + } + } + + private final BonsaiWorldState bws; + private final Object loadLock = new Object(); + private final ConcurrentHashMap accounts = new ConcurrentHashMap<>(); + private final ConcurrentHashMap storage = new ConcurrentHashMap<>(); + + FlatBaseSnapshot(final BonsaiWorldState bws) { + this.bws = bws; + } + + private Shell shell(final Address a) { + final Shell hit = accounts.get(a); // lock-free hot path + if (hit != null) { + return hit; + } + synchronized (loadLock) { + final Shell again = accounts.get(a); + if (again != null) { + return again; + } + final Account acct = bws.get(a); + final Shell shell = + acct == null + ? ABSENT + : new Shell(true, acct.getNonce(), acct.getBalance(), acct.getCode()); + accounts.put(a, shell); + return shell; + } + } + + @Override + public boolean accountExists(final Address address) { + return shell(address).exists; + } + + @Override + public long getNonce(final Address address) { + return shell(address).nonce; + } + + @Override + public Wei getBalance(final Address address) { + return shell(address).balance; + } + + @Override + public Bytes getCode(final Address address) { + return shell(address).code; + } + + @Override + public UInt256 getStorageValue(final Address address, final UInt256 slot) { + final SlotKey key = new SlotKey(address, slot); + final UInt256 hit = storage.get(key); // lock-free hot path + if (hit != null) { + return hit; + } + synchronized (loadLock) { + final UInt256 again = storage.get(key); + if (again != null) { + return again; + } + final Account acct = bws.get(address); + final UInt256 value = acct == null ? UInt256.ZERO : acct.getStorageValue(slot); + storage.put(key, value); + return value; + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvAccount.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvAccount.java new file mode 100644 index 0000000..97feae1 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvAccount.java @@ -0,0 +1,82 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.evm.fluent.SimpleAccount; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * An account facade over multi-version memory for a single Block-STM incarnation. Reads of a storage + * slot are routed through MVMemory (highest lower-index write, else base), recording the observed + * provenance into the execution's read set. Writes buffer in the inherited {@link SimpleAccount} + * overlay and are extracted as the write set after execution. + */ +@SuppressWarnings("MissingSummary") +public class MvAccount extends SimpleAccount { + + private final MvContext ctx; + private final Address addr; + + // Values this incarnation started from (for write-set diffing). + private final long seededNonce; + private final Wei seededBalance; + private final Hash seededCodeHash; + + MvAccount( + final MvContext ctx, + final Address address, + final long nonce, + final Wei balance, + final Bytes code) { + super(null, address, nonce, balance, code); + this.ctx = ctx; + this.addr = address; + this.seededNonce = nonce; + this.seededBalance = balance; + this.seededCodeHash = code == null || code.isEmpty() ? Hash.EMPTY : Hash.hash(code); + } + + @Override + public UInt256 getOriginalStorageValue(final UInt256 key) { + final MvKey k = MvKey.storage(addr, key); + final MvMemory.ReadResult rr = ctx.mv.read(k, ctx.txnIdx); + if (rr.isBlocked()) { + if (ctx.blockedOn < 0) { + ctx.blockedOn = rr.blockingTxn(); + } + return UInt256.ZERO; + } + if (!rr.descriptor().isBase()) { + ctx.record(k, rr.descriptor()); + final Object v = rr.value(); + return v == null ? UInt256.ZERO : (UInt256) v; + } + ctx.record(k, MvMemory.ReadDescriptor.BASE); + return ctx.base.getStorageValue(addr, key); + } + + long seededNonce() { + return seededNonce; + } + + Wei seededBalance() { + return seededBalance; + } + + Hash seededCodeHash() { + return seededCodeHash; + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvBaseLayer.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvBaseLayer.java new file mode 100644 index 0000000..bbb54d6 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvBaseLayer.java @@ -0,0 +1,149 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.fluent.SimpleWorld; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.tuweni.bytes.Bytes; + +/** + * The base (bottom) world layer for one Block-STM incarnation. It never holds writes: every EVM + * write lands in a child {@link SimpleWorld} layer obtained via {@link #updater()}. On a miss, this + * layer materialises an {@link MvAccount} by routing balance/nonce/code reads through MVMemory + * (highest lower-index write, else the immutable base state), recording provenance into the read + * set. Storage reads route lazily inside {@link MvAccount}. + */ +@SuppressWarnings("MissingSummary") +public class MvBaseLayer extends SimpleWorld { + + private final MvContext ctx; + private final Map materialized = new HashMap<>(); + private final Set

absent = new HashSet<>(); + + MvBaseLayer(final MvContext ctx) { + super(); + this.ctx = ctx; + } + + @Override + public Account get(final Address address) { + return getAccount(address); + } + + @Override + public MutableAccount getAccount(final Address address) { + final MvAccount m = materialized.get(address); + if (m != null) { + return m; + } + if (absent.contains(address)) { + return null; + } + final MvAccount created = materialize(address); + if (created == null) { + absent.add(address); + return null; + } + materialized.put(address, created); + return created; + } + + /** + * The seeded (pre-execution) account, for write-set diffing. + * + * @param address the account + * @return the materialised account, or null if it was absent / freshly created in a child layer + */ + MvAccount materializedAccount(final Address address) { + return materialized.get(address); + } + + private MvAccount materialize(final Address a) { + boolean exists = ctx.base.accountExists(a); + + // balance (beneficiary balance is excluded from conflict tracking) + final Wei balance; + if (ctx.isBeneficiary(a)) { + balance = ctx.base.getBalance(a); + } else { + final MvKey k = MvKey.balance(a); + final MvMemory.ReadResult rr = ctx.mv.read(k, ctx.txnIdx); + if (rr.isBlocked()) { + block(rr); + balance = Wei.ZERO; + } else if (!rr.descriptor().isBase()) { + ctx.record(k, rr.descriptor()); + balance = (Wei) rr.value(); + exists = true; + } else { + ctx.record(k, MvMemory.ReadDescriptor.BASE); + balance = ctx.base.getBalance(a); + } + } + + // nonce + final long nonce; + { + final MvKey k = MvKey.nonce(a); + final MvMemory.ReadResult rr = ctx.mv.read(k, ctx.txnIdx); + if (rr.isBlocked()) { + block(rr); + nonce = 0L; + } else if (!rr.descriptor().isBase()) { + ctx.record(k, rr.descriptor()); + nonce = (Long) rr.value(); + exists = true; + } else { + ctx.record(k, MvMemory.ReadDescriptor.BASE); + nonce = ctx.base.getNonce(a); + } + } + + // code + final Bytes code; + { + final MvKey k = MvKey.code(a); + final MvMemory.ReadResult rr = ctx.mv.read(k, ctx.txnIdx); + if (rr.isBlocked()) { + block(rr); + code = Bytes.EMPTY; + } else if (!rr.descriptor().isBase()) { + ctx.record(k, rr.descriptor()); + code = (Bytes) rr.value(); + exists = true; + } else { + ctx.record(k, MvMemory.ReadDescriptor.BASE); + code = ctx.base.getCode(a); + } + } + + if (!exists) { + // Absent account: reads recorded so a later creating write is caught by validation. + return null; + } + return new MvAccount(ctx, a, nonce, balance, code); + } + + private void block(final MvMemory.ReadResult rr) { + if (ctx.blockedOn < 0) { + ctx.blockedOn = rr.blockingTxn(); + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvContext.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvContext.java new file mode 100644 index 0000000..6f1902b --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvContext.java @@ -0,0 +1,55 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Per-execution capture state for one incarnation of one transaction. Confined to the single worker + * thread executing that incarnation, so it needs no synchronization internally. + * + *

Records the read set (key -> observed provenance) for later validation, and the address of the + * mining beneficiary, whose balance is excluded from conflict detection (its reward is applied + * additively during in-order commit, mirroring Besu's {@code traceBeforeRewardTransaction} trick). + */ +@SuppressWarnings("MissingSummary") +public final class MvContext { + + final int txnIdx; + final MvMemory mv; + final BaseReader base; + final Address beneficiary; + + /** key -> first observed provenance (dedup on key, preserve encounter order). */ + final Map reads = new LinkedHashMap<>(); + + /** set to the blocking txn index (>=0) the first time a read hit an ESTIMATE. */ + int blockedOn = -1; + + MvContext( + final int txnIdx, final MvMemory mv, final BaseReader base, final Address beneficiary) { + this.txnIdx = txnIdx; + this.mv = mv; + this.base = base; + this.beneficiary = beneficiary; + } + + void record(final MvKey key, final MvMemory.ReadDescriptor desc) { + reads.putIfAbsent(key, desc); + } + + boolean isBeneficiary(final Address a) { + return beneficiary != null && beneficiary.equals(a); + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvKey.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvKey.java new file mode 100644 index 0000000..b06eecf --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvKey.java @@ -0,0 +1,156 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Objects; + +import org.apache.tuweni.units.bigints.UInt256; + +/** + * A location in world state that Block-STM tracks for read/write-set conflict detection. Mirrors the + * Rust {@code Key} enum (Balance / Storage) generalised to the four EVM account facets that a + * transaction can read or write: BALANCE, NONCE, CODE and a STORAGE slot. + * + *

This is the unit over which MVMemory shards and over which read-set validation compares + * provenance. Two {@code MvKey}s are equal iff they refer to the same facet of the same account (and + * same slot for STORAGE). + */ +@SuppressWarnings("MissingSummary") +public final class MvKey { + + /** The account facet this key addresses. */ + public enum Kind { + /** Account balance. */ + BALANCE, + /** Account nonce. */ + NONCE, + /** Account code. */ + CODE, + /** A single storage slot. */ + STORAGE + } + + private final Kind kind; + private final Address address; + private final UInt256 slot; // null unless STORAGE + private final int hash; + + private MvKey(final Kind kind, final Address address, final UInt256 slot) { + this.kind = kind; + this.address = address; + this.slot = slot; + this.hash = Objects.hash(kind, address, slot); + } + + /** + * Balance key. + * + * @param address the account + * @return the key + */ + public static MvKey balance(final Address address) { + return new MvKey(Kind.BALANCE, address, null); + } + + /** + * Nonce key. + * + * @param address the account + * @return the key + */ + public static MvKey nonce(final Address address) { + return new MvKey(Kind.NONCE, address, null); + } + + /** + * Code key. + * + * @param address the account + * @return the key + */ + public static MvKey code(final Address address) { + return new MvKey(Kind.CODE, address, null); + } + + /** + * Storage-slot key. + * + * @param address the account + * @param slot the slot + * @return the key + */ + public static MvKey storage(final Address address, final UInt256 slot) { + return new MvKey(Kind.STORAGE, address, slot); + } + + /** + * The facet kind. + * + * @return kind + */ + public Kind kind() { + return kind; + } + + /** + * The account address. + * + * @return address + */ + public Address address() { + return address; + } + + /** + * Deterministic shard index, mirroring the Rust FNV-1a spread. + * + * @param shards number of shards + * @return shard index in [0, shards) + */ + public int shard(final int shards) { + long h = 0xcbf29ce484222325L; + h = fnv(h, kind.ordinal() + 1L); + h = fnv(h, address.hashCode() & 0xffffffffL); + h = fnv(h, slot == null ? 0L : slot.hashCode() & 0xffffffffL); + int s = (int) Long.remainderUnsigned(h, shards); + return s; + } + + private static long fnv(final long acc, final long b) { + long h = acc ^ b; + h *= 0x100000001b3L; + return h; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MvKey)) { + return false; + } + final MvKey k = (MvKey) o; + return kind == k.kind && address.equals(k.address) && Objects.equals(slot, k.slot); + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public String toString() { + return kind + "(" + address + (slot == null ? "" : "," + slot.toShortHexString()) + ")"; + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvMemory.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvMemory.java new file mode 100644 index 0000000..914df3a --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/MvMemory.java @@ -0,0 +1,266 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Multi-version shared memory (MVMemory). Faithful Java port of the AERE Rust {@code mvmemory.rs} + * (Gelashvili et al. 2022). + * + *

For every {@link MvKey}, MVMemory keeps an ordered map {@code txnIndex -> Cell}, where a cell + * is either a concrete value written by that transaction's latest incarnation, or an ESTIMATE + * placeholder (a marker that a lower transaction previously wrote here but is currently being + * re-executed, so its value is not trustworthy). + * + *

A read by transaction {@code i} for key {@code k} returns the entry of the HIGHEST transaction + * {@code j < i} that wrote {@code k}. That is what makes speculative parallel execution + * serial-equivalent: transaction {@code i} always observes the writes of all lower-indexed + * transactions that have run, and nothing from higher ones. + * + *

Storage is sharded across many independent locks keyed by a hash of the key, so + * non-conflicting transactions touch disjoint locks and run without contention. + */ +@SuppressWarnings("MissingSummary") +public class MvMemory { + + private static final int SHARDS = 256; + + /** What a read observed, recorded into the reader's read set for later validation. */ + public enum DescKind { + /** Nothing below the reader wrote this key: it came from base state. */ + BASE, + /** It was written by some (txnIndex, incarnation). */ + VERSION + } + + /** A read-set descriptor: BASE, or VERSION(txnIndex, incarnation). */ + public static final class ReadDescriptor { + /** Shared BASE singleton. */ + public static final ReadDescriptor BASE = new ReadDescriptor(DescKind.BASE, -1, -1); + + private final DescKind kind; + private final int txnIndex; + private final int incarnation; + + private ReadDescriptor(final DescKind kind, final int txnIndex, final int incarnation) { + this.kind = kind; + this.txnIndex = txnIndex; + this.incarnation = incarnation; + } + + static ReadDescriptor version(final int txnIndex, final int incarnation) { + return new ReadDescriptor(DescKind.VERSION, txnIndex, incarnation); + } + + /** @return true if this descriptor is a BASE read. */ + public boolean isBase() { + return kind == DescKind.BASE; + } + + boolean sameProvenance(final ReadDescriptor other) { + if (kind != other.kind) { + return false; + } + if (kind == DescKind.BASE) { + return true; + } + return txnIndex == other.txnIndex && incarnation == other.incarnation; + } + } + + /** Live read outcome: OK(descriptor,value) or BLOCKED(blockingTxn). */ + public static final class ReadResult { + final boolean blocked; + final int blockingTxn; + final ReadDescriptor descriptor; + final Object value; + + private ReadResult( + final boolean blocked, + final int blockingTxn, + final ReadDescriptor descriptor, + final Object value) { + this.blocked = blocked; + this.blockingTxn = blockingTxn; + this.descriptor = descriptor; + this.value = value; + } + + static ReadResult ok(final ReadDescriptor d, final Object v) { + return new ReadResult(false, -1, d, v); + } + + static ReadResult blocked(final int blockingTxn) { + return new ReadResult(true, blockingTxn, null, null); + } + + /** @return true if this read hit an ESTIMATE and the reader must park. */ + public boolean isBlocked() { + return blocked; + } + + /** @return the blocking txn index (valid iff {@link #isBlocked()}). */ + public int blockingTxn() { + return blockingTxn; + } + + /** @return the observed provenance descriptor (valid iff not blocked). */ + public ReadDescriptor descriptor() { + return descriptor; + } + + /** @return the observed value, or null for a BASE read (caller falls back to base state). */ + public Object value() { + return value; + } + } + + private static final class Cell { + final boolean estimate; + final int incarnation; + final Object value; + + private Cell(final boolean estimate, final int incarnation, final Object value) { + this.estimate = estimate; + this.incarnation = incarnation; + this.value = value; + } + + static Cell value(final int inc, final Object v) { + return new Cell(false, inc, v); + } + + static final Cell ESTIMATE = new Cell(true, -1, null); + } + + private final ReentrantLock[] locks = new ReentrantLock[SHARDS]; + // shard -> (key -> (txnIndex -> Cell)) + private final Map>[] shards; + + /** New empty MVMemory. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public MvMemory() { + shards = new Map[SHARDS]; + for (int i = 0; i < SHARDS; i++) { + locks[i] = new ReentrantLock(); + shards[i] = new HashMap<>(); + } + } + + /** + * Read the value visible to {@code txnIdx}: the highest entry strictly below it. + * + * @param key the key + * @param txnIdx the reading transaction index + * @return the read result + */ + public ReadResult read(final MvKey key, final int txnIdx) { + final int s = key.shard(SHARDS); + locks[s].lock(); + try { + final TreeMap versions = shards[s].get(key); + if (versions != null) { + final Map.Entry e = versions.lowerEntry(txnIdx); + if (e != null) { + final Cell cell = e.getValue(); + if (cell.estimate) { + return ReadResult.blocked(e.getKey()); + } + return ReadResult.ok(ReadDescriptor.version(e.getKey(), cell.incarnation), cell.value); + } + } + return ReadResult.ok(ReadDescriptor.BASE, null); + } finally { + locks[s].unlock(); + } + } + + /** + * Install {@code txnIdx}'s new write set (incarnation {@code inc}); remove stale entries the + * previous incarnation wrote but this one no longer does. Returns true if the written key-SET + * changed (a key added or removed) — Block-STM's "wrote a new path", which triggers revalidation + * of the suffix. + * + * @param txnIdx the writing transaction + * @param inc the incarnation + * @param writes the new write set + * @param prevKeys the keys written by the previous incarnation + * @return whether the written key-set changed + */ + public boolean applyWriteSet( + final int txnIdx, + final int inc, + final Map writes, + final Set prevKeys) { + for (final Map.Entry w : writes.entrySet()) { + final MvKey k = w.getKey(); + final int s = k.shard(SHARDS); + locks[s].lock(); + try { + shards[s].computeIfAbsent(k, kk -> new TreeMap<>()).put(txnIdx, Cell.value(inc, w.getValue())); + } finally { + locks[s].unlock(); + } + } + boolean removedAny = false; + for (final MvKey k : prevKeys) { + if (!writes.containsKey(k)) { + final int s = k.shard(SHARDS); + locks[s].lock(); + try { + final TreeMap versions = shards[s].get(k); + if (versions != null) { + versions.remove(txnIdx); + } + } finally { + locks[s].unlock(); + } + removedAny = true; + } + } + boolean addedAny = false; + for (final MvKey k : writes.keySet()) { + if (!prevKeys.contains(k)) { + addedAny = true; + break; + } + } + return addedAny || removedAny; + } + + /** + * Convert all of {@code txnIdx}'s current writes (over {@code keys}) into ESTIMATE placeholders, + * so any concurrent higher transaction that reads one of these keys blocks on {@code txnIdx} + * instead of consuming a soon-to-be-wrong value. + * + * @param txnIdx the aborted transaction + * @param keys its written keys + */ + public void markEstimates(final int txnIdx, final Set keys) { + for (final MvKey k : keys) { + final int s = k.shard(SHARDS); + locks[s].lock(); + try { + final TreeMap versions = shards[s].get(k); + if (versions != null && versions.containsKey(txnIdx)) { + versions.put(txnIdx, Cell.ESTIMATE); + } + } finally { + locks[s].unlock(); + } + } + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/Scheduler.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/Scheduler.java new file mode 100644 index 0000000..a967bbd --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockstm/Scheduler.java @@ -0,0 +1,390 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.blockstm; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +/** + * The Block-STM collaborative scheduler (Gelashvili et al. 2022, Algorithm 3). Faithful Java port of + * the AERE Rust {@code scheduler.rs}. + * + *

Hands out EXECUTION and VALIDATION tasks to a worker pool. Two cursors ({@code executionIdx}, + * {@code validationIdx}) drive it; validation of a lower index always takes priority over execution + * of a higher one. When a transaction aborts or writes a new path, the relevant cursor is decreased + * so the affected range is revisited. The block is done when both cursors have passed the end AND no + * task is in flight AND no decrease happened during the check. Dependencies are handled without + * condvars via parking on the blocker's dependency list. + */ +@SuppressWarnings("MissingSummary") +public class Scheduler { + + /** Task type handed to a worker. */ + public enum TaskKind { + /** Run incarnation {@code inc} of transaction {@code idx}. */ + EXECUTION, + /** Re-check {@code idx}'s read set. */ + VALIDATION, + /** No task available right now; retry. */ + NO_TASK, + /** Block fully executed & validated. */ + DONE + } + + /** A unit of work, or a control signal. */ + public static final class Task { + /** Shared NO_TASK singleton. */ + public static final Task NO_TASK = new Task(TaskKind.NO_TASK, -1, -1); + /** Shared DONE singleton. */ + public static final Task DONE = new Task(TaskKind.DONE, -1, -1); + + final TaskKind kind; + final int idx; + final int inc; + + Task(final TaskKind kind, final int idx, final int inc) { + this.kind = kind; + this.idx = idx; + this.inc = inc; + } + + /** @return the task kind */ + public TaskKind kind() { + return kind; + } + + /** @return the transaction index */ + public int idx() { + return idx; + } + + /** @return the incarnation */ + public int inc() { + return inc; + } + } + + private enum StatusKind { + READY_TO_EXECUTE, + EXECUTING, + SUSPENDED, + EXECUTED, + ABORTING + } + + private static final class Status { + StatusKind kind; + int inc; + + Status(final StatusKind kind, final int inc) { + this.kind = kind; + this.inc = inc; + } + } + + private final int numTxns; + private final AtomicInteger executionIdx = new AtomicInteger(0); + private final AtomicInteger validationIdx = new AtomicInteger(0); + private final AtomicInteger decreaseCnt = new AtomicInteger(0); + private final AtomicInteger numActiveTasks = new AtomicInteger(0); + private final AtomicBoolean done = new AtomicBoolean(false); + private final Status[] status; + private final ReentrantLock[] statusLocks; + private final List[] dependencies; + private final ReentrantLock[] depLocks; + + /** + * New scheduler over {@code numTxns} transactions. + * + * @param numTxns transaction count + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public Scheduler(final int numTxns) { + this.numTxns = numTxns; + this.status = new Status[numTxns]; + this.statusLocks = new ReentrantLock[numTxns]; + this.dependencies = new List[numTxns]; + this.depLocks = new ReentrantLock[numTxns]; + for (int i = 0; i < numTxns; i++) { + status[i] = new Status(StatusKind.READY_TO_EXECUTE, 0); + statusLocks[i] = new ReentrantLock(); + dependencies[i] = new ArrayList<>(); + depLocks[i] = new ReentrantLock(); + } + } + + /** @return true once the block is fully executed & validated. */ + public boolean done() { + return done.get(); + } + + private void decreaseExecutionIdx(final int target) { + executionIdx.getAndUpdate(cur -> Math.min(cur, target)); + decreaseCnt.incrementAndGet(); + } + + private void decreaseValidationIdx(final int target) { + validationIdx.getAndUpdate(cur -> Math.min(cur, target)); + decreaseCnt.incrementAndGet(); + } + + private void checkDone() { + final int observed = decreaseCnt.get(); + if (executionIdx.get() >= numTxns + && validationIdx.get() >= numTxns + && numActiveTasks.get() == 0 + && decreaseCnt.get() == observed) { + done.set(true); + } + } + + private Task tryIncarnate(final int idx) { + if (idx < numTxns) { + statusLocks[idx].lock(); + try { + if (status[idx].kind == StatusKind.READY_TO_EXECUTE) { + final int inc = status[idx].inc; + status[idx].kind = StatusKind.EXECUTING; + return new Task(TaskKind.EXECUTION, idx, inc); + } + } finally { + statusLocks[idx].unlock(); + } + } + numActiveTasks.decrementAndGet(); + return null; + } + + private Task nextVersionToExecute() { + if (executionIdx.get() >= numTxns) { + checkDone(); + return null; + } + numActiveTasks.incrementAndGet(); + final int idx = executionIdx.getAndIncrement(); + return tryIncarnate(idx); + } + + private Task nextVersionToValidate() { + if (validationIdx.get() >= numTxns) { + checkDone(); + return null; + } + numActiveTasks.incrementAndGet(); + final int idx = validationIdx.getAndIncrement(); + if (idx < numTxns) { + statusLocks[idx].lock(); + try { + if (status[idx].kind == StatusKind.EXECUTED) { + return new Task(TaskKind.VALIDATION, idx, status[idx].inc); + } + } finally { + statusLocks[idx].unlock(); + } + } + numActiveTasks.decrementAndGet(); + return null; + } + + /** + * Main entry point workers spin on. + * + * @return a task, NO_TASK (retry), or DONE + */ + public Task nextTask() { + int idle = 0; + while (true) { + if (done.get()) { + return Task.DONE; + } + final int v = validationIdx.get(); + final int e = executionIdx.get(); + Task task = (v < e) ? nextVersionToValidate() : nextVersionToExecute(); + if (task == null && v < e) { + // The preferred lower-index validation had nothing runnable (its target is still + // EXECUTING on another worker). Rather than busy-draining validationIdx while pending + // independent EXECUTION tasks sit ungrabbed (which starved real parallel work and made + // the execution phase anti-scale with worker count), try to grab a pending execution. + // This is scheduling-order only: executing any READY incarnation in any order is already + // permitted by Block-STM, and read-set validation still re-checks and re-executes on any + // conflict, so the committed result stays bit-identical to sequential. + task = nextVersionToExecute(); + } + if (task != null) { + return task; + } + if (done.get()) { + return Task.DONE; + } + // Nothing runnable right now: back off so idle workers stop ping-ponging the shared + // cursors/locks across cores (that cache-line contention slows the workers doing real + // execution). Pure CPU hint / brief park; does not alter task selection or ordering. + if (++idle < 128) { + Thread.onSpinWait(); + } else { + java.util.concurrent.locks.LockSupport.parkNanos(Math.min((idle - 127) * 512L, 50_000L)); + } + } + } + + /** + * Register that {@code idx} depends on {@code blocking} (it read {@code blocking}'s ESTIMATE). + * + * @param idx the reader + * @param blocking the blocker + * @return true if {@code idx} was parked (caller drops the task); false if {@code blocking} + * already finished (caller re-executes {@code idx}) + */ + public boolean addDependency(final int idx, final int blocking) { + statusLocks[blocking].lock(); + try { + if (status[blocking].kind == StatusKind.EXECUTED) { + return false; + } + depLocks[blocking].lock(); + try { + dependencies[blocking].add(idx); + } finally { + depLocks[blocking].unlock(); + } + statusLocks[idx].lock(); + try { + if (status[idx].kind == StatusKind.EXECUTING) { + status[idx].kind = StatusKind.SUSPENDED; + } + } finally { + statusLocks[idx].unlock(); + } + } finally { + statusLocks[blocking].unlock(); + } + numActiveTasks.decrementAndGet(); + return true; + } + + private void resume(final int idx) { + statusLocks[idx].lock(); + try { + if (status[idx].kind == StatusKind.SUSPENDED) { + status[idx].kind = StatusKind.READY_TO_EXECUTE; + } + } finally { + statusLocks[idx].unlock(); + } + } + + private void setReadyAfterAbort(final int idx) { + statusLocks[idx].lock(); + try { + if (status[idx].kind == StatusKind.ABORTING) { + status[idx].kind = StatusKind.READY_TO_EXECUTE; + status[idx].inc = status[idx].inc + 1; + } + } finally { + statusLocks[idx].unlock(); + } + } + + private void setExecuted(final int idx, final int inc) { + statusLocks[idx].lock(); + try { + status[idx].kind = StatusKind.EXECUTED; + status[idx].inc = inc; + } finally { + statusLocks[idx].unlock(); + } + } + + /** + * Called after an execution task completes. + * + * @param idx transaction + * @param inc incarnation + * @param wroteNewPath whether the written key-set changed vs the previous incarnation + * @return a follow-up task (inheriting the active-task unit) or NO_TASK (releasing it) + */ + public Task finishExecution(final int idx, final int inc, final boolean wroteNewPath) { + setExecuted(idx, inc); + + final List deps; + depLocks[idx].lock(); + try { + deps = new ArrayList<>(dependencies[idx]); + dependencies[idx].clear(); + } finally { + depLocks[idx].unlock(); + } + if (!deps.isEmpty()) { + final int minDep = Collections.min(deps); + for (final int d : deps) { + resume(d); + } + decreaseExecutionIdx(minDep); + } + + if (validationIdx.get() > idx) { + if (wroteNewPath) { + decreaseValidationIdx(idx); + numActiveTasks.decrementAndGet(); + return Task.NO_TASK; + } else { + return new Task(TaskKind.VALIDATION, idx, inc); + } + } else { + numActiveTasks.decrementAndGet(); + return Task.NO_TASK; + } + } + + /** + * Attempt to win the right to abort {@code idx} at {@code inc}. + * + * @param idx transaction + * @param inc incarnation + * @return true iff this caller flipped EXECUTED(inc) -> ABORTING(inc) + */ + public boolean tryValidationAbort(final int idx, final int inc) { + statusLocks[idx].lock(); + try { + if (status[idx].kind == StatusKind.EXECUTED && status[idx].inc == inc) { + status[idx].kind = StatusKind.ABORTING; + return true; + } + return false; + } finally { + statusLocks[idx].unlock(); + } + } + + /** + * Called after a validation task. + * + * @param idx transaction + * @param aborted whether validation failed AND this caller won {@link #tryValidationAbort} + * @return a follow-up execution task (inheriting the active-task unit) or NO_TASK (releasing it) + */ + public Task finishValidation(final int idx, final boolean aborted) { + if (aborted) { + setReadyAfterAbort(idx); + decreaseValidationIdx(idx + 1); + if (executionIdx.get() > idx) { + final Task t = tryIncarnate(idx); + return t == null ? Task.NO_TASK : t; + } + } + numActiveTasks.decrementAndGet(); + return Task.NO_TASK; + } +} diff --git a/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarket.java b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarket.java new file mode 100644 index 0000000..e7c8c97 --- /dev/null +++ b/anchor/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarket.java @@ -0,0 +1,252 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0004-aere-basefee-floor.patch. + */ +package org.hyperledger.besu.ethereum.mainnet.feemarket; + +import org.hyperledger.besu.config.GenesisConfig; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.feemarket.TransactionPriceCalculator; + +import java.util.Optional; + +import org.apache.tuweni.units.bigints.UInt256s; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LondonFeeMarket implements BaseFeeMarket { + private static final Logger LOG = LoggerFactory.getLogger(LondonFeeMarket.class); + + static final Wei DEFAULT_BASEFEE_INITIAL_VALUE = GenesisConfig.BASEFEE_AT_GENESIS_DEFAULT_VALUE; + static final long DEFAULT_BASEFEE_MAX_CHANGE_DENOMINATOR = 8L; + static final long DEFAULT_SLACK_COEFFICIENT = 2L; + + private static final Wei DEFAULT_BASEFEE_FLOOR = Wei.of(7L); + + // AERE base-fee-floor fork (fork-gated, no-op until an operator activates it). + // + // Raises the consensus EIP-1559 base-fee floor from the emergent 7-wei integer-division floor + // to a configurable value (default 1 Gwei) at and after a chosen fork block, so the public RPC + // can serve a TRUE, block-hash-verifiable baseFeePerGas instead of a proxy-faked constant. + // + // Activation is read once at fee-market construction (node start) from a JVM system property + // with an environment-variable fallback, mirroring how the AERE PQC/Block-STM fork reads its + // own JVM-level configuration: + // -Daere.basefee.floor.forkBlock= (env AERE_BASEFEE_FLOOR_FORKBLOCK) default Long.MAX_VALUE (UNSET => no-op) + // -Daere.basefee.floor.value= (env AERE_BASEFEE_FLOOR_VALUE) default 1_000_000_000 (1 Gwei) + // + // UNSET (default) means the fork block is Long.MAX_VALUE, so no real block ever reaches it and + // every code path below is byte-identical to upstream London behaviour (7-wei emergent floor). + static final long DEFAULT_AERE_BASEFEE_FLOOR_VALUE = 1_000_000_000L; // 1 Gwei in wei + private static final String AERE_FLOOR_FORK_BLOCK_PROPERTY = "aere.basefee.floor.forkBlock"; + private static final String AERE_FLOOR_FORK_BLOCK_ENV = "AERE_BASEFEE_FLOOR_FORKBLOCK"; + private static final String AERE_FLOOR_VALUE_PROPERTY = "aere.basefee.floor.value"; + private static final String AERE_FLOOR_VALUE_ENV = "AERE_BASEFEE_FLOOR_VALUE"; + + protected final Wei baseFeeInitialValue; + private final long londonForkBlockNumber; + private final TransactionPriceCalculator txPriceCalculator; + private final Wei baseFeeFloor; + + // Activation block for the AERE base-fee floor. Long.MAX_VALUE == UNSET == no-op. + private final long aereBaseFeeFloorForkBlock; + // Floor value applied at and after aereBaseFeeFloorForkBlock (wei). + private final long aereBaseFeeFloorValue; + + LondonFeeMarket(final long londonForkBlockNumber, final Optional baseFeePerGasOverride) { + this(TransactionPriceCalculator.eip1559(), londonForkBlockNumber, baseFeePerGasOverride); + } + + LondonFeeMarket( + final TransactionPriceCalculator txPriceCalculator, + final long londonForkBlockNumber, + final Optional baseFeePerGasOverride) { + // Production path: resolve the AERE base-fee-floor fork activation from JVM configuration. + this( + txPriceCalculator, + londonForkBlockNumber, + baseFeePerGasOverride, + readAereFloorForkBlockConfig(), + readAereFloorValueConfig()); + } + + // Package-private constructor taking explicit AERE floor-fork parameters. Used by the + // production constructor above (after resolving JVM config) and by unit tests so the fork can be + // exercised deterministically without mutating global JVM system properties / environment. + LondonFeeMarket( + final TransactionPriceCalculator txPriceCalculator, + final long londonForkBlockNumber, + final Optional baseFeePerGasOverride, + final long aereBaseFeeFloorForkBlock, + final long aereBaseFeeFloorValue) { + this.txPriceCalculator = txPriceCalculator; + this.londonForkBlockNumber = londonForkBlockNumber; + this.baseFeeInitialValue = baseFeePerGasOverride.orElse(DEFAULT_BASEFEE_INITIAL_VALUE); + this.baseFeeFloor = baseFeeInitialValue.isZero() ? Wei.ZERO : DEFAULT_BASEFEE_FLOOR; + this.aereBaseFeeFloorForkBlock = aereBaseFeeFloorForkBlock; + this.aereBaseFeeFloorValue = aereBaseFeeFloorValue; + } + + private static long readAereFloorForkBlockConfig() { + return readLongConfig( + AERE_FLOOR_FORK_BLOCK_PROPERTY, AERE_FLOOR_FORK_BLOCK_ENV, Long.MAX_VALUE); + } + + private static long readAereFloorValueConfig() { + return readLongConfig( + AERE_FLOOR_VALUE_PROPERTY, AERE_FLOOR_VALUE_ENV, DEFAULT_AERE_BASEFEE_FLOOR_VALUE); + } + + private static long readLongConfig( + final String systemProperty, final String envVar, final long defaultValue) { + String raw = System.getProperty(systemProperty); + if (raw == null || raw.isBlank()) { + raw = System.getenv(envVar); + } + if (raw == null || raw.isBlank()) { + return defaultValue; + } + try { + final long parsed = Long.parseLong(raw.trim()); + if (parsed < 0L) { + LOG.warn( + "Ignoring negative {}/{} value '{}', using default {}", + systemProperty, + envVar, + raw, + defaultValue); + return defaultValue; + } + return parsed; + } catch (final NumberFormatException e) { + LOG.warn( + "Ignoring unparseable {}/{} value '{}', using default {}", + systemProperty, + envVar, + raw, + defaultValue); + return defaultValue; + } + } + + // Clamp a computed baseFee up to the active AERE floor at and after the fork block. Before the + // fork block (and when UNSET, since Long.MAX_VALUE is never reached) this returns baseFee + // unchanged, keeping pre-fork behaviour byte-identical to upstream. + private Wei applyAereBaseFeeFloor(final long blockNumber, final Wei baseFee) { + if (blockNumber >= aereBaseFeeFloorForkBlock) { + return UInt256s.max(baseFee, Wei.of(aereBaseFeeFloorValue)); + } + return baseFee; + } + + // Effective tx-pool minimum-fee floor. Pre-fork (UNSET) this is exactly the legacy baseFeeFloor + // (7 wei, or 0 on a zero-basefee chain), so tx-pool acceptance is byte-identical. Once an + // operator activates the AERE base-fee-floor fork, the tx-pool floor rises to the active AERE + // floor so tx-pool pricing agrees with the >= 1 Gwei header baseFee produced by computeBaseFee. + // A zero-basefee chain (baseFeeFloor == 0) is never raised. + private Wei effectiveBaseFeeFloor() { + if (baseFeeFloor.isZero() || aereBaseFeeFloorForkBlock == Long.MAX_VALUE) { + return baseFeeFloor; + } + return UInt256s.max(baseFeeFloor, Wei.of(aereBaseFeeFloorValue)); + } + + @Override + public long getBasefeeMaxChangeDenominator() { + return DEFAULT_BASEFEE_MAX_CHANGE_DENOMINATOR; + } + + @Override + public Wei getInitialBasefee() { + return baseFeeInitialValue; + } + + @Override + public long getSlackCoefficient() { + return DEFAULT_SLACK_COEFFICIENT; + } + + @Override + public TransactionPriceCalculator getTransactionPriceCalculator() { + return txPriceCalculator; + } + + @Override + public boolean satisfiesFloorTxFee(final Transaction txn) { + // ensure effective baseFee is at least above floor + return txn.getGasPrice() + .map(Optional::of) + .orElse(txn.getMaxFeePerGas()) + .filter(fee -> fee.greaterOrEqualThan(effectiveBaseFeeFloor())) + .isPresent(); + } + + @Override + public Wei computeBaseFee( + final long blockNumber, + final Wei parentBaseFee, + final long parentBlockGasUsed, + final long targetGasUsed) { + if (londonForkBlockNumber == blockNumber) { + return applyAereBaseFeeFloor(blockNumber, getInitialBasefee()); + } + + long gasDelta; + Wei feeDelta, baseFee; + if (parentBlockGasUsed == targetGasUsed) { + return applyAereBaseFeeFloor(blockNumber, parentBaseFee); + } else if (parentBlockGasUsed > targetGasUsed) { + gasDelta = parentBlockGasUsed - targetGasUsed; + final long denominator = getBasefeeMaxChangeDenominator(); + feeDelta = + UInt256s.max( + parentBaseFee.multiply(gasDelta).divide(targetGasUsed).divide(denominator), Wei.ONE); + baseFee = parentBaseFee.add(feeDelta); + } else { + gasDelta = targetGasUsed - parentBlockGasUsed; + final long denominator = getBasefeeMaxChangeDenominator(); + feeDelta = parentBaseFee.multiply(gasDelta).divide(targetGasUsed).divide(denominator); + baseFee = parentBaseFee.subtract(feeDelta); + } + LOG.trace( + "block #{} parentBaseFee: {} parentGasUsed: {} parentGasTarget: {} baseFee: {}", + blockNumber, + parentBaseFee, + parentBlockGasUsed, + targetGasUsed, + baseFee); + return applyAereBaseFeeFloor(blockNumber, baseFee); + } + + @Override + public ValidationMode baseFeeValidationMode(final long blockNumber) { + return londonForkBlockNumber == blockNumber ? ValidationMode.INITIAL : ValidationMode.ONGOING; + } + + @Override + public ValidationMode gasLimitValidationMode(final long blockNumber) { + return londonForkBlockNumber == blockNumber ? ValidationMode.INITIAL : ValidationMode.ONGOING; + } + + @Override + public boolean isBeforeForkBlock(final long blockNumber) { + return londonForkBlockNumber > blockNumber; + } +} diff --git a/anchor/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java b/anchor/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java new file mode 100644 index 0000000..65292ac --- /dev/null +++ b/anchor/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java @@ -0,0 +1,1225 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.core; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.hyperledger.besu.crypto.Hash.keccak256; +import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SecureRandomProvider; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Log; +import org.hyperledger.besu.datatypes.LogTopic; +import org.hyperledger.besu.datatypes.LogsBloomFilter; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.VersionedHash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; + +import java.math.BigInteger; +import java.security.InvalidAlgorithmParameterException; +import java.security.KeyPairGenerator; +import java.security.SecureRandom; +import java.security.Security; +import java.security.spec.ECGenParameterSpec; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Random; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPrivateKey; +import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPublicKey; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +public class BlockDataGenerator { + + static { + Security.addProvider(new BouncyCastleProvider()); + } + + private final Random random; + private final KeyPairGenerator keyPairGenerator; + private final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance(); + private Supplier blockOptionsSupplier = BlockOptions::create; + + public BlockDataGenerator(final int seed) { + this.random = new Random(seed); + keyPairGenerator = createKeyPairGenerator(seed); + } + + public BlockDataGenerator() { + this(1); + } + + public void setBlockOptionsSupplier(final Supplier blockOptionsSupplier) { + this.blockOptionsSupplier = blockOptionsSupplier; + } + + private KeyPairGenerator createKeyPairGenerator(final long seed) { + final KeyPairGenerator localKeyPairGenerator; + try { + localKeyPairGenerator = + KeyPairGenerator.getInstance( + SignatureAlgorithm.ALGORITHM, signatureAlgorithm.getProvider()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + final ECGenParameterSpec ecGenParameterSpec = + new ECGenParameterSpec(signatureAlgorithm.getCurveName()); + try { + final SecureRandom secureRandom = SecureRandomProvider.createSecureRandom(); + secureRandom.setSeed(seed); + localKeyPairGenerator.initialize(ecGenParameterSpec, secureRandom); + } catch (final InvalidAlgorithmParameterException e) { + throw new RuntimeException(e); + } + return localKeyPairGenerator; + } + + /** + * Wrapper class to return both Block and its generated BlockAccessList. This ensures that the BAL + * hash in the header matches the actual BAL data. + */ + public static class BlockWithAccessList { + private final Block block; + private final Optional blockAccessList; + + public BlockWithAccessList(final Block block, final Optional blockAccessList) { + this.block = block; + this.blockAccessList = blockAccessList; + } + + public Block getBlock() { + return block; + } + + public Optional getBlockAccessList() { + return blockAccessList; + } + } + + /** + * Generates a sequence of blocks with some accounts and account storage pre-populated with random + * data. + */ + private List blockSequence( + final int count, + final long nextBlock, + final Hash parent, + final WorldStateArchive worldStateArchive, + final List

accountsToSetup, + final List storageKeys) { + final List seq = new ArrayList<>(count); + + final MutableWorldState worldState = worldStateArchive.getWorldState(); + + long nextBlockNumber = nextBlock; + Hash parentHash = parent; + + for (int i = 0; i < count; i++) { + final WorldUpdater stateUpdater = worldState.updater(); + if (i == 0) { + // Set up some accounts + accountsToSetup.forEach(stateUpdater::createAccount); + stateUpdater.commit(); + } else { + // Mutate accounts + accountsToSetup.forEach( + hash -> { + final MutableAccount a = stateUpdater.getAccount(hash); + a.incrementNonce(); + a.setBalance(Wei.of(positiveLong())); + storageKeys.forEach(key -> a.setStorageValue(key, UInt256.ONE)); + }); + stateUpdater.commit(); + } + final BlockOptions options = + blockOptionsSupplier + .get() + .setBlockNumber(nextBlockNumber) + .setParentHash(parentHash) + .setStateRoot(worldState.rootHash()); + final Block next = block(options); + seq.add(next); + parentHash = next.getHash(); + nextBlockNumber = nextBlockNumber + 1L; + worldState.persist(null); + } + + return seq; + } + + /** + * Generates a sequence of blocks with BlockAccessList, returning BlockWithAccessList objects. + * This method ensures that generated BAL hashes match the header balHash field. + * + * @param count number of blocks to generate + * @param nextBlock starting block number + * @param parent parent block hash + * @param worldStateArchive world state archive for state management + * @param accountsToSetup accounts to setup in the world state + * @param storageKeys storage keys to populate + * @return list of BlockWithAccessList containing blocks and their associated BALs + */ + public List blockSequenceWithAccessList( + final int count, + final long nextBlock, + final Hash parent, + final WorldStateArchive worldStateArchive, + final List
accountsToSetup, + final List storageKeys) { + final List seq = new ArrayList<>(count); + + final MutableWorldState worldState = worldStateArchive.getWorldState(); + + long nextBlockNumber = nextBlock; + Hash parentHash = parent; + + for (int i = 0; i < count; i++) { + final WorldUpdater stateUpdater = worldState.updater(); + if (i == 0) { + // Set up some accounts + accountsToSetup.forEach(stateUpdater::createAccount); + stateUpdater.commit(); + } else { + // Mutate accounts + accountsToSetup.forEach( + hash -> { + final MutableAccount a = stateUpdater.getAccount(hash); + a.incrementNonce(); + a.setBalance(Wei.of(positiveLong())); + storageKeys.forEach(key -> a.setStorageValue(key, UInt256.ONE)); + }); + stateUpdater.commit(); + } + final BlockOptions options = + blockOptionsSupplier + .get() + .setBlockNumber(nextBlockNumber) + .setParentHash(parentHash) + .setStateRoot(worldState.rootHash()); + final BlockWithAccessList next = blockWithAccessList(options); + seq.add(next); + parentHash = next.getBlock().getHash(); + nextBlockNumber = nextBlockNumber + 1L; + worldState.persist(null); + } + + return seq; + } + + /** + * Generates a sequence of blocks with BlockAccessList starting from a previous block. + * + * @param previousBlock the block to build upon + * @param count number of blocks to generate + * @return list of BlockWithAccessList + */ + public List blockSequenceWithAccessList( + final Block previousBlock, final int count) { + final WorldStateArchive worldState = createInMemoryWorldStateArchive(); + final Hash parentHash = previousBlock.getHeader().getHash(); + final long blockNumber = previousBlock.getHeader().getNumber() + 1; + return blockSequenceWithAccessList( + count, + blockNumber, + parentHash, + worldState, + Collections.emptyList(), + Collections.emptyList()); + } + + /** + * Generates a sequence of blocks with BlockAccessList starting from genesis. + * + * @param count number of blocks to generate + * @return list of BlockWithAccessList + */ + public List blockSequenceWithAccessList(final int count) { + final WorldStateArchive worldState = createInMemoryWorldStateArchive(); + final long blockNumber = BlockHeader.GENESIS_BLOCK_NUMBER; + final Hash parentHash = Hash.ZERO; + return blockSequenceWithAccessList( + count, + blockNumber, + parentHash, + worldState, + Collections.emptyList(), + Collections.emptyList()); + } + + public List createRandomAccounts(final MutableWorldState worldState, final int count) { + return createRandomAccounts(worldState, count, .5f, .75f); + } + + public List createRandomContractAccountsWithNonEmptyStorage( + final MutableWorldState worldState, final int count) { + return createRandomAccounts(worldState, count, 1f, 1f); + } + + private List createRandomAccounts( + final MutableWorldState worldState, + final int count, + final float percentContractAccounts, + final float percentContractAccountsWithNonEmptyStorage) { + final WorldUpdater updater = worldState.updater(); + final List accounts = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + final MutableAccount account = updater.getOrCreate(address()); + if (random.nextFloat() < percentContractAccounts) { + // Some percentage of accounts are contract accounts + account.setCode(bytesValue(5, 50)); + if (random.nextFloat() < percentContractAccountsWithNonEmptyStorage) { + // Add some storage for contract accounts + final int storageValues = random.nextInt(20) + 10; + for (int j = 0; j < storageValues; j++) { + account.setStorageValue(uint256(), uint256()); + } + } + } + account.setNonce(random.nextLong()); + account.setBalance(Wei.of(positiveLong())); + + accounts.add(account); + } + updater.commit(); + worldState.persist(null); + return accounts; + } + + public List blockSequence(final int count) { + final WorldStateArchive worldState = createInMemoryWorldStateArchive(); + return blockSequence(count, worldState, Collections.emptyList(), Collections.emptyList()); + } + + public List blockSequence(final Block previousBlock, final int count) { + final WorldStateArchive worldState = createInMemoryWorldStateArchive(); + final Hash parentHash = previousBlock.getHeader().getHash(); + final long blockNumber = previousBlock.getHeader().getNumber() + 1; + return blockSequence( + count, + blockNumber, + parentHash, + worldState, + Collections.emptyList(), + Collections.emptyList()); + } + + public List blockSequence( + final int count, + final WorldStateArchive worldStateArchive, + final List
accountsToSetup, + final List storageKeys) { + final long blockNumber = BlockHeader.GENESIS_BLOCK_NUMBER; + final Hash parentHash = Hash.ZERO; + return blockSequence( + count, blockNumber, parentHash, worldStateArchive, accountsToSetup, storageKeys); + } + + public Block genesisBlock() { + return genesisBlock(blockOptionsSupplier.get()); + } + + public Block genesisBlock(final BlockOptions options) { + options + .setBlockNumber(BlockHeader.GENESIS_BLOCK_NUMBER) + .setStateRoot(Hash.EMPTY_TRIE_HASH) + .setParentHash(Hash.ZERO); + return block(options); + } + + public Block block(final BlockOptions options) { + return blockWithAccessList(options).getBlock(); + } + + public Block block() { + return block(blockOptionsSupplier.get()); + } + + /** + * Creates a block with its associated BlockAccessList if generation is enabled. This method + * generates the BAL before creating the header to ensure the balHash in the header matches the + * actual BAL hash. + * + * @param options block generation options + * @return BlockWithAccessList containing the block and optional BAL + */ + public BlockWithAccessList blockWithAccessList(final BlockOptions options) { + final long blockNumber = options.getBlockNumber(positiveLong()); + final BlockBody body = + blockNumber == BlockHeader.GENESIS_BLOCK_NUMBER ? BlockBody.empty() : body(options); + + Optional generatedBal = Optional.empty(); + + // Generate BAL before creating header if enabled + if (options.shouldGenerateBlockAccessList()) { + final BlockAccessList bal = + options.getBlockAccessList().orElseGet(() -> blockAccessList(1 + random.nextInt(5))); + generatedBal = Optional.of(bal); + // Set the balHash in options so it's included in the header + options.setBalHash(Hash.wrap(keccak256(RLP.encode(bal::writeTo)))); + } + + final BlockHeader header = header(blockNumber, body, options); + return new BlockWithAccessList(new Block(header, body), generatedBal); + } + + public BlockOptions nextBlockOptions(final Block afterBlock) { + return blockOptionsSupplier + .get() + .setBlockNumber(afterBlock.getHeader().getNumber() + 1) + .setParentHash(afterBlock.getHash()); + } + + public Block nextBlock(final Block afterBlock) { + final BlockOptions options = nextBlockOptions(afterBlock); + return block(options); + } + + public BlockHeader header(final long blockNumber, final BlockBody blockBody) { + return header(blockNumber, blockBody, new BlockOptions()); + } + + public BlockHeader header(final long blockNumber) { + return header(blockNumber, body(), blockOptionsSupplier.get()); + } + + public BlockHeader header() { + return header(positiveLong(), body(), blockOptionsSupplier.get()); + } + + public BlockHeader header(final long number, final BlockBody body, final BlockOptions options) { + final int gasLimit = random.nextInt() & Integer.MAX_VALUE; + final int gasUsed = Math.max(0, gasLimit - 1); + final long blockNonce = random.nextLong(); + + final BlockHeaderBuilder blockHeaderBuilder = + BlockHeaderBuilder.create() + .parentHash(options.getParentHash(hash())) + .ommersHash(BodyValidation.ommersHash(body.getOmmers())) + .coinbase(options.getCoinbase(address())) + .stateRoot(options.getStateRoot(hash())) + .transactionsRoot(BodyValidation.transactionsRoot(body.getTransactions())) + .receiptsRoot(options.getReceiptsRoot(hash())) + .logsBloom(options.getLogsBloom(logsBloom())) + .difficulty(options.getDifficulty(Difficulty.of(uint256(4)))) + .number(number) + .gasLimit(gasLimit) + .gasUsed(options.getGasUsed(gasUsed)) + .timestamp( + options + .getTimestamp() + .orElse(Instant.now().truncatedTo(ChronoUnit.SECONDS).getEpochSecond())) + .extraData(options.getExtraData(bytes32())) + .mixHash(hash()) + .nonce(blockNonce) + .withdrawalsRoot(options.getWithdrawalsRoot(null)) + .requestsHash(options.getRequestsHash(null)) + .blockHeaderFunctions( + options.getBlockHeaderFunctions(new MainnetBlockHeaderFunctions())); + + // Set balHash from options (already calculated if BAL was generated) + blockHeaderBuilder.balHash(options.getBalHash(null)); + + options.getBaseFee(Optional.of(Wei.of(uint256(2)))).ifPresent(blockHeaderBuilder::baseFee); + return blockHeaderBuilder.buildBlockHeader(); + } + + public BlockBody body() { + return body(blockOptionsSupplier.get()); + } + + public BlockBody body(final BlockOptions options) { + final List ommers = new ArrayList<>(); + if (options.hasOmmers()) { + final int ommerCount = random.nextInt(3); + for (int i = 0; i < ommerCount; i++) { + ommers.add(ommer()); + } + } + final List defaultTxs = new ArrayList<>(options.transactionCount()); + if (options.hasTransactions()) { + for (int i = 0; i < options.transactionCount(); i++) { + defaultTxs.add(transaction(options.getTransactionTypes())); + } + } + + return new BlockBody( + options.getTransactions(defaultTxs), ommers, options.getWithdrawals(Optional.empty())); + } + + private BlockHeader ommer() { + return header(positiveLong(), body(BlockOptions.create().hasOmmers(false))); + } + + private TransactionType transactionType() { + return transactionType( + TransactionType.FRONTIER, TransactionType.ACCESS_LIST, TransactionType.EIP1559); + } + + private TransactionType transactionType(final TransactionType... transactionTypes) { + return transactionTypes[random.nextInt(transactionTypes.length)]; + } + + public Transaction transaction() { + return transaction(transactionType()); + } + + public Transaction transaction(final TransactionType... transactionTypes) { + return transaction(transactionType(transactionTypes)); + } + + public Transaction transaction(final TransactionType transactionType) { + return transaction(transactionType, bytes32(), address()); + } + + public Transaction transaction(final Bytes payload) { + return transaction(transactionType(), payload); + } + + public Transaction transaction(final TransactionType transactionType, final Bytes payload) { + return transaction(transactionType, payload, address()); + } + + public Transaction transaction( + final TransactionType transactionType, final Bytes payload, final Address to) { + return switch (transactionType) { + case FRONTIER -> frontierTransaction(payload, to); + case EIP1559 -> eip1559Transaction(payload, to); + case ACCESS_LIST -> accessListTransaction(payload, to); + case BLOB -> blobTransaction(payload, to); + case DELEGATE_CODE -> null; + // AIP-20 transactions need ML-DSA keys; the random generator does not produce them (same as 7702) + case AERE_PQ -> null; + }; + } + + private Transaction accessListTransaction(final Bytes payload, final Address to) { + return Transaction.builder() + .type(TransactionType.ACCESS_LIST) + .nonce(random.nextLong()) + .gasPrice(Wei.wrap(bytesValue(4))) + .gasLimit(positiveLong()) + .to(to) + .value(Wei.wrap(bytes32())) + .payload(payload) + .accessList(accessList()) + .chainId(BigInteger.ONE) + .signAndBuild(generateKeyPair()); + } + + private List accessList() { + final List
accessedAddresses = + Stream.generate(this::address).limit(1L + random.nextInt(3)).toList(); + final List accessedStorage = new ArrayList<>(); + for (int i = 0; i < accessedAddresses.size(); ++i) { + accessedStorage.add( + new AccessListEntry( + accessedAddresses.get(i), Stream.generate(this::bytes32).limit(2L * i).toList())); + } + return accessedStorage; + } + + private Transaction eip1559Transaction(final Bytes payload, final Address to) { + return Transaction.builder() + .type(TransactionType.EIP1559) + .nonce(random.nextLong()) + .maxPriorityFeePerGas(Wei.wrap(bytesValue(4))) + .maxFeePerGas(Wei.wrap(bytesValue(4))) + .gasLimit(positiveLong()) + .to(to) + .value(Wei.of(positiveLong())) + .payload(payload) + .chainId(BigInteger.ONE) + .signAndBuild(generateKeyPair()); + } + + private Transaction blobTransaction(final Bytes payload, final Address to) { + return Transaction.builder() + .type(TransactionType.BLOB) + .nonce(random.nextLong()) + .maxPriorityFeePerGas(Wei.wrap(bytesValue(4))) + .maxFeePerGas(Wei.wrap(bytesValue(4))) + .gasLimit(positiveLong()) + .to(to) + .value(Wei.of(positiveLong())) + .payload(payload) + .chainId(BigInteger.ONE) + .maxFeePerBlobGas(Wei.of(1)) + .versionedHashes(List.of(VersionedHash.DEFAULT_VERSIONED_HASH)) + .signAndBuild(generateKeyPair()); + } + + private Transaction frontierTransaction(final Bytes payload, final Address to) { + return Transaction.builder() + .type(TransactionType.FRONTIER) + .nonce(random.nextLong()) + .gasPrice(Wei.wrap(bytesValue(4))) + .gasLimit(positiveLong()) + .to(to) + .value(Wei.wrap(bytes32())) + .payload(payload) + .chainId(BigInteger.ONE) + .signAndBuild(generateKeyPair()); + } + + public Set transactions(final int n, final TransactionType... transactionTypes) { + return Stream.generate(() -> transaction(transactionTypes)) + .parallel() + .limit(n) + .collect(toUnmodifiableSet()); + } + + public Set transactions(final int n) { + return transactions( + n, TransactionType.FRONTIER, TransactionType.ACCESS_LIST, TransactionType.EIP1559); + } + + public Set transactionsWithAllTypes() { + return transactionsWithAllTypes(0); + } + + public Set transactionsWithAllTypes(final int atLeast) { + checkArgument(atLeast >= 0); + final HashSet remainingTransactionTypes = + new HashSet<>( + Set.of(TransactionType.FRONTIER, TransactionType.ACCESS_LIST, TransactionType.EIP1559)); + final HashSet transactions = new HashSet<>(); + while (transactions.size() < atLeast || !remainingTransactionTypes.isEmpty()) { + final Transaction newTransaction = transaction(); + transactions.add(newTransaction); + remainingTransactionTypes.remove(newTransaction.getType()); + } + return transactions; + } + + public TransactionReceipt receipt(final long cumulativeGasUsed) { + return new TransactionReceipt( + transactionType(), + random.nextInt(2), + cumulativeGasUsed, + Arrays.asList(log(), log()), + Optional.empty()); + } + + public TransactionReceipt receipt(final Bytes revertReason) { + return new TransactionReceipt( + transactionType(), + random.nextInt(2), + positiveLong(), + Arrays.asList(log(), log()), + Optional.of(revertReason)); + } + + public TransactionReceipt receipt() { + return receipt(positiveLong()); + } + + public TransactionReceipt receipt(final List logs) { + return new TransactionReceipt( + transactionType(), random.nextInt(2), positiveLong(), logs, Optional.empty()); + } + + public UInt256 storageKey() { + return uint256(); + } + + public List receipts(final Block block) { + final long totalGas = block.getHeader().getGasUsed(); + final int receiptCount = block.getBody().getTransactions().size(); + + final List receipts = new ArrayList<>(receiptCount); + for (int i = 0; i < receiptCount; i++) { + receipts.add(receipt((totalGas * (i + 1)) / (receiptCount))); + } + + return receipts; + } + + public List logs(final int logsCount, final int topicsPerLog) { + return Stream.generate(() -> log(topicsPerLog)).limit(logsCount).toList(); + } + + public Log log() { + return log(0); + } + + public Log log(final int topicCount) { + final List topics = Stream.generate(this::logTopic).limit(topicCount).toList(); + return new Log(address(), bytesValue(5, 15), topics); + } + + private LogTopic logTopic() { + return LogTopic.wrap(bytes32()); + } + + public Bytes32 bytes32() { + return Bytes32.wrap(bytes(Bytes32.SIZE)); + } + + public Bytes bytesValue(final int size) { + return Bytes.wrap(bytes(size)); + } + + public Bytes bytesValue() { + return bytesValue(1, 20); + } + + public Bytes bytesValue(final int minSize, final int maxSize) { + checkArgument(minSize >= 0); + checkArgument(maxSize >= 0); + checkArgument(maxSize > minSize); + final int size = random.nextInt(maxSize - minSize) + minSize; + return Bytes.wrap(bytes(size)); + } + + /** + * Creates a UInt256 with a value that fits within maxByteSize + * + * @param maxByteSize The byte size to cap this value to + * @return the maximum size + */ + private UInt256 uint256(final int maxByteSize) { + checkArgument(maxByteSize <= 32); + return UInt256.fromBytes(Bytes32.wrap(bytes(32, 32 - maxByteSize))); + } + + private UInt256 uint256() { + return UInt256.fromBytes(bytes32()); + } + + private long positiveLong() { + final long l = random.nextLong(); + return l < 0 ? Math.abs(l + 1) : l; + } + + public Hash hash() { + return Hash.wrap(bytes32()); + } + + public Address address() { + return Address.wrap(bytesValue(Address.SIZE)); + } + + public LogsBloomFilter logsBloom() { + return new LogsBloomFilter(Bytes.of(bytes(LogsBloomFilter.BYTE_SIZE))); + } + + private byte[] bytes(final int size) { + return bytes(size, 0); + } + + /** + * Creates a byte sequence with leading zeros. + * + * @param size The size of the byte array to return + * @param zerofill The number of lower-order bytes to fill with zero (creating a smaller big + * endian integer value) + * @return the array of bytes. + */ + private byte[] bytes(final int size, final int zerofill) { + final byte[] bytes = new byte[size]; + random.nextBytes(bytes); + Arrays.fill(bytes, 0, zerofill, (byte) 0x0); + return bytes; + } + + public KeyPair generateKeyPair() { + final java.security.KeyPair rawKeyPair = keyPairGenerator.generateKeyPair(); + final BCECPrivateKey privateKey = (BCECPrivateKey) rawKeyPair.getPrivate(); + final BCECPublicKey publicKey = (BCECPublicKey) rawKeyPair.getPublic(); + + final BigInteger privateKeyValue = privateKey.getD(); + + // Ethereum does not use encoded public keys like bitcoin - see + // https://en.bitcoin.it/wiki/Elliptic_Curve_Digital_Signature_Algorithm for details + // Additionally, as the first bit is a constant prefix (0x04) we ignore this value + final byte[] publicKeyBytes = publicKey.getQ().getEncoded(false); + final BigInteger publicKeyValue = + new BigInteger(1, Arrays.copyOfRange(publicKeyBytes, 1, publicKeyBytes.length)); + + return new KeyPair( + signatureAlgorithm.createPrivateKey(privateKeyValue), + signatureAlgorithm.createPublicKey(publicKeyValue)); + } + + /** + * Generates a random BlockAccessList with a specified number of account changes. + * + * @param accountCount the number of accounts to include + * @return a BlockAccessList with random data + */ + public BlockAccessList blockAccessList(final int accountCount) { + final List accountChanges = new ArrayList<>(accountCount); + + for (int i = 0; i < accountCount; i++) { + accountChanges.add(accountChanges()); + } + + return new BlockAccessList(accountChanges); + } + + /** + * Generates a random BlockAccessList with 1-5 account changes. + * + * @return a BlockAccessList with random data + */ + public BlockAccessList blockAccessList() { + return blockAccessList(1 + random.nextInt(5)); + } + + /** + * Generates a single AccountChanges with random data. + * + * @return an AccountChanges with random storage changes, reads, balance, nonce, and code changes + */ + private BlockAccessList.AccountChanges accountChanges() { + final Address address = address(); + final int storageChangeCount = random.nextInt(4); + final int storageReadCount = random.nextInt(3); + final int balanceChangeCount = random.nextInt(3) + 1; + final int nonceChangeCount = random.nextInt(3) + 1; + final int codeChangeCount = random.nextInt(2); + + final List storageChanges = new ArrayList<>(); + for (int i = 0; i < storageChangeCount; i++) { + storageChanges.add(slotChanges(i)); + } + + final List storageReads = new ArrayList<>(); + for (int i = 0; i < storageReadCount; i++) { + storageReads.add(new BlockAccessList.SlotRead(storageSlotKey())); + } + + final List balanceChanges = new ArrayList<>(); + for (int i = 0; i < balanceChangeCount; i++) { + balanceChanges.add(new BlockAccessList.BalanceChange(i, Wei.of(positiveLong()))); + } + + final List nonceChanges = new ArrayList<>(); + for (int i = 0; i < nonceChangeCount; i++) { + nonceChanges.add(new BlockAccessList.NonceChange(i, positiveLong())); + } + + final List codeChanges = new ArrayList<>(); + for (int i = 0; i < codeChangeCount; i++) { + codeChanges.add(new BlockAccessList.CodeChange(i, bytesValue(10, 100))); + } + + return new BlockAccessList.AccountChanges( + address, storageChanges, storageReads, balanceChanges, nonceChanges, codeChanges); + } + + /** + * Generates SlotChanges with random storage modifications. + * + * @param txIndex the transaction index for these changes + * @return SlotChanges with random data + */ + private BlockAccessList.SlotChanges slotChanges(final int txIndex) { + final StorageSlotKey slot = storageSlotKey(); + final int changeCount = 1 + random.nextInt(3); + final List changes = new ArrayList<>(); + + for (int i = 0; i < changeCount; i++) { + changes.add(new BlockAccessList.StorageChange(txIndex + i, uint256())); + } + + return new BlockAccessList.SlotChanges(slot, changes); + } + + /** + * Generates a random StorageSlotKey. + * + * @return a StorageSlotKey with a random UInt256 value + */ + private StorageSlotKey storageSlotKey() { + return new StorageSlotKey(uint256()); + } + + /** + * Generates an empty BlockAccessList. + * + * @return an empty BlockAccessList + */ + public BlockAccessList emptyBlockAccessList() { + return new BlockAccessList(Collections.emptyList()); + } + + public static class BlockOptions { + private OptionalLong blockNumber = OptionalLong.empty(); + private Optional parentHash = Optional.empty(); + private Optional stateRoot = Optional.empty(); + private Optional difficulty = Optional.empty(); + private final List transactions = new ArrayList<>(); + private final List ommers = new ArrayList<>(); + + private Optional>> withdrawals = Optional.empty(); + private Optional>> requests = Optional.empty(); + private Optional extraData = Optional.empty(); + private Optional blockHeaderFunctions = Optional.empty(); + private Optional receiptsRoot = Optional.empty(); + private Optional gasUsed = Optional.empty(); + private Optional logsBloom = Optional.empty(); + private Optional timestamp = Optional.empty(); + private boolean hasOmmers = true; + private boolean hasTransactions = true; + private int transactionCount = 2; + private TransactionType[] transactionTypes = { + TransactionType.FRONTIER, TransactionType.ACCESS_LIST, TransactionType.EIP1559 + }; + private Optional
coinbase = Optional.empty(); + private Optional> maybeBaseFee = Optional.empty(); + + private Optional withdrawalsRoot = Optional.empty(); + private Optional requestsHash = Optional.empty(); + private Optional balHash = Optional.empty(); + + private Optional> maybeMaxFeePerBlobGas = Optional.empty(); + + private Optional blockAccessList = Optional.empty(); + private boolean generateBlockAccessList = false; + + public static BlockOptions create() { + return new BlockOptions(); + } + + public List getTransactions(final List defaultValue) { + return transactions.isEmpty() ? defaultValue : transactions; + } + + public TransactionType[] getTransactionTypes() { + return transactionTypes; + } + + public List getOmmers(final List defaultValue) { + return ommers.isEmpty() ? defaultValue : ommers; + } + + public long getBlockNumber(final long defaultValue) { + return blockNumber.orElse(defaultValue); + } + + public Hash getParentHash(final Hash defaultValue) { + return parentHash.orElse(defaultValue); + } + + public Hash getStateRoot(final Hash defaultValue) { + return stateRoot.orElse(defaultValue); + } + + public Difficulty getDifficulty(final Difficulty defaultValue) { + return difficulty.orElse(defaultValue); + } + + public Bytes getExtraData(final Bytes32 defaultValue) { + return extraData.orElse(defaultValue); + } + + public BlockHeaderFunctions getBlockHeaderFunctions(final BlockHeaderFunctions defaultValue) { + return blockHeaderFunctions.orElse(defaultValue); + } + + public Hash getReceiptsRoot(final Hash defaultValue) { + return receiptsRoot.orElse(defaultValue); + } + + public long getGasUsed(final long defaultValue) { + return gasUsed.orElse(defaultValue); + } + + public LogsBloomFilter getLogsBloom(final LogsBloomFilter defaultValue) { + return logsBloom.orElse(defaultValue); + } + + public Optional getTimestamp() { + return timestamp; + } + + public Hash getWithdrawalsRoot(final Hash defaultValue) { + return withdrawalsRoot.orElse(defaultValue); + } + + public Optional> getWithdrawals( + final Optional> defaultValue) { + return withdrawals.orElse(defaultValue); + } + + public Hash getRequestsHash(final Hash defaultValue) { + return requestsHash.orElse(defaultValue); + } + + public Hash getBalHash(final Hash defaultValue) { + return balHash.orElse(defaultValue); + } + + public Optional> getRequests(final Optional> defaultValue) { + return requests.orElse(defaultValue); + } + + public boolean hasTransactions() { + return hasTransactions; + } + + public int transactionCount() { + return transactionCount; + } + + public boolean hasOmmers() { + return hasOmmers; + } + + public Optional getBlockAccessList() { + return blockAccessList; + } + + public boolean shouldGenerateBlockAccessList() { + return generateBlockAccessList; + } + + public BlockOptions addTransaction(final Transaction... tx) { + transactions.addAll(Arrays.asList(tx)); + return this; + } + + public BlockOptions addOmmers(final BlockHeader... headers) { + ommers.addAll(Arrays.asList(headers)); + return this; + } + + public BlockOptions addTransaction(final Collection txs) { + return addTransaction(txs.toArray(new Transaction[] {})); + } + + public BlockOptions setWithdrawals(final Optional> withdrawals) { + this.withdrawals = Optional.of(withdrawals); + if (withdrawals.isPresent()) { + final List withdrawalList = withdrawals.get(); + final ArrayList bytesList = new ArrayList<>(withdrawalList.size()); + withdrawalList.forEach( + w -> { + final BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput(); + w.writeTo(rlpOutput); + bytesList.add(rlpOutput.encoded()); + }); + this.withdrawalsRoot = Optional.of(Util.getRootFromListOfBytes(bytesList)); + } else { + this.withdrawalsRoot = Optional.empty(); + } + return this; + } + + public BlockOptions setRequests(final Optional> requests) { + this.requests = Optional.of(requests); + return this; + } + + public BlockOptions setBlockNumber(final long blockNumber) { + this.blockNumber = OptionalLong.of(blockNumber); + return this; + } + + public BlockOptions setParentHash(final Hash parentHash) { + this.parentHash = Optional.of(parentHash); + return this; + } + + public BlockOptions setStateRoot(final Hash stateRoot) { + this.stateRoot = Optional.of(stateRoot); + return this; + } + + public BlockOptions setDifficulty(final Difficulty difficulty) { + this.difficulty = Optional.of(difficulty); + return this; + } + + public BlockOptions setExtraData(final Bytes extraData) { + this.extraData = Optional.of(extraData); + return this; + } + + public BlockOptions setBlockHeaderFunctions(final BlockHeaderFunctions blockHeaderFunctions) { + this.blockHeaderFunctions = Optional.of(blockHeaderFunctions); + return this; + } + + public BlockOptions setReceiptsRoot(final Hash receiptsRoot) { + this.receiptsRoot = Optional.of(receiptsRoot); + return this; + } + + public BlockOptions setGasUsed(final long gasUsed) { + this.gasUsed = Optional.of(gasUsed); + return this; + } + + public BlockOptions setLogsBloom(final LogsBloomFilter logsBloom) { + this.logsBloom = Optional.of(logsBloom); + return this; + } + + public BlockOptions hasTransactions(final boolean hasTransactions) { + this.hasTransactions = hasTransactions; + return this; + } + + public BlockOptions transactionCount(final int transactionCount) { + this.transactionCount = transactionCount; + return this; + } + + public BlockOptions transactionTypes(final TransactionType... transactionTypes) { + this.transactionTypes = transactionTypes; + return this; + } + + public BlockOptions hasOmmers(final boolean hasOmmers) { + this.hasOmmers = hasOmmers; + return this; + } + + public BlockOptions setTimestamp(final Long timestamp) { + this.timestamp = Optional.of(timestamp); + return this; + } + + public BlockOptions setCoinbase(final Address coinbase) { + this.coinbase = Optional.of(coinbase); + return this; + } + + public Address getCoinbase(final Address defaultValue) { + return coinbase.orElse(defaultValue); + } + + public Optional getBaseFee(final Optional defaultValue) { + return maybeBaseFee.orElse(defaultValue); + } + + public BlockOptions setBaseFee(final Optional baseFee) { + this.maybeBaseFee = Optional.of(baseFee); + return this; + } + + public BlockOptions setWithdrawalsRoot(final Hash withdrawalsRoot) { + this.withdrawalsRoot = Optional.of(withdrawalsRoot); + return this; + } + + public BlockOptions setRequestsHash(final Hash requestsHash) { + this.requestsHash = Optional.of(requestsHash); + return this; + } + + public BlockOptions setBalHash(final Hash balHash) { + this.balHash = Optional.of(balHash); + return this; + } + + public Optional getMaxFeePerBlobGas(final Optional defaultValue) { + return maybeMaxFeePerBlobGas.orElse(defaultValue); + } + + public BlockOptions setMaxFeePerBlobGas(final Optional maxFeePerBlobGas) { + this.maybeMaxFeePerBlobGas = Optional.of(maxFeePerBlobGas); + return this; + } + + public BlockOptions setBlockAccessList(final BlockAccessList blockAccessList) { + this.blockAccessList = Optional.of(blockAccessList); + this.generateBlockAccessList = false; + return this; + } + + public BlockOptions setEmptyBlockAccessList() { + this.blockAccessList = Optional.of(new BlockAccessList(Collections.emptyList())); + this.generateBlockAccessList = false; + return this; + } + + public BlockOptions clearBlockAccessList() { + this.blockAccessList = Optional.empty(); + this.generateBlockAccessList = false; + return this; + } + + /** + * Enables automatic BAL generation when creating the block. The balHash will be automatically + * calculated and set in the header. + * + * @return this BlockOptions for chaining + */ + public BlockOptions withGeneratedBlockAccessList() { + this.generateBlockAccessList = true; + return this; + } + + /** + * Enables automatic BAL generation with a specific number of accounts. + * + * @param accountCount the number of accounts to generate in the BAL + * @return this BlockOptions for chaining + */ + public BlockOptions withGeneratedBlockAccessList(final int accountCount) { + this.generateBlockAccessList = true; + return this; + } + + /** + * Disables automatic BAL generation. + * + * @return this BlockOptions for chaining + */ + public BlockOptions withoutGeneratedBlockAccessList() { + this.generateBlockAccessList = false; + return this; + } + } +} diff --git a/anchor/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java b/anchor/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java new file mode 100644 index 0000000..03a72ea --- /dev/null +++ b/anchor/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java @@ -0,0 +1,241 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.core; + +import org.hyperledger.besu.crypto.CodeDelegationSignature; +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.VersionedHash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.encoding.CodeDelegationTransactionEncoder; +import org.hyperledger.besu.ethereum.core.kzg.BlobsWithCommitments; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class TransactionTestFixture { + private static final SignatureAlgorithm SIGNATURE_ALGORITHM = + SignatureAlgorithmFactory.getInstance(); + private static final KeyPair KEY_PAIR = SIGNATURE_ALGORITHM.generateKeyPair(); + private static final org.hyperledger.besu.datatypes.CodeDelegation CODE_DELEGATION = + createSignedCodeDelegation(BigInteger.ZERO, Address.ZERO, 0, KEY_PAIR); + + private TransactionType transactionType = TransactionType.FRONTIER; + + private long nonce = 0; + + private Optional gasPrice = Optional.empty(); + + private long gasLimit = 5000; + + private Optional
to = Optional.empty(); + private Address sender = Address.fromHexString(String.format("%020x", 1)); + + private Wei value = Wei.of(4); + + private Bytes payload = Bytes.EMPTY; + + private Optional chainId = Optional.of(BigInteger.valueOf(1337)); + + private Optional maxPriorityFeePerGas = Optional.empty(); + private Optional maxFeePerGas = Optional.empty(); + private Optional maxFeePerBlobGas = Optional.empty(); + + private Optional> accessListEntries = Optional.empty(); + private Optional> versionedHashes = Optional.empty(); + + private Optional blobs = Optional.empty(); + private Optional v = Optional.empty(); + private Optional> codeDelegations = + Optional.empty(); + + public Transaction createTransaction(final KeyPair keys) { + final Transaction.Builder builder = Transaction.builder(); + builder + .type(transactionType) + .gasLimit(gasLimit) + .nonce(nonce) + .payload(payload) + .value(value) + .sender(sender); + + switch (transactionType) { + case FRONTIER: + builder.gasPrice(gasPrice.orElse(Wei.of(5000))); + break; + case ACCESS_LIST: + builder.gasPrice(gasPrice.orElse(Wei.of(5000))); + builder.accessList(accessListEntries.orElse(List.of())); + break; + case EIP1559: + builder.maxPriorityFeePerGas(maxPriorityFeePerGas.orElse(Wei.of(500))); + builder.maxFeePerGas(maxFeePerGas.orElse(Wei.of(5000))); + builder.accessList(accessListEntries.orElse(List.of())); + break; + case BLOB: + builder.maxPriorityFeePerGas(maxPriorityFeePerGas.orElse(Wei.of(500))); + builder.maxFeePerGas(maxFeePerGas.orElse(Wei.of(5000))); + builder.accessList(accessListEntries.orElse(List.of())); + builder.maxFeePerBlobGas(maxFeePerBlobGas.orElse(Wei.ONE)); + if (blobs.isPresent()) { + builder.kzgBlobs( + blobs.get().getBlobType(), + blobs.get().getKzgCommitments(), + blobs.get().getBlobs(), + blobs.get().getKzgProofs()); + } else versionedHashes.ifPresent(builder::versionedHashes); + break; + case DELEGATE_CODE: + builder.maxPriorityFeePerGas(maxPriorityFeePerGas.orElse(Wei.of(500))); + builder.maxFeePerGas(maxFeePerGas.orElse(Wei.of(5000))); + builder.accessList(accessListEntries.orElse(List.of())); + builder.codeDelegations(codeDelegations.orElse(List.of(CODE_DELEGATION))); + break; + case AERE_PQ: + throw new IllegalArgumentException( + "AIP-20 transactions need ML-DSA keys; this fixture does not create them"); + } + + to.ifPresent(builder::to); + chainId.ifPresent(builder::chainId); + v.ifPresent(builder::v); + + return builder.signAndBuild(keys); + } + + public TransactionTestFixture type(final TransactionType transactionType) { + this.transactionType = transactionType; + return this; + } + + public TransactionTestFixture nonce(final long nonce) { + this.nonce = nonce; + return this; + } + + public TransactionTestFixture gasPrice(final Wei gasPrice) { + this.gasPrice = Optional.ofNullable(gasPrice); + return this; + } + + public TransactionTestFixture gasLimit(final long gasLimit) { + this.gasLimit = gasLimit; + return this; + } + + public TransactionTestFixture to(final Optional
to) { + this.to = to; + return this; + } + + public TransactionTestFixture sender(final Address sender) { + this.sender = sender; + return this; + } + + public TransactionTestFixture value(final Wei value) { + this.value = value; + return this; + } + + public TransactionTestFixture payload(final Bytes payload) { + this.payload = payload; + return this; + } + + public TransactionTestFixture chainId(final Optional chainId) { + this.chainId = chainId; + return this; + } + + public TransactionTestFixture maxPriorityFeePerGas(final Optional maxPriorityFeePerGas) { + this.maxPriorityFeePerGas = maxPriorityFeePerGas; + return this; + } + + public TransactionTestFixture maxFeePerGas(final Optional maxFeePerGas) { + this.maxFeePerGas = maxFeePerGas; + return this; + } + + public TransactionTestFixture maxFeePerBlobGas(final Optional maxFeePerBlobGas) { + this.maxFeePerBlobGas = maxFeePerBlobGas; + return this; + } + + public TransactionTestFixture accessList(final List accessListEntries) { + this.accessListEntries = Optional.ofNullable(accessListEntries); + return this; + } + + public TransactionTestFixture versionedHashes( + final Optional> versionedHashes) { + this.versionedHashes = versionedHashes; + return this; + } + + public TransactionTestFixture v(final Optional v) { + this.v = v; + return this; + } + + public TransactionTestFixture blobsWithCommitments(final Optional blobs) { + this.blobs = blobs; + return this; + } + + public TransactionTestFixture codeDelegations( + final List codeDelegations) { + this.codeDelegations = Optional.ofNullable(codeDelegations); + return this; + } + + public static CodeDelegation createSignedCodeDelegation( + final BigInteger chainId, final Address address, final long nonce, final KeyPair keys) { + BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput(); + CodeDelegationTransactionEncoder.encodeSingleCodeDelegationWithoutSignature( + new org.hyperledger.besu.ethereum.core.CodeDelegation(chainId, address, nonce, null), + rlpOutput); + + final Hash hash = + Hash.hash( + Bytes.concatenate( + org.hyperledger.besu.ethereum.core.CodeDelegation.MAGIC, rlpOutput.encoded())); + + final var signature = SIGNATURE_ALGORITHM.sign(Bytes32.wrap(hash.getBytes()), keys); + + return new org.hyperledger.besu.ethereum.core.CodeDelegation( + chainId, + address, + nonce, + CodeDelegationSignature.create(signature.getR(), signature.getS(), signature.getRecId())); + } +} diff --git a/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/AerePqTransactionTest.java b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/AerePqTransactionTest.java new file mode 100644 index 0000000..6f6da53 --- /dev/null +++ b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/AerePqTransactionTest.java @@ -0,0 +1,210 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.hyperledger.besu.datatypes.AerePqAlgorithm; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.BlobType; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.core.encoding.EncodingContext; +import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder; +import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder; +import org.hyperledger.besu.ethereum.mainnet.AerePqTransactionSupport; +import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionValidator; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; +import org.hyperledger.besu.ethereum.mainnet.ValidationResult; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; +import org.hyperledger.besu.evm.gascalculator.OsakaGasCalculator; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.Test; + +/** + * AIP-20 conformance on the Besu-derived client, against the vectors produced by the reference tooling + * (`tools/pqtx/pqtx.mjs vectors`, @noble/post-quantum): the same bytes must decode to the same sender and + * hash, verify under Bouncy Castle's ML-DSA, be charged the AIP-20 gas, and every negative vector must be + * refused for the reason the vector names. Two independent ML-DSA implementations agreeing on the same + * signature is the cross-client property the AIP requires. + */ +class AerePqTransactionTest { + + private static final BigInteger CHAIN_ID = BigInteger.valueOf(28001); + + private static String vector(final String id) throws IOException { + try (InputStream in = + AerePqTransactionTest.class.getResourceAsStream("/aere/pqtx/" + id + ".json")) { + assertThat(in).as("vector " + id + " on the test classpath").isNotNull(); + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static String field(final String json, final String name) { + final Matcher m = Pattern.compile("\"" + name + "\"\\s*:\\s*\"([^\"]*)\"").matcher(json); + assertThat(m.find()).as("field " + name).isTrue(); + return m.group(1); + } + + private static Transaction decode(final String rawHex) { + return TransactionDecoder.decodeOpaqueBytes( + Bytes.fromHexString(rawHex), EncodingContext.POOLED_TRANSACTION); + } + + private static MainnetTransactionValidator validator() { + return new MainnetTransactionValidator( + new OsakaGasCalculator(), + GasLimitCalculator.constant(), + FeeMarket.london(0L), + true, + Optional.of(CHAIN_ID), + Set.of(TransactionType.EIP1559, TransactionType.AERE_PQ), + Set.of(BlobType.KZG_CELL_PROOFS), + Integer.MAX_VALUE); + } + + @Test + void validVectorsDecodeToTheReferenceSenderAndHashAndRoundTrip() throws IOException { + for (final int alg : new int[] {1, 2, 3}) { + final String v = vector("valid-ml-dsa-" + alg); + final Transaction tx = decode(field(v, "raw")); + assertThat(tx.getType()).isEqualTo(TransactionType.AERE_PQ); + assertThat(tx.getPqAuthorization()).isPresent(); + assertThat(tx.getPqAuthorization().get().algorithm()) + .isEqualTo(AerePqAlgorithm.byId(alg).orElseThrow()); + assertThat(tx.getSignature()).as("no secp256k1 signature on a type 0x50 tx").isNull(); + assertThat(tx.getSender()).isEqualTo(Address.fromHexString(field(v, "sender"))); + assertThat(tx.getHash()).isEqualTo(Hash.fromHexString(field(v, "hash"))); + assertThat(tx.getPqSigningHash()).isEqualTo(Bytes.fromHexString(field(v, "signingHash"))); + assertThat(tx.getChainId()).contains(CHAIN_ID); + assertThat(tx.getR()).isNull(); + assertThat(tx.getYParity()).isNull(); + final Bytes reEncoded = TransactionEncoder.encodeOpaqueBytes(tx, EncodingContext.BLOCK_BODY); + assertThat(reEncoded).as("round trip, alg " + alg).isEqualTo(Bytes.fromHexString(field(v, "raw"))); + // a detached copy keeps everything that identifies the transaction + final Transaction copy = tx.detachedCopy(); + assertThat(copy.getHash()).isEqualTo(tx.getHash()); + assertThat(copy.getSender()).isEqualTo(tx.getSender()); + assertThat(copy).isEqualTo(tx); + } + } + + @Test + void validVectorsVerifyUnderBouncyCastleAndPassTheValidator() throws IOException { + for (final int alg : new int[] {1, 2, 3}) { + final Transaction tx = decode(field(vector("valid-ml-dsa-" + alg), "raw")); + final ValidationResult auth = + AerePqTransactionSupport.validateAuthorization(tx); + assertThat(auth.isValid()) + .withFailMessage(() -> "ML-DSA verification, alg " + alg + ": " + auth.getErrorMessage()) + .isTrue(); + final ValidationResult full = + validator() + .validate( + tx, Optional.of(Wei.of(1_000_000_000L)), Optional.of(Wei.ZERO), TransactionValidationParams.processingBlock()); + assertThat(full.isValid()) + .withFailMessage(() -> "validator, alg " + alg + ": " + full.getErrorMessage()) + .isTrue(); + } + } + + @Test + void aip20GasIsChargedForTheAuthorization() throws IOException { + final Transaction tx = decode(field(vector("valid-ml-dsa-2"), "raw")); + final long intrinsic = new OsakaGasCalculator().transactionIntrinsicGasCost(tx, 0L); + // 21,000 + 16 * (1,952 + 3,309) + 75,000 = 180,176 (AIP-20 worked example, empty calldata) + assertThat(intrinsic).isEqualTo(180_176L); + final Transaction tx87 = decode(field(vector("valid-ml-dsa-3"), "raw")); + assertThat(new OsakaGasCalculator().transactionIntrinsicGasCost(tx87, 0L)) + .isEqualTo(21_000L + 16L * (2_592L + 4_627L) + 100_000L); + } + + @Test + void aCorruptedSignatureIsInvalidSignature() throws IOException { + for (final int alg : new int[] {1, 2, 3}) { + final Transaction tx = decode(field(vector("bad-signature-ml-dsa-" + alg), "raw")); + final ValidationResult r = + AerePqTransactionSupport.validateAuthorization(tx); + assertThat(r.isValid()).isFalse(); + assertThat(r.getInvalidReason()).isEqualTo(TransactionInvalidReason.INVALID_SIGNATURE); + } + } + + @Test + void theWrongChainIdIsRefusedByTheValidator() throws IOException { + final Transaction tx = decode(field(vector("wrong-chain-ml-dsa-2"), "raw")); + final ValidationResult r = + validator() + .validate(tx, Optional.of(Wei.of(1_000_000_000L)), Optional.of(Wei.ZERO), TransactionValidationParams.processingBlock()); + assertThat(r.isValid()).isFalse(); + assertThat(r.getInvalidReason()).isEqualTo(TransactionInvalidReason.WRONG_CHAIN_ID); + } + + @Test + void aShortSignatureAndAnUnknownAlgorithmAreRefusedBeforeAnyCryptography() throws IOException { + for (final int alg : new int[] {1, 2, 3}) { + final String raw = field(vector("short-signature-ml-dsa-" + alg), "raw"); + assertThatThrownBy(() -> decode(raw)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("byte signature"); + } + final String unknown = field(vector("unknown-alg-0x10"), "raw"); + assertThatThrownBy(() -> decode(unknown)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown post-quantum algorithm identifier 16"); + } + + @Test + void theTypeIsNeverActiveWithoutTheForkBlockProperty() { + // the test JVM does not set aere.pqtx.forkBlock, so the rule is "never": the same as a binary without it + assertThat(AerePqTransactionSupport.forkBlock()).isEqualTo(Long.MAX_VALUE); + assertThat(AerePqTransactionSupport.isActiveAt(Long.MAX_VALUE - 1)).isFalse(); + assertThat(AerePqTransactionSupport.inactiveMessage(1)).contains("not enabled"); + } + + @Test + void aClassicalTransactionCarriesNoAuthorizationAndStillWorks() { + final Transaction tx = + Transaction.builder() + .type(TransactionType.EIP1559) + .chainId(CHAIN_ID) + .nonce(0) + .maxPriorityFeePerGas(Wei.ONE) + .maxFeePerGas(Wei.ONE) + .gasLimit(21_000) + .to(Address.ZERO) + .value(Wei.ZERO) + .payload(Bytes.EMPTY) + .signAndBuild( + org.hyperledger.besu.crypto.SignatureAlgorithmFactory.getInstance().generateKeyPair()); + assertThat(tx.getPqAuthorization()).isEmpty(); + assertThat(tx.getR()).isNotNull(); + assertThat(new OsakaGasCalculator().transactionIntrinsicGasCost(tx, 0L)).isEqualTo(21_000L); + } +} diff --git a/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD432GasCapUnsignedTest.java b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD432GasCapUnsignedTest.java new file mode 100644 index 0000000..7dcf917 --- /dev/null +++ b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD432GasCapUnsignedTest.java @@ -0,0 +1,103 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Aere Network notice (Apache-2.0 section 4(b)): this file was added by Aere Network, 2026-09-28 (D-432). + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.BlobType; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; +import org.hyperledger.besu.evm.gascalculator.OsakaGasCalculator; + +import java.math.BigInteger; +import java.util.Optional; +import java.util.Set; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.Test; + +/** + * AERE D-432: the EIP-7825 transaction gas cap is compared UNSIGNED. The decoder reads gasLimit with readLongScalar, so a limit + * of 2^63 or more is a negative long here; a signed compare let it through the cap, while the second client refuses it at + * decoding. Found by the AIP-20 differential fuzzer (aips/aip20-conformitate/fuzz-pqtx). + */ +class AereD432GasCapUnsignedTest { + + private static final long CAP = 1L << 24; + private static final BigInteger CHAIN = BigInteger.valueOf(28001); + private static final KeyPair KEYS = SignatureAlgorithmFactory.getInstance().generateKeyPair(); + + private static MainnetTransactionValidator validator() { + final GasLimitCalculator glc = + new GasLimitCalculator() { + @Override + public long nextGasLimit(final long currentGasLimit, final long targetGasLimit, final long newBlockNumber) { + return currentGasLimit; + } + + @Override + public long transactionGasLimitCap() { + return CAP; + } + }; + return new MainnetTransactionValidator( + new OsakaGasCalculator(), glc, FeeMarket.london(0L), true, Optional.of(CHAIN), Set.of(TransactionType.EIP1559), + Set.of(BlobType.KZG_CELL_PROOFS), 2 * 24576); + } + + private static ValidationResult judge(final long gasLimit) { + final Transaction tx = + Transaction.builder() + .type(TransactionType.EIP1559) + .chainId(CHAIN) + .nonce(0) + .maxPriorityFeePerGas(Wei.ONE) + .maxFeePerGas(Wei.of(2_000_000_000L)) + .gasLimit(gasLimit) + .to(Address.ZERO) + .value(Wei.ZERO) + .payload(Bytes.EMPTY) + .signAndBuild(KEYS); + return validator().validate(tx, Optional.of(Wei.of(1_000_000_000L)), Optional.empty(), TransactionValidationParams.processingBlock()); + } + + @Test + void aGasLimitOf2To63OrMoreIsRefusedByTheCapNotLetThroughAsANegativeLong() { + // 2^63, 2^63 + 5 and 2^64 - 1, as readLongScalar hands them over + for (final long g : new long[] {Long.MIN_VALUE, Long.MIN_VALUE + 5, -1L}) { + final ValidationResult r = judge(g); + assertThat(r.isValid()).withFailMessage("gasLimit %s admitted", Long.toUnsignedString(g)).isFalse(); + assertThat(r.getInvalidReason()) + .withFailMessage("gasLimit %s refused for %s, not the cap", Long.toUnsignedString(g), r.getInvalidReason()) + .isEqualTo(TransactionInvalidReason.EXCEEDS_TRANSACTION_GAS_LIMIT); + } + } + + @Test + void theCapItselfIsAdmittedAndOneAboveIsRefused() { + // positive control: the rule is armed at exactly 2^24 and the fixture is otherwise valid + assertThat(judge(CAP).isValid()).withFailMessage(() -> "gasLimit = cap refused: " + judge(CAP).getErrorMessage()).isTrue(); + assertThat(judge(CAP + 1).getInvalidReason()).isEqualTo(TransactionInvalidReason.EXCEEDS_TRANSACTION_GAS_LIMIT); + } +} diff --git a/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD433BftWithdrawalsTest.java b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD433BftWithdrawalsTest.java new file mode 100644 index 0000000..6c935b7 --- /dev/null +++ b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/AereD433BftWithdrawalsTest.java @@ -0,0 +1,75 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Aere Network notice (Apache-2.0 section 4(b)): this file was added by Aere Network, 2026-09-28 (D-433), as the regression + * test of the ported upstream fixes hyperledger/besu ffe8104ee2 (GHSA-p4h2-gvh4-pv6j) and b7f1387cb4. + */ +package org.hyperledger.besu.ethereum.mainnet; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.GWei; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.Withdrawal; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.units.bigints.UInt64; +import org.junit.jupiter.api.Test; + +/** + * AERE D-433: on a BFT chain (QBFT on 2800 and 28001, Shanghai active from genesis) withdrawals are not applicable, and the + * BFT schedule installs NotApplicableWithdrawals while keeping Shanghai's WithdrawalsProcessor. Before the port the validator + * returned true for anything, so a single Byzantine proposer could put withdrawals in a block and every honest Besu node would + * credit them as native balance. The port (upstream ffe8104ee2) refuses a non-empty list and a withdrawals root other than the + * empty trie root. + */ +class AereD433BftWithdrawalsTest { + + private static final WithdrawalsValidator V = new WithdrawalsValidator.NotApplicableWithdrawals(); + + @Test + void aByzantineWithdrawalListAndItsRootAreRefused() { + final List injected = + List.of(new Withdrawal(UInt64.valueOf(1), UInt64.valueOf(2), Address.fromHexString("0x000000000000000000000000000000000000c0de"), GWei.of(11))); + assertThat(V.validateWithdrawals(Optional.of(injected))).withFailMessage("a non-empty withdrawal list was accepted").isFalse(); + final BlockDataGenerator gen = new BlockDataGenerator(); + final Block withRoot = + gen.block(BlockDataGenerator.BlockOptions.create().setWithdrawals(Optional.of(injected)).setWithdrawalsRoot(Hash.ZERO)); + assertThat(V.validateWithdrawalsRoot(withRoot)).withFailMessage("a withdrawals root other than the empty trie was accepted").isFalse(); + // upstream b7f1387cb4: a root in the header with NO withdrawals field in the body is malformed, even when the root is the empty one + final Block noBody = gen.block(BlockDataGenerator.BlockOptions.create().setWithdrawals(Optional.empty()).setWithdrawalsRoot(Hash.EMPTY_TRIE_HASH)); + assertThat(V.validateWithdrawalsRoot(noBody)).withFailMessage("a header root without body withdrawals was accepted").isFalse(); + } + + @Test + void theEmptyFormsAreStillAccepted() { + // positive control: the blocks the chain actually carries (empty list, empty-trie root) and pre-Shanghai (absent) + assertThat(V.validateWithdrawals(Optional.of(Collections.emptyList()))).isTrue(); + assertThat(V.validateWithdrawals(Optional.empty())).isTrue(); + final BlockDataGenerator gen = new BlockDataGenerator(); + assertThat( + V.validateWithdrawalsRoot( + gen.block( + BlockDataGenerator.BlockOptions.create() + .setWithdrawals(Optional.of(Collections.emptyList())) + .setWithdrawalsRoot(Hash.EMPTY_TRIE_HASH)))) + .isTrue(); + } +} diff --git a/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarketAereFloorTest.java b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarketAereFloorTest.java new file mode 100644 index 0000000..b36bfd7 --- /dev/null +++ b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/feemarket/LondonFeeMarketAereFloorTest.java @@ -0,0 +1,129 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.feemarket; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.feemarket.TransactionPriceCalculator; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +/** + * Tests for the AERE fork-gated base-fee floor in {@link LondonFeeMarket}. + * + *

Proves: (a) fork UNSET is byte-behaviour-identical to upstream London (empty blocks decay to / + * stay at the legacy 7-wei floor, no 1-Gwei clamp); (b) once the fork block N is set with a 1-Gwei + * floor, empty blocks at/after N floor at EXACTLY 1 Gwei and never below; (c) the jump to >= 1 Gwei + * happens exactly at block N (block N-1 is still legacy); (d) an over-target block still rises above + * 1 Gwei via the normal EIP-1559 formula. + */ +public class LondonFeeMarketAereFloorTest { + + private static final long ONE_GWEI = 1_000_000_000L; + private static final long GAS_TARGET = 15_000_000L; + private static final long FORK_BLOCK = 100L; + + /** London fee market with an explicit AERE floor-fork configuration (bypasses global JVM config). */ + private static LondonFeeMarket market(final long aereForkBlock, final long aereFloorValueWei) { + return new LondonFeeMarket( + TransactionPriceCalculator.eip1559(), + 0L, // londonForkBlockNumber (kept off the tested block range) + Optional.empty(), + aereForkBlock, + aereFloorValueWei); + } + + // (a) UNSET fork => byte-identical to upstream: empty blocks decay to and stay at 7 wei, and no + // value is ever clamped up to 1 Gwei. + @Test + public void forkUnset_emptyBlocksDecayToSevenWei_noClampToOneGwei() { + final LondonFeeMarket unset = market(Long.MAX_VALUE, ONE_GWEI); + + // Decay many empty blocks from well above the floor: must settle at EXACTLY 7 wei. + Wei baseFee = Wei.of(10_000L); + for (int i = 0; i < 400; i++) { + baseFee = unset.computeBaseFee(500L + i, baseFee, 0L, GAS_TARGET); + } + assertThat(baseFee).isEqualTo(Wei.of(7L)); + + // Once at 7 wei it stays at 7 wei (never below), never jumps to 1 Gwei. + assertThat(unset.computeBaseFee(9_000L, Wei.of(7L), 0L, GAS_TARGET)).isEqualTo(Wei.of(7L)); + + // A mid-range empty-block value is NOT clamped up (proves the fork is a genuine no-op): + // 1000 - 1000/8 = 1000 - 125 = 875 wei. + assertThat(unset.computeBaseFee(9_000L, Wei.of(1_000L), 0L, GAS_TARGET)) + .isEqualTo(Wei.of(875L)); + } + + // (b) Fork ACTIVE => empty blocks at/after N floor at EXACTLY 1 Gwei and never drop below. + @Test + public void forkActive_emptyBlocksFloorAtExactlyOneGwei_neverBelow() { + final LondonFeeMarket forked = market(FORK_BLOCK, ONE_GWEI); + + // Sanity: raw EIP-1559 decrease from 1 Gwei on an empty block would be 875M; clamp pins it. + assertThat(forked.computeBaseFee(FORK_BLOCK + 1, Wei.of(ONE_GWEI), 0L, GAS_TARGET)) + .isEqualTo(Wei.of(ONE_GWEI)); + + // Many consecutive empty blocks after the fork: stays EXACTLY at 1 Gwei each block. + Wei baseFee = Wei.of(ONE_GWEI); + for (int i = 0; i < 50; i++) { + baseFee = forked.computeBaseFee(FORK_BLOCK + 1 + i, baseFee, 0L, GAS_TARGET); + assertThat(baseFee).isEqualTo(Wei.of(ONE_GWEI)); + } + + // Even a pathologically low parent baseFee (7 wei) is clamped up to exactly 1 Gwei post-fork. + assertThat(forked.computeBaseFee(FORK_BLOCK + 500, Wei.of(7L), 0L, GAS_TARGET)) + .isEqualTo(Wei.of(ONE_GWEI)); + } + + // (c) The jump to >= 1 Gwei happens exactly at the fork block N (N-1 is still legacy). + @Test + public void forkActivation_jumpHappensExactlyAtForkBlock() { + final LondonFeeMarket forked = market(FORK_BLOCK, ONE_GWEI); + + // Block N-1 is pre-fork: legacy behaviour, a 7-wei empty-block parent stays at 7 wei. + assertThat(forked.computeBaseFee(FORK_BLOCK - 1, Wei.of(7L), 0L, GAS_TARGET)) + .isEqualTo(Wei.of(7L)); + + // Block N itself jumps to >= 1 Gwei even though the parent baseFee was only 7 wei. + final Wei atForkBlock = forked.computeBaseFee(FORK_BLOCK, Wei.of(7L), 0L, GAS_TARGET); + assertThat(atForkBlock).isEqualTo(Wei.of(ONE_GWEI)); + assertThat(atForkBlock.greaterOrEqualThan(Wei.of(ONE_GWEI))).isTrue(); + } + + // (d) An over-target (active) block still rises above 1 Gwei via the normal EIP-1559 formula, + // and the clamp does not distort above-floor values (identical to the unset market's raw result). + @Test + public void forkActive_overTargetBlockRisesAboveOneGweiNormally() { + final LondonFeeMarket forked = market(FORK_BLOCK, ONE_GWEI); + final LondonFeeMarket unset = market(Long.MAX_VALUE, ONE_GWEI); + + final long parentGasUsed = 2 * GAS_TARGET; // over target => baseFee increases + final Wei parent = Wei.of(ONE_GWEI); + + // EIP-1559 rise: 1Gwei + 1Gwei * (2T - T)/T / 8 = 1Gwei + 1Gwei/8 = 1_125_000_000 wei. + final Wei rose = forked.computeBaseFee(FORK_BLOCK + 5, parent, parentGasUsed, GAS_TARGET); + assertThat(rose).isEqualTo(Wei.of(1_125_000_000L)); + assertThat(rose.greaterOrEqualThan(Wei.of(ONE_GWEI))).isTrue(); + assertThat(rose).isNotEqualTo(Wei.of(ONE_GWEI)); // strictly above the floor + + // Above-floor values are unaffected by the clamp: identical to the unset (no-op) market. + assertThat(rose) + .isEqualTo(unset.computeBaseFee(FORK_BLOCK + 5, parent, parentGasUsed, GAS_TARGET)); + } +} diff --git a/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmBenchmarkTest.java b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmBenchmarkTest.java new file mode 100644 index 0000000..aac192a --- /dev/null +++ b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmBenchmarkTest.java @@ -0,0 +1,620 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.parallelization; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.MINING_BENEFICIARY; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hyperledger.besu.config.GenesisConfig; +import org.hyperledger.besu.config.JsonUtil; +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.BlockProcessingResult; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockBody; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.ExecutionContextTestFixture; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.mainnet.BlockProcessor; +import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.blockstm.AereBlockStmEngine; +import org.hyperledger.besu.ethereum.mainnet.blockstm.AereBlockStmPreprocessing; +import org.hyperledger.besu.ethereum.mainnet.blockstm.BlockStmStats; +import org.hyperledger.besu.ethereum.trie.pathbased.common.provider.WorldStateQueryParams; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; + +import java.io.InputStream; +import java.lang.management.ManagementFactory; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * AERE Block-STM benchmark + verification harness (chain-independent, block-processor grain). + * + *

Proves serial-equivalence (parallel state root == sequential state root) for an independent + * workload AND a genuine read/write-conflict workload (all transfers to one shared recipient, which + * forces the multi-version scheduler to abort and re-execute), and measures parallel-vs-sequential + * block execution time / TPS across worker counts. All numbers are written to {@code REPORT}. + */ +public class AereBlockStmBenchmarkTest extends AbstractParallelBlockProcessorIntegrationTest { + + private static final String BASE_GENESIS = + "/org/hyperledger/besu/ethereum/mainnet/parallelization/genesis-it.json"; + private static final Path REPORT = + Path.of(System.getProperty("java.io.tmpdir"), "blockstm_bench_results.txt"); + + // Compute-heavy runtime: PUSH1 0 CALLDATALOAD (iters); loop { KECCAK256 32 bytes of mem; MSTORE }; + // then SSTORE slot0=1; STOP. Iteration count is supplied per-tx via 32-byte calldata. + private static final String COMPUTE_CODE = + "0x6000355b60206000206000526001900380600357600160005500"; + + private static final int MAX_SENDERS = 1024; + private static final Wei BASE_FEE = Wei.of(5); + + private static GenesisConfig customGenesis; + private static KeyPair[] senders; + private static Address[] senderAddr; + private final List lines = new ArrayList<>(); + + private static Address recipient(final int i) { + return Address.fromHexString(String.format("0x%040x", 0x10000000L + i)); + } + + private static final Address SHARED_RECIPIENT = + Address.fromHexString(String.format("0x%040x", 0x0DEAD001L)); + + private static Address computeContract(final int i) { + return Address.fromHexString(String.format("0x%040x", 0x20000000L + i)); + } + + @BeforeAll + static void buildGenesis() throws Exception { + final String baseText; + try (InputStream is = AereBlockStmBenchmarkTest.class.getResourceAsStream(BASE_GENESIS)) { + baseText = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + final ObjectNode root = JsonUtil.objectNodeFromString(baseText); + final ObjectNode alloc = (ObjectNode) root.get("alloc"); + + senders = new KeyPair[MAX_SENDERS]; + senderAddr = new Address[MAX_SENDERS]; + final String richBalance = "0x" + new BigInteger("1000000000000000000000000").toString(16); + for (int i = 0; i < MAX_SENDERS; i++) { + final String pkHex = String.format("%064x", i + 1); + final KeyPair kp = ParallelBlockProcessorTestSupport.generateKeyPair(pkHex); + senders[i] = kp; + senderAddr[i] = Util.publicKeyToAddress(kp.getPublicKey()); + final ObjectNode acct = JsonUtil.createEmptyObjectNode(); + acct.put("balance", richBalance); + alloc.set(senderAddr[i].getBytes().toUnprefixedHexString(), acct); + + final ObjectNode contract = JsonUtil.createEmptyObjectNode(); + contract.put("balance", "0"); + contract.put("code", COMPUTE_CODE); + alloc.set(computeContract(i).getBytes().toUnprefixedHexString(), contract); + } + customGenesis = GenesisConfig.fromConfig(root); + Files.write( + REPORT, + ("AERE BLOCK-STM BENCHMARK / VERIFICATION REPORT\n" + + "generated on a scratch host (chain-independent block-processor grain)\n" + + "cpus(reported)=" + Runtime.getRuntime().availableProcessors() + + " loadAvg(start)=" + + String.format("%.2f", ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage()) + + "\n===============================================================\n") + .getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + } + + private void log(final String s) { + lines.add(s); + System.out.println("[BLOCKSTM-BENCH] " + s); + } + + private void flush() throws Exception { + Files.write( + REPORT, + (String.join("\n", lines) + "\n").getBytes(StandardCharsets.UTF_8), + StandardOpenOption.APPEND); + lines.clear(); + } + + @Override + protected ExecutionContextTestFixture createFreshContext() { + return ExecutionContextTestFixture.builder(customGenesis) + .dataStorageFormat(DataStorageFormat.BONSAI) + .build(); + } + + @Override + protected String getVariantName() { + return "AERE Block-STM Benchmark"; + } + + @Override + protected ParallelTransactionPreprocessing createParallelPreprocessing( + final MainnetTransactionProcessor transactionProcessor) { + return new ParallelTransactionPreprocessing( + transactionProcessor, Runnable::run, getBalConfiguration()); + } + + // ---- workloads ---------------------------------------------------------- + + private Transaction[] independentTransfers(final int n) { + final Transaction[] txs = new Transaction[n]; + for (int i = 0; i < n; i++) { + txs[i] = + createTransferTransaction( + 0, 1L, 100_000L, 0L, 5L, recipient(i).toHexString(), senders[i]); + } + return txs; + } + + private Transaction[] lowConflictTransfers(final int n, final int pool) { + // Distinct senders credit recipients drawn from a small pool -> occasional write-conflict on a + // shared recipient balance (ERC20-style low contention). pool read/write conflict on that balance. + final Transaction[] txs = new Transaction[n]; + for (int i = 0; i < n; i++) { + txs[i] = + createTransferTransaction( + 0, 1L, 100_000L, 0L, 5L, SHARED_RECIPIENT.toHexString(), senders[i]); + } + return txs; + } + + private Transaction[] independentCompute(final int n, final long iters) { + final Bytes payload = UInt256.valueOf(iters).toBytes(); + final Transaction[] txs = new Transaction[n]; + for (int i = 0; i < n; i++) { + txs[i] = + Transaction.builder() + .type(TransactionType.EIP1559) + .nonce(0) + .maxPriorityFeePerGas(Wei.ZERO) + .maxFeePerGas(Wei.of(5)) + .gasLimit(30_000_000L) + .to(computeContract(i)) + .value(Wei.ZERO) + .payload(payload) + .chainId(BigInteger.valueOf(42)) + .signAndBuild(senders[i]); + } + return txs; + } + + // ---- block plumbing (configurable gas limit) ---------------------------- + + private Block buildBlock( + final BlockHeader parent, + final Hash stateRoot, + final long gasLimit, + final Transaction[] txs) { + final BlockHeader header = + new BlockHeaderTestFixture() + .number(parent.getNumber() + 1L) + .parentHash(parent.getHash()) + .coinbase(MINING_BENEFICIARY) + .stateRoot(stateRoot) + .gasLimit(gasLimit) + .baseFeePerGas(BASE_FEE) + .buildHeader(); + final BlockBody body = + new BlockBody(Arrays.asList(txs), Collections.emptyList(), Optional.empty()); + return new Block(header, body); + } + + private Hash discover(final long gasLimit, final Transaction[] txs) { + final ExecutionContextTestFixture ctx = createFreshContext(); + final BlockHeader parent = ctx.getBlockchain().getChainHeadHeader(); + final MutableWorldState ws = + ctx.getStateArchive() + .getWorldState(WorldStateQueryParams.withBlockHeaderAndUpdateNodeHead(parent)) + .orElseThrow(); + final Block block = buildBlock(parent, Hash.ZERO, gasLimit, txs); + final BlockProcessor seq = createSequentialProcessor(ctx); + final BlockProcessingResult r = + seq.processBlock(ctx.getProtocolContext(), ctx.getBlockchain(), ws, block); + if (r.isSuccessful()) { + return ws.rootHash(); + } + // Header stateRoot was ZERO, so processing "failed" on the root-mismatch check; recover the + // real (calculated) root from the error message: the first non-zero 32-byte hex it contains. + final String msg = r.errorMessage.orElseThrow(); + final java.util.regex.Matcher m = + java.util.regex.Pattern.compile("0x[0-9a-fA-F]{64}").matcher(msg); + while (m.find()) { + final Hash h = Hash.fromHexString(m.group()); + if (!h.equals(Hash.ZERO)) { + return h; + } + } + throw new AssertionError("could not recover state root from: " + msg); + } + + private BlockProcessor stockParallel(final ExecutionContextTestFixture ctx) { + final ProtocolSpec spec = + ctx.getProtocolSchedule() + .getByBlockHeader(new BlockHeaderTestFixture().number(0L).buildHeader()); + return new MainnetParallelBlockProcessor( + spec.getTransactionProcessor(), + spec.getTransactionReceiptFactory(), + Wei.ZERO, + BlockHeader::getCoinbase, + true, + ctx.getProtocolSchedule(), + getBalConfiguration(), + new NoOpMetricsSystem()); + } + + /** One timed sequential run. Returns [rootHex, nanos]. */ + private Object[] runSeq(final long gasLimit, final Hash stateRoot, final Transaction[] txs) { + final ExecutionContextTestFixture ctx = createFreshContext(); + final BlockHeader parent = ctx.getBlockchain().getChainHeadHeader(); + final MutableWorldState ws = + ctx.getStateArchive() + .getWorldState(WorldStateQueryParams.withBlockHeaderAndUpdateNodeHead(parent)) + .orElseThrow(); + final Block block = buildBlock(parent, stateRoot, gasLimit, txs); + final BlockProcessor seq = createSequentialProcessor(ctx); + final long t0 = System.nanoTime(); + final BlockProcessingResult r = + seq.processBlock(ctx.getProtocolContext(), ctx.getBlockchain(), ws, block); + final long dt = System.nanoTime() - t0; + assertTrue(r.isSuccessful(), "seq failed: " + r.errorMessage.orElse("?")); + return new Object[] {ws.rootHash(), dt, r}; + } + + /** One timed AERE Block-STM parallel run at {@code workers}. Returns [rootHex, nanos, nbParallel]. */ + private Object[] runPar( + final long gasLimit, final Hash stateRoot, final int workers, final Transaction[] txs) { + final ExecutionContextTestFixture ctx = createFreshContext(); + final BlockHeader parent = ctx.getBlockchain().getChainHeadHeader(); + final MutableWorldState ws = + ctx.getStateArchive() + .getWorldState(WorldStateQueryParams.withBlockHeaderAndUpdateNodeHead(parent)) + .orElseThrow(); + final Block block = buildBlock(parent, stateRoot, gasLimit, txs); + final ProtocolSpec spec = + ctx.getProtocolSchedule() + .getByBlockHeader(new BlockHeaderTestFixture().number(0L).buildHeader()); + final BlockProcessor par = stockParallel(ctx); + final AereBlockStmPreprocessing pf = + new AereBlockStmPreprocessing(spec.getTransactionProcessor(), workers); + final long t0 = System.nanoTime(); + final BlockProcessingResult r = + par.processBlock(ctx.getProtocolContext(), ctx.getBlockchain(), ws, block, pf); + final long dt = System.nanoTime() - t0; + assertTrue(r.isSuccessful(), "par failed: " + r.errorMessage.orElse("?")); + return new Object[] {ws.rootHash(), dt, r.getNbParallelizedTransactions().orElse(0)}; + } + + /** Drive the engine directly to read scheduler statistics (executions / aborts / verified). */ + private BlockStmStats engineStats(final long gasLimit, final int workers, final Transaction[] txs) { + final ExecutionContextTestFixture ctx = createFreshContext(); + final BlockHeader parent = ctx.getBlockchain().getChainHeadHeader(); + final Block block = buildBlock(parent, Hash.ZERO, gasLimit, txs); + final AereBlockStmEngine engine = + new AereBlockStmEngine( + ctx.getProtocolSchedule() + .getByBlockHeader(new BlockHeaderTestFixture().number(0L).buildHeader()) + .getTransactionProcessor(), + workers); + final BlockHashLookup zero = (frame, number) -> Hash.ZERO; + engine.runAsyncBlock( + ctx.getProtocolContext(), + block.getHeader(), + Arrays.asList(txs), + MINING_BENEFICIARY, + zero, + Wei.ZERO, + Runnable::run, + Optional.empty(), + Optional.of(parent)); + return engine.stats(); + } + + private static long median(final long[] a) { + final long[] c = a.clone(); + Arrays.sort(c); + return c[c.length / 2]; + } + + // ---- the suite ---------------------------------------------------------- + + @Test + void blockStmVerifyAndBenchmark() throws Exception { + final int cpus = Runtime.getRuntime().availableProcessors(); + final int[] workerCounts = {1, 2, 4, 8, 16}; + + // ===================== CORRECTNESS ===================== + log(""); + log("## PART 1 - CORRECTNESS (parallel root == sequential root)"); + + // 1a. Independent transfers: expect zero aborts, executions == n. + { + final int n = 300; + final Transaction[] txs = independentTransfers(n); + final Hash root = discover(30_000_000L, txs); + final Object[] seq = runSeq(30_000_000L, root, txs); + final Object[] par = runPar(30_000_000L, root, cpus, txs); + final Hash seqRoot = (Hash) seq[0]; + final Hash parRoot = (Hash) par[0]; + final int nbPar = (Integer) par[2]; + final BlockStmStats st = engineStats(30_000_000L, cpus, txs); + assertThat(parRoot).as("independent parallel root == sequential").isEqualTo(seqRoot); + log(String.format( + " [independent n=%d w=%d] parRoot==seqRoot: %s root=%s", n, cpus, + parRoot.equals(seqRoot), seqRoot.toHexString())); + log(String.format( + " nbParallelized=%d verified=%s execs=%d (n=%d) aborts=%d committedParallel=%d", + nbPar, st.verified(), st.totalExecutions(), st.numTxns(), st.totalAborts(), + st.committedParallel())); + assertThat(st.verified()).as("independent must verify").isTrue(); + assertThat(st.totalAborts()).as("independent must have zero aborts").isZero(); + assertThat(st.totalExecutions()).as("independent: one execution per tx").isEqualTo(n); + assertThat(nbPar).as("parallel path exercised").isGreaterThan(0); + } + + // 1b. Conflicting transfers (shared recipient): expect aborts>0, executions>n, root still equal. + { + final int n = 300; + final Transaction[] txs = conflictingTransfers(n); + final Hash root = discover(30_000_000L, txs); + final Object[] seq = runSeq(30_000_000L, root, txs); + final Object[] par = runPar(30_000_000L, root, cpus, txs); + final Hash seqRoot = (Hash) seq[0]; + final Hash parRoot = (Hash) par[0]; + final int nbPar = (Integer) par[2]; + final BlockStmStats st = engineStats(30_000_000L, cpus, txs); + assertThat(parRoot).as("CONFLICT parallel root == sequential").isEqualTo(seqRoot); + log(String.format( + " [conflict n=%d w=%d] parRoot==seqRoot: %s root=%s", n, cpus, + parRoot.equals(seqRoot), seqRoot.toHexString())); + log(String.format( + " nbParallelized=%d verified=%s execs=%d (n=%d, re-exec=%d) aborts=%d committedParallel=%d", + nbPar, st.verified(), st.totalExecutions(), st.numTxns(), + st.totalExecutions() - st.numTxns(), st.totalAborts(), st.committedParallel())); + assertThat(st.verified()).as("conflict must verify").isTrue(); + assertThat(st.totalAborts()).as("conflict must trigger re-execution (aborts>0)").isGreaterThan(0); + assertThat(st.totalExecutions()).as("conflict re-executes (execs>n)").isGreaterThan(n); + } + + // 1c. Independent compute contract calls: root equal, zero aborts. + { + final int n = 128; + final long iters = 400; + final Transaction[] txs = independentCompute(n, iters); + final Hash root = discover(3_000_000_000L, txs); + final Object[] seq = runSeq(3_000_000_000L, root, txs); + final Object[] par = runPar(3_000_000_000L, root, cpus, txs); + final Hash seqRoot = (Hash) seq[0]; + final Hash parRoot = (Hash) par[0]; + final BlockStmStats st = engineStats(3_000_000_000L, cpus, txs); + assertThat(parRoot).as("compute parallel root == sequential").isEqualTo(seqRoot); + log(String.format( + " [compute n=%d w=%d iters=%d] parRoot==seqRoot: %s root=%s", n, cpus, iters, + parRoot.equals(seqRoot), seqRoot.toHexString())); + log(String.format( + " verified=%s execs=%d (n=%d) aborts=%d committedParallel=%d", + st.verified(), st.totalExecutions(), st.numTxns(), st.totalAborts(), + st.committedParallel())); + assertThat(st.verified()).isTrue(); + assertThat(st.totalAborts()).isZero(); + } + + // 1d. Low-conflict transfers (recipient pool): root equal, verified (aborts allowed but rare). + { + final int n = 300; + final Transaction[] txs = lowConflictTransfers(n, 270); + final Hash root = discover(30_000_000L, txs); + final Object[] seq = runSeq(30_000_000L, root, txs); + final Object[] par = runPar(30_000_000L, root, cpus, txs); + final Hash seqRoot = (Hash) seq[0]; + final Hash parRoot = (Hash) par[0]; + final int nbPar = (Integer) par[2]; + final BlockStmStats st = engineStats(30_000_000L, cpus, txs); + assertThat(parRoot).as("low-conflict parallel root == sequential").isEqualTo(seqRoot); + log(String.format( + " [low-conflict n=%d w=%d pool=270] parRoot==seqRoot: %s root=%s", n, cpus, + parRoot.equals(seqRoot), seqRoot.toHexString())); + log(String.format( + " nbParallelized=%d verified=%s execs=%d (n=%d, re-exec=%d) aborts=%d committedParallel=%d", + nbPar, st.verified(), st.totalExecutions(), st.numTxns(), + st.totalExecutions() - st.numTxns(), st.totalAborts(), st.committedParallel())); + assertThat(st.verified()).as("low-conflict must verify").isTrue(); + } + flush(); + + // ===================== THROUGHPUT ===================== + final int warmup = 3; + final int measure = 5; + + // Workload A: independent transfers. + runThroughput( + "PART 2A - THROUGHPUT: independent transfers", 1000, 30_000_000L, + independentTransfers(1000), workerCounts, warmup, measure); + + // Workload A': independent compute-heavy calls (execution-bound; best case for parallelism). + runThroughput( + "PART 2B - THROUGHPUT: independent compute calls (iters=800)", 256, 3_000_000_000L, + independentCompute(256, 800), workerCounts, warmup, measure); + + // Workload C: high-conflict transfers (all to one recipient) - expect no speedup. + runThroughput( + "PART 2C - THROUGHPUT: high-conflict transfers (shared recipient)", 500, 30_000_000L, + conflictingTransfers(500), workerCounts, warmup, measure); + + // Workload D: low-conflict transfers (recipient pool=900 of 1000) - ERC20-style ~10% contention. + runThroughput( + "PART 2D - THROUGHPUT: low-conflict transfers (recipient pool=900)", 1000, 30_000_000L, + lowConflictTransfers(1000, 900), workerCounts, warmup, measure); + + // ===================== ISOLATED EXECUTION PHASE ===================== + // The real Block-STM test: time ONLY the parallel execution phase (engine.runAsyncBlock, i.e. + // the flat-snapshot base reads + concurrent EVM execution + MV bookkeeping), measured INSIDE the + // engine (execPhaseNanos), with the serial Bonsai trie commit excluded entirely. Execution-bound + // independent compute calls so there is real work to parallelize; scaling vs w=1 is the proof. + runExecPhase( + "PART 3A - EXECUTION PHASE (isolated): independent compute calls (iters=800)", + 256, 3_000_000_000L, independentCompute(256, 800), workerCounts, warmup, measure); + runExecPhase( + "PART 3B - EXECUTION PHASE (isolated): independent compute calls (iters=3000, heavy)", + 256, 3_000_000_000L, independentCompute(256, 3000), workerCounts, warmup, measure); + + log(""); + log("loadAvg(end)=" + + String.format("%.2f", ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage())); + log("=============================== END ==============================="); + flush(); + } + + private void runThroughput( + final String title, + final int n, + final long gasLimit, + final Transaction[] txs, + final int[] workerCounts, + final int warmup, + final int measure) + throws Exception { + log(""); + log("## " + title + " (n=" + n + " txs)"); + final Hash root = discover(gasLimit, txs); + + // Sequential baseline. + for (int i = 0; i < warmup; i++) { + runSeq(gasLimit, root, txs); + } + final long[] seqT = new long[measure]; + for (int i = 0; i < measure; i++) { + seqT[i] = (Long) runSeq(gasLimit, root, txs)[1]; + } + final long seqMed = median(seqT); + final double seqMs = seqMed / 1e6; + final double seqTps = n / (seqMed / 1e9); + log(String.format( + " SEQUENTIAL (MainnetBlockProcessor): %8.3f ms %10.0f tx/s", seqMs, seqTps)); + log(" worker | median ms | tx/s | speedup vs seq | scaling vs w=1"); + log(" -------+------------+----------+----------------+---------------"); + + double par1Ms = Double.NaN; + for (final int w : workerCounts) { + for (int i = 0; i < warmup; i++) { + runPar(gasLimit, root, w, txs); + } + final long[] pt = new long[measure]; + int nbPar = 0; + for (int i = 0; i < measure; i++) { + final Object[] r = runPar(gasLimit, root, w, txs); + pt[i] = (Long) r[1]; + nbPar = (Integer) r[2]; + } + final long med = median(pt); + final double ms = med / 1e6; + final double tps = n / (med / 1e9); + if (w == 1) { + par1Ms = ms; + } + log(String.format( + " %6d | %10.3f | %8.0f | %13.2fx | %12.2fx (nbParallel=%d)", + w, ms, tps, seqMs / ms, par1Ms / ms, nbPar)); + } + flush(); + } + + /** + * Measures the ISOLATED parallel execution phase across worker counts. Drives the engine directly + * (runAsyncBlock) and reads {@code execPhaseNanos} recorded INSIDE the engine (worker pool start -> + * termination), so the flat-snapshot mint and the downstream serial Bonsai trie commit are both + * excluded. Scaling vs w=1 of this number is the direct test of whether Block-STM execution + * parallelism works once the per-worker snapshot overhead is removed. + */ + private void runExecPhase( + final String title, + final int n, + final long gasLimit, + final Transaction[] txs, + final int[] workerCounts, + final int warmup, + final int measure) + throws Exception { + log(""); + log("## " + title + " (n=" + n + " txs)"); + log(" worker | exec-phase ms | snapshot ms | exec tx/s | scaling vs w=1 | verified"); + log(" -------+----------------+-------------+--------------+----------------+---------"); + + double w1Ms = Double.NaN; + for (final int w : workerCounts) { + for (int i = 0; i < warmup; i++) { + engineStats(gasLimit, w, txs); + } + final long[] execT = new long[measure]; + final long[] snapT = new long[measure]; + boolean verified = true; + for (int i = 0; i < measure; i++) { + final BlockStmStats st = engineStats(gasLimit, w, txs); + execT[i] = st.execPhaseNanos(); + snapT[i] = st.snapshotNanos(); + verified = verified && st.verified(); + } + final long execMed = median(execT); + final long snapMed = median(snapT); + final double execMs = execMed / 1e6; + final double snapMs = snapMed / 1e6; + final double tps = n / (execMed / 1e9); + if (w == 1) { + w1Ms = execMs; + } + assertThat(verified).as("exec-phase run must verify at w=" + w).isTrue(); + log(String.format( + " %6d | %14.3f | %11.3f | %12.0f | %13.2fx | %s", + w, execMs, snapMs, tps, w1Ms / execMs, verified)); + } + flush(); + } +} diff --git a/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmSmokeTest.java b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmSmokeTest.java new file mode 100644 index 0000000..a8e3db9 --- /dev/null +++ b/anchor/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/parallelization/AereBlockStmSmokeTest.java @@ -0,0 +1,130 @@ +/* + * Copyright contributors to AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.mainnet.parallelization; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_2; +import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_3; +import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_4; +import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_1_KEYPAIR; +import static org.hyperledger.besu.ethereum.mainnet.parallelization.ParallelBlockProcessorTestSupport.ACCOUNT_GENESIS_2_KEYPAIR; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.BlockProcessingResult; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.ExecutionContextTestFixture; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor.PreprocessingFunction; +import org.hyperledger.besu.ethereum.mainnet.BlockProcessor; +import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.blockstm.AereBlockStmPreprocessing; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Smoke test proving AERE Block-STM parallel execution is serial-equivalent: a block processed + * through the AERE Block-STM engine (a stock {@link MainnetParallelBlockProcessor} driven by {@link + * AereBlockStmPreprocessing}) yields a state root bit-identical to sequential {@code + * MainnetBlockProcessor} execution of the same block, and the parallel path is actually exercised. + */ +public class AereBlockStmSmokeTest extends AbstractParallelBlockProcessorIntegrationTest { + + @Override + protected String getVariantName() { + return "AERE Block-STM"; + } + + @Override + protected ParallelTransactionPreprocessing createParallelPreprocessing( + final MainnetTransactionProcessor transactionProcessor) { + // Not used by this test; provided to satisfy the abstract base. + return new ParallelTransactionPreprocessing( + transactionProcessor, Runnable::run, getBalConfiguration()); + } + + private BlockProcessor stockParallelProcessor(final ExecutionContextTestFixture ctx) { + final ProtocolSpec spec = + ctx.getProtocolSchedule() + .getByBlockHeader(new BlockHeaderTestFixture().number(0L).buildHeader()); + return new MainnetParallelBlockProcessor( + spec.getTransactionProcessor(), + spec.getTransactionReceiptFactory(), + Wei.ZERO, + BlockHeader::getCoinbase, + true, + ctx.getProtocolSchedule(), + getBalConfiguration(), + new NoOpMetricsSystem()); + } + + @Test + @DisplayName("AERE Block-STM parallel root equals sequential root for independent transfers") + void blockStmRootMatchesSequential() { + final Transaction tx1 = + createTransferTransaction( + 0, 1_000_000_000_000_000_000L, 300_000L, 0L, 5L, ACCOUNT_2, ACCOUNT_GENESIS_1_KEYPAIR); + final Transaction tx2 = + createTransferTransaction( + 0, 2_000_000_000_000_000_000L, 300_000L, 0L, 5L, ACCOUNT_3, ACCOUNT_GENESIS_2_KEYPAIR); + final Transaction tx3 = + createTransferTransaction( + 1, 3_000_000_000_000_000_000L, 300_000L, 0L, 5L, ACCOUNT_4, ACCOUNT_GENESIS_1_KEYPAIR); + + final Wei baseFee = Wei.of(5); + final Transaction[] txs = {tx1, tx2, tx3}; + final Hash stateRoot = discoverStateRoot(baseFee, txs); + + // Sequential + final ExecutionContextTestFixture seqCtx = createFreshContext(); + final MutableWorldState seqWs = seqCtx.getStateArchive().getWorldState(); + final Block seqBlock = createBlock(seqCtx, stateRoot, baseFee, txs); + final BlockProcessor seqProcessor = createSequentialProcessor(seqCtx); + final BlockProcessingResult seqResult = + seqProcessor.processBlock( + seqCtx.getProtocolContext(), seqCtx.getBlockchain(), seqWs, seqBlock); + assertTrue(seqResult.isSuccessful(), "sequential processing failed"); + + // Parallel via AERE Block-STM + final ExecutionContextTestFixture parCtx = createFreshContext(); + final MutableWorldState parWs = parCtx.getStateArchive().getWorldState(); + final Block parBlock = createBlock(parCtx, stateRoot, baseFee, txs); + final ProtocolSpec spec = + parCtx + .getProtocolSchedule() + .getByBlockHeader(new BlockHeaderTestFixture().number(0L).buildHeader()); + final BlockProcessor parProcessor = stockParallelProcessor(parCtx); + final PreprocessingFunction blockStm = + new AereBlockStmPreprocessing( + spec.getTransactionProcessor(), Runtime.getRuntime().availableProcessors()); + final BlockProcessingResult parResult = + parProcessor.processBlock( + parCtx.getProtocolContext(), parCtx.getBlockchain(), parWs, parBlock, blockStm); + assertTrue( + parResult.isSuccessful(), + "AERE Block-STM processing failed: " + parResult.errorMessage.orElse("(no message)")); + + // Serial-equivalence: bit-identical 32-byte state root. + assertThat(parWs.rootHash()) + .as("AERE Block-STM parallel state root must equal sequential") + .isEqualTo(seqWs.rootHash()); + + // The parallel path was actually exercised (outcomes consumed, not per-tx sequential fallback). + assertParallelizationRecordedInResults(seqResult, parResult, txs.length); + } +} diff --git a/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java new file mode 100644 index 0000000..92ec415 --- /dev/null +++ b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -0,0 +1,941 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-cap-prestatus-rlpx-connections.patch. + */ +package org.hyperledger.besu.ethereum.eth.manager; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.EthProtocol; +import org.hyperledger.besu.ethereum.eth.SnapProtocol; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer.DisconnectCallback; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerSelector; +import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; +import org.hyperledger.besu.ethereum.eth.sync.ChainHeadTracker; +import org.hyperledger.besu.ethereum.eth.sync.SnapServerChecker; +import org.hyperledger.besu.ethereum.eth.sync.SyncMode; +import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; +import org.hyperledger.besu.ethereum.forkid.ForkId; +import org.hyperledger.besu.ethereum.forkid.ForkIdManager; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.peers.Peer; +import org.hyperledger.besu.ethereum.p2p.peers.PeerId; +import org.hyperledger.besu.ethereum.p2p.rlpx.RlpxAgent; +import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.PeerClientName; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.PeerInfo; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; +import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.Counter; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; +import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider; +import org.hyperledger.besu.util.Subscribers; + +import java.time.Clock; +import java.time.Duration; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalNotification; +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EthPeers implements PeerSelector { + private static final Logger LOG = LoggerFactory.getLogger(EthPeers.class); + public static final Comparator TOTAL_DIFFICULTY = + Comparator.comparing((final EthPeerImmutableAttributes p) -> p.estimatedTotalDifficulty()); + + public static final Comparator CHAIN_HEIGHT = + Comparator.comparing((final EthPeerImmutableAttributes p) -> p.estimatedChainHeight()); + + public static final Comparator MOST_USEFUL_PEER = + Comparator.comparing((final EthPeerImmutableAttributes p) -> p.reputationScore()) + .thenComparing(CHAIN_HEIGHT); + + public static final Comparator TOTAL_DIFFICULTY_THEN_HEIGHT = + TOTAL_DIFFICULTY.thenComparing(CHAIN_HEIGHT); + + public static final Comparator LEAST_TO_MOST_BUSY = + Comparator.comparing(EthPeerImmutableAttributes::outstandingRequests) + .thenComparing(EthPeerImmutableAttributes::lastRequestTimestamp); + public static final int NODE_ID_LENGTH = 64; + public static final int USEFULL_PEER_SCORE_THRESHOLD = 102; + + private final Map activeConnections = new ConcurrentHashMap<>(); + + /** + * Lower bound for the pre-STATUS (incomplete) connection cap, so that even very small {@code + * --max-peers} values still tolerate a reasonable number of concurrent inbound handshakes. AERE + * D-434: ported from upstream hyperledger/besu e540ae2394 (GHSA-6r9q-wjp4-34gh). + */ + private static final int INCOMPLETE_CONNECTIONS_CAP_FLOOR = 10; + + /** + * Maximum number of connections that have completed the devp2p HELLO but not yet the eth STATUS + * handshake that we retain. Bounds file-descriptor and heap growth from peers that connect and + * never send STATUS, which are otherwise invisible to {@code --max-peers} accounting. + */ + private final int maxIncompleteConnections; + + private final Cache incompleteConnections; + private final Clock clock; + private final List permissioningProviders; + private final int maxMessageSize; + private final Subscribers connectCallbacks = Subscribers.create(); + private final Subscribers disconnectCallbacks = Subscribers.create(); + private final Collection pendingRequests = new CopyOnWriteArrayList<>(); + private final int peerUpperBound; + private final int maxRemotelyInitiatedConnections; + private final Boolean randomPeerPriority; + private final Bytes nodeIdMask = Bytes.random(NODE_ID_LENGTH); + private final Supplier currentProtocolSpecSupplier; + private final SyncMode syncMode; + private final ForkIdManager forkIdManager; + private final int snapServerTargetNumber; + private final boolean shouldLimitRemoteConnections; + + private Comparator bestPeerComparator; + private final Bytes localNodeId; + private RlpxAgent rlpxAgent; + + private final Counter connectedPeersCounter; + private ChainHeadTracker tracker; + private SnapServerChecker snapServerChecker; + private boolean snapServerPeersNeeded = false; + private Supplier trailingPeerRequirementsSupplier = + () -> TrailingPeerRequirements.UNRESTRICTED; + + public EthPeers( + final Supplier currentProtocolSpecSupplier, + final Clock clock, + final MetricsSystem metricsSystem, + final int maxMessageSize, + final List permissioningProviders, + final Bytes localNodeId, + final int peerUpperBound, + final int maxRemotelyInitiatedConnections, + final Boolean randomPeerPriority, + final SyncMode syncMode, + final ForkIdManager forkIdManager) { + this.currentProtocolSpecSupplier = currentProtocolSpecSupplier; + this.clock = clock; + this.permissioningProviders = permissioningProviders; + this.maxMessageSize = maxMessageSize; + this.bestPeerComparator = TOTAL_DIFFICULTY_THEN_HEIGHT; + this.localNodeId = localNodeId; + this.peerUpperBound = peerUpperBound; + this.maxRemotelyInitiatedConnections = maxRemotelyInitiatedConnections; + this.randomPeerPriority = randomPeerPriority; + LOG.trace("MaxPeers: {}, Max Remote: {}", peerUpperBound, maxRemotelyInitiatedConnections); + this.syncMode = syncMode; + this.forkIdManager = forkIdManager; + this.snapServerTargetNumber = + peerUpperBound / 2; // 50% of peers should be snap servers while snap syncing + this.shouldLimitRemoteConnections = maxRemotelyInitiatedConnections < peerUpperBound; + this.maxIncompleteConnections = Math.max(peerUpperBound * 2, INCOMPLETE_CONNECTIONS_CAP_FLOOR); + this.incompleteConnections = + CacheBuilder.newBuilder() + .maximumSize(maxIncompleteConnections) + .expireAfterWrite(Duration.ofSeconds(20L)) + .concurrencyLevel(1) + .removalListener(this::onCacheRemoval) + .build(); + + metricsSystem.createIntegerGauge( + BesuMetricCategory.ETHEREUM, + "peer_count", + "The current number of peers connected", + activeConnections::size); + metricsSystem.createIntegerGauge( + BesuMetricCategory.ETHEREUM, + "peer_count_snap_server", + "The current number of peers connected that serve snap data", + () -> + (int) streamAvailablePeers().filter(EthPeerImmutableAttributes::isServingSnap).count()); + metricsSystem.createIntegerGauge( + BesuMetricCategory.PEERS, + "pending_peer_requests_current", + "Number of peer requests currently pending because peers are busy", + pendingRequests::size); + metricsSystem.createIntegerGauge( + BesuMetricCategory.ETHEREUM, + "peer_limit", + "The maximum number of peers this node allows to connect", + () -> peerUpperBound); + metricsSystem.createIntegerGauge( + BesuMetricCategory.ETHEREUM, + "peer_count_incomplete", + "The current number of connections that have not yet completed the eth STATUS handshake", + () -> (int) incompleteConnections.size()); + + connectedPeersCounter = + metricsSystem.createCounter( + BesuMetricCategory.PEERS, "connected_total", "Total number of peers connected"); + + final LabelledSuppliedMetric peerClientLabelledGauge = + metricsSystem.createLabelledSuppliedGauge( + BesuMetricCategory.PEERS, + "peer_count_by_client", + "The number of clients connected by client", + "client"); + + for (final var clientName : PeerClientName.values()) { + peerClientLabelledGauge.labels( + () -> countConnectedPeersByClientName(clientName), clientName.getDisplayName()); + } + } + + private double countConnectedPeersByClientName(final PeerClientName clientName) { + return streamAllActiveConnections() + .map(PeerConnection::getPeerInfo) + .map(PeerInfo::getClientName) + .filter(clientName::equals) + .count(); + } + + public void registerNewConnection( + final PeerConnection newConnection, final List peerValidators) { + final Bytes id = newConnection.getPeer().getId(); + synchronized (this) { + EthPeer ethPeer = activeConnections.get(id); + if (ethPeer == null) { + final Optional peerInList = + incompleteConnections.asMap().values().stream() + .filter(p -> p.getId().equals(id)) + .findFirst(); + ethPeer = + peerInList.orElseGet( + () -> + new EthPeer( + newConnection, + this::ethPeerStatusExchanged, + peerValidators, + maxMessageSize, + clock, + permissioningProviders, + localNodeId)); + } + incompleteConnections.put(newConnection, ethPeer); + } + } + + @NotNull + private List getIncompleteConnections(final Bytes id) { + return incompleteConnections.asMap().keySet().stream() + .filter(nrc -> nrc.getPeer().getId().equals(id)) + .collect(Collectors.toList()); + } + + public boolean registerDisconnect(final PeerConnection connection) { + final EthPeer peer = peer(connection); + return registerDisconnect(peer, connection); + } + + private boolean registerDisconnect(final EthPeer peer, final PeerConnection connection) { + incompleteConnections.invalidate(connection); + boolean removed = false; + if (peer == null) { + LOG.atTrace() + .setMessage("attempt to remove null peer with connection {}") + .addArgument(connection) + .log(); + return false; + } + if (peer.getConnection().equals(connection)) { + final Bytes id = peer.getId(); + if (!peerHasIncompleteConnection(id)) { + removed = activeConnections.remove(id, peer); + disconnectCallbacks.forEach(callback -> callback.onDisconnect(peer)); + peer.handleDisconnect(); + abortPendingRequestsAssignedToDisconnectedPeers(); + if (peer.getReputation().getScore() > USEFULL_PEER_SCORE_THRESHOLD) { + LOG.atDebug().setMessage("Disconnected USEFUL peer {}").addArgument(peer).log(); + } else { + LOG.atDebug() + .setMessage("Disconnected EthPeer {}") + .addArgument(peer.getLoggableId()) + .log(); + } + } + } + reattemptPendingPeerRequests(); + return removed; + } + + private boolean peerHasIncompleteConnection(final Bytes id) { + return getIncompleteConnections(id).stream().anyMatch(conn -> !conn.isDisconnected()); + } + + private void abortPendingRequestsAssignedToDisconnectedPeers() { + synchronized (this) { + for (final PendingPeerRequest request : pendingRequests) { + if (request.getAssignedPeer().map(EthPeer::isDisconnected).orElse(false)) { + request.abort(); + } + } + } + } + + public EthPeer peer(final PeerConnection connection) { + final EthPeer ethPeer = incompleteConnections.getIfPresent(connection); + return ethPeer != null ? ethPeer : activeConnections.get(connection.getPeer().getId()); + } + + public PendingPeerRequest executePeerRequest( + final PeerRequest request, final long minimumBlockNumber, final Optional peer) { + final long actualMinBlockNumber; + if (minimumBlockNumber > 0 && currentProtocolSpecSupplier.get().isPoS()) { + // if on PoS do not enforce a min block number, since the estimated chain height of the remote + // peer is not updated anymore. + actualMinBlockNumber = 0; + } else { + actualMinBlockNumber = minimumBlockNumber; + } + final PendingPeerRequest pendingPeerRequest = + new PendingPeerRequest(this, request, actualMinBlockNumber, peer); + synchronized (this) { + if (!pendingPeerRequest.attemptExecution()) { + pendingRequests.add(pendingPeerRequest); + } + } + return pendingPeerRequest; + } + + public void dispatchMessage( + final EthPeer peer, final EthMessage ethMessage, final String protocolName) { + final Optional maybeRequestManager = peer.dispatch(ethMessage, protocolName); + if (maybeRequestManager.isPresent() && peer.hasAvailableRequestCapacity()) { + reattemptPendingPeerRequests(); + } + } + + public void dispatchMessage(final EthPeer peer, final EthMessage ethMessage) { + dispatchMessage(peer, ethMessage, EthProtocol.NAME); + } + + @VisibleForTesting + void reattemptPendingPeerRequests() { + synchronized (this) { + final Iterator iterator = pendingRequests.iterator(); + while (iterator.hasNext() + && streamAvailablePeers() + .anyMatch(EthPeerImmutableAttributes::hasAvailableRequestCapacity)) { + final PendingPeerRequest request = iterator.next(); + if (request.attemptExecution()) { + pendingRequests.remove(request); + } + } + } + } + + public long subscribeConnect(final ConnectCallback callback) { + return connectCallbacks.subscribe(callback); + } + + public void unsubscribeConnect(final long id) { + connectCallbacks.unsubscribe(id); + } + + public void subscribeDisconnect(final DisconnectCallback callback) { + disconnectCallbacks.subscribe(callback); + } + + public int peerCount() { + removeDisconnectedPeers(); + return activeConnections.size(); + } + + public int getMaxPeers() { + return peerUpperBound; + } + + public Stream streamAllPeers() { + return activeConnections.values().stream().map(EthPeerImmutableAttributes::from); + } + + private void removeDisconnectedPeers() { + activeConnections + .values() + .forEach( + ep -> { + if (ep.isDisconnected()) { + registerDisconnect(ep, ep.getConnection()); + } + }); + } + + public Stream streamAvailablePeers() { + return streamAllPeers().filter(peer -> !peer.isDisconnected()); + } + + public Stream streamBestPeers() { + return streamAvailablePeers() + .filter(EthPeerImmutableAttributes::isFullyValidated) + .sorted(getBestPeerComparator().reversed()); + } + + public Optional bestPeer() { + return streamAvailablePeers() + .max(getBestPeerComparator()) + .map(EthPeerImmutableAttributes::ethPeer); + } + + public Optional bestPeerWithHeightEstimate() { + return bestPeerMatchingCriteria(p -> p.isFullyValidated() && p.hasEstimatedChainHeight()); + } + + public Optional bestPeerMatchingCriteria( + final Predicate matchesCriteria) { + return streamAvailablePeers() + .filter(matchesCriteria) + .max(getBestPeerComparator()) + .map(EthPeerImmutableAttributes::ethPeer); + } + + public void setBestPeerComparator(final Comparator comparator) { + LOG.info("Updating the default best peer comparator"); + bestPeerComparator = comparator; + } + + public Comparator getBestPeerComparator() { + return bestPeerComparator; + } + + public void setRlpxAgent(final RlpxAgent rlpxAgent) { + this.rlpxAgent = rlpxAgent; + } + + public Stream streamAllActiveConnections() { + return activeConnections.values().stream() + .map(EthPeer::getConnection) + .filter(c -> !c.isDisconnected()); + } + + public Stream streamAllConnections() { + return Stream.concat( + activeConnections.values().stream().map(EthPeer::getConnection), + incompleteConnections.asMap().keySet().stream()) + .distinct() + .filter(c -> !c.isDisconnected()); + } + + public Optional gatePeerConnection(final Peer peer, final boolean inbound) { + + if (peer.getForkId().isPresent()) { + final ForkId forkId = peer.getForkId().get(); + if (!forkIdManager.peerCheck(forkId)) { + LOG.atDebug() + .setMessage("Wrong fork id, not trying to connect to peer {}") + .addArgument(peer::getId) + .log(); + + return Optional.of(DisconnectReason.USELESS_PEER_BY_CHAIN_COMPARATOR); + } + } + + final Bytes id = peer.getId(); + if (alreadyConnectedOrConnecting(inbound, id)) { + LOG.atTrace() + .setMessage("not connecting to peer {} - already connected") + .addArgument(peer.getLoggableId()) + .log(); + return Optional.of(DisconnectReason.ALREADY_CONNECTED); + } + + if (peerCount() < getMaxPeers() || needMoreSnapServers() || canExceedPeerLimits(id)) { + return Optional.empty(); + } else { + return Optional.of(DisconnectReason.TOO_MANY_PEERS); + } + } + + private boolean alreadyConnectedOrConnecting(final boolean inbound, final Bytes id) { + final EthPeer ethPeer = activeConnections.get(id); + if (ethPeer != null && !ethPeer.isDisconnected()) { + return true; + } + final List incompleteConnections = getIncompleteConnections(id); + return incompleteConnections.stream() + .anyMatch(c -> !c.isDisconnected() && (!inbound || (inbound && c.inboundInitiated()))); + } + + public void disconnectWorstUselessPeer() { + streamAvailablePeers() + .filter(p -> !canExceedPeerLimits(p.ethPeer().getId())) + .min(getBestPeerComparator()) + .ifPresent( + peer -> { + LOG.atDebug() + .setMessage( + "disconnecting peer {}. Waiting for better peers. Current {} of max {}") + .addArgument(peer.ethPeer().getLoggableId()) + .addArgument(this::peerCount) + .addArgument(this::getMaxPeers) + .log(); + peer.ethPeer() + .disconnect(DisconnectMessage.DisconnectReason.USELESS_PEER_BY_CHAIN_COMPARATOR); + }); + } + + public void setChainHeadTracker(final ChainHeadTracker tracker) { + this.tracker = tracker; + } + + public void setSnapServerChecker(final SnapServerChecker checker) { + this.snapServerChecker = checker; + } + + public void snapServerPeersNeeded(final boolean b) { + this.snapServerPeersNeeded = b; + } + + public void setTrailingPeerRequirementsSupplier( + final Supplier tprSupplier) { + this.trailingPeerRequirementsSupplier = tprSupplier; + } + + // Part of the PeerSelector interface, to be split apart later + @Override + public Optional getPeer(final Predicate filter) { + return streamAvailablePeers() + .filter(filter) + .filter(EthPeerImmutableAttributes::hasAvailableRequestCapacity) + .filter(EthPeerImmutableAttributes::isFullyValidated) + .max(getBestPeerComparator()) + .map(EthPeerImmutableAttributes::ethPeer); + } + + // Part of the PeerSelector interface, to be split apart later + @Override + public CompletableFuture waitForPeer( + final Predicate filter) { + final CompletableFuture future = new CompletableFuture<>(); + LOG.debug("Waiting for peer matching filter. {} peers currently connected.", peerCount()); + // check for an existing peer matching the filter and use that if one is found + Optional maybePeer = getPeer(filter); + if (maybePeer.isPresent()) { + LOG.debug("Found peer matching filter already connected!"); + future.complete(maybePeer.get()); + } else { + // no existing peer matches our filter. Subscribe to new connections until we find one + LOG.debug("Subscribing to new peer connections to wait until one matches filter"); + final long subscriptionId = + subscribeConnect( + (peer) -> { + if (!future.isDone() && filter.test(EthPeerImmutableAttributes.from(peer))) { + LOG.debug("Found new peer matching filter!"); + future.complete(peer); + } else { + LOG.debug("New peer does not match filter"); + } + }); + future.handle( + (peer, throwable) -> { + LOG.debug("Unsubscribing from new peer connections with ID {}", subscriptionId); + unsubscribeConnect(subscriptionId); + return null; + }); + } + + return future; + } + + // Part of the PeerSelector interface, to be split apart later + @Override + public Optional getPeerByPeerId(final PeerId peerId) { + return Optional.ofNullable(activeConnections.get(peerId.getId())); + } + + @FunctionalInterface + public interface ConnectCallback { + void onPeerConnected(EthPeer newPeer); + } + + @Override + public String toString() { + if (activeConnections.isEmpty()) { + return "0 EthPeers {}"; + } + final String connectionsList = + activeConnections.values().stream() + .sorted() + .map(EthPeer::toString) + .collect(Collectors.joining(", \n")); + return activeConnections.size() + " EthPeers {\n" + connectionsList + '}'; + } + + private void ethPeerStatusExchanged(final EthPeer peer) { + // We have a connection to a peer that is on the right chain and is willing to connect to us. + // Find out what the EthPeer block height is and whether it can serve snap data (if we are doing + // snap sync) + LOG.debug("Peer {} status exchanged", peer); + assert tracker != null : "ChainHeadTracker must be set before EthPeers can be used"; + CompletableFuture future = tracker.getBestHeaderFromPeer(peer); + + future.whenComplete( + (peerHeadBlockHeader, error) -> { + if (peerHeadBlockHeader == null) { + LOG.debug( + "Failed to retrieve chain head info. Disconnecting {}... {}", + peer.getLoggableId(), + error); + peer.disconnect( + DisconnectMessage.DisconnectReason.USELESS_PEER_FAILED_TO_RETRIEVE_CHAIN_HEAD); + } else { + + // we can check trailing peers now + final TrailingPeerRequirements trailingPeerRequirements = + trailingPeerRequirementsSupplier.get(); + if (trailingPeerRequirements != null) { + if (peer.chainState().getEstimatedHeight() + < trailingPeerRequirements.getMinimumHeightToBeUpToDate()) { + if (!(getNumTrailingPeers(trailingPeerRequirements.getMinimumHeightToBeUpToDate()) + < trailingPeerRequirements.getMaxTrailingPeers())) { + LOG.atTrace() + .setMessage( + "Adding trailing peer {} would exceed max trailing peers {}. Disconnecting...") + .addArgument(peer.getLoggableId()) + .addArgument(trailingPeerRequirements.getMaxTrailingPeers()) + .log(); + peer.disconnect( + DisconnectMessage.DisconnectReason.USELESS_PEER_EXCEEDS_TRAILING_PEERS); + return; + } + } + } + + peer.chainState().updateHeightEstimate(peerHeadBlockHeader.getNumber()); + CompletableFuture isServingSnapFuture; + if (syncMode == SyncMode.SNAP) { + // even if we have finished the snap sync, we still want to know if the peer is a snap + // server + isServingSnapFuture = + CompletableFuture.runAsync( + () -> { + try { + checkIsSnapServer(peer, peerHeadBlockHeader); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + isServingSnapFuture = CompletableFuture.completedFuture(null); + } + isServingSnapFuture.thenRun( + () -> { + if (!peer.getConnection().isDisconnected() && addPeerToEthPeers(peer)) { + connectedPeersCounter.inc(); + connectCallbacks.forEach(cb -> cb.onPeerConnected(peer)); + } + }); + } + }); + } + + private void checkIsSnapServer(final EthPeer peer, final BlockHeader peersHeadBlockHeader) { + if (peer.getAgreedCapabilities().contains(SnapProtocol.SNAP1)) { + if (snapServerChecker != null) { + // set that peer is a snap server for doing the test + peer.setIsServingSnap(true); + Boolean isServer; + try { + isServer = snapServerChecker.check(peer, peersHeadBlockHeader).get(6L, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.atTrace() + .setMessage("Error checking if peer {} is a snap server. Setting to false.") + .addArgument(peer.getLoggableId()) + .log(); + peer.setIsServingSnap(false); + return; + } + peer.setIsServingSnap(isServer); + LOG.atTrace() + .setMessage("{}: peer {}") + .addArgument(isServer ? "Is a snap server" : "Is NOT a snap server") + .addArgument(peer.getLoggableId()) + .log(); + } + } + } + + private int comparePeerPriorities(final EthPeer p1, final EthPeer p2) { + final PeerConnection a = p1.getConnection(); + final PeerConnection b = p2.getConnection(); + final boolean aCanExceedPeerLimits = canExceedPeerLimits(a.getPeer().getId()); + final boolean bCanExceedPeerLimits = canExceedPeerLimits(b.getPeer().getId()); + if (aCanExceedPeerLimits && !bCanExceedPeerLimits) { + return -1; + } else if (bCanExceedPeerLimits && !aCanExceedPeerLimits) { + return 1; + } else { + return randomPeerPriority + ? compareByMaskedNodeId(a, b) + : compareConnectionInitiationTimes(a, b); + } + } + + private boolean canExceedPeerLimits(final Bytes peerId) { + if (rlpxAgent == null) { + return false; + } + return rlpxAgent.canExceedConnectionLimits(peerId); + } + + private int compareConnectionInitiationTimes(final PeerConnection a, final PeerConnection b) { + return Math.toIntExact(a.getInitiatedAt() - b.getInitiatedAt()); + } + + private int compareByMaskedNodeId(final PeerConnection a, final PeerConnection b) { + return a.getPeer().getId().xor(nodeIdMask).compareTo(b.getPeer().getId().xor(nodeIdMask)); + } + + private void enforceRemoteConnectionLimits() { + if (!shouldLimitRemoteConnections || peerCount() < maxRemotelyInitiatedConnections) { + // Nothing to do + return; + } + + getActivePrioritizedPeers() + .filter(p -> p.getConnection().inboundInitiated()) + .filter(p -> !canExceedPeerLimits(p.getId())) + .skip(maxRemotelyInitiatedConnections) + .forEach( + conn -> { + LOG.trace( + "Too many remotely initiated connections. Disconnect low-priority connection: {}, maxRemote={}", + conn, + maxRemotelyInitiatedConnections); + conn.disconnect(DisconnectMessage.DisconnectReason.TOO_MANY_PEERS); + }); + } + + private Stream getActivePrioritizedPeers() { + return activeConnections.values().stream() + .filter(p -> !p.isDisconnected()) + .sorted(this::comparePeerPriorities); + } + + private void enforceConnectionLimits() { + if (peerCount() < peerUpperBound) { + // Nothing to do - we're under our limits + return; + } + getActivePrioritizedPeers() + .skip(peerUpperBound) + .map(EthPeer::getConnection) + .filter(c -> !canExceedPeerLimits(c.getPeer().getId())) + .forEach( + conn -> { + LOG.trace( + "Too many connections. Disconnect low-priority connection: {}, maxConnections={}", + conn, + peerUpperBound); + conn.disconnect(DisconnectMessage.DisconnectReason.TOO_MANY_PEERS); + }); + } + + private boolean inboundInitiatedConnectionLimitExceeded() { + return shouldLimitRemoteConnections + && countUntrustedRemotelyInitiatedConnections() > maxRemotelyInitiatedConnections; + } + + private long countUntrustedRemotelyInitiatedConnections() { + return activeConnections.values().stream() + .map(EthPeer::getConnection) + .filter(PeerConnection::inboundInitiated) + .filter(c -> !c.isDisconnected()) + .filter(conn -> !canExceedPeerLimits(conn.getPeer().getId())) + .count(); + } + + // AERE D-434 (e540ae2394, GHSA-6r9q-wjp4-34gh): close pre-STATUS connections evicted from the + // bounded incomplete-connection cache so they cannot leak file descriptors or heap. + @VisibleForTesting + void onCacheRemoval(final RemovalNotification removalNotification) { + // Only react to evictions (size cap or expiry). Explicit invalidations (e.g. on a normal + // disconnect) already close the connection through their own path. + if (!removalNotification.wasEvicted()) { + return; + } + final PeerConnection evictedConnection = removalNotification.getKey(); + final EthPeer peer = removalNotification.getValue(); + if (evictedConnection == null || evictedConnection.isDisconnected()) { + return; + } + + final boolean isCurrentConnectionOfPeer = + peer != null && evictedConnection.equals(peer.getConnection()); + if (isCurrentConnectionOfPeer && peer.statusHasBeenReceived()) { + // The peer completed (or is completing) the eth STATUS handshake and is being promoted to an + // active connection; its incomplete-cache entry is expiring naturally. Leave the live + // connection alone - it is (or will be) tracked in activeConnections. + return; + } + + // Either a superseded connection (the peer reconnected with a different connection), or a + // connection that completed the devp2p HELLO but never sent eth STATUS and has now been evicted + // (20s expiry, or pushed out of the bounded cache). Close the socket so evicted pre-STATUS + // connections cannot leak file descriptors or heap while remaining invisible to --max-peers. + DisconnectReason reason = + isCurrentConnectionOfPeer + ? DisconnectMessage.DisconnectReason.TIMEOUT + : DisconnectMessage.DisconnectReason.ALREADY_CONNECTED; + + LOG.atTrace() + .setMessage( + "Closing pre-STATUS connection {} evicted from incomplete-connection cache, reason {}") + .addArgument(evictedConnection::getPeerInfo) + .addArgument(reason) + .log(); + + evictedConnection.disconnect(reason); + } + + @VisibleForTesting + int incompleteConnectionCount() { + return (int) incompleteConnections.size(); + } + + boolean addPeerToEthPeers(final EthPeer peer) { + // We have a connection to a peer that is on the right chain and is willing to connect to us. + // Figure out whether we want to add it to the active connections. + final PeerConnection connection = peer.getConnection(); + if (activeConnections.containsValue(peer)) { + return false; + } + + final Bytes id = peer.getId(); + if (!randomPeerPriority) { + + if (peerCount() >= peerUpperBound) { + final long numSnapServers = numberOfSnapServers(); + final boolean inboundLimitExceeded = inboundInitiatedConnectionLimitExceeded(); + // three reasons why we would disconnect an existing peer to accommodate the new peer + if (canExceedPeerLimits(id) + || (snapServerPeersNeeded + && numSnapServers < snapServerTargetNumber + && peer.isServingSnap()) + || (inboundLimitExceeded && !peer.getConnection().inboundInitiated())) { + + final boolean filterOutSnapServers = + snapServerPeersNeeded && (numSnapServers <= snapServerTargetNumber); + + // find and disconnect the least useful peer we can disconnect + activeConnections.values().stream() + .filter(p -> !canExceedPeerLimits(p.getId())) + .map(EthPeerImmutableAttributes::from) + .filter(filterOutSnapServers ? p -> !p.isServingSnap() : p -> true) + .filter(inboundLimitExceeded ? p -> p.isInboundInitiated() : p -> true) + .min(MOST_USEFUL_PEER) + .ifPresentOrElse( + pe -> { + pe.ethPeer().disconnect(DisconnectMessage.DisconnectReason.TOO_MANY_PEERS); + LOG.atTrace() + .setMessage("Disconnecting peer {} to be replaced by prioritised peer {}") + .addArgument(pe.ethPeer().getLoggableId()) + .addArgument(peer.getLoggableId()) + .log(); + }, + () -> // disconnect the least useful peer + activeConnections.values().stream() + .filter(p -> !canExceedPeerLimits(p.getId())) + .map(EthPeerImmutableAttributes::from) + .min(MOST_USEFUL_PEER) + .ifPresent( + p -> { + p.ethPeer() + .disconnect(DisconnectMessage.DisconnectReason.TOO_MANY_PEERS); + LOG.atTrace() + .setMessage( + "Disconnecting peer {} to be replaced by prioritised peer {}") + .addArgument(p.ethPeer().getLoggableId()) + .addArgument(peer.getLoggableId()) + .log(); + })); + } else { + LOG.atTrace() + .setMessage( + "Too many peers. Disconnect peer {} with connection: {}, max connections {}") + .addArgument(peer.getLoggableId()) + .addArgument(connection) + .addArgument(peerUpperBound) + .log(); + connection.disconnect(DisconnectMessage.DisconnectReason.TOO_MANY_PEERS); + return false; + } + } + + final boolean added = (activeConnections.putIfAbsent(id, peer) == null); + if (added) { + LOG.atTrace() + .setMessage("Added peer {} with connection {} to activeConnections") + .addArgument(id) + .addArgument(connection) + .log(); + } else { + LOG.atTrace() + .setMessage("Did not add peer {} with connection {} to activeConnections") + .addArgument(id) + .addArgument(connection) + .log(); + } + return added; + + } else { + // randomPeerPriority! Add the peer and if there are too many connections fix it + // TODO: random peer priority does not care yet about snap server peers -> check later + activeConnections.putIfAbsent(id, peer); + enforceRemoteConnectionLimits(); + enforceConnectionLimits(); + return activeConnections.containsKey(id); + } + } + + private long getNumTrailingPeers(final long minimumHeightToBeUpToDate) { + return streamAvailablePeers() + .filter(p -> p.estimatedChainHeight() < minimumHeightToBeUpToDate) + .count(); + } + + private boolean needMoreSnapServers() { + return snapServerPeersNeeded && numberOfSnapServers() < snapServerTargetNumber; + } + + private long numberOfSnapServers() { + return activeConnections.values().stream().filter(EthPeer::isServingSnap).count(); + } +} diff --git a/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/BlockPropagationManager.java b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/BlockPropagationManager.java new file mode 100644 index 0000000..dbcc8fb --- /dev/null +++ b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/BlockPropagationManager.java @@ -0,0 +1,870 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0008-aere-pq-aip22.patch. + */ +package org.hyperledger.besu.ethereum.eth.sync; + +import static org.hyperledger.besu.util.FutureUtils.exceptionallyCompose; + +import org.hyperledger.besu.consensus.merge.ForkchoiceEvent; +import org.hyperledger.besu.consensus.merge.UnverifiedForkchoiceListener; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.chain.BadBlockCause; +import org.hyperledger.besu.ethereum.chain.BadBlockManager; +import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthMessage; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.peertask.InvalidPeerTaskResponseException; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResponseCode; +import org.hyperledger.besu.ethereum.eth.manager.peertask.PeerTaskExecutorResult; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetBodiesFromPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetHeadersFromPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.peertask.task.GetHeadersFromPeerTask.Direction; +import org.hyperledger.besu.ethereum.eth.messages.EthProtocolMessages; +import org.hyperledger.besu.ethereum.eth.messages.NewBlockHashesMessage; +import org.hyperledger.besu.ethereum.eth.messages.NewBlockHashesMessage.NewBlockHash; +import org.hyperledger.besu.ethereum.eth.messages.NewBlockMessage; +import org.hyperledger.besu.ethereum.eth.sync.state.PendingBlocksManager; +import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.ethereum.eth.sync.tasks.PersistBlockTask; +import org.hyperledger.besu.ethereum.mainnet.BlockHeaderValidator; +import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; +import org.hyperledger.besu.ethereum.rlp.RLPException; +import org.hyperledger.besu.plugin.data.AddedBlockContext.EventType; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.collect.Range; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BlockPropagationManager implements UnverifiedForkchoiceListener { + private static final Logger LOG = LoggerFactory.getLogger(BlockPropagationManager.class); + private static final long TIMEOUT = Duration.ofSeconds(60L).toMillis(); + private final SynchronizerConfiguration config; + private final ProtocolSchedule protocolSchedule; + private final ProtocolContext protocolContext; + private final EthContext ethContext; + private final SyncState syncState; + private final MetricsSystem metricsSystem; + private final BlockBroadcaster blockBroadcaster; + + private final AtomicBoolean started = new AtomicBoolean(false); + private final ProcessingBlocksManager processingBlocksManager; + private final PendingBlocksManager pendingBlocksManager; + private Optional onBlockAddedSId = Optional.empty(); + private Optional newBlockSId; + private Optional newBlockHashesSId; + + BlockPropagationManager( + final SynchronizerConfiguration config, + final ProtocolSchedule protocolSchedule, + final ProtocolContext protocolContext, + final EthContext ethContext, + final SyncState syncState, + final PendingBlocksManager pendingBlocksManager, + final MetricsSystem metricsSystem, + final BlockBroadcaster blockBroadcaster) { + this( + config, + protocolSchedule, + protocolContext, + ethContext, + syncState, + pendingBlocksManager, + metricsSystem, + blockBroadcaster, + new ProcessingBlocksManager()); + } + + BlockPropagationManager( + final SynchronizerConfiguration config, + final ProtocolSchedule protocolSchedule, + final ProtocolContext protocolContext, + final EthContext ethContext, + final SyncState syncState, + final PendingBlocksManager pendingBlocksManager, + final MetricsSystem metricsSystem, + final BlockBroadcaster blockBroadcaster, + final ProcessingBlocksManager processingBlocksManager) { + this.config = config; + this.protocolSchedule = protocolSchedule; + this.protocolContext = protocolContext; + this.ethContext = ethContext; + this.metricsSystem = metricsSystem; + this.blockBroadcaster = blockBroadcaster; + this.syncState = syncState; + this.pendingBlocksManager = pendingBlocksManager; + this.syncState.subscribeTTDReached(this::reactToTTDReachedEvent); + this.processingBlocksManager = processingBlocksManager; + } + + public void start() { + if (started.compareAndSet(false, true)) { + setupListeners(); + } else { + throw new IllegalStateException( + "Attempt to start an already started " + this.getClass().getSimpleName() + "."); + } + } + + public void stop() { + if (started.get()) { + clearListeners(); + started.set(false); + } else { + LOG.debug("Attempted to stop when we are not even running..."); + } + } + + public boolean isRunning() { + return started.get(); + } + + private void setupListeners() { + onBlockAddedSId = + Optional.of(protocolContext.getBlockchain().observeBlockAdded(this::onBlockAdded)); + newBlockSId = + Optional.of( + ethContext + .getEthMessages() + .subscribe(EthProtocolMessages.NEW_BLOCK, this::handleNewBlockFromNetwork)); + newBlockHashesSId = + Optional.of( + ethContext + .getEthMessages() + .subscribe( + EthProtocolMessages.NEW_BLOCK_HASHES, this::handleNewBlockHashesFromNetwork)); + } + + private void clearListeners() { + onBlockAddedSId.ifPresent(id -> protocolContext.getBlockchain().removeObserver(id)); + newBlockSId.ifPresent( + id -> ethContext.getEthMessages().unsubscribe(id, EthProtocolMessages.NEW_BLOCK)); + newBlockHashesSId.ifPresent( + id -> ethContext.getEthMessages().unsubscribe(id, EthProtocolMessages.NEW_BLOCK_HASHES)); + onBlockAddedSId = Optional.empty(); + newBlockSId = Optional.empty(); + newBlockHashesSId = Optional.empty(); + } + + private void onBlockAdded(final BlockAddedEvent blockAddedEvent) { + // Check to see if any of our pending blocks are now ready for import + LOG.atTrace() + .setMessage("Block added event type {} for block {}. Current status {}") + .addArgument(blockAddedEvent::getEventType) + .addArgument(blockAddedEvent.getHeader()::toLogString) + .addArgument(this) + .log(); + + // If there is no children to process, maybe try non announced blocks + if (!maybeProcessPendingChildrenBlocks(blockAddedEvent.getHeader())) { + LOG.atTrace() + .setMessage("There are no pending blocks ready to import for block {}") + .addArgument(blockAddedEvent.getHeader()::toLogString) + .log(); + maybeProcessNonAnnouncedBlocks(blockAddedEvent); + } + + if (blockAddedEvent.getEventType().equals(EventType.HEAD_ADVANCED)) { + final long head = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long cutoff = head + config.getBlockPropagationRange().lowerEndpoint(); + pendingBlocksManager.purgeBlocksOlderThan(cutoff); + } + } + + /** + * Process pending Children if any + * + * @param header the header of the block to process the children of + * @return true if block has any pending child + */ + private boolean maybeProcessPendingChildrenBlocks(final BlockHeader header) { + final List readyForImport; + synchronized (pendingBlocksManager) { + // Remove block from pendingBlocks list + pendingBlocksManager.deregisterPendingBlock(header); + + // Import any pending blocks that are children of the newly added block + readyForImport = pendingBlocksManager.childrenOf(header.getHash()); + } + + if (!readyForImport.isEmpty()) { + + LOG.atTrace() + .setMessage("Ready to import pending blocks found [{}] for block {}") + .addArgument( + () -> + readyForImport.stream().map(Block::toLogString).collect(Collectors.joining(", "))) + .addArgument(header::toLogString) + .log(); + + final Supplier>> importBlocksTask = + PersistBlockTask.forUnorderedBlocks( + protocolSchedule, + protocolContext, + ethContext, + readyForImport, + HeaderValidationMode.FULL, + metricsSystem); + ethContext + .getScheduler() + .scheduleSyncWorkerTask(importBlocksTask) + .whenComplete( + (r, t) -> { + if (r != null) { + LOG.info( + "Imported {} pending blocks: {}", + r.size(), + r.stream().map(b -> b.getHeader().getNumber()).collect(Collectors.toList())); + } + if (t != null) { + LOG.error("Error importing pending blocks", t); + } + }); + } + return !readyForImport.isEmpty(); + } + + private void maybeProcessNonAnnouncedBlocks(final BlockAddedEvent newBlock) { + final long localHeadBlockNumber = protocolContext.getBlockchain().getChainHeadBlockNumber(); + + if (newBlock.getHeader().getNumber() > localHeadBlockNumber) { + pendingBlocksManager + .lowestAnnouncedBlock() + .map(ProcessableBlockHeader::getNumber) + .ifPresent( + minAnnouncedBlockNumber -> { + final long distance = minAnnouncedBlockNumber - localHeadBlockNumber; + LOG.trace( + "Found lowest announced block {} with distance {}", + minAnnouncedBlockNumber, + distance); + + final long firstNonAnnouncedBlockNumber = newBlock.getHeader().getNumber() + 1; + + if (distance < config.getBlockPropagationRange().upperEndpoint() + && minAnnouncedBlockNumber > firstNonAnnouncedBlockNumber) { + + if (processingBlocksManager.addNonAnnouncedBlocks(firstNonAnnouncedBlockNumber)) { + retrieveNonAnnouncedBlock(firstNonAnnouncedBlockNumber); + } + } + }); + } + } + + private void handleNewBlockFromNetwork(final EthMessage message) { + final Blockchain blockchain = protocolContext.getBlockchain(); + final NewBlockMessage newBlockMessage = NewBlockMessage.readFrom(message.getData()); + try { + final Block block = newBlockMessage.block(protocolSchedule); + LOG.atTrace() + .setMessage("New block from network {} from peer {}. Current status {}") + .addArgument(block::toLogString) + .addArgument(message::getPeer) + .addArgument(this) + .log(); + + final Difficulty totalDifficulty = newBlockMessage.totalDifficulty(protocolSchedule); + + message.getPeer().chainState().updateForAnnouncedBlock(block.getHeader(), totalDifficulty); + + // Return early if we don't care about this block + final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + if (!shouldImportBlockAtHeight( + block.getHeader().getNumber(), localChainHeight, bestChainHeight)) { + LOG.atTrace() + .setMessage( + "Do not import new block from network {}, current chain heights are: local {}, best {}") + .addArgument(block::toLogString) + .addArgument(localChainHeight) + .addArgument(bestChainHeight) + .log(); + return; + } + if (pendingBlocksManager.contains(block.getHash())) { + LOG.atTrace() + .setMessage("New block from network {} is already pending") + .addArgument(block::toLogString) + .log(); + return; + } + if (blockchain.contains(block.getHash())) { + LOG.atTrace() + .setMessage("New block from network {} is already present") + .addArgument(block::toLogString) + .log(); + return; + } + + importOrSavePendingBlock(block, message.getPeer().nodeId()); + } catch (final RLPException e) { + LOG.debug( + "Malformed NEW_BLOCK message received from peer (BREACH_OF_PROTOCOL), disconnecting: {}", + message.getPeer(), + e); + message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL_MALFORMED_MESSAGE_RECEIVED); + } + } + + private void handleNewBlockHashesFromNetwork(final EthMessage message) { + final Blockchain blockchain = protocolContext.getBlockchain(); + final NewBlockHashesMessage newBlockHashesMessage = + NewBlockHashesMessage.readFrom(message.getData()); + try { + // Register announced blocks + final List announcedBlocks = + Lists.newArrayList(newBlockHashesMessage.getNewHashes()); + LOG.atTrace() + .setMessage("New block hashes from network {} from peer {}. Current status {}") + .addArgument(() -> toLogString(announcedBlocks)) + .addArgument(message::getPeer) + .addArgument(this) + .log(); + + for (final NewBlockHash announcedBlock : announcedBlocks) { + message.getPeer().registerKnownBlock(announcedBlock.hash()); + message.getPeer().registerHeight(announcedBlock.hash(), announcedBlock.number()); + } + + // Filter announced blocks for blocks we care to import + final long localChainHeight = protocolContext.getBlockchain().getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + final List relevantAnnouncements = + announcedBlocks.stream() + .filter(a -> shouldImportBlockAtHeight(a.number(), localChainHeight, bestChainHeight)) + .collect(Collectors.toList()); + + // Filter for blocks we don't yet know about + final List newBlocks = new ArrayList<>(); + for (final NewBlockHash announcedBlock : relevantAnnouncements) { + if (pendingBlocksManager.contains(announcedBlock.hash())) { + LOG.trace("New block hash from network {} is already pending", announcedBlock); + continue; + } + if (processingBlocksManager.alreadyImporting(announcedBlock.hash())) { + LOG.trace("New block hash from network {} is already importing", announcedBlock); + continue; + } + if (blockchain.contains(announcedBlock.hash())) { + LOG.trace("New block hash from network {} was already imported", announcedBlock); + continue; + } + if (processingBlocksManager.addRequestedBlock(announcedBlock.hash())) { + newBlocks.add(announcedBlock); + } else { + LOG.trace("New block hash from network {} was already requested", announcedBlock); + } + } + + // Process known blocks we care about + for (final NewBlockHash newBlock : newBlocks) { + processAnnouncedBlock(message.getPeer(), newBlock); + } + } catch (final RLPException e) { + LOG.debug( + "Malformed NEW_BLOCK_HASHES message received from peer (BREACH_OF_PROTOCOL), disconnecting: {}", + message.getPeer(), + e); + message.getPeer().disconnect(DisconnectReason.BREACH_OF_PROTOCOL_MALFORMED_MESSAGE_RECEIVED); + } + } + + private CompletableFuture retrieveNonAnnouncedBlock(final long blockNumber) { + LOG.trace("Retrieve non announced block {} from peers", blockNumber); + return getBlockFromPeers(Optional.empty(), blockNumber, Optional.empty()); + } + + private CompletableFuture processAnnouncedBlock( + final EthPeer peer, final NewBlockHash blockHash) { + LOG.trace("Retrieve announced block by header {} from peers", blockHash); + return getBlockFromPeers(Optional.of(peer), blockHash.number(), Optional.of(blockHash.hash())); + } + + private void requestParentBlock(final Block block) { + final BlockHeader blockHeader = block.getHeader(); + if (processingBlocksManager.addRequestedBlock(blockHeader.getParentHash())) { + retrieveParentBlock(blockHeader); + } else { + LOG.debug("Parent block with hash {} is already requested", blockHeader.getParentHash()); + } + } + + private CompletableFuture retrieveParentBlock(final BlockHeader blockHeader) { + final long targetParentBlockNumber = blockHeader.getNumber() - 1L; + final Hash targetParentBlockHash = blockHeader.getParentHash(); + LOG.info("Retrieving parent {} of block {}", targetParentBlockHash, blockHeader.toLogString()); + return getBlockFromPeers( + Optional.empty(), targetParentBlockNumber, Optional.of(targetParentBlockHash)); + } + + private CompletableFuture getBlockFromPeers( + final Optional preferredPeer, + final long blockNumber, + final Optional maybeBlockHash) { + return repeatableGetBlockFromPeer(preferredPeer, blockNumber, maybeBlockHash) + .whenComplete( + (block, throwable) -> { + if (block != null) { + LOG.atDebug() + .setMessage("Successfully retrieved block {}") + .addArgument(block::toLogString) + .log(); + processingBlocksManager.registerReceivedBlock(block); + } else { + if (throwable != null) { + LOG.warn( + "Failed to retrieve block " + + logBlockNumberMaybeHash(blockNumber, maybeBlockHash), + throwable); + } else { + // this could happen if we give up at some point since we find that it make no + // sense to retry + LOG.atDebug() + .setMessage("Block {} not retrieved") + .addArgument(() -> logBlockNumberMaybeHash(blockNumber, maybeBlockHash)) + .log(); + } + processingBlocksManager.registerFailedGetBlock(blockNumber, maybeBlockHash); + } + }); + } + + private CompletableFuture repeatableGetBlockFromPeer( + final Optional preferredPeer, + final long blockNumber, + final Optional maybeBlockHash) { + return exceptionallyCompose( + scheduleGetBlockFromPeers(preferredPeer, blockNumber, maybeBlockHash), + handleGetBlockErrors(blockNumber, maybeBlockHash)) + .thenCompose(r -> maybeRepeatGetBlock(blockNumber, maybeBlockHash)); + } + + private Function> handleGetBlockErrors( + final long blockNumber, final Optional maybeBlockHash) { + return throwable -> { + LOG.atDebug() + .setMessage("Temporary failure retrieving block {} from peers with error {}") + .addArgument(() -> logBlockNumberMaybeHash(blockNumber, maybeBlockHash)) + .addArgument(throwable) + .log(); + return CompletableFuture.completedFuture(null); + }; + } + + private CompletableFuture maybeRepeatGetBlock( + final long blockNumber, final Optional maybeBlockHash) { + final MutableBlockchain blockchain = protocolContext.getBlockchain(); + final Optional maybeBlock = + maybeBlockHash + .map(hash -> blockchain.getBlockByHash(hash)) + .orElseGet(() -> blockchain.getBlockByNumber(blockNumber)); + + // check if we got this block by other means + if (maybeBlock.isPresent()) { + final Block block = maybeBlock.get(); + LOG.atDebug() + .setMessage("No need to retry to get block {} since it is already present") + .addArgument(block::toLogString) + .log(); + return CompletableFuture.completedFuture(block); + } + + final long localChainHeight = blockchain.getChainHeadBlockNumber(); + final long bestChainHeight = syncState.bestChainHeight(localChainHeight); + if (!shouldImportBlockAtHeight(blockNumber, localChainHeight, bestChainHeight)) { + LOG.atDebug() + .setMessage("Not retrying to get block {} since we are too far from local chain head {}") + .addArgument(() -> logBlockNumberMaybeHash(blockNumber, maybeBlockHash)) + .addArgument(blockchain.getChainHead()::toLogString) + .log(); + return CompletableFuture.completedFuture(null); + } + + LOG.atDebug() + .setMessage("Retrying to get block {}") + .addArgument(() -> logBlockNumberMaybeHash(blockNumber, maybeBlockHash)) + .log(); + + return ethContext + .getScheduler() + .scheduleSyncWorkerTask( + () -> repeatableGetBlockFromPeer(Optional.empty(), blockNumber, maybeBlockHash)); + } + + private CompletableFuture scheduleGetBlockFromPeers( + final Optional maybePreferredPeer, + final long blockNumber, + final Optional maybeBlockHash) { + + return ethContext + .getScheduler() + .scheduleServiceTask(() -> getBlockHeader(maybePreferredPeer, blockNumber, maybeBlockHash)) + .thenApply((blockHeader) -> getBlock(maybePreferredPeer, blockHeader)) + .thenCompose( + blockExecutorResult -> + importOrSavePendingBlock( + blockExecutorResult.result().get().getFirst(), + blockExecutorResult.ethPeers().getLast().nodeId())) + .orTimeout(TIMEOUT, TimeUnit.MILLISECONDS); + } + + private CompletableFuture getBlockHeader( + final Optional maybePreferredPeer, + final long blockNumber, + final Optional maybeBlockHash) { + GetHeadersFromPeerTask headersFromPeerTask = + new GetHeadersFromPeerTask( + maybeBlockHash.orElse(null), + blockNumber, + 1, + 0, + Direction.FORWARD, + Math.max(1, ethContext.getEthPeers().peerCount()), + protocolSchedule); + if (maybePreferredPeer.isPresent()) { + PeerTaskExecutorResult> executorResult = + ethContext + .getPeerTaskExecutor() + .executeAgainstPeer(headersFromPeerTask, maybePreferredPeer.get()); + if (executorResult.result().isPresent() + && executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { + return CompletableFuture.completedFuture(executorResult.result().get().getFirst()); + } + } + PeerTaskExecutorResult> executorResult = + ethContext.getPeerTaskExecutor().execute(headersFromPeerTask); + if (executorResult.result().isEmpty() + || executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS) { + return CompletableFuture.failedFuture(new InvalidPeerTaskResponseException()); + } else { + return CompletableFuture.completedFuture(executorResult.result().get().getFirst()); + } + } + + private PeerTaskExecutorResult> getBlock( + final Optional maybePreferredPeer, final BlockHeader blockHeader) { + GetBodiesFromPeerTask bodiesTask = + new GetBodiesFromPeerTask(List.of(blockHeader), protocolSchedule); + if (maybePreferredPeer.isPresent()) { + PeerTaskExecutorResult> executorResult = + ethContext.getPeerTaskExecutor().executeAgainstPeer(bodiesTask, maybePreferredPeer.get()); + if (executorResult.result().isPresent() + && executorResult.responseCode() == PeerTaskExecutorResponseCode.SUCCESS) { + return executorResult; + } + } + PeerTaskExecutorResult> executorResult = + ethContext.getPeerTaskExecutor().execute(bodiesTask); + if (executorResult.result().isEmpty() + || executorResult.responseCode() != PeerTaskExecutorResponseCode.SUCCESS) { + throw new RuntimeException(new InvalidPeerTaskResponseException()); + } else { + return executorResult; + } + } + + private void broadcastBlock(final Block block, final BlockHeader parent) { + final Difficulty totalDifficulty = + protocolContext + .getBlockchain() + .getTotalDifficultyByHash(parent.getHash()) + .get() + .add(block.getHeader().getDifficulty()); + blockBroadcaster.propagate(block, totalDifficulty); + } + + @VisibleForTesting + CompletableFuture importOrSavePendingBlock(final Block block, final Bytes nodeId) { + // Synchronize to avoid race condition where block import event fires after the + // blockchain.contains() check and before the block is registered, causing onBlockAdded() to be + // invoked for the parent of this block before we are able to register it. + LOG.atTrace() + .setMessage("Import or save pending block {}") + .addArgument(block::toLogString) + .log(); + + if (!protocolContext.getBlockchain().contains(block.getHeader().getParentHash())) { + // Block isn't connected to local chain, save it to pending blocks collection + if (savePendingBlock(block, nodeId)) { + // if block is saved as pending, try to resolve it + maybeProcessPendingBlocks(block); + } + return CompletableFuture.completedFuture(block); + } + + if (!processingBlocksManager.addImportingBlock(block.getHash())) { + LOG.atTrace() + .setMessage("We're already importing this block {}") + .addArgument(block::toLogString) + .log(); + return CompletableFuture.completedFuture(block); + } + + if (protocolContext.getBlockchain().contains(block.getHash())) { + LOG.atTrace() + .setMessage("We've already imported this block {}") + .addArgument(block::toLogString) + .log(); + processingBlocksManager.registerBlockImportDone(block.getHash()); + return CompletableFuture.completedFuture(block); + } + + final BlockHeader parent = + protocolContext + .getBlockchain() + .getBlockHeader(block.getHeader().getParentHash()) + .orElseThrow( + () -> + new IllegalArgumentException( + "Incapable of retrieving header from non-existent parent of " + + block.toLogString())); + final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader()); + final BlockHeaderValidator blockHeaderValidator = protocolSpec.getBlockHeaderValidator(); + final BadBlockManager badBlockManager = protocolContext.getBadBlockManager(); + return ethContext + .getScheduler() + .scheduleSyncWorkerTask( + () -> + validateAndProcessPendingBlock( + blockHeaderValidator, block, parent, badBlockManager)); + } + + /** + * Save the given block. + * + * @param block the block to track + * @param nodeId node that sent the block + * @return true if the block was added (was not previously present) + */ + private boolean savePendingBlock(final Block block, final Bytes nodeId) { + synchronized (pendingBlocksManager) { + if (pendingBlocksManager.registerPendingBlock(block, nodeId)) { + LOG.info( + "Saved announced block for future import {} - {} saved block(s)", + block.toLogString(), + pendingBlocksManager.size()); + return true; + } + return false; + } + } + + /** + * Try to request the lowest ancestor for the given pending block or process the descendants if + * the ancestor is already in the chain + */ + private void maybeProcessPendingBlocks(final Block block) { + // Try to get the lowest ancestor pending for this block, so we can import it + final Optional lowestPending = pendingBlocksManager.pendingAncestorBlockOf(block); + if (lowestPending.isPresent()) { + final Block lowestPendingBlock = lowestPending.get(); + // If the parent of the lowest ancestor is not in the chain, request it. + if (!protocolContext + .getBlockchain() + .contains(lowestPendingBlock.getHeader().getParentHash())) { + requestParentBlock(lowestPendingBlock); + } else { + LOG.trace("Parent block is already in the chain"); + // if the parent is already imported, process its children + maybeProcessPendingChildrenBlocks(lowestPendingBlock.getHeader()); + } + } + } + + private CompletableFuture validateAndProcessPendingBlock( + final BlockHeaderValidator blockHeaderValidator, + final Block block, + final BlockHeader parent, + final BadBlockManager badBlockManager) { + final HeaderValidationMode validationMode = HeaderValidationMode.FULL; + if (blockHeaderValidator.validateHeader( + block.getHeader(), parent, protocolContext, validationMode)) { + // AERE D-412 (2026-09-26), Rule C: a valid header is not, from AIP-22 stage 5, a proof that a + // quorum agreed the block (it carries only the proposer's single Falcon seal). A validator does + // not ADOPT a propagated block at the tip of its own consensus height without a post-quantum + // commit quorum for it; the gate is inert below H_hdr and for non-validators, and the sync + // downloader (a different path) is untouched, so this is fork choice, not validation. + if (!PropagatedBlockAdoptionGate.mayAdopt(block.getHeader(), parent)) { + processingBlocksManager.registerBlockImportDone(block.getHash()); + LOG.info( + "AERE D-412: not adopting propagated block {} at the tip without a post-quantum commit " + + "quorum; this node's own consensus decides this height", + block.toLogString()); + return CompletableFuture.completedFuture(block); + } + ethContext.getScheduler().scheduleSyncWorkerTask(() -> broadcastBlock(block, parent)); + return runImportTask(block); + } else { + processingBlocksManager.registerBlockImportDone(block.getHash()); + final String description = String.format("Failed header validation (%s)", validationMode); + badBlockManager.addBadBlock(block, BadBlockCause.fromValidationFailure(description)); + LOG.warn( + "Added to bad block manager for invalid header, failed to import announced block {}", + block.toLogString()); + return CompletableFuture.completedFuture(block); + } + } + + private CompletableFuture runImportTask(final Block block) { + final PersistBlockTask importTask = + PersistBlockTask.create( + protocolSchedule, + protocolContext, + ethContext, + block, + HeaderValidationMode.NONE, + metricsSystem); + return importTask + .run() + .whenComplete( + (result, throwable) -> { + processingBlocksManager.registerBlockImportDone(block.getHash()); + if (throwable != null) { + LOG.warn("Failed to import announced block {}", block.toLogString()); + } + }); + } + + // Only import blocks within a certain range of our head and sync target + private boolean shouldImportBlockAtHeight( + final long blockNumber, final long localHeight, final long bestChainHeight) { + final long distanceFromLocalHead = blockNumber - localHeight; + final long distanceFromBestPeer = blockNumber - bestChainHeight; + final Range importRange = config.getBlockPropagationRange(); + return importRange.contains(distanceFromLocalHead) + && importRange.contains(distanceFromBestPeer); + } + + private String toLogString(final Collection newBlockHashs) { + return newBlockHashs.stream() + .map(NewBlockHash::toString) + .collect(Collectors.joining(", ", "[", "]")); + } + + private void reactToTTDReachedEvent(final boolean ttdReached) { + if (started.get() && ttdReached) { + LOG.info("Block propagation was running, then ttd reached"); + } else if (!started.get()) { + start(); + } + } + + @Override + public String toString() { + return "BlockPropagationManager{" + + processingBlocksManager + + ", pendingBlocksManager=" + + pendingBlocksManager + + '}'; + } + + private String logBlockNumberMaybeHash( + final long blockNumber, final Optional maybeBlockHash) { + return blockNumber + maybeBlockHash.map(h -> " (" + h + ")").orElse(""); + } + + @Override + public void onNewUnverifiedForkchoice(final ForkchoiceEvent event) { + if (event.hasValidFinalizedBlockHash()) { + stop(); + } + } + + static class ProcessingBlocksManager { + private final Set importingBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set requestedBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set requestedNonAnnouncedBlocks = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + + boolean addRequestedBlock(final Hash hash) { + return requestedBlocks.add(hash); + } + + public boolean addNonAnnouncedBlocks(final long blockNumber) { + return requestedNonAnnouncedBlocks.add(blockNumber); + } + + public boolean alreadyImporting(final Hash hash) { + return importingBlocks.contains(hash); + } + + public synchronized void registerReceivedBlock(final Block block) { + requestedBlocks.remove(block.getHash()); + requestedNonAnnouncedBlocks.remove(block.getHeader().getNumber()); + } + + public synchronized void registerFailedGetBlock( + final long blockNumber, final Optional maybeBlockHash) { + requestedNonAnnouncedBlocks.remove(blockNumber); + maybeBlockHash.ifPresent(requestedBlocks::remove); + } + + public boolean addImportingBlock(final Hash hash) { + return importingBlocks.add(hash); + } + + public void registerBlockImportDone(final Hash hash) { + importingBlocks.remove(hash); + } + + @Override + public synchronized String toString() { + return "ProcessingBlocksManager{" + + "importingBlocks=" + + importingBlocks + + ", requestedBlocks=" + + requestedBlocks + + ", requestedNonAnnouncedBlocks=" + + requestedNonAnnouncedBlocks + + '}'; + } + } +} diff --git a/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncState.java b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncState.java new file mode 100644 index 0000000..9d5cdad --- /dev/null +++ b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncState.java @@ -0,0 +1,372 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-syncstate-in-sync-lock.patch. + */ +package org.hyperledger.besu.ethereum.eth.sync.state; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.chain.ChainHead; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.DefaultSyncStatus; +import org.hyperledger.besu.ethereum.core.Synchronizer; +import org.hyperledger.besu.ethereum.core.Synchronizer.InSyncListener; +import org.hyperledger.besu.ethereum.eth.manager.ChainHeadEstimate; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthPeers; +import org.hyperledger.besu.ethereum.eth.sync.common.checkpoint.Checkpoint; +import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadStatus; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; +import org.hyperledger.besu.plugin.data.SyncStatus; +import org.hyperledger.besu.plugin.services.BesuEvents.InitialSyncCompletionListener; +import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener; +import org.hyperledger.besu.plugin.services.BesuEvents.TTDReachedListener; +import org.hyperledger.besu.util.Subscribers; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +public class SyncState { + + private final Blockchain blockchain; + private final EthPeers ethPeers; + + // Ensures checkInSync() re-evaluation gives a consistent view of sync status. A + // standalone lock is used instead of the object monitor to prevent checkInSync() + // causing a deadlock while synchronized on the object monitor. + // (AERE D-426: upstream hyperledger/besu#11041, ported) + private final Object inSyncLock = new Object(); + + private final AtomicLong inSyncSubscriberId = new AtomicLong(); + private final Map inSyncTrackers = new ConcurrentHashMap<>(); + private final Subscribers syncStatusListeners = Subscribers.create(); + private final Subscribers ttdReachedListeners = Subscribers.create(); + + private final Subscribers completionListenerSubscribers = + Subscribers.create(); + + private volatile long chainHeightListenerId; + private volatile Optional syncTarget = Optional.empty(); + private Optional worldStateDownloadStatus = Optional.empty(); + private Optional newPeerListenerId; + private Optional reachedTerminalDifficulty = Optional.empty(); + private final Optional checkpoint; + private volatile boolean isInitialSyncPhaseDone; + + private volatile boolean isResyncNeeded; + + private Optional

maybeAccountToRepair = Optional.empty(); + + public SyncState(final Blockchain blockchain, final EthPeers ethPeers) { + this(blockchain, ethPeers, false, Optional.empty()); + } + + public SyncState( + final Blockchain blockchain, + final EthPeers ethPeers, + final boolean hasInitialSyncPhase, + final Optional checkpoint) { + this.blockchain = blockchain; + this.ethPeers = ethPeers; + isInitialSyncPhaseDone = !hasInitialSyncPhase; + + blockchain.observeBlockAdded( + event -> { + if (event.isNewCanonicalHead()) { + checkInSync(); + } + }); + + // Add new peer listener to prevent permissioned PoA network stalling on start-up. + // https://github.com/hyperledger/besu/issues/528 + newPeerListenerId = + Optional.of( + ethPeers.subscribeConnect( + newPeer -> { + if (newPeer.readyForRequests()) { + checkInSync(); + } + })); + this.checkpoint = checkpoint; + } + + /** + * Add a listener that will be notified when this node's sync status changes. A node is considered + * in-sync if the local chain height is no more than {@code SYNC_TOLERANCE} behind the highest + * estimated remote chain height. + * + * @param listener The callback to invoke when the sync status changes + * @return An {@code Unsubscriber} that can be used to stop listening for these events + */ + public long subscribeInSync(final InSyncListener listener) { + return subscribeInSync(listener, Synchronizer.DEFAULT_IN_SYNC_TOLERANCE); + } + + /** + * Add a listener that will be notified when this node's sync status changes. A node is considered + * in-sync if the local chain height is no more than {@code syncTolerance} behind the highest + * estimated remote chain height. + * + * @param listener The callback to invoke when the sync status changes + * @param syncTolerance The tolerance used to determine whether this node is in-sync. A value of + * zero means that the node is considered in-sync only when the local chain height is greater + * than or equal to the best estimated remote chain height. + * @return An {@code Unsubscriber} that can be used to stop listening for these events + */ + public long subscribeInSync(final InSyncListener listener, final long syncTolerance) { + final InSyncTracker inSyncTracker = InSyncTracker.create(listener, syncTolerance); + final long id = inSyncSubscriberId.incrementAndGet(); + inSyncTrackers.put(id, inSyncTracker); + + return id; + } + + public boolean unsubscribeInSync(final long subscriberId) { + return inSyncTrackers.remove(subscriberId) != null; + } + + public long subscribeSyncStatus(final SyncStatusListener listener) { + return syncStatusListeners.subscribe(listener); + } + + public long subscribeTTDReached(final TTDReachedListener listener) { + return ttdReachedListeners.subscribe(listener); + } + + public long subscribeCompletionReached(final InitialSyncCompletionListener listener) { + return completionListenerSubscribers.subscribe(listener); + } + + public boolean unsubscribeSyncStatus(final long listenerId) { + return syncStatusListeners.unsubscribe(listenerId); + } + + public boolean unsubscribeTTDReached(final long listenerId) { + return ttdReachedListeners.unsubscribe(listenerId); + } + + public boolean unsubscribeInitialConditionReached(final long listenerId) { + return completionListenerSubscribers.unsubscribe(listenerId); + } + + public Optional syncStatus() { + return syncStatus(syncTarget); + } + + public Optional syncTarget() { + return syncTarget; + } + + public void setSyncTarget(final EthPeer peer, final BlockHeader commonAncestor) { + final SyncTarget syncTarget = new SyncTarget(peer, commonAncestor); + replaceSyncTarget(Optional.of(syncTarget)); + } + + public void setSyncProgress( + final long startingBlock, final long currentBlock, final long highestBlock) { + final SyncStatus status = + new DefaultSyncStatus( + startingBlock, currentBlock, highestBlock, Optional.empty(), Optional.empty()); + syncStatusListeners.forEach(c -> c.onSyncStatusChanged(Optional.of(status))); + } + + public void setWorldStateDownloadStatus(final WorldStateDownloadStatus worldStateDownloadStatus) { + this.worldStateDownloadStatus = Optional.ofNullable(worldStateDownloadStatus); + } + + public boolean isInSync() { + return isInSync(Synchronizer.DEFAULT_IN_SYNC_TOLERANCE); + } + + public boolean isInSync(final long syncTolerance) { + return isInSync( + getLocalChainHead(), getSyncTargetChainHead(), getBestPeerChainHead(), syncTolerance); + } + + public void setReachedTerminalDifficulty(final boolean stoppedAtTerminalDifficulty) { + this.reachedTerminalDifficulty = Optional.of(stoppedAtTerminalDifficulty); + ttdReachedListeners.forEach(listener -> listener.onTTDReached(stoppedAtTerminalDifficulty)); + } + + public Optional hasReachedTerminalDifficulty() { + if (isInitialSyncPhaseDone) { + return reachedTerminalDifficulty; + } + return Optional.of(Boolean.FALSE); + } + + private boolean isInSync( + final ChainHead localChain, + final Optional syncTargetChain, + final Optional bestPeerChain, + final long syncTolerance) { + return isInitialSyncPhaseDone + && reachedTerminalDifficulty.orElse(true) + // Sync target may be temporarily empty while we switch sync targets during a sync, so + // check both the sync target and our best peer to determine if we're in sync or not + && isInSync(localChain, syncTargetChain, syncTolerance) + && isInSync(localChain, bestPeerChain, syncTolerance); + } + + private boolean isInSync( + final ChainHead localChain, + final Optional remoteChain, + final long syncTolerance) { + return remoteChain + .map(remoteState -> InSyncTracker.isInSync(localChain, remoteState, syncTolerance)) + .orElse(true); + } + + private ChainHead getLocalChainHead() { + return blockchain.getChainHead(); + } + + private Optional getSyncTargetChainHead() { + return syncTarget.map(SyncTarget::peer).map(EthPeer::chainStateSnapshot); + } + + public Optional getBestPeerChainHead() { + return ethPeers.bestPeerWithHeightEstimate().map(EthPeer::chainStateSnapshot); + } + + public void disconnectSyncTarget(final DisconnectReason reason) { + syncTarget.ifPresent(syncTarget -> syncTarget.peer().disconnect(reason)); + } + + public void clearSyncTarget() { + replaceSyncTarget(Optional.empty()); + } + + private synchronized void replaceSyncTarget(final Optional newTarget) { + if (syncTarget.equals(newTarget)) { + // Nothing to do + return; + } + syncTarget.ifPresent(this::removeEstimatedHeightListener); + syncTarget = newTarget; + newTarget.ifPresent(this::addEstimatedHeightListener); + publishSyncStatus(newTarget); + checkInSync(); + } + + private void publishSyncStatus(final Optional newTarget) { + final Optional syncStatus = syncStatus(newTarget); + syncStatusListeners.forEach(c -> c.onSyncStatusChanged(syncStatus)); + } + + private Optional syncStatus(final Optional maybeTarget) { + return maybeTarget.map( + target -> { + final long chainHeadBlockNumber = blockchain.getChainHeadBlockNumber(); + final long commonAncestor = target.commonAncestor().getNumber(); + final long highestKnownBlock = bestChainHeight(chainHeadBlockNumber); + return new DefaultSyncStatus( + commonAncestor, + chainHeadBlockNumber, + highestKnownBlock, + worldStateDownloadStatus.flatMap(WorldStateDownloadStatus::getPulledStates), + worldStateDownloadStatus.flatMap(WorldStateDownloadStatus::getKnownStates)); + }); + } + + private void removeEstimatedHeightListener(final SyncTarget target) { + target.removePeerChainEstimatedHeightListener(chainHeightListenerId); + } + + private void addEstimatedHeightListener(final SyncTarget target) { + chainHeightListenerId = + target.addPeerChainEstimatedHeightListener(estimatedHeight -> checkInSync()); + } + + public long getLocalChainHeight() { + return blockchain.getChainHeadBlockNumber(); + } + + public long bestChainHeight() { + final long localChainHeight = blockchain.getChainHeadBlockNumber(); + return bestChainHeight(localChainHeight); + } + + public long bestChainHeight(final long localChainHeight) { + return Math.max( + localChainHeight, + ethPeers + .bestPeerWithHeightEstimate() + .map(p -> p.chainState().getEstimatedHeight()) + .orElse(localChainHeight)); + } + + /** Evaluates whether this node is in sync and notifies any tracker whose verdict changed. */ + private void checkInSync() { + synchronized (inSyncLock) { + final ChainHead localChain = getLocalChainHead(); + final Optional syncTargetChain = getSyncTargetChainHead(); + final Optional bestPeerChain = getBestPeerChainHead(); + + // Remove listener when we've found a peer. + newPeerListenerId.ifPresent( + listenerId -> { + ethPeers.unsubscribeConnect(listenerId); + newPeerListenerId = Optional.empty(); + }); + + inSyncTrackers + .values() + .forEach( + (syncTracker) -> syncTracker.checkState(localChain, syncTargetChain, bestPeerChain)); + } + } + + public Optional getCheckpoint() { + return checkpoint; + } + + public boolean isInitialSyncPhaseDone() { + return isInitialSyncPhaseDone; + } + + public void markInitialSyncPhaseAsDone() { + isInitialSyncPhaseDone = true; + isResyncNeeded = false; + completionListenerSubscribers.forEach(InitialSyncCompletionListener::onInitialSyncCompleted); + } + + public boolean isResyncNeeded() { + return isResyncNeeded; + } + + public void markResyncNeeded() { + isResyncNeeded = true; + } + + public Optional
getAccountToRepair() { + return maybeAccountToRepair; + } + + public void markAccountToRepair(final Optional
address) { + maybeAccountToRepair = address; + } + + public void markInitialSyncRestart() { + isInitialSyncPhaseDone = false; + completionListenerSubscribers.forEach(InitialSyncCompletionListener::onInitialSyncRestart); + } +} diff --git a/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java new file mode 100644 index 0000000..077f2be --- /dev/null +++ b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java @@ -0,0 +1,494 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.eth.transactions; + +import static org.hyperledger.besu.ethereum.core.kzg.CKZG4844Helper.CELL_PROOFS_PER_BLOB; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.ACCESS_LIST_ENTRY_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.ACCESS_LIST_STORAGE_KEY_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOBS_WITH_COMMITMENTS_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOB_PROOF_BUNDLE_SIZE_V0; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOB_PROOF_BUNDLE_SIZE_V1; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.CODE_DELEGATION_ENTRY_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.EIP1559_AND_EIP4844_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.KZG_PROOF_CONTAINER_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.KZG_PROOF_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_ACCESS_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_CHAIN_ID_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_TO_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.PAYLOAD_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.PENDING_TRANSACTION_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.calculateListShallowSize; + +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Tracks the additional metadata associated with transactions to enable prioritization for mining + * and deciding which transactions to drop when the transaction pool reaches its size limit. + */ +public abstract class PendingTransaction + implements org.hyperledger.besu.datatypes.PendingTransaction { + public static final Byte MAX_SCORE = Byte.MAX_VALUE; + private static final int NOT_INITIALIZED = -1; + private static final AtomicLong TRANSACTIONS_ADDED = new AtomicLong(); + private final Transaction transaction; + private final long addedAt; + private final long sequence; // Allows prioritization based on order transactions are added + private volatile byte score; + + private int memorySize = NOT_INITIALIZED; + + private PendingTransaction( + final Transaction transaction, final byte score, final long addedAt, final long sequence) { + this.transaction = transaction; + this.addedAt = addedAt; + this.sequence = sequence; + this.score = score; + } + + private PendingTransaction(final Transaction transaction, final byte score, final long addedAt) { + this(transaction, score, addedAt, TRANSACTIONS_ADDED.getAndIncrement()); + } + + public static PendingTransaction newPendingTransaction( + final Transaction transaction, + final boolean isLocal, + final boolean hasPriority, + final byte score) { + return newPendingTransaction( + transaction, isLocal, hasPriority, score, System.currentTimeMillis()); + } + + public static PendingTransaction newPendingTransaction( + final Transaction transaction, + final boolean isLocal, + final boolean hasPriority, + final byte score, + final long addedAt) { + if (isLocal) { + if (hasPriority) { + return new Local.Priority(transaction, score, addedAt); + } + return new Local(transaction, score, addedAt); + } + if (hasPriority) { + return new Remote.Priority(transaction, score, addedAt); + } + return new Remote(transaction, score, addedAt); + } + + @Override + public Transaction getTransaction() { + return transaction; + } + + public Wei getGasPrice() { + return transaction.getGasPrice().orElse(Wei.ZERO); + } + + public long getSequence() { + return sequence; + } + + public long getNonce() { + return transaction.getNonce(); + } + + public Address getSender() { + return transaction.getSender(); + } + + public Hash getHash() { + return transaction.getHash(); + } + + @Override + public long getAddedAt() { + return addedAt; + } + + @Override + public int memorySize() { + if (memorySize == NOT_INITIALIZED) { + memorySize = computeMemorySize(); + } + return memorySize; + } + + public byte getScore() { + return score; + } + + public void decrementScore() { + // use temp var to avoid non-atomic update of volatile var + final byte newScore = (byte) (score - 1); + + // check to avoid underflow + if (newScore < score) { + score = newScore; + } + } + + public abstract PendingTransaction detachedCopy(); + + private int computeMemorySize() { + return switch (transaction.getType()) { + case FRONTIER -> computeFrontierMemorySize(); + case ACCESS_LIST -> computeAccessListMemorySize(); + case EIP1559 -> computeEIP1559MemorySize(); + case BLOB -> computeBlobMemorySize(); + case DELEGATE_CODE -> computeDelegateCodeMemorySize(); + case AERE_PQ -> computeAerePqMemorySize(); + } + + PENDING_TRANSACTION_SHALLOW_SIZE; + } + + private int computeFrontierMemorySize() { + return FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE + + computePayloadMemorySize() + + computeToMemorySize() + + computeChainIdMemorySize(); + } + + private int computeAccessListMemorySize() { + return FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE + + computePayloadMemorySize() + + computeToMemorySize() + + computeChainIdMemorySize() + + computeAccessListEntriesMemorySize(); + } + + private int computeEIP1559MemorySize() { + return EIP1559_AND_EIP4844_SHALLOW_SIZE + + computePayloadMemorySize() + + computeToMemorySize() + + computeChainIdMemorySize() + + computeAccessListEntriesMemorySize(); + } + + private int computeBlobMemorySize() { + return computeEIP1559MemorySize() + + OPTIONAL_SHALLOW_SIZE // for the versionedHashes field + + computeBlobWithCommitmentsMemorySize(); + } + + private int computeDelegateCodeMemorySize() { + return computeEIP1559MemorySize() + computeCodeDelegationListMemorySize(); + } + + // AIP-20: an EIP-1559 shape plus the post-quantum authorization (record + two Bytes wrappers + the + // raw key and signature bytes, up to 7,219 of them for ML-DSA-87). + private static final int AERE_PQ_AUTHORIZATION_SHALLOW_SIZE = 64; + + private int computeAerePqMemorySize() { + return computeEIP1559MemorySize() + + OPTIONAL_SHALLOW_SIZE + + transaction + .getPqAuthorization() + .map(a -> AERE_PQ_AUTHORIZATION_SHALLOW_SIZE + a.authorizationByteCount()) + .orElse(0); + } + + private int computeBlobWithCommitmentsMemorySize() { + final int blobCount = transaction.getBlobCount(); + + return switch (transaction.getBlobsWithCommitments().get().getBlobType()) { + case KZG_PROOF -> + OPTIONAL_SHALLOW_SIZE + + BLOBS_WITH_COMMITMENTS_SIZE + + calculateListShallowSize(blobCount) // list of KZGProofBundle + + BLOB_PROOF_BUNDLE_SIZE_V0 * blobCount; + case KZG_CELL_PROOFS -> + OPTIONAL_SHALLOW_SIZE + + BLOBS_WITH_COMMITMENTS_SIZE + + calculateListShallowSize(blobCount) // list of KZGProofBundle + + (BLOB_PROOF_BUNDLE_SIZE_V1 + + KZG_PROOF_CONTAINER_SHALLOW_SIZE + + KZG_PROOF_SIZE * CELL_PROOFS_PER_BLOB) + * blobCount; + }; + } + + private int computePayloadMemorySize() { + return !transaction.getPayload().isEmpty() + ? PAYLOAD_SHALLOW_SIZE + transaction.getPayload().size() + : 0; + } + + private int computeToMemorySize() { + if (transaction.getTo().isPresent()) { + return OPTIONAL_TO_SIZE; + } + return 0; + } + + private int computeChainIdMemorySize() { + if (transaction.getChainId().isPresent()) { + return OPTIONAL_CHAIN_ID_SIZE; + } + return 0; + } + + private int computeAccessListEntriesMemorySize() { + return transaction + .getAccessList() + .map( + al -> { + int totalSize = OPTIONAL_ACCESS_LIST_SHALLOW_SIZE; + totalSize += al.size() * ACCESS_LIST_ENTRY_SHALLOW_SIZE; + totalSize += + al.stream().map(AccessListEntry::storageKeys).mapToInt(List::size).sum() + * ACCESS_LIST_STORAGE_KEY_SIZE; + return totalSize; + }) + .orElse(0); + } + + private int computeCodeDelegationListMemorySize() { + return transaction + .getCodeDelegationList() + .map( + cd -> { + int totalSize = OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE; + totalSize += cd.size() * CODE_DELEGATION_ENTRY_SIZE; + return totalSize; + }) + .orElse(0); + } + + public static List toTransactionList( + final Collection transactionsInfo) { + return transactionsInfo.stream().map(PendingTransaction::getTransaction).toList(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + PendingTransaction that = (PendingTransaction) o; + + return sequence == that.sequence; + } + + @Override + public int hashCode() { + return 31 * Long.hashCode(sequence); + } + + @Override + public String toString() { + return "Hash=" + + transaction.getHash() + + ", nonce=" + + transaction.getNonce() + + ", sender=" + + transaction.getSender() + + ", addedAt=" + + addedAt + + ", sequence=" + + sequence + + ", isLocal=" + + isReceivedFromLocalSource() + + ", hasPriority=" + + hasPriority() + + ", score=" + + score + + '}'; + } + + @Override + public String toTraceLog() { + return "{sequence: " + + sequence + + ", addedAt: " + + addedAt + + ", isLocal=" + + isReceivedFromLocalSource() + + ", hasPriority=" + + hasPriority() + + ", score=" + + score + + ", " + + transaction.toTraceLog() + + "}"; + } + + public static class Local extends PendingTransaction { + + public Local(final Transaction transaction, final byte score, final long addedAt) { + super(transaction, score, addedAt); + } + + public Local(final Transaction transaction) { + this(transaction, MAX_SCORE, System.currentTimeMillis()); + } + + private Local(final long sequence, final byte score, final Transaction transaction) { + super(transaction, score, System.currentTimeMillis(), sequence); + } + + @Override + public PendingTransaction detachedCopy() { + return new Local(getSequence(), getScore(), getTransaction().detachedCopy()); + } + + @Override + public boolean isReceivedFromLocalSource() { + return true; + } + + @Override + public boolean hasPriority() { + return false; + } + + public static class Priority extends Local { + public Priority(final Transaction transaction) { + this(transaction, MAX_SCORE, System.currentTimeMillis()); + } + + public Priority(final Transaction transaction, final byte score, final long addedAt) { + super(transaction, score, addedAt); + } + + public Priority(final long sequence, final byte score, final Transaction transaction) { + super(sequence, score, transaction); + } + + @Override + public PendingTransaction detachedCopy() { + return new Priority(getSequence(), getScore(), getTransaction().detachedCopy()); + } + + @Override + public boolean hasPriority() { + return true; + } + } + } + + public static class Remote extends PendingTransaction { + + public Remote(final Transaction transaction, final byte score, final long addedAt) { + super(transaction, score, addedAt); + } + + public Remote(final Transaction transaction) { + this(transaction, MAX_SCORE, System.currentTimeMillis()); + } + + private Remote(final long sequence, final byte score, final Transaction transaction) { + super(transaction, score, System.currentTimeMillis(), sequence); + } + + @Override + public PendingTransaction detachedCopy() { + return new Remote(getSequence(), getScore(), getTransaction().detachedCopy()); + } + + @Override + public boolean isReceivedFromLocalSource() { + return false; + } + + @Override + public boolean hasPriority() { + return false; + } + + public static class Priority extends Remote { + public Priority(final Transaction transaction) { + this(transaction, MAX_SCORE, System.currentTimeMillis()); + } + + public Priority(final Transaction transaction, final byte score, final long addedAt) { + super(transaction, score, addedAt); + } + + public Priority(final long sequence, final byte score, final Transaction transaction) { + super(sequence, score, transaction); + } + + @Override + public PendingTransaction detachedCopy() { + return new Priority(getSequence(), getScore(), getTransaction().detachedCopy()); + } + + @Override + public boolean hasPriority() { + return true; + } + } + } + + /** + * The memory size of an object is calculated using the PendingTransactionEstimatedMemorySizeTest + * look there for the details of the calculation and to adapt the code when any of the related + * class changes its structure. + */ + public interface MemorySize { + int FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE = 896; + int EIP1559_AND_EIP4844_SHALLOW_SIZE = 1008; + int OPTIONAL_TO_SIZE = 104; + int OPTIONAL_CHAIN_ID_SIZE = 80; + int PAYLOAD_SHALLOW_SIZE = 32; + int ACCESS_LIST_STORAGE_KEY_SIZE = 32; + int ACCESS_LIST_ENTRY_SHALLOW_SIZE = 160; + int OPTIONAL_ACCESS_LIST_SHALLOW_SIZE = 40; + int OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE = 40; + int CODE_DELEGATION_ENTRY_SIZE = 512; + int OPTIONAL_SHALLOW_SIZE = 16; + int KZG_PROOF_CONTAINER_SHALLOW_SIZE = 24; + int KZG_PROOF_SIZE = 112; + int BLOBS_WITH_COMMITMENTS_SIZE = 48; + int BLOB_PROOF_BUNDLE_SIZE_V0 = 131536; + int BLOB_PROOF_BUNDLE_SIZE_V1 = 393576; + int PENDING_TRANSACTION_SHALLOW_SIZE = 40; + + /** + * This calculation is for an immutable List + * + * @param length the length of the list + * @return the memory shallow size of the list and the contained array + */ + static int calculateListShallowSize(final int length) { + // list object shallow size is 24. + // array w/o elements size is 16. + // for every 2 objects in the array add 8 + return 40 + ((length + 1) / 2) * 8; + } + } +} diff --git a/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java new file mode 100644 index 0000000..1ad11b0 --- /dev/null +++ b/anchor/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java @@ -0,0 +1,1008 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.eth.transactions; + +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MAX_SCORE; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolStructuredLogUtils.logInvalid; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolStructuredLogUtils.logStart; +import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolStructuredLogUtils.logStop; +import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.CHAIN_HEAD_NOT_AVAILABLE; +import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE; +import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.INTERNAL_ERROR; +import static org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason.TRANSACTION_ALREADY_KNOWN; +import static org.hyperledger.besu.ethereum.trie.pathbased.common.provider.WorldStateQueryParams.withBlockHeaderAndNoUpdateNodeHead; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.VersionedHash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; +import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.encoding.EncodingContext; +import org.hyperledger.besu.ethereum.core.kzg.BlobProofBundle; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidator; +import org.hyperledger.besu.ethereum.mainnet.ValidationResult; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.mainnet.transactionpool.TransactionPoolPreProcessor; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; +import org.hyperledger.besu.ethereum.trie.MerkleTrieException; +import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.fluent.SimpleAccount; +import org.hyperledger.besu.plugin.data.AddedBlockContext.EventType; +import org.hyperledger.besu.util.Subscribers; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimaps; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Maintains the set of pending transactions received from JSON-RPC or other nodes. Transactions are + * removed automatically when they are included in a block on the canonical chain and re-added if a + * re-org removes them from the canonical chain again. + * + *

This class is safe for use across multiple threads. + */ +public class TransactionPool implements BlockAddedObserver { + private static final Logger LOG = LoggerFactory.getLogger(TransactionPool.class); + private final Supplier pendingTransactionsSupplier; + private final BlobCache cacheForBlobsOfTransactionsAddedToABlock; + private volatile PendingTransactions pendingTransactions = new DisabledPendingTransactions(); + private final ProtocolSchedule protocolSchedule; + private final ProtocolContext protocolContext; + private final EthContext ethContext; + private final TransactionBroadcaster transactionBroadcaster; + private final TransactionPoolMetrics metrics; + private final TransactionPoolConfiguration configuration; + private final AtomicBoolean isPoolEnabled = new AtomicBoolean(false); + private final PendingTransactionsListenersProxy pendingTransactionsListenersProxy = + new PendingTransactionsListenersProxy(); + private volatile OptionalLong subscribeConnectId = OptionalLong.empty(); + private final SaveRestoreManager saveRestoreManager = new SaveRestoreManager(); + private final Set

localSenders = ConcurrentHashMap.newKeySet(); + private final EthScheduler.OrderedProcessor blockAddedEventOrderedProcessor; + private final ListMultimap mapOfBlobsInTransactionPool = + Multimaps.synchronizedListMultimap( + Multimaps.newListMultimap(new HashMap<>(), () -> new ArrayList<>(1))); + + public TransactionPool( + final Supplier pendingTransactionsSupplier, + final ProtocolSchedule protocolSchedule, + final ProtocolContext protocolContext, + final TransactionBroadcaster transactionBroadcaster, + final EthContext ethContext, + final TransactionPoolMetrics metrics, + final TransactionPoolConfiguration configuration, + final BlobCache blobCache) { + this.pendingTransactionsSupplier = pendingTransactionsSupplier; + this.protocolSchedule = protocolSchedule; + this.protocolContext = protocolContext; + this.ethContext = ethContext; + this.transactionBroadcaster = transactionBroadcaster; + this.metrics = metrics; + this.configuration = configuration; + this.blockAddedEventOrderedProcessor = + ethContext.getScheduler().createOrderedProcessor(this::processBlockAddedEvent); + this.cacheForBlobsOfTransactionsAddedToABlock = blobCache; + initializeBlobMetrics(); + subscribePendingTransactions(this::mapBlobsOnTransactionAdded); + subscribeDroppedTransactions( + (transaction, reason) -> unmapBlobsOnTransactionDropped(transaction)); + subscribeDroppedTransactions(transactionBroadcaster); + } + + @VisibleForTesting + void handleConnect(final EthPeer peer) { + transactionBroadcaster.relayTransactionPoolTo( + peer, pendingTransactions.getPendingTransactions()); + } + + public ValidationResult addTransactionViaApi( + final Transaction transaction) { + + final boolean hasPriority = isPriorityTransaction(transaction, true); + final var result = addTransaction(transaction, true, hasPriority, MAX_SCORE); + if (result.isValid()) { + localSenders.add(transaction.getSender()); + transactionBroadcaster.onTransactionsAdded(List.of(transaction)); + } else { + logInvalid(transaction, result, true, hasPriority); + } + return result; + } + + public Map> addRemoteTransactions( + final Collection transactions) { + final long started = System.nanoTime(); + final int initialCount = transactions.size(); + final List addedTransactions = new ArrayList<>(initialCount); + LOG.trace("Adding {} remote transactions", initialCount); + + final var validationResults = + sortedBySenderAndNonce(transactions) + .collect( + Collectors.toMap( + Transaction::getHash, + transaction -> { + final boolean hasPriority = isPriorityTransaction(transaction, false); + final var result = addTransaction(transaction, false, hasPriority, MAX_SCORE); + if (result.isValid()) { + addedTransactions.add(transaction); + } else { + logInvalid(transaction, result, false, hasPriority); + } + return result; + }, + (transaction1, transaction2) -> transaction1)); + + if (isEnabled()) { + TransactionPoolStructuredLogUtils.logStats(pendingTransactions); + } + + LOG.atTrace() + .setMessage("Added {} transactions to the pool in {}, {} not added, current pool stats {}") + .addArgument(addedTransactions::size) + .addArgument(() -> Duration.ofNanos(System.nanoTime() - started)) + .addArgument(() -> initialCount - addedTransactions.size()) + .addArgument(pendingTransactions::logStats) + .log(); + + if (!addedTransactions.isEmpty()) { + transactionBroadcaster.onTransactionsAdded(addedTransactions); + } + return validationResults; + } + + private ValidationResult addTransaction( + final Transaction baseTransaction, + final boolean isLocal, + final boolean hasPriority, + final byte score) { + + if (pendingTransactions.containsTransaction(baseTransaction)) { + LOG.atTrace() + .setMessage("Discard already present transaction {}") + .addArgument(baseTransaction::toTraceLog) + .log(); + // We already have this transaction, don't even validate it. + metrics.incrementRejected(isLocal, hasPriority, TRANSACTION_ALREADY_KNOWN, "txpool"); + return ValidationResult.invalid(TRANSACTION_ALREADY_KNOWN); + } + + // Apply any necessary fork related pre-processing before submitting the transaction to the pool + Transaction transaction = + getTransactionPoolPreProcessor() + .map(preProcessor -> preProcessor.prepareTransaction(baseTransaction, isLocal)) + .orElse(baseTransaction); + + final ValidationResultAndAccount validationResult = + validateTransaction(transaction, isLocal, hasPriority); + + if (validationResult.result.isValid()) { + final TransactionAddedResult status = + pendingTransactions.addTransaction( + PendingTransaction.newPendingTransaction(transaction, isLocal, hasPriority, score), + validationResult.maybeAccount); + if (status.isSuccess()) { + LOG.atTrace() + .setMessage("Added {} transaction {}") + .addArgument(() -> isLocal ? "local" : "remote") + .addArgument(transaction::toTraceLog) + .log(); + } else { + final var rejectReason = + status + .maybeInvalidReason() + .orElseGet( + () -> { + LOG.warn("Missing invalid reason for status {}", status); + return INTERNAL_ERROR; + }); + LOG.atTrace() + .setMessage("Transaction {} rejected reason {}") + .addArgument(transaction::toTraceLog) + .addArgument(rejectReason) + .log(); + metrics.incrementRejected(isLocal, hasPriority, rejectReason, "txpool"); + return ValidationResult.invalid(rejectReason); + } + } else { + LOG.atTrace() + .setMessage("Discard invalid transaction {}, reason {}, because {}") + .addArgument(transaction::toTraceLog) + .addArgument(validationResult.result::getInvalidReason) + .addArgument(validationResult.result::getErrorMessage) + .log(); + metrics.incrementRejected( + isLocal, hasPriority, validationResult.result.getInvalidReason(), "txpool"); + } + + return validationResult.result; + } + + private Optional getMaxGasPrice(final Transaction transaction) { + return transaction.getGasPrice().map(Optional::of).orElse(transaction.getMaxFeePerGas()); + } + + private boolean isMaxGasPriceBelowConfiguredMinGasPrice(final Transaction transaction) { + return getMaxGasPrice(transaction) + .map(g -> g.lessThan(configuration.getMinGasPrice())) + .orElse(true); + } + + private Stream sortedBySenderAndNonce(final Collection transactions) { + return transactions.stream() + .sorted( + Comparator.comparing((Transaction tx) -> tx.getSender().getBytes()) + .thenComparing(Transaction::getNonce)); + } + + private boolean isPriorityTransaction(final Transaction transaction, final boolean isLocal) { + if (isLocal && !configuration.getNoLocalPriority()) { + // unless no-local-priority option is specified, senders of local sent txs are prioritized + return true; + } + // otherwise check if the sender belongs to the priority list + return configuration.getPrioritySenders().contains(transaction.getSender()); + } + + public long subscribePendingTransactions(final PendingTransactionAddedListener listener) { + return pendingTransactionsListenersProxy.onAddedListeners.subscribe(listener); + } + + public void unsubscribePendingTransactions(final long id) { + pendingTransactionsListenersProxy.onAddedListeners.unsubscribe(id); + } + + public long subscribeDroppedTransactions(final PendingTransactionDroppedListener listener) { + return pendingTransactionsListenersProxy.onDroppedListeners.subscribe(listener); + } + + public void unsubscribeDroppedTransactions(final long id) { + pendingTransactionsListenersProxy.onDroppedListeners.unsubscribe(id); + } + + @Override + public void onBlockAdded(final BlockAddedEvent event) { + if (isPoolEnabled.get()) { + if (event.getEventType().equals(EventType.HEAD_ADVANCED) + || event.getEventType().equals(EventType.CHAIN_REORG)) { + + blockAddedEventOrderedProcessor.submit(event); + } + } + } + + private void processBlockAddedEvent(final BlockAddedEvent e) { + final long started = System.currentTimeMillis(); + pendingTransactions.manageBlockAdded( + e.getHeader(), + e.getAddedTransactions(), + e.getRemovedTransactions(), + protocolSchedule.getByBlockHeader(e.getHeader()).getFeeMarket()); + reAddTransactions(e.getRemovedTransactions()); + LOG.atTrace() + .setMessage("Block added event {} processed in {}ms") + .addArgument(e) + .addArgument(() -> System.currentTimeMillis() - started) + .log(); + } + + private void reAddTransactions(final List reAddTransactions) { + if (!reAddTransactions.isEmpty()) { + // if adding a blob tx, and it is missing its blob, is a re-org and we should restore the blob + // from cache. + var txsByOrigin = + reAddTransactions.stream() + .map(t -> pendingTransactions.restoreBlob(t).orElse(t)) + .collect(Collectors.partitioningBy(tx -> isLocalSender(tx.getSender()))); + var reAddLocalTxs = txsByOrigin.get(true); + var reAddRemoteTxs = txsByOrigin.get(false); + if (!reAddLocalTxs.isEmpty()) { + logReAddedTransactions(reAddLocalTxs, "local"); + sortedBySenderAndNonce(reAddLocalTxs).forEach(this::addTransactionViaApi); + } + if (!reAddRemoteTxs.isEmpty()) { + logReAddedTransactions(reAddRemoteTxs, "remote"); + addRemoteTransactions(reAddRemoteTxs); + } + } + } + + private static void logReAddedTransactions( + final List reAddedTxs, final String source) { + LOG.atTrace() + .setMessage("Re-adding {} {} transactions from a block event: {}") + .addArgument(reAddedTxs::size) + .addArgument(source) + .addArgument( + () -> + reAddedTxs.stream().map(Transaction::toTraceLog).collect(Collectors.joining("; "))) + .log(); + } + + private TransactionValidator getTransactionValidator() { + return protocolSchedule + .getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader()) + .getTransactionValidatorFactory() + .get(); + } + + private Optional getTransactionPoolPreProcessor() { + return protocolSchedule + .getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader()) + .getTransactionPoolPreProcessor(); + } + + private ValidationResultAndAccount validateTransaction( + final Transaction transaction, final boolean isLocal, final boolean hasPriority) { + + final BlockHeader chainHeadBlockHeader = getChainHeadBlockHeader().orElse(null); + if (chainHeadBlockHeader == null) { + LOG.atWarn() + .setMessage("rejecting transaction {} due to chain head not available yet") + .addArgument(transaction::getHash) + .log(); + return ValidationResultAndAccount.invalid(CHAIN_HEAD_NOT_AVAILABLE); + } + + final FeeMarket feeMarket = + protocolSchedule.getByBlockHeader(chainHeadBlockHeader).getFeeMarket(); + final TransactionInvalidReason priceInvalidReason = + validatePrice(transaction, isLocal, hasPriority, feeMarket); + if (priceInvalidReason != null) { + return ValidationResultAndAccount.invalid(priceInvalidReason); + } + + final ValidationResult basicValidationResult = + getTransactionValidator() + .validate( + transaction, + chainHeadBlockHeader.getBaseFee(), + Optional.of( + Wei.ZERO), // TransactionValidationParams.transactionPool() allows underpriced + // txs + TransactionValidationParams.transactionPool()); + if (!basicValidationResult.isValid()) { + return new ValidationResultAndAccount(basicValidationResult); + } + + if (hasPriority + && strictReplayProtectionShouldBeEnforcedLocally(chainHeadBlockHeader) + && transaction.getChainId().isEmpty()) { + // Strict replay protection is enabled but the tx is not replay-protected + return ValidationResultAndAccount.invalid( + TransactionInvalidReason.REPLAY_PROTECTED_SIGNATURE_REQUIRED); + } + if (transaction.getGasLimit() > chainHeadBlockHeader.getGasLimit()) { + return ValidationResultAndAccount.invalid( + TransactionInvalidReason.EXCEEDS_BLOCK_GAS_LIMIT, + String.format( + "Transaction gas limit of %s exceeds block gas limit of %s", + transaction.getGasLimit(), chainHeadBlockHeader.getGasLimit())); + } + if (transaction.getType().equals(TransactionType.EIP1559) && !feeMarket.implementsBaseFee()) { + return ValidationResultAndAccount.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + "EIP-1559 transaction are not allowed yet"); + } else if (transaction.getType().equals(TransactionType.BLOB) + && transaction.getBlobsWithCommitments().isEmpty()) { + return ValidationResultAndAccount.invalid( + TransactionInvalidReason.INVALID_BLOBS, "Blob transaction must have at least one blob"); + } + // AIP-20: not admitted to the pool before the activation height (the next block is judged). + if (transaction.getType().supportsPqAuthorization() + && !org.hyperledger.besu.ethereum.mainnet.AerePqTransactionSupport.isActiveAt( + chainHeadBlockHeader.getNumber() + 1)) { + return ValidationResultAndAccount.invalid( + TransactionInvalidReason.INVALID_TRANSACTION_FORMAT, + org.hyperledger.besu.ethereum.mainnet.AerePqTransactionSupport.inactiveMessage( + chainHeadBlockHeader.getNumber() + 1)); + } + + // Call the transaction validator plugin + final Optional maybePluginInvalid = + configuration + .getTransactionPoolValidatorService() + .createTransactionValidator() + .validateTransaction(transaction, isLocal, hasPriority); + if (maybePluginInvalid.isPresent()) { + return ValidationResultAndAccount.invalid( + TransactionInvalidReason.PLUGIN_TX_POOL_VALIDATOR, maybePluginInvalid.get()); + } + + try (final var worldState = + protocolContext + .getWorldStateArchive() + .getWorldState(withBlockHeaderAndNoUpdateNodeHead(chainHeadBlockHeader)) + .orElseThrow()) { + if (worldState instanceof BonsaiWorldState bonsaiWorldState) { + bonsaiWorldState.disableCacheMerkleTrieLoader(); + } + final Account senderAccount = worldState.get(transaction.getSender()); + return new ValidationResultAndAccount( + senderAccount, + getTransactionValidator() + .validateForSender( + transaction, senderAccount, TransactionValidationParams.transactionPool())); + } catch (MerkleTrieException ex) { + LOG.debug( + "MerkleTrieException while validating transaction for sender {}", + transaction.getSender()); + return ValidationResultAndAccount.invalid(CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE); + } catch (Exception ex) { + return ValidationResultAndAccount.invalid(CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE); + } + } + + private TransactionInvalidReason validatePrice( + final Transaction transaction, + final boolean isLocal, + final boolean hasPriority, + final FeeMarket feeMarket) { + + if (isLocal) { + if (!configuration.getTxFeeCap().isZero() + && getMaxGasPrice(transaction).get().greaterThan(configuration.getTxFeeCap())) { + return TransactionInvalidReason.TX_FEECAP_EXCEEDED; + } + } + if (hasPriority) { + // allow priority transactions to be below minGas as long as the gas price is above the + // configured floor + if (!feeMarket.satisfiesFloorTxFee(transaction)) { + return TransactionInvalidReason.GAS_PRICE_TOO_LOW; + } + } else { + if (isMaxGasPriceBelowConfiguredMinGasPrice(transaction)) { + LOG.atTrace() + .setMessage("Discard transaction {} below min gas price {}") + .addArgument(transaction::toTraceLog) + .addArgument(configuration::getMinGasPrice) + .log(); + return TransactionInvalidReason.GAS_PRICE_TOO_LOW; + } + } + return null; + } + + private boolean strictReplayProtectionShouldBeEnforcedLocally( + final BlockHeader chainHeadBlockHeader) { + return configuration.getStrictTransactionReplayProtectionEnabled() + && protocolSchedule.getChainId().isPresent() + && transactionReplaySupportedAtBlock(chainHeadBlockHeader); + } + + private boolean transactionReplaySupportedAtBlock(final BlockHeader blockHeader) { + return protocolSchedule.getByBlockHeader(blockHeader).isReplayProtectionSupported(); + } + + public Optional getTransactionByHash(final Hash hash) { + return pendingTransactions.getTransactionByHash(hash); + } + + private Optional getChainHeadBlockHeader() { + final MutableBlockchain blockchain = protocolContext.getBlockchain(); + + // Optimistically get the block header for the chain head without taking a lock, + // but revert to the safe implementation if it returns an empty optional. (It's + // possible the chain head has been updated but the block is still being persisted + // to storage/cache under the lock). + return blockchain + .getBlockHeader(blockchain.getChainHeadHash()) + .or(() -> blockchain.getBlockHeaderSafe(blockchain.getChainHeadHash())); + } + + private boolean isLocalSender(final Address sender) { + return localSenders.contains(sender); + } + + public int count() { + return pendingTransactions.size(); + } + + public Collection getPendingTransactions() { + return pendingTransactions.getPendingTransactions(); + } + + /** + * Returns all pending transactions for the given sender, sorted by nonce in ascending order. + * + * @param sender the sender address + * @return transactions for the sender sorted by nonce ascending, or an empty list if none exist + */ + public SenderPendingTransactionsData getPendingTransactionsFor(final Address sender) { + return pendingTransactions.getPendingTransactionsFor(sender); + } + + public Map getPendingTransactionsBySender() { + return pendingTransactions.getPendingTransactionsBySender(); + } + + public OptionalLong getNextNonceForSender(final Address address) { + return pendingTransactions.getNextNonceForSender(address); + } + + public long maxSize() { + return pendingTransactions.maxSize(); + } + + public void evictOldTransactions() { + pendingTransactions.evictOldTransactions(); + } + + public void selectTransactions( + final PendingTransactions.PendingTransactionsSelector transactionSelector) { + pendingTransactions.selectTransactions(transactionSelector); + } + + public String logStats() { + return pendingTransactions.logStats(); + } + + public PendingTransactions.Status getStatus() { + return pendingTransactions.getStatus(); + } + + @VisibleForTesting + Class pendingTransactionsImplementation() { + return pendingTransactions.getClass(); + } + + @VisibleForTesting + SaveRestoreManager getSaveRestoreManager() { + return saveRestoreManager; + } + + public interface TransactionBatchAddedListener { + + void onTransactionsAdded(Collection transactions); + } + + private static class ValidationResultAndAccount { + final ValidationResult result; + final Optional maybeAccount; + + ValidationResultAndAccount( + final Account account, final ValidationResult result) { + this.result = result; + this.maybeAccount = + Optional.ofNullable(account) + .map( + acct -> new SimpleAccount(acct.getAddress(), acct.getNonce(), acct.getBalance())); + } + + ValidationResultAndAccount(final ValidationResult result) { + this.result = result; + this.maybeAccount = Optional.empty(); + } + + static ValidationResultAndAccount invalid( + final TransactionInvalidReason reason, final String message) { + return new ValidationResultAndAccount(ValidationResult.invalid(reason, message)); + } + + static ValidationResultAndAccount invalid(final TransactionInvalidReason reason) { + return new ValidationResultAndAccount(ValidationResult.invalid(reason)); + } + } + + public CompletableFuture setEnabled() { + if (!isEnabled()) { + logStart(configuration, getChainHeadBlockHeader()); + pendingTransactions = pendingTransactionsSupplier.get(); + pendingTransactionsListenersProxy.subscribe(); + isPoolEnabled.set(true); + subscribeConnectId = + OptionalLong.of(ethContext.getEthPeers().subscribeConnect(this::handleConnect)); + return saveRestoreManager + .loadFromDisk() + .exceptionally( + t -> { + LOG.error("Error while restoring transaction pool from disk", t); + return null; + }); + } + return CompletableFuture.completedFuture(null); + } + + /** + * Disables the transaction pool. + * + *

When called, this method: + * + *

    + *
  • Marks the pool as disabled and unsubscribes listeners/events. + *
  • Clears the blob map to prevent stale or duplicate entries (blobs are restored from disk + * when the pool is re-enabled). + *
  • Persists current pending transactions to disk asynchronously. + *
  • Replaces the pending transactions with a disabled placeholder. + *
+ * + * @return a {@link CompletableFuture} that completes once the save-to-disk operation finishes + */ + public CompletableFuture setDisabled() { + if (isEnabled()) { + isPoolEnabled.set(false); + subscribeConnectId.ifPresent(ethContext.getEthPeers()::unsubscribeConnect); + pendingTransactionsListenersProxy.unsubscribe(); + mapOfBlobsInTransactionPool.clear(); + final CompletableFuture saveOperation = + saveRestoreManager + .saveToDisk(pendingTransactions) + .exceptionally( + t -> { + LOG.error("Error while saving transaction pool to disk", t); + return null; + }); + pendingTransactions = new DisabledPendingTransactions(); + logStop(); + return saveOperation; + } + return CompletableFuture.completedFuture(null); + } + + private void mapBlobsOnTransactionAdded(final Transaction transaction) { + final var maybeBlobsWithCommitments = transaction.getBlobsWithCommitments(); + if (maybeBlobsWithCommitments.isEmpty()) { + return; + } + final List blobProofBundles = + maybeBlobsWithCommitments.get().getBlobProofBundles(); + + blobProofBundles.forEach(bq -> mapOfBlobsInTransactionPool.put(bq.getVersionedHash(), bq)); + } + + private void unmapBlobsOnTransactionDropped(final Transaction transaction) { + final var maybeBlobsWithCommitments = transaction.getBlobsWithCommitments(); + if (maybeBlobsWithCommitments.isEmpty()) { + return; + } + final List blobProofBundles = + maybeBlobsWithCommitments.get().getBlobProofBundles(); + + blobProofBundles.forEach(bq -> mapOfBlobsInTransactionPool.remove(bq.getVersionedHash(), bq)); + } + + public BlobProofBundle getBlobProofBundle(final VersionedHash vh) { + try { + // returns an empty list if the key is not present, so getFirst() will throw + return mapOfBlobsInTransactionPool.get(vh).getFirst(); + } catch (NoSuchElementException e) { + // do nothing + } + return cacheForBlobsOfTransactionsAddedToABlock.get(vh); + } + + public boolean isEnabled() { + return isPoolEnabled.get(); + } + + public int getBlobCacheSize() { + return (int) cacheForBlobsOfTransactionsAddedToABlock.size(); + } + + public int getBlobMapSize() { + return mapOfBlobsInTransactionPool.size(); + } + + private void initializeBlobMetrics() { + metrics.createBlobCacheSizeMetric(this::getBlobCacheSize); + metrics.createBlobMapSizeMetric(this::getBlobMapSize); + } + + class PendingTransactionsListenersProxy { + private final Subscribers onAddedListeners = + Subscribers.create(); + private final Subscribers onDroppedListeners = + Subscribers.create(); + + private volatile long onAddedListenerId; + private volatile long onDroppedListenerId; + + void subscribe() { + onAddedListenerId = pendingTransactions.subscribePendingTransactions(this::onAdded); + onDroppedListenerId = + pendingTransactions.subscribeDroppedTransactions( + (transaction, reason) -> onDropped(transaction, reason)); + } + + void unsubscribe() { + pendingTransactions.unsubscribePendingTransactions(onAddedListenerId); + pendingTransactions.unsubscribeDroppedTransactions(onDroppedListenerId); + } + + private void onDropped(final Transaction transaction, final RemovalReason reason) { + onDroppedListeners.forEach(listener -> listener.onTransactionDropped(transaction, reason)); + } + + private void onAdded(final Transaction transaction) { + onAddedListeners.forEach(listener -> listener.onTransactionAdded(transaction)); + } + } + + class SaveRestoreManager { + private final Semaphore diskAccessLock = new Semaphore(1, true); + private final AtomicReference> writeInProgress = + new AtomicReference<>(CompletableFuture.completedFuture(null)); + private final AtomicReference> readInProgress = + new AtomicReference<>(CompletableFuture.completedFuture(null)); + private final AtomicBoolean isCancelled = new AtomicBoolean(false); + + @VisibleForTesting + Semaphore getDiskAccessLock() { + return diskAccessLock; + } + + CompletableFuture saveToDisk(final PendingTransactions pendingTransactionsToSave) { + cancelInProgressReadOperation(); + return serializeAndDedupOperation( + () -> executeSaveToDisk(pendingTransactionsToSave), writeInProgress); + } + + CompletableFuture loadFromDisk() { + return serializeAndDedupOperation(this::executeLoadFromDisk, readInProgress); + } + + private void cancelInProgressReadOperation() { + if (!readInProgress.get().isDone()) { + LOG.debug("Cancelling in progress read operation"); + isCancelled.set(true); + try { + waitUntilReadOperationIsCancelled(); + LOG.debug("In progress read operation cancelled"); + } catch (InterruptedException | ExecutionException e) { + LOG.warn("Error while cancelling in progress read operation", e); + throw new RuntimeException(e); + } + } + } + + private void waitUntilReadOperationIsCancelled() + throws InterruptedException, ExecutionException { + readInProgress.get().get(); + } + + private CompletableFuture serializeAndDedupOperation( + final Runnable operation, + final AtomicReference> operationInProgress) { + if (configuration.getEnableSaveRestore()) { + try { + if (diskAccessLock.tryAcquire( + configuration.getUnstable().getSaveRestoreTimeout().toMillis(), + TimeUnit.MILLISECONDS)) { + + isCancelled.set(false); + operationInProgress.set( + CompletableFuture.runAsync(operation) + .whenComplete((res, err) -> diskAccessLock.release())); + return operationInProgress.get(); + } else { + return CompletableFuture.failedFuture( + new TimeoutException("Timeout waiting for disk access lock")); + } + } catch (InterruptedException ie) { + return CompletableFuture.failedFuture(ie); + } + } + return CompletableFuture.completedFuture(null); + } + + private void executeSaveToDisk(final PendingTransactions pendingTransactionsToSave) { + final File saveFile = configuration.getSaveFile(); + final boolean appending = saveFile.exists(); + try (final BufferedWriter bw = + new BufferedWriter(new FileWriter(saveFile, StandardCharsets.US_ASCII, appending))) { + final var allTxs = pendingTransactionsToSave.getPendingTransactions(); + LOG.info( + "{} {} transactions to file {}", + appending ? "Appending" : "Saving", + allTxs.size(), + saveFile); + + final long processedTxCount = + allTxs.parallelStream() + .takeWhile(unused -> !isCancelled.get()) + .map( + ptx -> { + final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); + ptx.getTransaction().writeTo(rlp, EncodingContext.POOLED_TRANSACTION); + return ptx.getScore() + + (ptx.isReceivedFromLocalSource() ? "l" : "r") + + rlp.encoded().toBase64String(); + }) + .mapToInt( + line -> { + synchronized (bw) { + try { + bw.write(line); + bw.newLine(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return 1; + }) + .sum(); + + if (isCancelled.get()) { + LOG.info( + "{} {} transactions to file {}, before operation was cancelled", + appending ? "Appended" : "Saved", + processedTxCount, + saveFile); + } else { + LOG.info( + "{} {} transactions to file {}", + appending ? "Appended" : "Saved", + processedTxCount, + saveFile); + } + } catch (IOException e) { + LOG.error("Error while saving txpool content to disk", e); + } + } + + private void executeLoadFromDisk() { + if (configuration.getEnableSaveRestore()) { + final File saveFile = configuration.getSaveFile(); + if (saveFile.exists()) { + LOG.info("Loading transaction pool content from file {}", saveFile); + try (final BufferedReader br = + new BufferedReader(new FileReader(saveFile, StandardCharsets.US_ASCII))) { + final Map stats = + br.lines() + .takeWhile(unused -> !isCancelled.get()) + .map( + line -> { + final var scoreStr = parseScore(line); + final byte score = + scoreStr.isEmpty() ? MAX_SCORE : Byte.parseByte(scoreStr); + final boolean isLocal = line.charAt(scoreStr.length()) == 'l'; + final Transaction tx = + Transaction.readFrom( + RLP.input( + Bytes.fromBase64String( + line.substring(scoreStr.length() + 1))), + EncodingContext.POOLED_TRANSACTION); + final boolean hasPriority = isPriorityTransaction(tx, isLocal); + final ValidationResult result = + addTransaction(tx, isLocal, hasPriority, score); + return result.isValid() ? "OK" : result.getInvalidReason().name(); + }) + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + + br.close(); + + final var added = stats.getOrDefault("OK", 0L); + final var processedLines = stats.values().stream().mapToLong(Long::longValue).sum(); + + LOG.debug("Restored transactions stats {}", stats); + + if (isCancelled.get()) { + LOG.info( + "Added {} transactions of {} loaded from file {}, before operation was cancelled", + added, + processedLines, + saveFile); + removeProcessedLines(saveFile, processedLines); + } else { + LOG.info( + "Added {} transactions of {} loaded from file {}, deleting file", + added, + processedLines, + saveFile); + saveFile.delete(); + } + } catch (IOException e) { + LOG.error("Error while loading txpool content from disk", e); + } + } + } + } + + private String parseScore(final String line) { + int i = 0; + final var sbScore = new StringBuilder(); + while ("1234567890-".indexOf(line.charAt(i)) >= 0) { + sbScore.append(line.charAt(i++)); + } + return sbScore.toString(); + } + + private void removeProcessedLines(final File saveFile, final long processedLines) + throws IOException { + + LOG.debug("Removing processed lines from save file"); + + // Create temporary file with default secure permissions + final var tmp = + Files.createTempFile(saveFile.getParentFile().toPath(), saveFile.getName(), ".tmp"); + + try (final BufferedReader reader = + Files.newBufferedReader(saveFile.toPath(), StandardCharsets.US_ASCII); + final BufferedWriter writer = Files.newBufferedWriter(tmp, StandardCharsets.US_ASCII)) { + reader + .lines() + .skip(processedLines) + .forEach( + line -> { + try { + writer.write(line); + writer.newLine(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + saveFile.delete(); + Files.move(tmp, saveFile.toPath()); + } + } +} diff --git a/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncStateTest.java b/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncStateTest.java new file mode 100644 index 0000000..5b58648 --- /dev/null +++ b/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncStateTest.java @@ -0,0 +1,877 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-syncstate-in-sync-lock.patch. + */ +package org.hyperledger.besu.ethereum.eth.sync.state; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.core.Synchronizer; +import org.hyperledger.besu.ethereum.core.Synchronizer.InSyncListener; +import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.eth.manager.ChainHeadEstimate; +import org.hyperledger.besu.ethereum.eth.manager.ChainState; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthPeers; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestBuilder; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; +import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.sync.common.checkpoint.Checkpoint; +import org.hyperledger.besu.ethereum.eth.sync.common.checkpoint.ImmutableCheckpoint; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.plugin.data.SyncStatus; +import org.hyperledger.besu.plugin.services.BesuEvents.InitialSyncCompletionListener; +import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener; +import org.hyperledger.besu.plugin.services.BesuEvents.TTDReachedListener; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SyncStateTest { + + private static final Difficulty standardDifficultyPerBlock = Difficulty.ONE; + private static final long OUR_CHAIN_HEAD_NUMBER = 3; + private static final Difficulty OUR_CHAIN_DIFFICULTY = + standardDifficultyPerBlock.multiply(OUR_CHAIN_HEAD_NUMBER); + private static final long TARGET_CHAIN_DELTA = 20; + private static final long TARGET_CHAIN_HEIGHT = OUR_CHAIN_HEAD_NUMBER + TARGET_CHAIN_DELTA; + private static final Difficulty TARGET_DIFFICULTY = + standardDifficultyPerBlock.multiply(TARGET_CHAIN_HEIGHT); + + private final InSyncListener inSyncListener = mock(InSyncListener.class); + private final InSyncListener inSyncListenerExact = mock(InSyncListener.class); + private final SyncStatusListener syncStatusListener = mock(SyncStatusListener.class); + + private final BlockDataGenerator gen = new BlockDataGenerator(1); + private final Block genesisBlock = + gen.genesisBlock(new BlockOptions().setDifficulty(Difficulty.ZERO)); + private final MutableBlockchain blockchain = + InMemoryKeyValueStorageProvider.createInMemoryBlockchain(genesisBlock); + + @Captor ArgumentCaptor> syncStatusCaptor; + + private EthProtocolManager ethProtocolManager; + private EthPeers ethPeers; + private RespondingEthPeer syncTargetPeer; + private RespondingEthPeer otherPeer; + private SyncState syncState; + + @BeforeEach + public void setUp() { + ethProtocolManager = + EthProtocolManagerTestBuilder.builder() + .setBlockchain(blockchain) + .setWorldStateArchive(mock(WorldStateArchive.class)) + .build(); + ethPeers = spy(ethProtocolManager.ethContext().getEthPeers()); + syncTargetPeer = createPeer(TARGET_CHAIN_HEIGHT); + otherPeer = createPeer(0); + + advanceLocalChain(OUR_CHAIN_HEAD_NUMBER); + + syncState = new SyncState(blockchain, ethPeers); + syncState.subscribeInSync(inSyncListener); + syncState.subscribeInSync(inSyncListenerExact, 0); + syncState.subscribeSyncStatus(syncStatusListener); + } + + @Test + public void isInSync_noPeers() { + otherPeer.disconnect(DisconnectReason.REQUESTED); + syncTargetPeer.disconnect(DisconnectReason.REQUESTED); + syncState.clearSyncTarget(); + assertThat(syncState.isInSync()).isTrue(); + } + + @Test + public void isInSync_singlePeerWithWorseChainBetterHeight() { + updateChainState( + otherPeer.getEthPeer(), TARGET_CHAIN_HEIGHT, OUR_CHAIN_DIFFICULTY.subtract(1L)); + final EthPeer peer = mockWorseChain(otherPeer.getEthPeer()); + doReturn(Optional.of(peer)).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.syncTarget()).isEmpty(); // Sanity check + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isTrue(); + } + + @Test + public void isInSync_singlePeerWithWorseChainWorseHeight() { + updateChainState( + otherPeer.getEthPeer(), OUR_CHAIN_HEAD_NUMBER - 1L, OUR_CHAIN_DIFFICULTY.subtract(1L)); + final EthPeer peer = mockWorseChain(otherPeer.getEthPeer()); + doReturn(Optional.of(peer)).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.syncTarget()).isEmpty(); // Sanity check + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isTrue(); + } + + @Test + public void isInSync_singlePeerWithBetterChainWorseHeight() { + updateChainState(otherPeer.getEthPeer(), OUR_CHAIN_HEAD_NUMBER - 1L, TARGET_DIFFICULTY); + final EthPeer peer = mockBetterChain(otherPeer.getEthPeer()); + doReturn(Optional.of(peer)).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.syncTarget()).isEmpty(); // Sanity check + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isTrue(); + } + + @Test + public void isInSync_singlePeerWithBetterChainBetterHeight() { + updateChainState(otherPeer.getEthPeer(), TARGET_CHAIN_HEIGHT, TARGET_DIFFICULTY); + final EthPeer peer = mockBetterChain(otherPeer.getEthPeer()); + doReturn(Optional.of(peer)).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.syncTarget()).isEmpty(); // Sanity check + assertThat(syncState.isInSync()).isFalse(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA - 1)).isFalse(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA)).isTrue(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA + 1)).isTrue(); + } + + @Test + public void isInSync_syncTargetWithBetterHeight() { + otherPeer.disconnect(DisconnectReason.REQUESTED); + setupOutOfSyncState(); + + assertThat(syncState.syncTarget()).isPresent(); // Sanity check + assertThat(syncState.isInSync()).isFalse(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA - 1)).isFalse(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA)).isTrue(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA + 1)).isTrue(); + } + + @Test + public void isInSync_syncTargetWithWorseHeight() { + otherPeer.disconnect(DisconnectReason.REQUESTED); + final long heightDifference = 20L; + advanceLocalChain(TARGET_CHAIN_HEIGHT + heightDifference); + setupOutOfSyncState(); + + assertThat(syncState.syncTarget()).isPresent(); // Sanity check + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isTrue(); + } + + @Test + public void isInSync_outOfSyncWithTargetAndOutOfSyncWithBestPeer() { + setupOutOfSyncState(); + updateChainState(otherPeer.getEthPeer(), TARGET_CHAIN_HEIGHT, TARGET_DIFFICULTY); + doReturn(Optional.of(otherPeer.getEthPeer())).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.isInSync()).isFalse(); + assertThat(syncState.isInSync(0)).isFalse(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA - 1)).isFalse(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA)).isTrue(); + assertThat(syncState.isInSync(TARGET_CHAIN_DELTA + 1)).isTrue(); + } + + @Test + public void isInSync_inSyncWithTargetOutOfSyncWithBestPeer() { + setupOutOfSyncState(); + advanceLocalChain(TARGET_CHAIN_HEIGHT); + final long heightDifference = 20L; + updateChainState( + otherPeer.getEthPeer(), + TARGET_CHAIN_HEIGHT + heightDifference, + TARGET_DIFFICULTY.add(heightDifference)); + doReturn(Optional.of(otherPeer.getEthPeer())).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.isInSync()).isFalse(); + assertThat(syncState.isInSync(0)).isFalse(); + assertThat(syncState.isInSync(heightDifference - 1)).isFalse(); + assertThat(syncState.isInSync(heightDifference)).isTrue(); + assertThat(syncState.isInSync(heightDifference + 1)).isTrue(); + } + + @Test + public void isInSync_inSyncWithTargetInSyncWithBestPeer() { + setupOutOfSyncState(); + advanceLocalChain(TARGET_CHAIN_HEIGHT); + updateChainState(otherPeer.getEthPeer(), TARGET_CHAIN_HEIGHT, TARGET_DIFFICULTY); + doReturn(Optional.of(otherPeer.getEthPeer())).when(ethPeers).bestPeerWithHeightEstimate(); + + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isTrue(); + } + + @Test + public void shouldSwitchToInSyncWhenNoBetterPeersAreAvailable() { + setupOutOfSyncState(); + + otherPeer.disconnect(DisconnectReason.REQUESTED); + syncTargetPeer.disconnect(DisconnectReason.REQUESTED); + syncState.clearSyncTarget(); + + verify(inSyncListener).onInSyncStatusChange(true); + verify(inSyncListenerExact).onInSyncStatusChange(true); + verifyNoMoreInteractions(inSyncListener); + verifyNoMoreInteractions(inSyncListenerExact); + } + + @Test + public void shouldBecomeInSyncWhenOurBlockchainCatchesUp() { + setupOutOfSyncState(); + + // Update to just within the default sync threshold + advanceLocalChain(TARGET_CHAIN_HEIGHT - Synchronizer.DEFAULT_IN_SYNC_TOLERANCE); + // We should register as in-sync with default tolerance, out-of-sync with exact tolerance + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isFalse(); + verify(inSyncListener).onInSyncStatusChange(true); + verify(inSyncListenerExact, never()).onInSyncStatusChange(true); + + // Advance one more block + advanceLocalChain(TARGET_CHAIN_HEIGHT - Synchronizer.DEFAULT_IN_SYNC_TOLERANCE + 1); + // We should register as in-sync with default tolerance, out-of-sync with exact tolerance + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isFalse(); + verifyNoMoreInteractions(inSyncListener); + verify(inSyncListenerExact, never()).onInSyncStatusChange(true); + + // Catch all the way up + advanceLocalChain(TARGET_CHAIN_HEIGHT); + // We should register as in-sync + assertThat(syncState.isInSync()).isTrue(); + assertThat(syncState.isInSync(0)).isTrue(); + verifyNoMoreInteractions(inSyncListener); + verify(inSyncListenerExact).onInSyncStatusChange(true); + } + + @Test + public void addInSyncListener_whileOutOfSync() { + setupOutOfSyncState(); + + // Add listener + InSyncListener newListener = mock(InSyncListener.class); + syncState.subscribeInSync(newListener); + verify(newListener, never()).onInSyncStatusChange(false); + verify(newListener, never()).onInSyncStatusChange(true); + + // Catch all the way up + advanceLocalChain(TARGET_CHAIN_HEIGHT); + + // Fall out of sync + updateChainState( + syncTargetPeer.getEthPeer(), + TARGET_CHAIN_HEIGHT + Synchronizer.DEFAULT_IN_SYNC_TOLERANCE + 1L, + TARGET_DIFFICULTY.add(10L)); + + final ArgumentCaptor inSyncEventCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(newListener, times(3)).onInSyncStatusChange(inSyncEventCaptor.capture()); + + final List syncChanges = inSyncEventCaptor.getAllValues(); + assertThat(syncChanges.get(0)).isEqualTo(false); + assertThat(syncChanges.get(1)).isEqualTo(true); + assertThat(syncChanges.get(2)).isEqualTo(false); + } + + @Test + public void addInSyncListener_whileInSync() { + setupOutOfSyncState(); + // Catch all the way up + advanceLocalChain(TARGET_CHAIN_HEIGHT); + + // Add listener + InSyncListener newListener = mock(InSyncListener.class); + syncState.subscribeInSync(newListener); + verify(newListener, never()).onInSyncStatusChange(false); + verify(newListener, never()).onInSyncStatusChange(true); + // Fall out of sync + updateChainState( + syncTargetPeer.getEthPeer(), + TARGET_CHAIN_HEIGHT + Synchronizer.DEFAULT_IN_SYNC_TOLERANCE + 1L, + TARGET_DIFFICULTY.add(10L)); + verify(newListener).onInSyncStatusChange(false); + verify(newListener, never()).onInSyncStatusChange(true); + + // Catch up + advanceLocalChain(TARGET_CHAIN_HEIGHT + 1L); + verify(newListener).onInSyncStatusChange(false); + verify(newListener).onInSyncStatusChange(true); + } + + @Test + public void removeInSyncListener_doesntReceiveSubsequentEvents() { + final long syncTolerance = Synchronizer.DEFAULT_IN_SYNC_TOLERANCE + 1L; + setupOutOfSyncState(); + + // Add listener + InSyncListener newListener = mock(InSyncListener.class); + final long subscriberId = syncState.subscribeInSync(newListener, syncTolerance); + verify(newListener, never()).onInSyncStatusChange(anyBoolean()); + + // Remove listener + syncState.unsubscribeInSync(subscriberId); + + // Catch all the way up + advanceLocalChain(TARGET_CHAIN_HEIGHT); + + // We should not register the in-sync event + verify(newListener, never()).onInSyncStatusChange(anyBoolean()); + + // Fall out of sync + updateChainState( + syncTargetPeer.getEthPeer(), + TARGET_CHAIN_HEIGHT + syncTolerance + 1L, + TARGET_DIFFICULTY.add(10L)); + + // We should not register the sync event + verify(newListener, never()).onInSyncStatusChange(anyBoolean()); + + // Other listeners should keep running + verify(inSyncListenerExact, times(2)).onInSyncStatusChange(false); + verify(inSyncListenerExact).onInSyncStatusChange(true); + } + + @Test + public void removeInSyncListener_addAdditionalListenerBeforeRemoving() { + final long syncTolerance = Synchronizer.DEFAULT_IN_SYNC_TOLERANCE + 1L; + setupOutOfSyncState(); + + // Add listener + InSyncListener listenerToRemove = mock(InSyncListener.class); + InSyncListener otherListener = mock(InSyncListener.class); + final long subscriberId = syncState.subscribeInSync(listenerToRemove, syncTolerance); + syncState.subscribeInSync(otherListener, syncTolerance); + + // Remove listener + syncState.unsubscribeInSync(subscriberId); + + // Catch all the way up + advanceLocalChain(TARGET_CHAIN_HEIGHT); + + // We should not register the in-sync event + verify(listenerToRemove, never()).onInSyncStatusChange(anyBoolean()); + + // Fall out of sync + updateChainState( + syncTargetPeer.getEthPeer(), + TARGET_CHAIN_HEIGHT + syncTolerance + 1L, + TARGET_DIFFICULTY.add(10L)); + + // We should not register the in-sync event + verify(listenerToRemove, never()).onInSyncStatusChange(anyBoolean()); + + final ArgumentCaptor inSyncEventCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(otherListener, times(3)).onInSyncStatusChange(inSyncEventCaptor.capture()); + + final List syncChanges = inSyncEventCaptor.getAllValues(); + assertThat(syncChanges.get(0)).isEqualTo(false); + assertThat(syncChanges.get(1)).isEqualTo(true); + assertThat(syncChanges.get(2)).isEqualTo(false); + + // Other listeners should keep running + verify(inSyncListenerExact).onInSyncStatusChange(true); + verify(inSyncListenerExact, times(2)).onInSyncStatusChange(false); + } + + @Test + public void removeInSyncListener_addAdditionalListenerAfterRemoving() { + final long syncTolerance = Synchronizer.DEFAULT_IN_SYNC_TOLERANCE + 1L; + setupOutOfSyncState(); + + // Add listener + InSyncListener listenerToRemove = mock(InSyncListener.class); + InSyncListener otherListener = mock(InSyncListener.class); + final long subscriberId = syncState.subscribeInSync(listenerToRemove, syncTolerance); + + // Remove listener + syncState.unsubscribeInSync(subscriberId); + + // Add new listener + syncState.subscribeInSync(otherListener, syncTolerance); + + // Catch all the way up + advanceLocalChain(TARGET_CHAIN_HEIGHT); + + // We should not register the sync event + verify(listenerToRemove, never()).onInSyncStatusChange(anyBoolean()); + + // Fall out of sync + updateChainState( + syncTargetPeer.getEthPeer(), + TARGET_CHAIN_HEIGHT + syncTolerance + 1L, + TARGET_DIFFICULTY.add(10L)); + + // We should not register the sync event + verify(listenerToRemove, never()).onInSyncStatusChange(anyBoolean()); + + final ArgumentCaptor inSyncEventCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(otherListener, times(3)).onInSyncStatusChange(inSyncEventCaptor.capture()); + + final List syncChanges = inSyncEventCaptor.getAllValues(); + assertThat(syncChanges.get(0)).isEqualTo(false); + assertThat(syncChanges.get(1)).isEqualTo(true); + assertThat(syncChanges.get(2)).isEqualTo(false); + + // Other listeners should keep running + verify(inSyncListenerExact, times(2)).onInSyncStatusChange(false); + verify(inSyncListenerExact).onInSyncStatusChange(true); + } + + @Test + public void syncStatusListener_receivesEventWhenSyncTargetSet() { + syncState.setSyncTarget(syncTargetPeer.getEthPeer(), blockchain.getBlockHeader(3L).get()); + + verify(syncStatusListener).onSyncStatusChanged(syncStatusCaptor.capture()); + + assertThat(syncStatusCaptor.getAllValues()).hasSize(1); + final Optional syncStatus = syncStatusCaptor.getValue(); + + assertThat(syncStatus).isPresent(); + assertThat(syncStatus.get().getStartingBlock()).isEqualTo(3L); + assertThat(syncStatus.get().getCurrentBlock()).isEqualTo(OUR_CHAIN_HEAD_NUMBER); + assertThat(syncStatus.get().getHighestBlock()).isEqualTo(TARGET_CHAIN_HEIGHT); + } + + @Test + public void syncStatusListener_receivesEventWhenSyncTargetCleared() { + syncState.setSyncTarget(syncTargetPeer.getEthPeer(), blockchain.getBlockHeader(3L).get()); + syncState.clearSyncTarget(); + + verify(syncStatusListener, times(2)).onSyncStatusChanged(syncStatusCaptor.capture()); + + List> events = syncStatusCaptor.getAllValues(); + assertThat(events).hasSize(2); + + // Check first value + final Optional syncingEvent = events.get(0); + assertThat(syncingEvent).isPresent(); + assertThat(syncingEvent.get().getStartingBlock()).isEqualTo(3L); + assertThat(syncingEvent.get().getCurrentBlock()).isEqualTo(OUR_CHAIN_HEAD_NUMBER); + assertThat(syncingEvent.get().getHighestBlock()).isEqualTo(TARGET_CHAIN_HEIGHT); + + // Check second value + final Optional clearedEvent = events.get(1); + assertThat(clearedEvent).isEmpty(); + } + + @Test + public void syncStatusListener_ignoreNoopChangesToSyncTarget() { + syncState.clearSyncTarget(); + syncState.setSyncTarget(syncTargetPeer.getEthPeer(), blockchain.getBlockHeader(3L).get()); + syncState.setSyncTarget(syncTargetPeer.getEthPeer(), blockchain.getBlockHeader(3L).get()); + syncState.clearSyncTarget(); + syncState.clearSyncTarget(); + + verify(syncStatusListener, times(2)).onSyncStatusChanged(syncStatusCaptor.capture()); + + List> events = syncStatusCaptor.getAllValues(); + assertThat(events).hasSize(2); + + // Check first value + final Optional syncingEvent = events.get(0); + assertThat(syncingEvent).isPresent(); + assertThat(syncingEvent.get().getStartingBlock()).isEqualTo(3L); + assertThat(syncingEvent.get().getCurrentBlock()).isEqualTo(OUR_CHAIN_HEAD_NUMBER); + assertThat(syncingEvent.get().getHighestBlock()).isEqualTo(TARGET_CHAIN_HEIGHT); + + // Check second value + final Optional clearedEvent = events.get(1); + assertThat(clearedEvent).isEmpty(); + } + + private RespondingEthPeer createPeer(final long blockHeight) { + return EthProtocolManagerTestUtil.createPeer(ethProtocolManager, blockHeight); + } + + private EthPeer mockWorseChain(final EthPeer peer) { + return mockChainIsBetterThan(peer, false); + } + + private EthPeer mockBetterChain(final EthPeer peer) { + return mockChainIsBetterThan(peer, true); + } + + private EthPeer mockChainIsBetterThan(final EthPeer peer, final boolean isBetter) { + final ChainState chainState = spy(peer.chainState()); + final ChainHeadEstimate chainStateSnapshot = spy(peer.chainStateSnapshot()); + lenient().doReturn(isBetter).when(chainState).chainIsBetterThan(any()); + lenient().doReturn(isBetter).when(chainStateSnapshot).chainIsBetterThan(any()); + final EthPeer mockedPeer = spy(peer); + lenient().doReturn(chainStateSnapshot).when(chainState).getSnapshot(); + lenient().doReturn(chainStateSnapshot).when(mockedPeer).chainStateSnapshot(); + lenient().doReturn(chainState).when(mockedPeer).chainState(); + return mockedPeer; + } + + private void setupOutOfSyncState() { + syncState.setSyncTarget(syncTargetPeer.getEthPeer(), blockchain.getGenesisBlock().getHeader()); + verify(inSyncListener).onInSyncStatusChange(false); + verify(inSyncListenerExact).onInSyncStatusChange(false); + } + + @Test + public void inSyncCheckDrivenByABlockImportDoesNotTakeTheSyncStateMonitor() { + // AERE D-426: upstream hyperledger/besu#11041, ported (unnamed lambda parameter spelled out for Java 21) + final List heldMonitorDuringCallback = new ArrayList<>(); + syncState.subscribeInSync( + ignored -> heldMonitorDuringCallback.add(Thread.holdsLock(syncState)), + Synchronizer.DEFAULT_IN_SYNC_TOLERANCE); + + // Fires the block-added observer and therefore calls checkInSync() on this thread. + advanceLocalChain(blockchain.getChainHeadBlockNumber() + 1); + + assertThat(heldMonitorDuringCallback) + .withFailMessage("a block import called checkInSync() while holding the sync state monitor") + .isNotEmpty() + .containsOnly(false); + } + + /** + * AERE D-426: the chain that froze validator 2 of testnet 28001 for 3 h 18 min on 2026-09-28. The chain downloader + * sets a sync target (replaceSyncTarget holds this monitor) and publishes the sync status; the BFT mining + * coordinator's listener stops the coordinator and waits for the BFT thread, which is importing a block, and the + * block-added observer calls checkInSync(). With checkInSync() on the monitor the import cannot finish while the + * listener waits, so each side waits for the other. + */ + @Test + public void syncStatusListenerWaitingOnABlockImportDoesNotDeadlockTheImport() throws Exception { + final CountDownLatch inCallback = new CountDownLatch(1); + final CountDownLatch blockAppended = new CountDownLatch(1); + final AtomicBoolean importFinishedWhileListenerWaited = new AtomicBoolean(); + final AtomicReference importFailure = new AtomicReference<>(); + syncState.subscribeSyncStatus( + ignored -> { + if (inCallback.getCount() == 0) { + return; + } + inCallback.countDown(); + try { + // stands for BftMiningCoordinator.stop() -> BftProcessor.awaitStop() + importFinishedWhileListenerWaited.set(blockAppended.await(5, TimeUnit.SECONDS)); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + final long headBefore = blockchain.getChainHeadBlockNumber(); + final Thread importer = + new Thread( + () -> { + try { + if (!inCallback.await(5, TimeUnit.SECONDS)) { + return; + } + // stands for QbftRound.importSealedBlock on the BFT thread + advanceLocalChain(headBefore + 1); + blockAppended.countDown(); + } catch (final Throwable t) { + importFailure.set(t); + } + }, + "aere-d426-importer"); + importer.start(); + + // publishes on this thread, as PipelineChainDownloader.startDownloadForSyncTarget does + syncState.setSyncTarget(syncTargetPeer.getEthPeer(), blockchain.getGenesisBlock().getHeader()); + importer.join(TimeUnit.SECONDS.toMillis(30)); + + assertThat(importer.isAlive()).withFailMessage("the importer thread never finished").isFalse(); + assertThat(importFailure.get()).withFailMessage("the import itself failed: %s", importFailure.get()).isNull(); + assertThat(inCallback.getCount()).withFailMessage("the sync status was never published").isZero(); + assertThat(blockchain.getChainHeadBlockNumber()).isEqualTo(headBefore + 1); + assertThat(importFinishedWhileListenerWaited) + .withFailMessage( + "a block import could not finish while a sync-status listener waited for it: " + + "checkInSync() is taking the monitor that replaceSyncTarget() holds (D-426)") + .isTrue(); + } + + private void advanceLocalChain(final long newChainHeight) { + while (blockchain.getChainHeadBlockNumber() < newChainHeight) { + final BlockHeader parent = blockchain.getChainHeadHeader(); + final Block block = + gen.block( + BlockOptions.create() + .setDifficulty(standardDifficultyPerBlock) + .setParentHash(parent.getHash()) + .setBlockNumber(parent.getNumber() + 1L) + .transactionCount(0)); + final List receipts = gen.receipts(block); + blockchain.appendBlock(block, receipts); + } + } + + /** + * Updates the chain state, such that the peer will end up with an estimated height of {@code + * blockHeight} and an estimated total difficulty of {@code totalDifficulty} + * + * @param peer The peer whose chain should be updated + * @param blockHeight The target estimated block height + * @param totalDifficulty The total difficulty + */ + private void updateChainState( + final EthPeer peer, final long blockHeight, final Difficulty totalDifficulty) { + // Chain state is updated based on the parent of the announced block + // So, increment block number by 1 and set block difficulty to zero + // in order to update to the values we want + final BlockHeader header = + new BlockHeaderTestFixture() + .number(blockHeight + 1L) + .difficulty(Difficulty.ZERO) + .buildHeader(); + peer.chainState().updateForAnnouncedBlock(header, totalDifficulty); + + // Sanity check this logic still holds + assertThat(peer.chainState().getEstimatedHeight()).isEqualTo(blockHeight); + assertThat(peer.chainState().getEstimatedTotalDifficulty()).isEqualTo(totalDifficulty); + } + + @Test + public void shouldSubscribeAndUnsubscribeTTDReachedListener() { + TTDReachedListener listener = mock(TTDReachedListener.class); + + long listenerId = syncState.subscribeTTDReached(listener); + assertThat(listenerId).isGreaterThanOrEqualTo(0); + + boolean unsubscribed = syncState.unsubscribeTTDReached(listenerId); + assertThat(unsubscribed).isTrue(); + + // Unsubscribing again should return false + boolean unsubscribedAgain = syncState.unsubscribeTTDReached(listenerId); + assertThat(unsubscribedAgain).isFalse(); + } + + @Test + public void shouldNotifyTTDReachedListeners() { + TTDReachedListener listener = mock(TTDReachedListener.class); + syncState.subscribeTTDReached(listener); + + syncState.setReachedTerminalDifficulty(true); + + verify(listener).onTTDReached(true); + } + + @Test + public void shouldNotifyMultipleTTDReachedListeners() { + TTDReachedListener listener1 = mock(TTDReachedListener.class); + TTDReachedListener listener2 = mock(TTDReachedListener.class); + + syncState.subscribeTTDReached(listener1); + syncState.subscribeTTDReached(listener2); + + syncState.setReachedTerminalDifficulty(false); + + verify(listener1).onTTDReached(false); + verify(listener2).onTTDReached(false); + } + + @Test + public void shouldSubscribeAndUnsubscribeCompletionListener() { + InitialSyncCompletionListener listener = mock(InitialSyncCompletionListener.class); + + long listenerId = syncState.subscribeCompletionReached(listener); + assertThat(listenerId).isGreaterThanOrEqualTo(0); + + boolean unsubscribed = syncState.unsubscribeInitialConditionReached(listenerId); + assertThat(unsubscribed).isTrue(); + + // Unsubscribing again should return false + boolean unsubscribedAgain = syncState.unsubscribeInitialConditionReached(listenerId); + assertThat(unsubscribedAgain).isFalse(); + } + + @Test + public void shouldNotifyCompletionListenersWhenSyncCompleted() { + SyncState syncStateWithInitialPhase = + new SyncState(blockchain, ethPeers, true, Optional.empty()); + InitialSyncCompletionListener listener = mock(InitialSyncCompletionListener.class); + + syncStateWithInitialPhase.subscribeCompletionReached(listener); + syncStateWithInitialPhase.markInitialSyncPhaseAsDone(); + + verify(listener).onInitialSyncCompleted(); + } + + @Test + public void shouldNotifyCompletionListenersOnRestart() { + SyncState syncStateWithInitialPhase = + new SyncState(blockchain, ethPeers, true, Optional.empty()); + InitialSyncCompletionListener listener = mock(InitialSyncCompletionListener.class); + + syncStateWithInitialPhase.subscribeCompletionReached(listener); + syncStateWithInitialPhase.markInitialSyncRestart(); + + verify(listener).onInitialSyncRestart(); + } + + @Test + public void shouldHandleCheckpointConstruction() { + Checkpoint checkpoint = + ImmutableCheckpoint.builder() + .blockNumber(100L) + .blockHash(Hash.ZERO) + .totalDifficulty(Difficulty.of(1000L)) + .build(); + SyncState syncStateWithCheckpoint = + new SyncState(blockchain, ethPeers, false, Optional.of(checkpoint)); + + assertThat(syncStateWithCheckpoint.getCheckpoint()).isPresent(); + assertThat(syncStateWithCheckpoint.getCheckpoint().get()).isEqualTo(checkpoint); + } + + @Test + public void shouldReturnEmptyCheckpointWhenNotSet() { + assertThat(syncState.getCheckpoint()).isEmpty(); + } + + @Test + public void shouldTrackInitialSyncPhaseState() { + SyncState syncStateWithPhase = new SyncState(blockchain, ethPeers, true, Optional.empty()); + + assertThat(syncStateWithPhase.isInitialSyncPhaseDone()).isFalse(); + + syncStateWithPhase.markInitialSyncPhaseAsDone(); + assertThat(syncStateWithPhase.isInitialSyncPhaseDone()).isTrue(); + + syncStateWithPhase.markInitialSyncRestart(); + assertThat(syncStateWithPhase.isInitialSyncPhaseDone()).isFalse(); + } + + @Test + public void shouldTrackResyncNeeded() { + assertThat(syncState.isResyncNeeded()).isFalse(); + + syncState.markResyncNeeded(); + assertThat(syncState.isResyncNeeded()).isTrue(); + + syncState.markInitialSyncPhaseAsDone(); + assertThat(syncState.isResyncNeeded()).isFalse(); + } + + @Test + public void shouldTrackAccountToRepair() { + assertThat(syncState.getAccountToRepair()).isEmpty(); + + Address testAddress = Address.fromHexString("0x1234567890123456789012345678901234567890"); + syncState.markAccountToRepair(Optional.of(testAddress)); + + assertThat(syncState.getAccountToRepair()).isPresent(); + assertThat(syncState.getAccountToRepair().get()).isEqualTo(testAddress); + + syncState.markAccountToRepair(Optional.empty()); + assertThat(syncState.getAccountToRepair()).isEmpty(); + } + + @Test + public void shouldReturnFalseForTTDBeforeInitialSyncComplete() { + SyncState syncStateWithPhase = new SyncState(blockchain, ethPeers, true, Optional.empty()); + + syncStateWithPhase.setReachedTerminalDifficulty(true); + + assertThat(syncStateWithPhase.hasReachedTerminalDifficulty()).isPresent(); + assertThat(syncStateWithPhase.hasReachedTerminalDifficulty().get()).isFalse(); + } + + @Test + public void shouldReturnTTDStatusAfterInitialSyncComplete() { + SyncState syncStateWithPhase = new SyncState(blockchain, ethPeers, true, Optional.empty()); + + syncStateWithPhase.setReachedTerminalDifficulty(true); + syncStateWithPhase.markInitialSyncPhaseAsDone(); + + assertThat(syncStateWithPhase.hasReachedTerminalDifficulty()).isPresent(); + assertThat(syncStateWithPhase.hasReachedTerminalDifficulty().get()).isTrue(); + } + + @Test + public void shouldNotifyMultipleSyncProgressUpdates() { + SyncStatusListener listener = mock(SyncStatusListener.class); + syncState.subscribeSyncStatus(listener); + + syncState.setSyncProgress(0, 10, 100); + syncState.setSyncProgress(0, 50, 100); + syncState.setSyncProgress(0, 100, 100); + + verify(listener, times(3)).onSyncStatusChanged(syncStatusCaptor.capture()); + + List> statuses = syncStatusCaptor.getAllValues(); + assertThat(statuses.get(0).get().getCurrentBlock()).isEqualTo(10); + assertThat(statuses.get(1).get().getCurrentBlock()).isEqualTo(50); + assertThat(statuses.get(2).get().getCurrentBlock()).isEqualTo(100); + } + + @Test + public void shouldReturnLocalChainHeight() { + assertThat(syncState.getLocalChainHeight()).isEqualTo(OUR_CHAIN_HEAD_NUMBER); + + advanceLocalChain(OUR_CHAIN_HEAD_NUMBER + 10); + assertThat(syncState.getLocalChainHeight()).isEqualTo(OUR_CHAIN_HEAD_NUMBER + 10); + } + + @Test + public void shouldReturnBestPeerChainHead() { + updateChainState(syncTargetPeer.getEthPeer(), TARGET_CHAIN_HEIGHT, TARGET_DIFFICULTY); + doReturn(Optional.of(syncTargetPeer.getEthPeer())).when(ethPeers).bestPeerWithHeightEstimate(); + + Optional bestPeer = syncState.getBestPeerChainHead(); + + assertThat(bestPeer).isPresent(); + assertThat(bestPeer.get().getEstimatedHeight()).isEqualTo(TARGET_CHAIN_HEIGHT); + } + + @Test + public void shouldReturnEmptyBestPeerChainHeadWhenNoPeers() { + syncTargetPeer.disconnect(DisconnectReason.REQUESTED); + otherPeer.disconnect(DisconnectReason.REQUESTED); + + Optional bestPeer = syncState.getBestPeerChainHead(); + + assertThat(bestPeer).isEmpty(); + } +} diff --git a/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java b/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java new file mode 100644 index 0000000..f3b9584 --- /dev/null +++ b/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/BaseTransactionPoolTest.java @@ -0,0 +1,362 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.core.TransactionTestFixture.createSignedCodeDelegation; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MAX_SCORE; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.BlobType; +import org.hyperledger.besu.datatypes.CodeDelegation; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlobTestFixture; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.TransactionTestFixture; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.core.kzg.BlobProofBundle; +import org.hyperledger.besu.ethereum.core.kzg.BlobsWithCommitments; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.util.TrustedSetupClassLoaderExtension; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.metrics.StubMetricsSystem; +import org.hyperledger.besu.testutil.DeterministicEthScheduler; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.stream.IntStream; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.BeforeEach; + +public class BaseTransactionPoolTest extends TrustedSetupClassLoaderExtension { + + protected static final SignatureAlgorithm SIGNATURE_ALGORITHM = + SignatureAlgorithmFactory.getInstance(); + protected static final KeyPair KEYS1 = SIGNATURE_ALGORITHM.generateKeyPair(); + protected static final KeyPair KEYS2 = SIGNATURE_ALGORITHM.generateKeyPair(); + protected static final Address SENDER1 = Util.publicKeyToAddress(KEYS1.getPublicKey()); + protected static final Address SENDER2 = Util.publicKeyToAddress(KEYS2.getPublicKey()); + protected static final CodeDelegation CODE_DELEGATION_SENDER_1 = + createSignedCodeDelegation(BigInteger.ONE, Address.ZERO, 0, KEYS1); + protected static final Wei DEFAULT_MIN_GAS_PRICE = Wei.of(50); + protected static final Wei DEFAULT_MIN_PRIORITY_FEE = Wei.ZERO; + private static final Random randomizeTxType = new Random(); + + protected final Transaction transaction0 = createTransaction(0); + protected final Transaction transaction1 = createTransaction(1); + protected final Transaction transaction2 = createTransaction(2); + protected final Transaction blobTransaction0 = createEIP4844Transaction(0, KEYS1, 1, 1); + + protected final EthScheduler ethScheduler = new DeterministicEthScheduler(); + protected final StubMetricsSystem metricsSystem = new StubMetricsSystem(); + protected final SenderBalanceChecker senderBalanceChecker = mock(SenderBalanceChecker.class); + + @BeforeEach + public void setupSenderBalanceChecker() { + when(senderBalanceChecker.hasEnoughBalanceFor(any())).thenReturn(true); + } + + protected Transaction createTransaction(final long nonce) { + return createTransaction(nonce, Wei.of(5000L), KEYS1); + } + + protected Transaction createTransaction(final long nonce, final KeyPair keys) { + return createTransaction(nonce, Wei.of(5000L), keys); + } + + protected Transaction createTransaction(final long nonce, final Wei maxGasPrice) { + return createTransaction(nonce, maxGasPrice, KEYS1); + } + + protected Transaction createTransaction(final long nonce, final int payloadSize) { + return createTransaction(nonce, Wei.of(5000L), payloadSize, KEYS1); + } + + protected Transaction createTransaction( + final long nonce, final Wei maxGasPrice, final KeyPair keys) { + return createTransaction(nonce, maxGasPrice, 0, keys); + } + + protected Transaction createEIP1559Transaction( + final long nonce, final KeyPair keys, final int gasFeeMultiplier) { + return createTransaction( + TransactionType.EIP1559, nonce, Wei.of(5000L).multiply(gasFeeMultiplier), 0, null, keys); + } + + protected Transaction createEIP4844Transaction( + final long nonce, final KeyPair keys, final int gasFeeMultiplier, final int blobCount) { + return createTransaction( + TransactionType.BLOB, + nonce, + Wei.of(5000L).multiply(gasFeeMultiplier), + Wei.of(5000L).multiply(gasFeeMultiplier).divide(10), + 0, + blobCount, + BlobType.KZG_PROOF, + null, + keys); + } + + protected Transaction createEIP7702Transaction( + final long nonce, + final KeyPair keys, + final int gasFeeMultiplier, + final List codeDelegations) { + return createTransaction( + TransactionType.DELEGATE_CODE, + nonce, + Wei.of(5000L).multiply(gasFeeMultiplier), + 0, + codeDelegations, + keys); + } + + protected Transaction createTransactionOfSize( + final long nonce, final Wei maxGasPrice, final int txSize, final KeyPair keys) { + + final TransactionType txType = + TransactionType.values()[ + randomizeTxType.nextInt(txSize < blobTransaction0.getSizeForAnnouncement() ? 3 : 4)]; + + final Transaction baseTx = + createTransaction( + txType, + nonce, + maxGasPrice, + maxGasPrice.divide(10), + 0, + 1, + BlobType.KZG_PROOF, + null, + keys); + final int payloadSize = txSize - baseTx.getSizeForAnnouncement(); + + return createTransaction( + txType, + nonce, + maxGasPrice, + maxGasPrice.divide(10), + payloadSize, + 1, + BlobType.KZG_PROOF, + null, + keys); + } + + protected Transaction createTransaction( + final long nonce, final Wei maxGasPrice, final int payloadSize, final KeyPair keys) { + + final TransactionType txType = TransactionType.values()[randomizeTxType.nextInt(4)]; + + return switch (txType) { + case FRONTIER, ACCESS_LIST, EIP1559 -> + createTransaction(txType, nonce, maxGasPrice, payloadSize, null, keys); + case BLOB -> + createTransaction( + txType, + nonce, + maxGasPrice, + maxGasPrice.divide(10), + payloadSize, + 1, + BlobType.KZG_PROOF, + null, + keys); + case DELEGATE_CODE -> + createTransaction( + txType, nonce, maxGasPrice, payloadSize, List.of(CODE_DELEGATION_SENDER_1), keys); + case AERE_PQ -> + throw new IllegalArgumentException( + "AIP-20 transactions need ML-DSA keys and are not generated by this test"); + }; + } + + protected Transaction createTransaction( + final TransactionType type, + final long nonce, + final Wei maxGasPrice, + final int payloadSize, + final List codeDelegations, + final KeyPair keys) { + return createTransaction( + type, + nonce, + maxGasPrice, + maxGasPrice.divide(10), + payloadSize, + 0, + BlobType.KZG_PROOF, + codeDelegations, + keys); + } + + protected Transaction createTransaction( + final TransactionType type, + final long nonce, + final Wei maxGasPrice, + final Wei maxPriorityFeePerGas, + final int payloadSize, + final int blobCount, + final BlobType blobType, + final List codeDelegations, + final KeyPair keys) { + return prepareTransaction( + type, + nonce, + maxGasPrice, + maxPriorityFeePerGas, + payloadSize, + blobCount, + blobType, + codeDelegations) + .createTransaction(keys); + } + + protected TransactionTestFixture prepareTransaction( + final TransactionType type, + final long nonce, + final Wei maxGasPrice, + final Wei maxPriorityFeePerGas, + final int payloadSize, + final int blobCount, + final BlobType blobType, + final List codeDelegations) { + + var tx = + new TransactionTestFixture() + .to(Optional.of(Address.fromHexString("0x634316eA0EE79c701c6F67C53A4C54cBAfd2316d"))) + .value(Wei.of(nonce)) + .nonce(nonce) + .type(type); + if (payloadSize > 0) { + var payloadBytes = Bytes.fromHexString("01".repeat(payloadSize)); + tx.payload(payloadBytes); + } + if (type.supports1559FeeMarket()) { + tx.maxFeePerGas(Optional.of(maxGasPrice)) + .maxPriorityFeePerGas(Optional.of(maxPriorityFeePerGas)); + if (type.supportsBlob() && blobCount > 0) { + tx.maxFeePerBlobGas(Optional.of(maxGasPrice)); + final var blobTestFixture = new BlobTestFixture(); + final var blobProofBundles = + IntStream.range(0, blobCount) + .mapToObj(i -> blobTestFixture.createBlobProofBundle(blobType)) + .toList(); + + final var versionedHashes = + blobProofBundles.stream().map(BlobProofBundle::getVersionedHash).toList(); + + final var blobsWithCommitments = new BlobsWithCommitments(blobProofBundles); + + tx.versionedHashes(Optional.of(versionedHashes)); + tx.blobsWithCommitments(Optional.of(blobsWithCommitments)); + } else if (type.supportsDelegateCode()) { + tx.codeDelegations(codeDelegations); + } + } else { + tx.gasPrice(maxGasPrice); + } + return tx; + } + + protected Transaction createTransactionReplacement( + final Transaction originalTransaction, final KeyPair keys) { + return createTransaction( + originalTransaction.getType(), + originalTransaction.getNonce(), + originalTransaction.getMaxGasPrice().multiply(2), + originalTransaction.getMaxGasPrice().multiply(2).divide(10), + 0, + 1, + originalTransaction + .getBlobsWithCommitments() + .map(BlobsWithCommitments::getBlobType) + .orElse(BlobType.KZG_PROOF), + originalTransaction.getCodeDelegationList().orElse(null), + keys); + } + + protected PendingTransaction createRemotePendingTransaction(final Transaction transaction) { + return new PendingTransaction.Remote(transaction); + } + + protected PendingTransaction createRemotePendingTransaction( + final Transaction transaction, final boolean hasPriority) { + return PendingTransaction.newPendingTransaction(transaction, false, hasPriority, MAX_SCORE); + } + + protected PendingTransaction createLocalPendingTransaction(final Transaction transaction) { + return new PendingTransaction.Local(transaction); + } + + protected void assertTransactionPending( + final PendingTransactions transactions, final Transaction t) { + assertThat(transactions.getTransactionByHash(t.getHash())).contains(t); + } + + protected void assertTransactionNotPending( + final PendingTransactions transactions, final Transaction t) { + assertThat(transactions.getTransactionByHash(t.getHash())).isEmpty(); + } + + protected void assertNoNextNonceForSender( + final PendingTransactions pendingTransactions, final Address sender) { + assertThat(pendingTransactions.getNextNonceForSender(sender)).isEmpty(); + } + + protected void assertNextNonceForSender( + final PendingTransactions pendingTransactions, final Address sender1, final int i) { + assertThat(pendingTransactions.getNextNonceForSender(sender1)).isPresent().hasValue(i); + } + + protected void addLocalTransactions( + final PendingTransactions sorter, final Account sender, final long... nonces) { + for (final long nonce : nonces) { + sorter.addTransaction( + createLocalPendingTransaction(createTransaction(nonce)), Optional.of(sender)); + } + } + + protected long getAddedCount( + final String source, final String priority, final AddReason addReason, final String layer) { + return metricsSystem.getCounterValue( + TransactionPoolMetrics.ADDED_COUNTER_NAME, source, priority, addReason.label(), layer); + } + + protected long getRemovedCount( + final String source, final String priority, final String operation, final String layer) { + return metricsSystem.getCounterValue( + TransactionPoolMetrics.REMOVED_COUNTER_NAME, source, priority, operation, layer); + } +} diff --git a/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java b/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java new file mode 100644 index 0000000..ccbf03c --- /dev/null +++ b/anchor/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayersTest.java @@ -0,0 +1,2246 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.ethereum.eth.transactions.layered; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; +import static org.hyperledger.besu.datatypes.TransactionType.ACCESS_LIST; +import static org.hyperledger.besu.datatypes.TransactionType.BLOB; +import static org.hyperledger.besu.datatypes.TransactionType.DELEGATE_CODE; +import static org.hyperledger.besu.datatypes.TransactionType.EIP1559; +import static org.hyperledger.besu.datatypes.TransactionType.FRONTIER; +import static org.hyperledger.besu.ethereum.core.TransactionTestFixture.createSignedCodeDelegation; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayeredRemovalReason.PoolRemovalReason.INVALIDATED; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.AuthorityAndNonce.NO_DELEGATIONS; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.AuthorityAndNonce.delegation; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.AuthorityAndNonce.toCodeDelegations; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S1; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S2; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S3; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.S4; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.SP1; +import static org.hyperledger.besu.ethereum.eth.transactions.layered.LayersTest.Sender.SP2; +import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.TX_EVALUATION_TOO_LONG; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.CodeDelegation; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MiningConfiguration; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; +import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; +import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.testutil.DeterministicEthScheduler; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class LayersTest extends BaseTransactionPoolTest { + private static final int MAX_PRIO_TRANSACTIONS = 3; + private static final int MAX_FUTURE_FOR_SENDER = 10; + private static final Wei BASE_FEE = Wei.ONE; + private static final Wei MIN_GAS_PRICE = BASE_FEE; + private static final byte MIN_SCORE = 125; + + private static final TransactionPoolConfiguration DEFAULT_TX_POOL_CONFIG = + ImmutableTransactionPoolConfiguration.builder() + .maxPrioritizedTransactions(MAX_PRIO_TRANSACTIONS) + .maxPrioritizedTransactionsByType(Map.of(BLOB, 1)) + .maxFutureBySender(MAX_FUTURE_FOR_SENDER) + .minScore(MIN_SCORE) + .pendingTransactionsLayerMaxCapacityBytes( + new PendingTransaction.Remote( + new BaseTransactionPoolTest().createEIP1559Transaction(0, KEYS1, 1)) + .memorySize() + * 3L) + .build(); + + private static final TransactionPoolConfiguration BLOB_TX_POOL_CONFIG = + ImmutableTransactionPoolConfiguration.builder() + .maxPrioritizedTransactions(MAX_PRIO_TRANSACTIONS) + .maxPrioritizedTransactionsByType(Map.of(BLOB, 1)) + .maxFutureBySender(MAX_FUTURE_FOR_SENDER) + .minScore(MIN_SCORE) + .pendingTransactionsLayerMaxCapacityBytes( + new PendingTransaction.Remote( + new BaseTransactionPoolTest().createEIP4844Transaction(0, KEYS1, 1, 1)) + .memorySize() + * 3L) + .build(); + + @ParameterizedTest + @MethodSource("providerAddTransactions") + void addTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerAddTransactionsMultipleSenders") + void addTransactionsMultipleSenders(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerRemoveTransactions") + void removeTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerInterleavedAddRemoveTransactions") + void interleavedAddRemoveTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerBlockAdded") + void removeConfirmedTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerNextNonceForSender") + void nextNonceForSender(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerSelectTransactions") + void selectTransactions(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerReorg") + void reorg(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerAsyncWorldStateUpdates") + void asyncWorldStateUpdates(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerPrioritySenders") + void prioritySenders(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerMaxPrioritizedByType") + void maxPrioritizedByType(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerPenalized") + void penalized(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerConfirmedEIP7702") + void confirmedEIP7702(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerSenderBalance") + void senderBalance(final Scenario scenario) { + assertScenario(scenario); + } + + @ParameterizedTest + @MethodSource("providerPendingBySender") + void pendingBySender(final Scenario scenario) { + assertScenario(scenario); + } + + private void assertScenario(final Scenario scenario) { + scenario.run(); + } + + static Stream providerAddTransactions() { + return Stream.of( + Arguments.of( + new Scenario("add first").addForSender(S1, 0).expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("add first sparse").addForSender(S1, 1).expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("fill prioritized") + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized reverse") + .addForSender(S1, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized mixed order 1 step by step") + .addForSender(S1, 2) + .expectedSparseForSender(S1, 2) + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSenders()), + Arguments.of( + new Scenario("fill prioritized mixed order 2") + .addForSender(S1, 0, 2, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("overflow to ready") + .addForSender(S1, 0, 1, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready reverse") + .addForSender(S1, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready mixed order 1") + .addForSender(S1, 3, 0, 2, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready mixed order 2") + .addForSender(S1, 0, 3, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to sparse") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse reverse") + .addForSender(S1, 6, 5, 4, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + // 4,5,6 are evicted since max capacity of sparse layer is 3 txs + .expectedReadyForSender(S1, 3) + .expectedDroppedForSender(S1, 4, 5, 6)), + Arguments.of( + new Scenario("overflow to sparse mixed order 1") + .addForSender(S1, 6, 0, 4, 1, 3, 2, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse mixed order 2") + .addForSender(S1, 0, 4, 6, 1, 5, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse mixed order 3") + .addForSender(S1, 0, 1, 2, 3, 5, 6, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("nonce gap to sparse 1") + .addForSender(S1, 0, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2)), + Arguments.of( + new Scenario("nonce gap to sparse 2") + .addForSender(S1, 0, 1, 2, 3, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSparseForSender(S1, 5)), + Arguments.of( + new Scenario("fill sparse 1") + .addForSender(S1, 2, 3, 5) + .expectedSparseForSender(S1, 2, 3, 5)), + Arguments.of( + new Scenario("fill sparse 2") + .addForSender(S1, 5, 3, 2) + .expectedSparseForSender(S1, 2, 3, 5)), + Arguments.of( + new Scenario("overflow sparse 1") + .addForSender(S1, 1, 2, 3, 4) + .expectedSparseForSender(S1, 1, 2, 3) + .expectedDroppedForSender(S1, 4)), + Arguments.of( + new Scenario("overflow sparse 2") + .addForSender(S1, 4, 2, 3, 1) + .expectedSparseForSender(S1, 1, 2, 3) + .expectedDroppedForSender(S1, 4)), + Arguments.of( + new Scenario("overflow sparse 3") + .addForSender(S1, 0, 4, 2, 3, 5) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2, 3, 4) + .expectedDroppedForSender(S1, 5))); + } + + static Stream providerAddTransactionsMultipleSenders() { + return Stream.of( + Arguments.of( + new Scenario("add first") + .addForSenders(S1, 0, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S1, 0)), + Arguments.of( + new Scenario("add first sparse") + .addForSenders(S1, 1, S2, 2) + .expectedSparseForSenders(S2, 2, S1, 1)), + Arguments.of( + new Scenario("fill prioritized 1") + .addForSender(S1, 0, 1, 2) + .addForSender(S2, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .expectedReadyForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized 2") + .addForSender(S2, 0, 1, 2) + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .expectedReadyForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized 3") + .addForSenders(S1, 0, S2, 0, S1, 1, S2, 1, S1, 2, S2, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .expectedReadyForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill prioritized mixed order") + .addForSenders(S1, 2, S2, 1) + .expectedPrioritizedForSenders() + .expectedReadyForSenders() + .expectedSparseForSenders(S2, 1, S1, 2) + .addForSenders(S2, 2, S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSenders() + .expectedSparseForSenders(S2, 1, S2, 2, S1, 2) + .addForSenders(S1, 1) + .expectedPrioritizedForSenders(S1, 0, S1, 1, S1, 2) + .expectedReadyForSenders() + .expectedSparseForSenders(S2, 1, S2, 2) + .addForSenders(S2, 0) + // only S2[0] is prioritized because there is no space to try to fill gaps + .expectedPrioritizedForSenders(S2, 0, S1, 0, S1, 1) + .expectedReadyForSender(S1, 2) + .expectedSparseForSenders(S2, 1, S2, 2) + // confirm some S1 txs will promote S2 txs + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 1) + .expectedReadyForSenders(S2, 2, S1, 2) + .expectedSparseForSenders() + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S2, 2) + .expectedReadyForSender(S1, 2) + .expectedSparseForSenders() + .confirmedForSenders(S2, 1) + .expectedPrioritizedForSenders(S2, 2, S1, 2) + .expectedReadyForSenders() + .expectedSparseForSenders()), + Arguments.of( + new Scenario("overflow to ready 1") + .addForSenders(S1, 0, S1, 1, S2, 0, S2, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready 2") + .addForSenders(S2, 0, S2, 1, S1, 0, S1, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready 3") + .addForSenders(S1, 0, S2, 0, S2, 1, S1, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready reverse 1") + .addForSenders(S1, 1, S1, 0, S2, 1, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready reverse 2") + .addForSenders(S1, 1, S2, 1, S2, 0, S1, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to ready mixed") + .addForSenders(S2, 1, S1, 0, S1, 1, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S2, 1, S1, 0) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to sparse") + .addForSenders(S1, 0, S1, 1, S2, 0, S2, 1, S3, 0, S3, 1, S3, 2) + .expectedPrioritizedForSender(S3, 0, 1, 2) + .expectedReadyForSenders(S2, 0, S2, 1, S1, 0) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to sparse reverse") + .addForSenders(S3, 2, S3, 1, S3, 0, S2, 1, S2, 0, S1, 1, S1, 0) + .expectedPrioritizedForSender(S3, 0, 1, 2) + .expectedReadyForSenders(S2, 0, S2, 1, S1, 0) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("overflow to sparse mixed") + .addForSenders(S2, 0, S3, 2, S1, 1) + .expectedPrioritizedForSender(S2, 0) + .expectedReadyForSenders() + .expectedSparseForSenders(S1, 1, S3, 2) + .addForSenders(S2, 1) + .expectedPrioritizedForSenders(S2, 0, S2, 1) + .expectedReadyForSenders() + .expectedSparseForSenders(S1, 1, S3, 2) + .addForSenders(S3, 0) + .expectedPrioritizedForSenders(S3, 0, S2, 0, S2, 1) + .expectedReadyForSenders() + .expectedSparseForSenders(S1, 1, S3, 2) + .addForSenders(S1, 0) + .expectedPrioritizedForSenders(S3, 0, S2, 0, S2, 1) + .expectedReadyForSenders(S1, 0, S1, 1) + .expectedSparseForSender(S3, 2) + .addForSenders(S3, 1) + // only S3[1] is prioritized because there is no space to try to fill gaps + .expectedPrioritizedForSenders(S3, 0, S3, 1, S2, 0) + .expectedReadyForSenders(S2, 1, S1, 0, S1, 1) + .expectedSparseForSender(S3, 2) + .addForSenders(S4, 0, S4, 1, S3, 3) + .expectedPrioritizedForSenders(S4, 0, S4, 1, S3, 0) + .expectedReadyForSenders(S3, 1, S2, 0, S2, 1) + .expectedSparseForSenders(S1, 0, S1, 1, S3, 2) + // ToDo: non optimal discard, worth to improve? + .expectedDroppedForSender(S3, 3)), + Arguments.of( + new Scenario("replacement cross layer") + .addForSenders(S2, 0, S3, 2, S1, 1, S2, 1, S3, 0, S1, 0, S3, 1) + // only S3[1] is prioritized because there is no space to try to fill gaps + .expectedPrioritizedForSenders(S3, 0, S3, 1, S2, 0) + .expectedReadyForSenders(S2, 1, S1, 0, S1, 1) + .expectedSparseForSender(S3, 2) + .replaceForSenders(S3, 2) // added in prioritized, but replacement in sparse + .expectedPrioritizedForSenders(S3, 0, S3, 1, S3, 2) + .expectedReadyForSenders(S2, 0, S2, 1, S1, 0) + .expectedSparseForSender(S1, 1))); + } + + static Stream providerRemoveTransactions() { + return Stream.of( + // when expected*ForSender(s) is not present, by default there is a check that the layers + // are empty + Arguments.of(new Scenario("remove not existing").removeForSender(S1, 0)), + Arguments.of(new Scenario("add/remove first").addForSender(S1, 0).removeForSender(S1, 0)), + Arguments.of( + new Scenario("add/remove first sparse").addForSender(S1, 1).removeForSender(S1, 1)), + Arguments.of( + new Scenario("fill/remove prioritized 1") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill/remove prioritized 2") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 2, 1, 0)), + Arguments.of( + new Scenario("fill/remove prioritized reverse 1") + .addForSender(S1, 2, 1, 0) + .removeForSender(S1, 0, 1, 2)), + Arguments.of( + new Scenario("fill/remove prioritized reverse 2") + .addForSender(S1, 2, 1, 0) + .removeForSender(S1, 2, 1, 0)), + Arguments.of( + new Scenario("fill/remove first prioritized") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 0) + .expectedSparseForSender(S1, 1, 2)), + Arguments.of( + new Scenario("fill/remove last prioritized") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1)), + Arguments.of( + new Scenario("fill/remove middle prioritized") + .addForSender(S1, 0, 1, 2) + .removeForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2)), + Arguments.of( + new Scenario("overflow to ready then remove 1") + .addForSender(S1, 0, 1, 2, 3) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready then remove 2") + .addForSender(S1, 0, 1, 2, 3, 4) + .removeForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3)), + Arguments.of( + new Scenario("overflow to ready then remove 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .removeForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSparseForSender(S1, 5)), + Arguments.of( + new Scenario("overflow to ready then remove 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .removeForSender(S1, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4, 5)), + Arguments.of( + new Scenario("overflow to ready then remove 5") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .removeForSender(S1, 0) + .expectedSparseForSender(S1, 1, 2, 3) + .expectedDroppedForSender(S1, 4, 5)), + Arguments.of( + new Scenario("overflow to sparse then remove 1") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .removeForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5)), + Arguments.of( + new Scenario("overflow to sparse then remove 2") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .removeForSender(S1, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSparseForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse then remove 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3, 4, 5) + .expectedDroppedForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse then remove 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .removeForSender(S1, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 8)), + Arguments.of( + new Scenario("overflow to sparse then remove 5") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .removeForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse then remove 6") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .removeForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse then remove 7") + .setAccountNonce(S1, 1) + .addForSender(S1, 1, 2, 3, 4, 5, 8) + .removeForSender(S1, 2) + .expectedPrioritizedForSender(S1, 1) + .expectedSparseForSender(S1, 3, 4, 5) + .expectedDroppedForSender(S1, 8))); + } + + static Stream providerInterleavedAddRemoveTransactions() { + return Stream.of( + Arguments.of( + new Scenario("interleaved add/remove 1") + .addForSender(S1, 0) + .removeForSender(S1, 0) + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("interleaved add/remove 2") + .addForSender(S1, 0) + .removeForSender(S1, 0) + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("interleaved add/remove 3") + .addForSender(S1, 0) + .removeForSender(S1, 0) + .addForSender(S1, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1))); + } + + static Stream providerBlockAdded() { + return Stream.of( + Arguments.of(new Scenario("confirmed not exist").confirmedForSenders(S1, 0)), + Arguments.of( + new Scenario("confirmed below existing lower nonce") + .addForSender(S1, 1) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1)), + Arguments.of( + new Scenario("confirmed only one existing") + .addForSender(S1, 0) + .confirmedForSenders(S1, 0)), + Arguments.of( + new Scenario("confirmed above existing") + .addForSender(S1, 0) + .confirmedForSenders(S1, 1)), + Arguments.of( + new Scenario("confirmed some existing 1") + .addForSender(S1, 0, 1) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1)), + Arguments.of( + new Scenario("confirmed some existing 2") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSender(S1, 2)), + Arguments.of( + new Scenario("overflow to ready and confirmed some existing 1") + .addForSender(S1, 0, 1, 2, 3) + .confirmedForSenders(S1, 3)), + Arguments.of( + new Scenario("overflow to ready and confirmed some existing 2") + .addForSender(S1, 0, 1, 2, 3) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3)), + Arguments.of( + new Scenario("overflow to ready and confirmed some existing 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSender(S1, 2, 3, 4) + .expectedReadyForSender(S1, 5)), + Arguments.of( + new Scenario("overflow to ready and confirmed all existing") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .confirmedForSenders(S1, 5)), + Arguments.of( + new Scenario("overflow to ready and confirmed above highest nonce") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .confirmedForSenders(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 1") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 2") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .confirmedForSenders(S1, 2) + .expectedPrioritizedForSender(S1, 3, 4, 5) + .expectedReadyForSender(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .confirmedForSenders(S1, 3) + .expectedPrioritizedForSender(S1, 4, 5, 6)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5, 6) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed some existing with gap") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all w/o gap") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .confirmedForSenders(S1, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all with gap") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all before gap 1") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 5) + .expectedSparseForSender(S1, 7, 8)), + Arguments.of( + new Scenario("overflow to sparse and confirmed all before gap 2") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7, 8) + .confirmedForSenders(S1, 6) + .expectedPrioritizedForSender(S1, 7, 8))); + } + + static Stream providerNextNonceForSender() { + return Stream.of( + Arguments.of(new Scenario("not exist").expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("first transaction") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedNextNonceForSenders(S1, 1)), + Arguments.of( + new Scenario("fill prioritized") + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("reverse fill prioritized") + .addForSender(S1, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("reverse fill prioritized 2") + .addForSender(S1, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedNextNonceForSenders(S1, 4)), + Arguments.of( + new Scenario("overflow to ready") + .addForSender(S1, 0, 1, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedNextNonceForSenders(S1, 4)), + Arguments.of( + new Scenario("fill ready") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedNextNonceForSenders(S1, 6)), + Arguments.of( + new Scenario("overflow to sparse") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6) + .expectedNextNonceForSenders(S1, 7)), + Arguments.of( + new Scenario("fill sparse") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedNextNonceForSenders(S1, 9)), + Arguments.of( + new Scenario("drop transaction") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedDroppedForSender(S1, 9) + .expectedNextNonceForSenders(S1, 9)), + Arguments.of( + new Scenario("first with gap") + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("sequence with gap 1") + .addForSender(S1, 0, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedNextNonceForSenders(S1, 1)), + Arguments.of( + new Scenario("sequence with gap 2") + .addForSender(S1, 0, 1, 2, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("sequence with gap 3") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 7) + .expectedNextNonceForSenders(S1, 6)), + Arguments.of( + new Scenario("sequence with gap 4") + .addForSender(S1, 0, 1, 2, 3, 4, 5, 6, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 8) + .expectedNextNonceForSenders(S1, 7)), + Arguments.of( + new Scenario("out of order sequence 1") + .addForSender(S1, 2, 0, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("out of order sequence 2") + .addForSender(S1, 2, 0, 4, 3, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 5)), + Arguments.of( + new Scenario("out of order sequence with gap 1") + .addForSender(S1, 2, 1) + .expectedSparseForSender(S1, 1, 2) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("out of order sequence with gap 2") + .addForSender(S1, 2, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedNextNonceForSenders(S1, 1)), + Arguments.of( + new Scenario("out of order sequence with gap 3") + .addForSender(S1, 2, 0, 1, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("out of order sequence with gap 4") + .addForSender(S1, 2, 0, 4, 1, 6, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSparseForSender(S1, 6) + .expectedNextNonceForSenders(S1, 5)), + Arguments.of( + new Scenario("no gap and confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2) + .expectedNextNonceForSenders(S1, 3)), + Arguments.of( + new Scenario("all confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 2) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("all confirmed 2") + .addForSender(S1, 3, 0, 4, 1, 5, 2) + .confirmedForSenders(S1, 8) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("all confirmed 3") + .addForSender(S1, 3, 0, 4, 1, 2, 5, 6, 7) + .confirmedForSenders(S1, 8) + .expectedNextNonceForSenders(S1, null)), + Arguments.of( + new Scenario("all confirmed step by step 1") + .addForSender(S1, 3) + .expectedSparseForSender(S1, 3) + .expectedNextNonceForSenders(S1, null) + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3) + .expectedNextNonceForSenders(S1, 1) + .addForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 1) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 2) + .addForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedNextNonceForSenders(S1, 5) + .addForSender(S1, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedNextNonceForSenders(S1, 6) + .addForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6) + .expectedNextNonceForSenders(S1, 7) + .addForSender(S1, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7) + .expectedNextNonceForSenders(S1, 8) + .addForSender(S1, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedNextNonceForSenders(S1, 9) + .addForSender(S1, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedDroppedForSender(S1, 9) + .expectedNextNonceForSenders(S1, 9) + .confirmedForSenders(S1, 9) + .expectedPrioritizedForSenders() + .expectedReadyForSenders() + .expectedSparseForSenders() + .expectedNextNonceForSenders(S1, null))); + } + + static Stream providerSelectTransactions() { + return Stream.of( + Arguments.of(new Scenario("no transactions").expectedSelectedTransactions()), + Arguments.of( + new Scenario("first transaction") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSelectedTransactions(S1, 0)), + Arguments.of( + new Scenario("fill prioritized") + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("reverse fill prioritized") + .addForSender(S1, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("reverse fill prioritized 2") + .addForSender(S1, 3, 2, 1, 0) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("overflow to ready") + .addForSender(S1, 0, 1, 2, 3) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("first with gap") + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1) + .expectedSelectedTransactions()), + Arguments.of( + new Scenario("sequence with gap 1") + .addForSender(S1, 0, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedSelectedTransactions(S1, 0)), + Arguments.of( + new Scenario("sequence with gap 2") + .addForSender(S1, 0, 1, 2, 4) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 4) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("out of order sequence 1") + .addForSender(S1, 2, 0, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("out of order sequence 2") + .addForSender(S1, 2, 0, 4, 3, 1) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2)), + Arguments.of( + new Scenario("out of order sequence with gap 1") + .addForSender(S1, 2, 1) + .expectedSparseForSender(S1, 1, 2) + .expectedSelectedTransactions()), + Arguments.of( + new Scenario("out of order sequence with gap 2") + .addForSender(S1, 2, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .expectedSelectedTransactions(S1, 0)), + Arguments.of( + new Scenario("no gap and confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1, 2) + .expectedSelectedTransactions(S1, 1, S1, 2)), + Arguments.of( + new Scenario("all confirmed 1") + .addForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 2) + .expectedSelectedTransactions()), + Arguments.of( + new Scenario("all confirmed step by step 1") + .addForSender(S1, 3) + .expectedSparseForSender(S1, 3) + .expectedSelectedTransactions() + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3) + .expectedSelectedTransactions(S1, 0) + .addForSender(S1, 4) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .expectedSparseForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0, S1, 1) + .addForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 7) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 8) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .addForSender(S1, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .expectedSparseForSender(S1, 6, 7, 8) + .expectedDroppedForSender(S1, 9) + .expectedSelectedTransactions(S1, 0, S1, 1, S1, 2) + .confirmedForSenders(S1, 9) + .expectedPrioritizedForSenders() + .expectedReadyForSenders() + .expectedSparseForSenders() + .expectedSelectedTransactions())); + } + + static Stream providerReorg() { + return Stream.of( + Arguments.of( + new Scenario("reorg") + .setAccountNonce(S1, 0) + .addForSender(S1, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .confirmedForSenders(S1, 1) + .expectedPrioritizedForSender(S1, 2) + .expectedNextNonceForSenders(S1, 3) + .addForSender(S1, 3) + .expectedPrioritizedForSender(S1, 2, 3) + .reorgForSenders(S1, 0) // rewind nonce due to reorg + .addForSender(S1, 0, 1) // re-add reorged txs + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3))); + } + + static Stream providerAsyncWorldStateUpdates() { + return Stream.of( + Arguments.of( + new Scenario("nonce forward just before confirmed block is processed") + .setAccountNonce(S1, 0) + .addForSender(S1, 0, 1, 2, 8, 9) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedSparseForSender(S1, 8, 9) + .expectedNextNonceForSenders(S1, 3) + // block has been imported and world state already has the new nonce, + // but block has not yet been processed by txpool + .setAccountNonce(S1, 5) + .addForSender(S1, 7) + .expectedPrioritizedForSenders() + .expectedSparseForSender(S1, 7, 8, 9))); + } + + static Stream providerPrioritySenders() { + return Stream.of( + Arguments.of( + new Scenario("priority first same fee") + .addForSenders(S1, 0, SP1, 0) + .expectedPrioritizedForSenders(SP1, 0, S1, 0)), + Arguments.of( + new Scenario("priority first lower fee") + .addForSenders(S2, 0, SP1, 0) + .expectedPrioritizedForSenders(SP1, 0, S2, 0)), + Arguments.of( + new Scenario("priority first higher fee") + .addForSenders(S1, 0, SP2, 0) + .expectedPrioritizedForSenders(SP2, 0, S1, 0)), + Arguments.of( + new Scenario("same priority order by fee") + .addForSenders(SP1, 0, SP2, 0) + .expectedPrioritizedForSenders(SP2, 0, SP1, 0)), + Arguments.of( + new Scenario("same priority order by fee") + .addForSenders(SP2, 0, SP1, 0) + .expectedPrioritizedForSenders(SP2, 0, SP1, 0)), + Arguments.of( + new Scenario("priority first overflow to ready") + .addForSender(S2, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .addForSender(SP1, 0) + .expectedPrioritizedForSenders(SP1, 0, S2, 0, S2, 1) + .expectedReadyForSender(S2, 2)), + Arguments.of( + new Scenario("priority first overflow to ready 2") + .addForSender(S2, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .addForSender(SP1, 0, 1, 2) + .expectedPrioritizedForSender(SP1, 0, 1, 2) + .expectedReadyForSender(S2, 0, 1, 2)), + Arguments.of( + new Scenario("multiple priority senders first overflow to ready") + .addForSender(S2, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .addForSenders(SP2, 0, SP1, 0) + .expectedPrioritizedForSenders(SP2, 0, SP1, 0, S2, 0) + .expectedReadyForSender(S2, 1, 2)), + Arguments.of( + new Scenario("priority with initial gap") + .addForSender(S2, 0) + .expectedPrioritizedForSender(S2, 0) + .addForSender(SP1, 1) // initial gap + .expectedPrioritizedForSender(S2, 0) + .expectedSparseForSender(SP1, 1) + .addForSender(SP1, 0) // fill gap + .expectedPrioritizedForSenders(SP1, 0, SP1, 1, S2, 0) + .expectedSparseForSenders()), + Arguments.of( + new Scenario("priority with initial gap overflow to ready") + .addForSender(S2, 0, 1) + .expectedPrioritizedForSender(S2, 0, 1) + .addForSender(SP1, 1) // initial gap + .expectedSparseForSender(SP1, 1) + .addForSender(SP1, 0) // fill gap + .expectedPrioritizedForSenders(SP1, 0, SP1, 1, S2, 0) + .expectedReadyForSender(S2, 1) + .expectedSparseForSenders()), + Arguments.of( + new Scenario("priority with initial gap overflow to ready when prioritized is full") + .addForSender(S2, 0, 1, 2) + .expectedPrioritizedForSender(S2, 0, 1, 2) + .addForSender(SP1, 1) // initial gap + .expectedSparseForSender(SP1, 1) + .addForSender(SP1, 0) // fill gap, but there is not enough space to promote + .expectedPrioritizedForSenders(SP1, 0, S2, 0, S2, 1) + .expectedReadyForSender(S2, 2) + .expectedSparseForSender(SP1, 1) + .confirmedForSenders( + SP1, 0) // asap there is new space the priority tx is promoted first + .expectedPrioritizedForSenders(SP1, 1, S2, 0, S2, 1) + .expectedReadyForSender(S2, 2) + .expectedSparseForSenders()), + Arguments.of( + new Scenario("overflow to ready promote priority first") + .addForSender(SP2, 0, 1, 2) + .expectedPrioritizedForSender(SP2, 0, 1, 2) + .addForSender(S2, 0) + .expectedReadyForSender(S2, 0) + .addForSender(SP1, 0) + .expectedReadyForSenders(SP1, 0, S2, 0) + .confirmedForSenders( + SP2, 0) // asap there is new space the priority tx is promoted first + .expectedPrioritizedForSenders(SP2, 1, SP2, 2, SP1, 0) + .expectedReadyForSender(S2, 0)), + Arguments.of( + new Scenario("priority first overflow to sparse") + .addForSender(SP2, 0, 1, 2) + .addForSender(S3, 0) + .expectedPrioritizedForSender(SP2, 0, 1, 2) + .expectedReadyForSender(S3, 0) + .addForSender(SP1, 0, 1, 2) + .expectedPrioritizedForSender(SP2, 0, 1, 2) + .expectedReadyForSender(SP1, 0, 1, 2) + .expectedSparseForSender(S3, 0)), + Arguments.of( + new Scenario("priority first overflow to sparse 2") + .addForSender(S2, 0, 1, 2) + .addForSender(S3, 0, 1, 2) + .expectedPrioritizedForSender(S3, 0, 1, 2) + .expectedReadyForSender(S2, 0, 1, 2) + .addForSender(SP1, 0) + .expectedPrioritizedForSenders(SP1, 0, S3, 0, S3, 1) + .expectedReadyForSenders(S3, 2, S2, 0, S2, 1) + .expectedSparseForSender(S2, 2)), + Arguments.of( + new Scenario("overflow to sparse promote priority first") + .addForSender(SP2, 0, 1, 2, 3, 4, 5) + .expectedPrioritizedForSender(SP2, 0, 1, 2) + .expectedReadyForSender(SP2, 3, 4, 5) + .addForSender(S3, 0) + .expectedSparseForSender(S3, 0) + .addForSender(SP1, 0) + .expectedSparseForSenders(SP1, 0, S3, 0) + .confirmedForSenders(SP2, 0) + .expectedPrioritizedForSender(SP2, 1, 2, 3) + .expectedReadyForSenders(SP2, 4, SP2, 5, SP1, 0) + .expectedSparseForSender(S3, 0)), + Arguments.of( + new Scenario("discard priority as last") + .addForSender(SP2, 0, 1, 2, 3, 4, 5) + .expectedPrioritizedForSender(SP2, 0, 1, 2) + .expectedReadyForSender(SP2, 3, 4, 5) + .addForSender(S3, 0) + .expectedSparseForSender(S3, 0) + .addForSender(SP1, 0, 1, 2) + .expectedSparseForSender(SP1, 0, 1, 2) + .expectedDroppedForSender(S3, 0))); + } + + static Stream providerMaxPrioritizedByType() { + return Stream.of( + Arguments.of( + new Scenario("first blob tx is prioritized", BLOB_TX_POOL_CONFIG) + .addForSender(S1, BLOB, 0) + .expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("multiple senders only first blob tx is prioritized", BLOB_TX_POOL_CONFIG) + .addForSender(S1, BLOB, 0) + .addForSender(S2, BLOB, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSender(S2, 0)), + Arguments.of( + new Scenario("same sender following blob txs are moved to ready", BLOB_TX_POOL_CONFIG) + .addForSender(S1, BLOB, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSender(S1, 1, 2)), + Arguments.of( + new Scenario("promoting txs respect prioritized count limit", BLOB_TX_POOL_CONFIG) + .addForSender(S1, BLOB, 0, 1, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSender(S1, 1, 2) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 1) + .expectedReadyForSender(S1, 2)), + Arguments.of( + new Scenario("filling gaps respect prioritized count limit", BLOB_TX_POOL_CONFIG) + .addForSender(S1, BLOB, 1) + .expectedSparseForSender(S1, 1) + .addForSender(S1, BLOB, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 1)), + Arguments.of( + new Scenario("promoting to ready is unbounded", BLOB_TX_POOL_CONFIG) + .addForSender(S1, BLOB, 0, 1, 2, 3, 4, 5, 6) + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSender(S1, 1, 2, 3) + .expectedSparseForSender(S1, 4, 5, 6) + .confirmedForSenders(S1, 3) + .expectedPrioritizedForSender(S1, 4) + .expectedReadyForSender(S1, 5, 6) + .expectedSparseForSenders())); + } + + static Stream providerPenalized() { + return Stream.of( + Arguments.of( + new Scenario("single sender, single tx") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .penalizeForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("single sender penalize last") + .addForSender(S1, 0, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .penalizeForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1)), + Arguments.of( + new Scenario("single sender penalize first") + .addForSender(S1, 0, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .penalizeForSender(S1, 0, 1) + // even if 0 has less score it is always the first for the sender + // since otherwise there is a nonce gap + .expectedPrioritizedForSender(S1, 0, 1)), + Arguments.of( + new Scenario("multiple senders, penalize top") + .addForSenders(S1, 0, S2, 0) + // remember S2 pays more fees + .expectedPrioritizedForSenders(S2, 0, S1, 0) + .penalizeForSender(S2, 0) + .expectedPrioritizedForSenders(S1, 0, S2, 0)), + Arguments.of( + new Scenario("multiple senders, penalize bottom") + .addForSenders(S1, 0, S2, 0) + .expectedPrioritizedForSenders(S2, 0, S1, 0) + .penalizeForSender(S1, 0) + .expectedPrioritizedForSenders(S2, 0, S1, 0)), + Arguments.of( + new Scenario("multiple senders, penalize middle") + .addForSenders(S1, 0, S2, 0, S3, 0) + .expectedPrioritizedForSenders(S3, 0, S2, 0, S1, 0) + .penalizeForSender(S2, 0) + .expectedPrioritizedForSenders(S3, 0, S1, 0, S2, 0)), + Arguments.of( + new Scenario("single sender, promote from ready") + .addForSender(S1, 0, 1, 2, 3, 4, 5) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3, 4, 5) + .penalizeForSender(S1, 3) + .confirmedForSenders(S1, 0) + // even if penalized 3 is promoted to avoid nonce gap + .expectedPrioritizedForSender(S1, 1, 2, 3) + .expectedReadyForSender(S1, 4, 5)), + Arguments.of( + new Scenario("multiple senders, overflow to ready") + .addForSenders(S1, 0, S2, 0, S3, 0) + .expectedPrioritizedForSenders(S3, 0, S2, 0, S1, 0) + .expectedReadyForSenders() + .penalizeForSender(S3, 0) + .addForSender(S1, 1) + .expectedPrioritizedForSenders(S2, 0, S1, 0, S1, 1) + // S3(0) is demoted to ready even if it is paying more fees, + // since has a lower score + .expectedReadyForSender(S3, 0)), + Arguments.of( + new Scenario("multiple senders, overflow to sparse") + .addForSenders(S1, 0, S2, 0, S3, 0, S1, 1, S2, 1, S3, 1) + .expectedPrioritizedForSenders(S3, 0, S3, 1, S2, 0) + .expectedReadyForSenders(S2, 1, S1, 0, S1, 1) + .penalizeForSender(S2, 1) + .addForSender(S2, 2) + .expectedReadyForSenders(S1, 0, S1, 1, S2, 1) + .expectedSparseForSender(S2, 2)), + Arguments.of( + new Scenario("remove below min score") + .addForSender(S1, 0) // score 127 + .expectedPrioritizedForSender(S1, 0) + .penalizeForSender(S1, 0) // score 126 + .expectedPrioritizedForSender(S1, 0) + .penalizeForSender(S1, 0) // score 125 + .expectedPrioritizedForSender(S1, 0) + .penalizeForSender(S1, 0) // score 124, removed since decreased score < MIN_SCORE + .expectedPrioritizedForSenders())); + } + + static Stream providerConfirmedEIP7702() { + return Stream.of( + Arguments.of( + new Scenario("code delegation tx only") + .addForSender(S1, delegation(S2, 0), 0) + .expectedPrioritizedForSender(S1, 0) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSenders()), + Arguments.of( + new Scenario("confirmed delegation over plain tx") + .addForSender(S2, 0) + .addForSender(S1, delegation(S2, 0), 0) + .expectedPrioritizedForSenders(S2, 0, S1, 0) + .confirmedForSenders(S1, 0) + // confirming the code delegation tx updates the nonce for S2, so his conflicting + // plain tx is removed + .expectedPrioritizedForSenders()), + Arguments.of( + new Scenario("confirmed plain tx over delegation") + .addForSender(S2, 0) + .addForSender(S1, delegation(S2, 0), 0) + .expectedPrioritizedForSenders(S2, 0, S1, 0) + .confirmedForSenders(S2, 0) + // verify the code delegation for S2 is still there, of course that delegation will + // fail, + // but is it not possible to remove it from the list + .expectedPrioritizedForSender(S1, 0)), + Arguments.of( + new Scenario("self code delegation") + .addForSender(S1, delegation(S1, 1), 0) + .expectedPrioritizedForSender(S1, 0) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSenders()), + Arguments.of( + new Scenario("self code delegation and plain tx") + .addForSender(S1, delegation(S1, 1), 0) + .addForSender(S1, 1) + .expectedPrioritizedForSender(S1, 0, 1) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSenders()), + Arguments.of( + new Scenario("self code delegation and plain tx in sparse") + .addForSender(S1, delegation(S1, 1), 0) + .addForSender(S1, 2) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 2) + .confirmedForSenders(S1, 0) + .expectedPrioritizedForSender(S1, 2) + .expectedSparseForSenders())); + } + + static Stream providerSenderBalance() { + return Stream.of( + Arguments.of( + new Scenario("not enough balance not added to prioritized") + .hasSenderEnoughBalance(S1, false) + .addForSender(S1, 0) + .expectedReadyForSender(S1, 0)), + Arguments.of( + new Scenario("first enough balance then not") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + // sender hasn't enough balance for 2nd tx, so it goes to ready + .hasSenderEnoughBalance(S1, false) + .addForSender(S1, 1) + .expectedReadyForSender(S1, 1)), + Arguments.of( + new Scenario("first no balance then yes") + // sender hasn't enough balance for tx, so it goes to ready + .hasSenderEnoughBalance(S1, false) + .addForSender(S1, 0) + .expectedReadyForSender(S1, 0) + // now sender has balance and new block is imported and the tx is promoted + .hasSenderEnoughBalance(S1, true) + .confirmedForSenders() + .expectedPrioritizedForSender(S1, 0) + .expectedReadyForSenders())); + } + + static Stream providerPendingBySender() { + return Stream.of( + Arguments.of(new Scenario("no txs for sender").expectedAllForSender(S1)), + Arguments.of( + new Scenario("only one in prio") + .addForSender(S1, 0) + .expectedPrioritizedForSender(S1, 0) + .expectedAllForSender(S1, 0)), + Arguments.of( + new Scenario("only one in ready") + .hasSenderEnoughBalance(S1, false) + .addForSender(S1, 0) + .expectedReadyForSender(S1, 0) + .expectedAllForSender(S1, 0)), + Arguments.of( + new Scenario("only one in sparse") + .addForSender(S1, 1) + .expectedSparseForSender(S1, 1) + .expectedAllForSender(S1, 1)), + Arguments.of( + new Scenario("only one in prio and sparse") + .addForSender(S1, 0, 3) + .expectedPrioritizedForSender(S1, 0) + .expectedSparseForSender(S1, 3) + .expectedAllForSender(S1, 0, 3)), + Arguments.of( + new Scenario("something in all layers") + .addForSender(S1, 0, 1, 2, 3, 6) + .expectedPrioritizedForSender(S1, 0, 1, 2) + .expectedReadyForSender(S1, 3) + .expectedSparseForSender(S1, 6) + .expectedAllForSender(S1, 0, 1, 2, 3, 6))); + } + + private static BlockHeader mockBlockHeader() { + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getBaseFee()).thenReturn(Optional.of(BASE_FEE)); + return blockHeader; + } + + private static boolean transactionReplacementTester( + final TransactionPoolConfiguration poolConfig, + final PendingTransaction pt1, + final PendingTransaction pt2) { + final TransactionPoolReplacementHandler transactionReplacementHandler = + new TransactionPoolReplacementHandler( + poolConfig.getPriceBump(), poolConfig.getBlobPriceBump()); + return transactionReplacementHandler.shouldReplace(pt1, pt2, mockBlockHeader()); + } + + static class Scenario extends BaseTransactionPoolTest implements Runnable { + + final String description; + final TransactionPoolConfiguration poolConfig; + final EvictCollectorLayer dropped; + final SparseTransactions sparse; + final ReadyTransactions ready; + final AbstractPrioritizedTransactions prio; + final LayeredPendingTransactions pending; + final NotificationsChecker notificationsChecker = new NotificationsChecker(); + final List actions = new ArrayList<>(); + List lastExpectedAll = new ArrayList<>(); + List lastExpectedPrioritized = new ArrayList<>(); + List lastExpectedReady = new ArrayList<>(); + List lastExpectedSparse = new ArrayList<>(); + List lastExpectedDropped = new ArrayList<>(); + + final EnumMap nonceBySender = new EnumMap<>(Sender.class); + + { + Arrays.stream(Sender.values()).forEach(e -> nonceBySender.put(e, 0L)); + } + + final EnumSet sendersWithReorg = EnumSet.noneOf(Sender.class); + + final EnumMap> liveTxsBySender = + new EnumMap<>(Sender.class); + + { + Arrays.stream(Sender.values()).forEach(e -> liveTxsBySender.put(e, new TreeMap<>())); + } + + final EnumMap> droppedTxsBySender = + new EnumMap<>(Sender.class); + + { + Arrays.stream(Sender.values()).forEach(e -> droppedTxsBySender.put(e, new TreeMap<>())); + } + + Scenario(final String description) { + this(description, DEFAULT_TX_POOL_CONFIG); + } + + Scenario(final String description, final TransactionPoolConfiguration poolConfig) { + this.description = description; + this.poolConfig = poolConfig; + + final TransactionPoolMetrics txPoolMetrics = new TransactionPoolMetrics(metricsSystem); + when(senderBalanceChecker.hasEnoughBalanceFor(any())).thenReturn(true); + + this.dropped = new EvictCollectorLayer(txPoolMetrics); + final EthScheduler ethScheduler = new DeterministicEthScheduler(); + this.sparse = + new SparseTransactions( + poolConfig, + ethScheduler, + this.dropped, + txPoolMetrics, + (pt1, pt2) -> transactionReplacementTester(poolConfig, pt1, pt2), + new BlobCache()); + + this.ready = + new ReadyTransactions( + poolConfig, + ethScheduler, + this.sparse, + txPoolMetrics, + (pt1, pt2) -> transactionReplacementTester(poolConfig, pt1, pt2), + new BlobCache()); + + this.prio = + new BaseFeePrioritizedTransactions( + poolConfig, + LayersTest::mockBlockHeader, + ethScheduler, + this.ready, + txPoolMetrics, + (pt1, pt2) -> transactionReplacementTester(poolConfig, pt1, pt2), + FeeMarket.london(0L), + new BlobCache(), + MiningConfiguration.newDefault().setMinTransactionGasPrice(MIN_GAS_PRICE), + senderBalanceChecker); + + this.pending = new LayeredPendingTransactions(poolConfig, this.prio, ethScheduler); + + this.pending.subscribePendingTransactions(notificationsChecker::collectAddNotification); + this.pending.subscribeDroppedTransactions( + (tx, reason) -> notificationsChecker.collectDropNotification(tx)); + } + + @Override + public void run() { + actions.forEach(Runnable::run); + assertExpectedPrioritized(prio, lastExpectedPrioritized); + assertExpectedReady(ready, lastExpectedReady); + assertExpectedSparse(sparse, lastExpectedSparse); + assertExpectedDropped(dropped, lastExpectedDropped); + } + + public Scenario addForSender(final Sender sender, final long... nonce) { + return addForSender(sender, EIP1559, NO_DELEGATIONS, nonce); + } + + public Scenario addForSender( + final Sender sender, final TransactionType type, final long... nonce) { + internalAddForSender(sender, type, NO_DELEGATIONS, nonce); + actions.add(notificationsChecker::assertExpectedNotifications); + return this; + } + + public Scenario addForSender( + final Sender sender, final AuthorityAndNonce[] authorityAndNonces, final long... nonce) { + return addForSender(sender, DELEGATE_CODE, authorityAndNonces, nonce); + } + + public Scenario addForSender( + final Sender sender, + final TransactionType type, + final AuthorityAndNonce[] authorityAndNonces, + final long... nonce) { + internalAddForSender(sender, type, authorityAndNonces, nonce); + actions.add(notificationsChecker::assertExpectedNotifications); + return this; + } + + private void internalAddForSender( + final Sender sender, + final TransactionType type, + final AuthorityAndNonce[] authorityAndNonces, + final long... nonce) { + actions.add( + () -> { + Arrays.stream(nonce) + .forEach( + n -> { + final var pendingTx = create(sender, type, authorityAndNonces, n); + final Account mockSender = mock(Account.class); + when(mockSender.getNonce()).thenReturn(nonceBySender.get(sender)); + pending.addTransaction(pendingTx, Optional.of(mockSender)); + notificationsChecker.addExpectedAddNotification(pendingTx); + }); + + // reorg case + if (sendersWithReorg.contains(sender)) { + // reorg is removing and re-adding all sender txs, so assert notifications accordingly + final var currentPendingTxs = + liveTxsBySender.get(sender).tailMap(nonce[nonce.length - 1], false).values(); + currentPendingTxs.forEach( + pt -> { + notificationsChecker.addExpectedAddNotification(pt); + notificationsChecker.addExpectedDropNotification(pt); + }); + sendersWithReorg.remove(sender); + } + + // reconciliation case + final var txsRemovedByReconciliation = + liveTxsBySender.get(sender).headMap(nonceBySender.get(sender), false).values(); + if (!txsRemovedByReconciliation.isEmpty()) { + // reconciliation is removing all sender txs, and re-adding only the ones with a + // larger nonce, so assert notifications accordingly + final var reconciledPendingTxs = + liveTxsBySender.get(sender).tailMap(nonce[nonce.length - 1], false).values(); + txsRemovedByReconciliation.forEach(notificationsChecker::addExpectedDropNotification); + reconciledPendingTxs.forEach( + pt -> { + notificationsChecker.addExpectedDropNotification(pt); + notificationsChecker.addExpectedAddNotification(pt); + }); + txsRemovedByReconciliation.clear(); + } + + handleDropped(); + }); + } + + private void handleDropped() { + // handle dropped tx due to layer or pool full + final var droppedTxs = dropped.getEvictedTransactions(); + droppedTxs.forEach(notificationsChecker::addExpectedDropNotification); + droppedTxs.stream() + .forEach( + pt -> { + liveTxsBySender.get(Sender.getByAddress(pt.getSender())).remove(pt.getNonce()); + droppedTxsBySender.get(Sender.getByAddress(pt.getSender())).put(pt.getNonce(), pt); + }); + } + + public Scenario addForSenders(final Object... args) { + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + internalAddForSender(sender, EIP1559, NO_DELEGATIONS, nonce); + } + actions.add(notificationsChecker::assertExpectedNotifications); + return this; + } + + public Scenario replaceForSender(final Sender sender, final long... nonce) { + internalReplaceForSender(sender, nonce); + actions.add(notificationsChecker::assertExpectedNotifications); + return this; + } + + private Scenario internalReplaceForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + Arrays.stream(nonce) + .forEach( + n -> { + final var maybeExistingTx = getMaybe(sender, n); + maybeExistingTx.ifPresentOrElse( + existingTx -> { + final var pendingTx = replace(sender, existingTx); + final Account mockSender = mock(Account.class); + when(mockSender.getNonce()).thenReturn(nonceBySender.get(sender)); + pending.addTransaction(pendingTx, Optional.of(mockSender)); + notificationsChecker.addExpectedAddNotification(pendingTx); + notificationsChecker.addExpectedDropNotification(existingTx); + }, + () -> + fail( + "Could not replace non-existing transaction with nonce " + + n + + " for sender " + + sender.name())); + }); + }); + return this; + } + + public Scenario replaceForSenders(final Object... args) { + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + internalReplaceForSender(sender, nonce); + } + actions.add(notificationsChecker::assertExpectedNotifications); + return this; + } + + public Scenario confirmedForSenders(final Object... args) { + actions.add( + () -> { + final Map maxConfirmedNonceBySender = new HashMap<>(); + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + maxConfirmedNonceBySender.put(sender, nonce); + nonceBySender.put(sender, nonce + 1); + + // if the confirmed tx contains delegations then update the confirmed nonce + // accordingly + getMaybe(sender, nonce) + .ifPresent( + confirmedTx -> + confirmedTx + .getTransaction() + .getCodeDelegationList() + .ifPresent( + codeDelegations -> + codeDelegations.forEach( + cd -> { + final var authority = + Sender.getByAddress(cd.authorizer().get()); + maxConfirmedNonceBySender.compute( + authority, + (unused, currentMax) -> + currentMax == null + ? cd.nonce() + : Math.max(currentMax, cd.nonce())); + nonceBySender.compute( + authority, + (unused, currentNonce) -> + currentNonce == null + ? cd.nonce() + 1 + : Math.max(currentNonce, cd.nonce()) + 1); + }))); + } + + maxConfirmedNonceBySender.entrySet().stream() + .forEach( + san -> { + final var sender = san.getKey(); + final var nonce = san.getValue(); + for (final var pendingTx : getAll(sender)) { + if (pendingTx.getNonce() <= nonce) { + notificationsChecker.addExpectedDropNotification( + liveTxsBySender.get(sender).remove(pendingTx.getNonce())); + } + } + }); + + prio.blockAdded( + FeeMarket.london(0L), + mockBlockHeader(), + maxConfirmedNonceBySender.entrySet().stream() + .collect( + Collectors.toMap(entry -> entry.getKey().address, Map.Entry::getValue))); + notificationsChecker.assertExpectedNotifications(); + }); + return this; + } + + public Scenario setAccountNonce(final Sender sender, final long nonce) { + actions.add(() -> nonceBySender.put(sender, nonce)); + return this; + } + + public Scenario reorgForSenders(final Object... args) { + actions.add( + () -> { + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + nonceBySender.put(sender, nonce); + sendersWithReorg.add(sender); + } + }); + return this; + } + + private PendingTransaction create( + final Sender sender, + final TransactionType type, + final AuthorityAndNonce[] authorityAndNonces, + final long nonce) { + if (liveTxsBySender.get(sender).containsKey(nonce)) { + fail( + "Transaction for sender " + sender.name() + " with nonce " + nonce + " already exists"); + } + final var newPendingTx = + switch (type) { + case FRONTIER -> createFrontierPendingTransaction(sender, nonce); + case ACCESS_LIST -> createAccessListPendingTransaction(sender, nonce); + case EIP1559 -> createEIP1559PendingTransaction(sender, nonce); + case BLOB -> createBlobPendingTransaction(sender, nonce); + case DELEGATE_CODE -> + createEIP7702PendingTransaction(sender, nonce, authorityAndNonces); + case AERE_PQ -> + throw new IllegalArgumentException( + "AIP-20 transactions need ML-DSA keys and are not generated by this test"); + }; + liveTxsBySender.get(sender).put(nonce, newPendingTx); + return newPendingTx; + } + + private PendingTransaction replace(final Sender sender, final PendingTransaction pendingTx) { + final var replaceTx = + createRemotePendingTransaction( + createTransactionReplacement(pendingTx.getTransaction(), sender.key), + sender.hasPriority); + liveTxsBySender.get(sender).replace(pendingTx.getNonce(), replaceTx); + return replaceTx; + } + + private Optional getMaybe(final Sender sender, final long nonce) { + return Optional.ofNullable(liveTxsBySender.get(sender).get(nonce)); + } + + private PendingTransaction get(final Sender sender, final long nonce) { + return getMaybe(sender, nonce).get(); + } + + private List getAll(final Sender sender) { + return List.copyOf(liveTxsBySender.get(sender).values()); + } + + private PendingTransaction createFrontierPendingTransaction( + final Sender sender, final long nonce) { + return createRemotePendingTransaction( + createTransaction(FRONTIER, nonce, Wei.ONE, 0, List.of(), sender.key), + sender.hasPriority); + } + + private PendingTransaction createAccessListPendingTransaction( + final Sender sender, final long nonce) { + return createRemotePendingTransaction( + createTransaction(ACCESS_LIST, nonce, Wei.ONE, 0, List.of(), sender.key), + sender.hasPriority); + } + + private PendingTransaction createEIP1559PendingTransaction( + final Sender sender, final long nonce) { + return createRemotePendingTransaction( + createEIP1559Transaction(nonce, sender.key, sender.gasFeeMultiplier), sender.hasPriority); + } + + private PendingTransaction createBlobPendingTransaction(final Sender sender, final long nonce) { + return createRemotePendingTransaction( + createEIP4844Transaction(nonce, sender.key, sender.gasFeeMultiplier, 1), + sender.hasPriority); + } + + private PendingTransaction createEIP7702PendingTransaction( + final Sender sender, final long nonce, final AuthorityAndNonce[] authorityAndNonces) { + return createRemotePendingTransaction( + createEIP7702Transaction( + nonce, sender.key, sender.gasFeeMultiplier, toCodeDelegations(authorityAndNonces)), + sender.hasPriority); + } + + public Scenario expectedAllForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + lastExpectedAll = expectedForSender(sender, nonce); + assertExpectedAll(prio, sender, lastExpectedAll); + }); + return this; + } + + public Scenario expectedPrioritizedForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + lastExpectedPrioritized = expectedForSender(sender, nonce); + assertExpectedPrioritized(prio, lastExpectedPrioritized); + }); + return this; + } + + public Scenario expectedReadyForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + lastExpectedReady = expectedForSender(sender, nonce); + assertExpectedReady(ready, lastExpectedReady); + }); + return this; + } + + public Scenario expectedSparseForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + lastExpectedSparse = expectedForSender(sender, nonce); + assertExpectedSparse(sparse, lastExpectedSparse); + }); + return this; + } + + public Scenario expectedDroppedForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + lastExpectedDropped = droppedForSender(sender, nonce); + assertExpectedDropped(dropped, lastExpectedDropped); + }); + return this; + } + + public Scenario expectedPrioritizedForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + actions.add( + () -> { + lastExpectedPrioritized = expectedForSenders(sender1, nonce1, sender2, args); + assertExpectedPrioritized(prio, lastExpectedPrioritized); + }); + return this; + } + + public Scenario expectedPrioritizedForSenders() { + actions.add( + () -> { + lastExpectedPrioritized = List.of(); + assertExpectedPrioritized(prio, lastExpectedPrioritized); + }); + return this; + } + + public Scenario expectedReadyForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + actions.add( + () -> { + lastExpectedReady = expectedForSenders(sender1, nonce1, sender2, args); + assertExpectedReady(ready, lastExpectedReady); + }); + return this; + } + + public Scenario expectedReadyForSenders() { + actions.add( + () -> { + lastExpectedReady = List.of(); + assertExpectedReady(ready, lastExpectedReady); + }); + return this; + } + + public Scenario expectedSparseForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + actions.add( + () -> { + lastExpectedSparse = expectedForSenders(sender1, nonce1, sender2, args); + assertExpectedSparse(sparse, lastExpectedSparse); + }); + return this; + } + + public Scenario expectedSparseForSenders() { + actions.add( + () -> { + lastExpectedSparse = List.of(); + assertExpectedSparse(sparse, lastExpectedSparse); + }); + return this; + } + + public Scenario expectedDroppedForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + actions.add( + () -> { + lastExpectedDropped = expectedForSenders(sender1, nonce1, sender2, args); + assertExpectedDropped(dropped, lastExpectedDropped); + }); + return this; + } + + public Scenario expectedDroppedForSenders() { + actions.add( + () -> { + lastExpectedDropped = List.of(); + assertExpectedDropped(dropped, lastExpectedDropped); + }); + return this; + } + + private void assertExpectedAll( + final AbstractPrioritizedTransactions prioLayer, + final Sender sender, + final List expected) { + final List allForSender = prioLayer.getAllBySender().get(sender.address); + if (expected.isEmpty()) { + assertThat(allForSender).describedAs("All").isNull(); + } else { + assertThat(allForSender).describedAs("All").containsExactlyElementsOf(expected); + } + } + + private void assertExpectedPrioritized( + final AbstractPrioritizedTransactions prioLayer, final List expected) { + assertThat(prioLayer.getByScore()) + .describedAs("Prioritized") + .containsExactlyElementsOf(expected); + } + + private void assertExpectedReady( + final ReadyTransactions readyLayer, final List expected) { + assertThat(readyLayer.getBySender().stream().flatMap(List::stream).toList()) + .describedAs("Ready") + .containsExactlyElementsOf(expected); + } + + private void assertExpectedSparse( + final SparseTransactions sparseLayer, final List expected) { + assertThat(sparseLayer.getBySender().stream().flatMap(List::stream).toList()) + .describedAs("Sparse") + .containsExactlyElementsOf(expected); + } + + private void assertExpectedDropped( + final EvictCollectorLayer evictCollector, final List expected) { + assertThat(evictCollector.getEvictedTransactions()) + .describedAs("Dropped") + .containsExactlyInAnyOrderElementsOf(expected); + } + + private List expectedForSenders( + final Sender sender1, final long nonce1, final Sender sender2, final Object... args) { + final List expected = new ArrayList<>(); + expected.add(get(sender1, nonce1)); + final List sendersAndNonce = new ArrayList<>(Arrays.asList(args)); + sendersAndNonce.add(0, sender2); + for (int i = 0; i < sendersAndNonce.size(); i = i + 2) { + final Sender sender = (Sender) sendersAndNonce.get(i); + final long nonce = (int) sendersAndNonce.get(i + 1); + expected.add(get(sender, nonce)); + } + return Collections.unmodifiableList(expected); + } + + private List expectedForSender(final Sender sender, final long... nonce) { + return Arrays.stream(nonce).mapToObj(n -> get(sender, n)).toList(); + } + + private List droppedForSender(final Sender sender, final long... nonce) { + return Arrays.stream(nonce).mapToObj(n -> droppedTxsBySender.get(sender).get(n)).toList(); + } + + public Scenario expectedNextNonceForSenders(final Object... args) { + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final Integer nullableInt = (Integer) args[i + 1]; + final OptionalLong nonce = + nullableInt == null ? OptionalLong.empty() : OptionalLong.of(nullableInt); + actions.add(() -> assertThat(prio.getNextNonceFor(sender.address)).isEqualTo(nonce)); + } + return this; + } + + public Scenario removeForSender(final Sender sender, final long... nonce) { + actions.add( + () -> { + Arrays.stream(nonce) + .forEach( + n -> { + final var maybeLiveTx = getMaybe(sender, n); + final var pendingTx = + maybeLiveTx.orElseGet(() -> create(sender, EIP1559, NO_DELEGATIONS, n)); + prio.remove(pendingTx, INVALIDATED); + maybeLiveTx.ifPresent( + liveTx -> { + notificationsChecker.addExpectedDropNotification(liveTx); + liveTxsBySender.get(sender).remove(liveTx.getNonce()); + droppedTxsBySender.get(sender).put(liveTx.getNonce(), liveTx); + }); + }); + handleDropped(); + notificationsChecker.assertExpectedNotifications(); + }); + return this; + } + + public Scenario penalizeForSender(final Sender sender, final long... nonce) { + actions.add( + () -> + Arrays.stream(nonce) + .forEach( + n -> { + final var senderTxs = prio.getAllFor(sender.address); + Arrays.stream(nonce) + .mapToObj( + n2 -> + senderTxs.stream().filter(pt -> pt.getNonce() == n2).findAny()) + .map(Optional::get) + .forEach( + penalizedTransaction -> + prio.penalize(penalizedTransaction, TX_EVALUATION_TOO_LONG)); + })); + return this; + } + + public Scenario expectedSelectedTransactions(final Object... args) { + actions.add( + () -> { + List expectedSelected = new ArrayList<>(); + for (int i = 0; i < args.length; i = i + 2) { + final Sender sender = (Sender) args[i]; + final long nonce = (int) args[i + 1]; + expectedSelected.add(get(sender, nonce)); + } + + assertThat(prio.getBySender().stream().flatMap(List::stream).toList()) + .containsExactlyElementsOf(expectedSelected); + }); + return this; + } + + public Scenario hasSenderEnoughBalance(final Sender sender, final boolean hasEnoughBalance) { + actions.add( + () -> { + reset(senderBalanceChecker); + when(senderBalanceChecker.hasEnoughBalanceFor( + argThat( + pendingTransaction -> + pendingTransaction + .getTransaction() + .getSender() + .equals(sender.address)))) + .thenReturn(hasEnoughBalance); + }); + return this; + } + + @Override + public String toString() { + return description; + } + } + + enum Sender { + S1(false, 1), + S2(false, 2), + S3(false, 3), + S4(false, 4), + SP1(true, 1), + SP2(true, 2); + + final KeyPair key; + final Address address; + final int gasFeeMultiplier; + + final boolean hasPriority; + + Sender(final boolean hasPriority, final int gasFeeMultiplier) { + this.key = SIGNATURE_ALGORITHM.generateKeyPair(); + this.address = Util.publicKeyToAddress(key.getPublicKey()); + this.gasFeeMultiplier = gasFeeMultiplier; + this.hasPriority = hasPriority; + } + + static Sender getByAddress(final Address address) { + return Arrays.stream(values()).filter(s -> s.address.equals(address)).findAny().get(); + } + } + + static class NotificationsChecker { + private final List collectedAddNotifications = + Collections.synchronizedList(new ArrayList<>()); + private final List collectedDropNotifications = + Collections.synchronizedList(new ArrayList<>()); + private final List expectedAddNotifications = new ArrayList<>(); + private final List expectedDropNotifications = new ArrayList<>(); + + void collectAddNotification(final Transaction tx) { + collectedAddNotifications.add(tx); + } + + void collectDropNotification(final Transaction tx) { + collectedDropNotifications.add(tx); + } + + void addExpectedAddNotification(final PendingTransaction tx) { + expectedAddNotifications.add(tx.getTransaction()); + } + + void addExpectedDropNotification(final PendingTransaction tx) { + expectedDropNotifications.add(tx.getTransaction()); + } + + void assertExpectedNotifications() { + assertAddNotifications(expectedAddNotifications); + assertDropNotifications(expectedDropNotifications); + } + + private void assertAddNotifications(final List expectedAddedTxs) { + await() + .untilAsserted( + () -> + assertThat(collectedAddNotifications) + .describedAs("Added notifications") + .containsExactlyInAnyOrderElementsOf(expectedAddedTxs)); + collectedAddNotifications.clear(); + expectedAddNotifications.clear(); + } + + private void assertDropNotifications(final List expectedDroppedTxs) { + await() + .untilAsserted( + () -> + assertThat(collectedDropNotifications) + .describedAs("Dropped notifications") + .containsExactlyInAnyOrderElementsOf(expectedDroppedTxs)); + collectedDropNotifications.clear(); + expectedDropNotifications.clear(); + } + } + + record AuthorityAndNonce(Sender sender, long nonce) { + static final AuthorityAndNonce[] NO_DELEGATIONS = new AuthorityAndNonce[0]; + + static AuthorityAndNonce[] delegation(final Sender sender, final long nonce) { + return new AuthorityAndNonce[] {new AuthorityAndNonce(sender, nonce)}; + } + + static CodeDelegation toCodeDelegation(final AuthorityAndNonce authorityAndNonce) { + return createSignedCodeDelegation( + BigInteger.ZERO, Address.ZERO, authorityAndNonce.nonce, authorityAndNonce.sender.key); + } + + static List toCodeDelegations(final AuthorityAndNonce[] authorityAndNonces) { + return Arrays.stream(authorityAndNonces).map(AuthorityAndNonce::toCodeDelegation).toList(); + } + } +} diff --git a/anchor/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java b/anchor/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java new file mode 100644 index 0000000..695f088 --- /dev/null +++ b/anchor/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPInput.java @@ -0,0 +1,435 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-bft-bounded-message-lists.patch. + */ +package org.hyperledger.besu.ethereum.rlp; + +import java.math.BigInteger; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.Bytes48; +import org.apache.tuweni.units.bigints.UInt256; +import org.apache.tuweni.units.bigints.UInt64; + +/** + * An input used to decode data in RLP encoding. + * + *

An RLP "value" is fundamentally an {@code Item} defined the following way: + * + * 

+ *   Item ::= List | Bytes
+ *   List ::= [ Item, ... , Item ]
+ *   Bytes ::= a binary value (comprised of an arbitrary number of bytes).
+ *
+ * + *

In other words, RLP encodes binary data organized in arbitrary nested lists. + * + *

A {@link RLPInput} thus provides methods to decode both lists and binary values. A list in the + * input is "entered" by calling {@link #enterList()} and left by calling {@link #leaveList()}. + * Binary values can be read directly with {@link #readBytes()}, but the {@link RLPInput} interface + * provides a wealth of convenience methods to read specific types of data that are in specific + * encoding. + * + *

Amongst the methods to read binary data, some methods are provided to read "scalar". A scalar + * should simply be understood as a positive integer that is encoded with no leading zeros. In other + * word, a method like {@link #readLongScalar()} does not expect an encoded value of exactly 8 bytes + * (by opposition to {@link #readLong}), but rather one that is "up to" 8 bytes. + * + * @see BytesValueRLPInput + */ +public interface RLPInput { + + /** + * Whether the input has been already fully decoded (has no more data to read). + * + * @return {@code false} if the input has more data to read, {@code true} otherwise. + */ + boolean isDone(); + + /** + * Whether the next element to read from this input is a list. + * + * @return {@code true} if the input is not done and the next item to read is a list. + */ + boolean nextIsList(); + + /** + * Whether the next element to read from this input is an RLP "null" (that is, {@link + * Bytes#EMPTY}). + * + * @return {@code true} if the input is not done and the next item to read is an empty value. + */ + boolean nextIsNull(); + + /** + * Returns the payload size of the next item + * + * @return the payload size of the next item + */ + int nextSize(); + + /** + * Returns the size of the current item + * + * @return the size of the current item + */ + int currentSize(); + + /** + * Returns the offset of the next item, counting from the start of the RLP Input as a whole. + * + * @return offset from buffer start + */ + int nextOffset(); + + /** + * Whether the input is at the end of a currently entered list, that is if {@link #leaveList()} + * should be the next method called. + * + * @return Whether all elements of the current list have been read but said list haven't been + * "left" yet. + */ + boolean isEndOfCurrentList(); + + /** + * Skips the next item to read in the input. + * + *

Note that if the next item is a list, the whole list is skipped. + */ + void skipNext(); + + /** + * If the next item to read is a list, enter that list, placing the input on the first item of + * that list. + * + * @return The number of item of the entered list. + * @throws RLPException if the next item to read from this input is not a list, or the input is + * corrupted. + */ + int enterList(); + + /** + * Exits the current list after all its items have been consumed. + * + *

Note that this method technically doesn't consume any input but must be called after having + * read the last element of a list. This allows it to ensure the structure of the input is indeed + * the one expected. + * + * @throws RLPException if the current list is not finished (it has more items). + */ + void leaveList(); + + /** + * Exits the current list, ignoring any remaining unconsumed elements. + * + *

Note that this method technically doesn't consume any input but must be called after having + * read the last element of a list. This allows it to ensure the structure of the input is indeed + * the one expected. + */ + void leaveListLenient(); + + /** + * Reads a non-negative scalar from the input and return it as a long value which is interpreted + * as an unsigned long. + * + * @return The next scalar item of this input as a long value. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is either too big to + * fit a long or has leading zeros. + */ + long readLongScalar(); + + /** + * Reads a scalar from the input and return is as an int value. + * + * @return The next scalar item of this input as an int value. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is either too big to + * fit a long or has leading zeros. + */ + int readIntScalar(); + + /** + * Reads a scalar from the input and return is as a {@link BigInteger}. + * + * @return The next scalar item of this input as a {@link BigInteger}. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item has leading zeros. + */ + BigInteger readBigIntegerScalar(); + + /** + * Reads a scalar from the input and return is as a {@link UInt64}. + * + * @return The next scalar item of this input as a {@link UInt64}. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is either too big to + * fit a {@link UInt64} or has leading zeros. + */ + UInt64 readUInt64Scalar(); + + /** + * Reads a scalar from the input and return is as a {@link UInt256}. + * + * @return The next scalar item of this input as a {@link UInt256}. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is either too big to + * fit a {@link UInt256} or has leading zeros. + */ + UInt256 readUInt256Scalar(); + + /** + * Reads the next item of this input (which must be exactly 1 byte) as a byte. + * + * @return The byte corresponding to the next item of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not a single byte + * long. + */ + byte readByte(); + + /** + * Reads the next item of this input (which must be exactly 2-bytes) as a (signed) short. + * + * @return The short corresponding to the next item of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not 2-bytes. + */ + short readShort(); + + /** + * Reads the next item of this input (which must be exactly 4-bytes) as a (signed) int. + * + * @return The int corresponding to the next item of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not 4-bytes. + */ + int readInt(); + + /** + * Reads the next item of this input (which must be exactly 8-bytes) as a (signed) long. + * + * @return The long corresponding to the next item of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not 8-bytes. + */ + long readLong(); + + /** + * Reads the next item of this input (which must be exactly 1 byte) as an unsigned byte. + * + * @return The value of the next item interpreted as an unsigned byte. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not a single byte + * long. + */ + default int readUnsignedByte() { + if (isZeroLengthString()) { + // Decode an empty string (0x80) as an unsigned byte with value 0 + readBytes(); + return 0; + } + return readByte() & 0xFF; + } + + /** + * Reads the next item of this input (which must be exactly 2-bytes) as an unsigned short. + * + * @return The value of the next item interpreted as an unsigned short. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not 2-bytes. + */ + default int readUnsignedShort() { + return readShort() & 0xFFFF; + } + + /** + * Reads the next item of this input (which must be exactly 4-bytes) as an unsigned int. + * + * @return The value of the next item interpreted as an unsigned int. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is not 4-bytes. + */ + default long readUnsignedInt() { + return (readInt()) & 0xFFFFFFFFL; + } + + /** + * Reads a scalar from the input and return is as an unsigned int contained in a long + * + * @return The next scalar item of this input as an unsigned int value as long + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is either too big to + * fit an unsigned int or has leading zeros. + */ + long readUnsignedIntScalar(); + + /** + * Reads a scalar from the input and return is as an unsigned byte contained in an int + * + * @return The next scalar item of this input as an unsigned byte value as int + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is either too big to + * fit an unsigned byte or has leading zeros. + */ + int readUnsignedByteScalar(); + + /** + * Reads an inet address from this input. + * + * @return The inet address corresponding to the next item of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or if the next item is neither 4 nor 16 + * bytes. + */ + InetAddress readInetAddress(); + + /** + * Reads the next item of this input (assuming it is not a list). + * + * @return The next item read of this input. + * @throws RLPException if the next item to read is a list or the input is at the end of its + * current list (and {@link #leaveList()} hasn't been called). + */ + Bytes readBytes(); + + /** + * Reads the next item of this input (assuming it is not a list) that must be exact 32 bytes. + * + * @return The next item read of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or the next element is not exactly 32 + * bytes. + */ + Bytes32 readBytes32(); + + /** + * Reads the next item of this input (assuming it is not a list) that must be exact 48 bytes. + * + * @return The next item read of this input. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or the next element is not exactly 48 + * bytes. + */ + Bytes48 readBytes48(); + + /** + * Reads the next item of this input (assuming it is not a list) and transforms it with the + * provided mapping function. + * + *

Note that the only benefit of this method over calling the mapper function on the result of + * {@link #readBytes()} is that any error thrown by the mapper will be wrapped by a {@link + * RLPException}, which can make error handling more convenient (having a particular decoded value + * not match what is expected is not fundamentally different from trying to read an unsigned short + * from an item with strictly more or less than 2 bytes). + * + * @param mapper The mapper to apply to the read value. + * @param The type of the result. + * @return The next item read from this input, mapped through {@code mapper}. + * @throws RLPException if the next item to read is a list, the input is at the end of its current + * list (and {@link #leaveList()} hasn't been called) or {@code mapper} throws an exception + * when applied. + */ + T readBytes(Function mapper); + + /** + * Returns the current element as a standalone RLP element. + * + *

This method is useful to extract self-contained RLP elements from a list, so they can be + * processed individually later. + * + * @return The current element as a standalone RLP input element. + */ + RLPInput readAsRlp(); + + /** + * Returns a raw {@link Bytes} representation of this RLP. + * + * @return The raw RLP. + */ + Bytes raw(); + + boolean isZeroLengthString(); + + /** Resets this RLP input to the start. */ + void reset(); + + /** + * Returns a raw {@link Bytes} representation of this RLP list. + * + * @return The raw RLP list. + */ + Bytes currentListAsBytes(); + + /** + * Reads a full list from the input given a method that knows how to read its elements. + * + * @param valueReader A method that can decode a single list element. + * @param The type of the elements of the decoded list. + * @return The next list of this input, where elements are decoded using {@code valueReader}. + * @throws RLPException is the next item to read is not a list, of if any error happens when + * applying {@code valueReader} to read elements of the list. + */ + default List readList(final Function valueReader) { + return readList(valueReader, Integer.MAX_VALUE); + } + + /** + * Reads a full list from the input, rejecting inputs whose element count exceeds {@code + * maxElements}. AERE D-434: ported from upstream hyperledger/besu dd4101c9ec. + * + * @param valueReader A method that can decode a single list element. + * @param maxElements The maximum number of elements the list may contain. + * @param The type of the elements of the decoded list. + * @return The next list of this input, where elements are decoded using {@code valueReader}. + * @throws RLPException if the next item is not a list, if it holds more than {@code maxElements} + * elements, or if any error happens when applying {@code valueReader} to read elements. + */ + default List readList(final Function valueReader, final int maxElements) { + final int size = enterList(); + if (size > maxElements) { + throw new RLPException( + String.format( + "List of %d elements exceeds the maximum permitted size of %d", size, maxElements)); + } + final List res = size == 0 ? List.of() : new ArrayList<>(size); + for (int i = 0; i < size; i++) { + try { + res.add(valueReader.apply(this)); + } catch (final RLPException e) { + throw e; + } catch (final Exception e) { + throw new RLPException( + String.format("Error applying element decoding function on element %d of the list", i), + e); + } + } + leaveList(); + return res; + } + + default Bytes currentListAsBytesNoCopy(final boolean moveToNextItem) { + throw new UnsupportedOperationException("Not supported by this input"); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java new file mode 100644 index 0000000..eadf4b3 --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java @@ -0,0 +1,1920 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-warm-access-hashdos-treeset.patch. + */ +package org.hyperledger.besu.evm.frame; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Collections.emptySet; + +import org.hyperledger.besu.collections.undo.UndoSet; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Log; +import org.hyperledger.besu.datatypes.VersionedHash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.evm.Code; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; +import org.hyperledger.besu.evm.internal.MemoryEntry; +import org.hyperledger.besu.evm.internal.OperandStack; +import org.hyperledger.besu.evm.internal.StorageEntry; +import org.hyperledger.besu.evm.internal.UnderflowException; +import org.hyperledger.besu.evm.operation.Operation; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; + +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.TreeSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Table; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * A container object for all the states associated with a message. + * + *

A message corresponds to an interaction between two accounts. A Transaction spawns at least + * one message when it's processed. Messages can also spawn messages depending on the code executed + * within a message. + * + *

Note that there is no specific Message object in the code base. Instead, message executions + * correspond to a {@code MessageFrame} and a specific AbstractMessageProcessor. Currently, there + * are two such AbstractMessageProcessor types: + * + *

Message Call ({@code MESSAGE_CALL}) + * + *

A message call consists of applying a set of changes to an account on behalf of another + * account. At the minimal end of changes is a value transfer between a sender and recipient + * account. If the recipient account contains code, that code is also executed. + * + *

Contract Creation ({@code CONTRACT_CREATION}) + * + *

A contract creation, as its name suggests, creates contract accounts. Contract initialization + * code and a value are supplied to initialize the contract account code and balance, respectively. + */ +public class MessageFrame { + + /** + * Message Frame State. + * + *

Message Frame Lifecycle

+ * + *

The diagram below presents the message frame lifecycle: + * + * 

+   *            ------------------------------------------------------
+   *            |                                                    |
+   *            |                                                    v
+   *            |               ---------------------     ---------------------
+   *            |               |                   |     |                   |
+   *            |               |    CODE_SUCCESS   | --> | COMPLETED_SUCCESS |
+   *            |               |                   |     |                   |
+   *            |               ---------------------     ---------------------
+   *            |                         ^
+   *            |                         |
+   *  ---------------------     ---------------------     ---------------------
+   *  |                   |     |                   | --> |                   |
+   *  |    NOT_STARTED    | --> |   CODE_EXECUTING  |     |   CODE_SUSPENDED  |
+   *  |                   |     |                   | <-- |                   |
+   *  ---------------------     ---------------------     ---------------------
+   *            |                         |
+   *            |                         |
+   *            |                         |                 ---------------------
+   *            |                         |                 |                   |
+   *            |                         |------------> |      REVERTED     |
+   *            |                         |                 |                   |
+   *            |                         |                 ---------------------
+   *            |                         |
+   *            |                         v
+   *            |               ---------------------     ---------------------
+   *            |               |                   |     |                   |
+   *            |-------------> |  EXCEPTIONAL_HALT | --> | COMPLETED_FAILURE |
+   *                            |                   |     |                   |
+   *                            ---------------------     ---------------------
+   *
+ * + * Message Not Started ({@link #NOT_STARTED}) + * + *

The message has not begun to execute yet. + * + *

Code Executing ({@link #CODE_EXECUTING}) + * + *

The message contains code and has begun executing it. The execution will continue until it + * is halted due to (1) spawning a child message (2) encountering an exceptional halting condition + * (2) completing successfully. + * + *

Code Suspended Execution ({@link #CODE_SUSPENDED}) + * + *

The message has spawned a child message and has suspended its execution until the child + * message has completed and notified its parent message. The message will then continue executing + * code ({@link #CODE_EXECUTING}) again. + * + *

Code Execution Completed Successfully ({@link #CODE_SUSPENDED}) + * + *

The code within the message has executed to completion successfully. + * + *

Message Exceptionally Halted ({@link #EXCEPTIONAL_HALT}) + * + *

The message execution has encountered an exceptional halting condition at some point during + * its execution. + * + *

Message Reverted ({@link #REVERT}) + * + *

The message execution has requested to revert state during execution. + * + *

Message Execution Failed ({@link #COMPLETED_FAILED}) + * + *

The message execution failed to execute successfully; most likely due to encountering an + * exceptional halting condition. At this point the message frame is finalized and the parent is + * notified. + * + *

Message Execution Completed Successfully ({@link #COMPLETED_SUCCESS}) + * + *

The message execution completed successfully and needs to finalized and propagated to the + * parent message that spawned it. + */ + public enum State { + + /** Message execution has not started. */ + NOT_STARTED, + + /** Code execution within the message is in progress. */ + CODE_EXECUTING, + + /** Code execution within the message has finished successfully. */ + CODE_SUCCESS, + + /** Code execution within the message has been suspended. */ + CODE_SUSPENDED, + + /** An exceptional halting condition has occurred. */ + EXCEPTIONAL_HALT, + + /** State changes were reverted during execution. */ + REVERT, + + /** The message execution has failed to complete successfully. */ + COMPLETED_FAILED, + + /** The message execution has completed successfully. */ + COMPLETED_SUCCESS, + } + + /** The message type the frame corresponds to. */ + public enum Type { + + /** A Contract creation message. */ + CONTRACT_CREATION, + + /** A message call message. */ + MESSAGE_CALL, + } + + /** The constant DEFAULT_MAX_STACK_SIZE. */ + public static final int DEFAULT_MAX_STACK_SIZE = 1024; + + // Global data fields. + private final WorldUpdater worldUpdater; + + // Metadata fields. + private final Type type; + private State state = State.NOT_STARTED; + // EIP-7778/EIP-8037: Flipped to true once code execution starts; used to distinguish a halt + // that fires during opcode execution (halt-burn counts toward block regular gas) from a halt + // raised pre-execution in the processor's start() (halt-burn must be excluded). + private boolean codeExecuted = false; + + // Machine state fields. + private long gasRemaining; + private int pc; + private final Memory memory = new Memory(); + private final OperandStack stack; + // EVM v2 stack: 4 longs per 256-bit word (index 0 = most significant, index 3 = least + // significant) + private final long[] stackDataV2; + private int stackTopV2; + private Bytes output = Bytes.EMPTY; + private Bytes returnData = Bytes.EMPTY; + private Code createdCode = null; + private final boolean isStatic; + + // Transaction state fields. + private final List logs = new ArrayList<>(); + private final Map refunds = new HashMap<>(); + + // Execution Environment fields. + private final Address recipient; + private final Address contract; + private final Bytes inputData; + private final Address sender; + private final Wei value; + private final Wei apparentValue; + private final Code code; + + private Optional revertReason; + + private final Map contextVariables; + + private Optional exceptionalHaltReason = Optional.empty(); + private Operation currentOperation; + private final Consumer completer; + private Optional maybeUpdatedMemory = Optional.empty(); + private Optional maybeUpdatedStorage = Optional.empty(); + + private final TxValues txValues; + + private Optional eip7928AccessList = Optional.empty(); + + /** The mark of the undoable collections at the creation of this message frame */ + private long undoMark; + + /** + * Builder builder. + * + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + private MessageFrame( + final boolean enableEvmV2, + final Type type, + final WorldUpdater worldUpdater, + final long initialGas, + final Address recipient, + final Address contract, + final Bytes inputData, + final Address sender, + final Wei value, + final Wei apparentValue, + final Code code, + final boolean isStatic, + final Consumer completer, + final Map contextVariables, + final Optional revertReason, + final TxValues txValues, + final Optional eip7928AccessList) { + + this.txValues = txValues; + this.type = type; + this.worldUpdater = worldUpdater; + this.gasRemaining = initialGas; + this.stack = new OperandStack(txValues.maxStackSize()); + this.stackDataV2 = enableEvmV2 ? new long[txValues.maxStackSize() * 4] : null; + this.stackTopV2 = 0; + this.pc = 0; + this.recipient = recipient; + this.contract = contract; + this.inputData = inputData; + this.sender = sender; + this.value = value; + this.apparentValue = apparentValue; + this.code = code; + this.isStatic = isStatic; + this.completer = completer; + this.contextVariables = contextVariables; + this.revertReason = revertReason; + this.eip7928AccessList = eip7928AccessList; + this.undoMark = txValues.transientStorage().mark(); + } + + /** + * Return the program counter. + * + * @return the program counter + */ + public int getPC() { + return pc; + } + + /** + * Set the program counter. + * + * @param pc The new program counter value + */ + public void setPC(final int pc) { + this.pc = pc; + } + + /** Deducts the remaining gas. */ + public void clearGasRemaining() { + this.gasRemaining = 0L; + } + + /** + * Decrement the amount of remaining gas. + * + * @param amount The amount of gas to deduct + * @return the amount of gas available, after deductions. + */ + public long decrementRemainingGas(final long amount) { + this.gasRemaining -= amount; + return this.gasRemaining; + } + + /** + * Return the amount of remaining gas. + * + * @return the amount of remaining gas + */ + public long getRemainingGas() { + return gasRemaining; + } + + /** + * Increment the amount of remaining gas. + * + * @param amount The amount of gas to increment + */ + public void incrementRemainingGas(final long amount) { + this.gasRemaining += amount; + } + + /** + * Set the amount of remaining gas. + * + * @param amount The amount of remaining gas + */ + public void setGasRemaining(final long amount) { + this.gasRemaining = amount; + } + + /** + * Return the output data. + * + * @return the output data + */ + public Bytes getOutputData() { + return output; + } + + /** + * Set the output data. + * + * @param output The output data + */ + public void setOutputData(final Bytes output) { + this.output = output; + } + + /** + * Sets the created code from CREATE* operations + * + * @param createdCode the code that was created + */ + public void setCreatedCode(final Code createdCode) { + this.createdCode = createdCode; + } + + /** + * gets the created code from CREATE* operations + * + * @return the code that was created + */ + public Code getCreatedCode() { + return createdCode; + } + + /** Clears the output data buffer. */ + public void clearOutputData() { + setOutputData(Bytes.EMPTY); + } + + /** + * Return the return data. + * + * @return the return data + */ + public Bytes getReturnData() { + return returnData; + } + + /** + * Set the return data. + * + * @param returnData The return data + */ + public void setReturnData(final Bytes returnData) { + this.returnData = returnData; + } + + /** Clear the return data buffer. */ + public void clearReturnData() { + setReturnData(Bytes.EMPTY); + } + + /** + * Returns the item at the specified offset in the stack. + * + * @param offset The item's position relative to the top of the stack + * @return The item at the specified offset in the stack + * @throws UnderflowException if the offset is out of range + */ + public Bytes getStackItem(final int offset) { + return stack.get(offset); + } + + /** + * Removes the item at the top of the stack. + * + * @return the item at the top of the stack + * @throws UnderflowException if the stack is empty + */ + public Bytes popStackItem() { + return stack.pop(); + } + + /** + * Removes the corresponding number of items from the top of the stack. + * + * @param n The number of items to pop off the stack + */ + public void popStackItems(final int n) { + stack.bulkPop(n); + } + + /** + * Pushes the corresponding item onto the top of the stack + * + * @param value The value to push onto the stack. + */ + public void pushStackItem(final Bytes value) { + stack.push(value); + } + + /** + * Sets the stack item at the specified offset from the top of the stack to the value + * + * @param offset The item's position relative to the top of the stack + * @param value The value to set the stack item to + * @throws IllegalStateException if the stack is too small + */ + public void setStackItem(final int offset, final Bytes value) { + stack.set(offset, value); + } + + /** + * Return the current stack size. + * + * @return The current stack size + */ + public int stackSize() { + return stack.size(); + } + + // region --- EVM v2 long[] stack operations --- + // --------------------------------------------------------------------------- + + /** + * Returns the backing long[] array of the operand stack. + * + * @return the raw data array + */ + public long[] stackDataV2() { + return stackDataV2; + } + + /** + * Returns the current stack top (item count, 0 = empty). + * + * @return the item count + */ + public int stackTopV2() { + return stackTopV2; + } + + /** + * Sets the stack top (item count). Used after StackMath operations. + * + * @param newTop the new item count + */ + public void setTopV2(final int newTop) { + this.stackTopV2 = newTop; + } + + /** + * Returns true if the stack has at least {@code n} items. + * + * @param n the number of items required + * @return true if the stack contains at least n items + */ + public boolean stackHasItems(final int n) { + return stackTopV2 >= n; + } + + // --------------------------------------------------------------------------- + // endregion + + /** + * Returns whether the message frame is static or not. + * + * @return {@code} true if the frame is static; otherwise {@code false} + */ + public boolean isStatic() { + return isStatic; + } + + /** + * Returns the memory size for specified memory access. + * + * @param offset The offset in memory + * @param length The length of the memory access + * @return the memory size for specified memory access + */ + public long calculateMemoryExpansion(final long offset, final long length) { + return memory.calculateNewActiveWords(offset, length); + } + + /** + * Expands memory to accommodate the specified memory access. + * + * @param offset The offset in memory + * @param length The length of the memory access + */ + public void expandMemory(final long offset, final long length) { + memory.ensureCapacityForBytes(offset, length); + } + + /** + * Returns the number of bytes in memory. + * + * @return the number of bytes in memory + */ + public long memoryByteSize() { + return memory.getActiveBytes(); + } + + /** + * Returns the number of words in memory. + * + * @return the number of words in memory + */ + public int memoryWordSize() { + return memory.getActiveWords(); + } + + /** + * Returns the revertReason as string + * + * @return the revertReason string + */ + public Optional getRevertReason() { + return revertReason; + } + + /** + * Sets revert reason. + * + * @param revertReason the revert reason + */ + public void setRevertReason(final Bytes revertReason) { + this.revertReason = Optional.ofNullable(revertReason); + } + + /** + * Read bytes in memory as mutable. Contents should not be considered stable outside the scope of + * the current operation. + * + * @param offset The offset in memory + * @param length The length of the bytes to read + * @return The bytes in the specified range + */ + public MutableBytes readMutableMemory(final long offset, final long length) { + return readMutableMemory(offset, length, false); + } + + /** + * Read bytes in memory without expanding the word capacity. + * + * @param offset The offset in memory + * @param length The length of the bytes to read + * @return The bytes in the specified range + */ + public Bytes shadowReadMemory(final long offset, final long length) { + return memory.getBytesWithoutGrowth(offset, length); + } + + /** + * Read bytes in memory . + * + * @param offset The offset in memory + * @param length The length of the bytes to read + * @return The bytes in the specified range + */ + public Bytes readMemory(final long offset, final long length) { + return readMutableMemory(offset, length, false).copy(); + } + + /** + * Read bytes in memory. Contents should not be considered stable outside the scope of the current + * operation. + * + * @param offset The offset in memory + * @param length The length of the bytes to read + * @param explicitMemoryRead true if triggered by a memory opcode, false otherwise + * @return The bytes in the specified range + */ + public MutableBytes readMutableMemory( + final long offset, final long length, final boolean explicitMemoryRead) { + final MutableBytes memBytes = memory.getMutableBytes(offset, length); + if (explicitMemoryRead) { + setUpdatedMemory(offset, memBytes); + } + return memBytes; + } + + /** + * Write byte to memory + * + * @param offset The offset in memory + * @param value The value to set in memory + * @param explicitMemoryUpdate true if triggered by a memory opcode, false otherwise + */ + public void writeMemory(final long offset, final byte value, final boolean explicitMemoryUpdate) { + memory.setByte(offset, value); + if (explicitMemoryUpdate) { + setUpdatedMemory(offset, Bytes.of(value)); + } + } + + /** + * Write bytes to memory + * + * @param offset The offset in memory + * @param length The length of the bytes to write + * @param value The value to write + */ + public void writeMemory(final long offset, final long length, final Bytes value) { + writeMemory(offset, length, value, false); + } + + /** + * Write bytes to memory + * + * @param offset The offset in memory + * @param length The length of the bytes to write + * @param value The value to write + * @param explicitMemoryUpdate true if triggered by a memory opcode, false otherwise + */ + public void writeMemory( + final long offset, final long length, final Bytes value, final boolean explicitMemoryUpdate) { + memory.setBytes(offset, length, value); + if (explicitMemoryUpdate) { + setUpdatedMemory(offset, 0, length, value); + } + } + + /** + * Copy the bytes from the value param into memory at the specified offset. In cases where the + * value does not have numBytes bytes the appropriate amount of zero bytes will be added before + * writing the value bytes. + * + * @param offset The offset in memory + * @param length The length of the bytes to write + * @param value The value to write + * @param explicitMemoryUpdate true if triggered by a memory opcode, false otherwise + */ + public void writeMemoryRightAligned( + final long offset, final long length, final Bytes value, final boolean explicitMemoryUpdate) { + memory.setBytesRightAligned(offset, length, value); + if (explicitMemoryUpdate) { + setUpdatedMemoryRightAligned(offset, length, value); + } + } + + /** + * Write bytes to memory + * + * @param offset The offset in memory to start writing + * @param sourceOffset The offset in the source value to start writing + * @param length The length of the bytes to write + * @param value The value to write + */ + public void writeMemory( + final long offset, final long sourceOffset, final long length, final Bytes value) { + writeMemory(offset, sourceOffset, length, value, false); + } + + /** + * Write bytes to memory + * + * @param offset The offset in memory to start writing + * @param sourceOffset The offset in the source value to start writing + * @param length The length of the bytes to write + * @param value The value to write + * @param explicitMemoryUpdate true if triggered by a memory opcode, false otherwise + */ + public void writeMemory( + final long offset, + final long sourceOffset, + final long length, + final Bytes value, + final boolean explicitMemoryUpdate) { + memory.setBytes(offset, sourceOffset, length, value); + if (explicitMemoryUpdate && length > 0) { + setUpdatedMemory(offset, sourceOffset, length, value); + } + } + + /** + * Copies bytes within memory. + * + *

Copying behaves as if the values are copied to an intermediate buffer before writing. + * + * @param dst The destination address + * @param src The source address + * @param length the number of bytes to copy + * @param explicitMemoryUpdate true if triggered by a memory opcode, false otherwise + */ + public void copyMemory( + final long dst, final long src, final long length, final boolean explicitMemoryUpdate) { + if (length > 0) { + memory.copy(dst, src, length); + if (explicitMemoryUpdate) { + setUpdatedMemory(dst, memory.getBytes(dst, length)); + } + } + } + + private void setUpdatedMemory( + final long offset, final long sourceOffset, final long length, final Bytes value) { + final long endIndex = sourceOffset + length; + if (sourceOffset >= 0 && endIndex > 0) { + final int srcSize = value.size(); + if (endIndex > srcSize) { + final MutableBytes paddedAnswer = MutableBytes.create((int) length); + if (sourceOffset < srcSize) { + value.slice((int) sourceOffset, (int) (srcSize - sourceOffset)).copyTo(paddedAnswer, 0); + } + setUpdatedMemory(offset, paddedAnswer.copy()); + } else { + setUpdatedMemory(offset, value.slice((int) sourceOffset, (int) length).copy()); + } + } + } + + private void setUpdatedMemoryRightAligned( + final long offset, final long length, final Bytes value) { + if (length > 0) { + final int srcSize = value.size(); + if (length > srcSize) { + final MutableBytes paddedAnswer = MutableBytes.create((int) length); + if ((long) 0 < srcSize) { + value.slice(0, srcSize).copyTo(paddedAnswer, (int) (length - srcSize)); + } + setUpdatedMemory(offset, paddedAnswer.copy()); + } else { + setUpdatedMemory(offset, value.slice(0, (int) length).copy()); + } + } + } + + private void setUpdatedMemory(final long offset, final Bytes value) { + maybeUpdatedMemory = Optional.of(new MemoryEntry(offset, value)); + } + + /** + * Storage was updated. + * + * @param storageAddress the storage address + * @param value the value + */ + public void storageWasUpdated(final UInt256 storageAddress, final Bytes value) { + maybeUpdatedStorage = Optional.of(new StorageEntry(storageAddress, value)); + } + + /** + * Accumulate a log. + * + * @param log The log to accumulate + */ + public void addLog(final Log log) { + logs.add(log); + } + + /** + * Accumulate logs. + * + * @param logs The logs to accumulate + */ + public void addLogs(final List logs) { + this.logs.addAll(logs); + } + + /** Clear the accumulated logs. */ + public void clearLogs() { + logs.clear(); + } + + /** + * Return the accumulated logs. + * + * @return the accumulated logs + */ + public List getLogs() { + return logs; + } + + /** + * Increment the gas refund. + * + * @param amount The amount to increment the refund + */ + public void incrementGasRefund(final long amount) { + this.txValues.gasRefunds().set(this.txValues.gasRefunds().get() + amount); + } + + /** Clear the accumulated gas refund. */ + public void clearGasRefund() { + this.txValues.gasRefunds().set(0L); + } + + /** + * Return the accumulated gas refund. + * + * @return accumulated gas refund + */ + public long getGasRefund() { + return txValues.gasRefunds().get(); + } + + /** + * Increment the state gas used (EIP-8037). This is NOT undone on revert since consumed gas is + * consumed regardless of execution outcome. + * + * @param amount The amount of state gas to add + */ + public void incrementStateGasUsed(final long amount) { + txValues.stateGasUsed().set(txValues.stateGasUsed().get() + amount); + } + + /** + * Return the accumulated state gas used (EIP-8037). + * + * @return accumulated state gas used + */ + public long getStateGasUsed() { + return txValues.stateGasUsed().get(); + } + + /** + * Gets the state gas reservoir (EIP-8037). + * + * @return the state gas reservoir amount + */ + public long getStateGasReservoir() { + return txValues.stateGasReservoir().get(); + } + + /** + * Sets the state gas reservoir (EIP-8037). + * + * @param amount the amount to set the reservoir to + */ + public void setStateGasReservoir(final long amount) { + txValues.stateGasReservoir().set(amount); + } + + /** + * Increments the state gas reservoir (EIP-8037). Used for state gas refunds. + * + * @param amount the amount to add to the reservoir + */ + public void incrementStateGasReservoir(final long amount) { + txValues.stateGasReservoir().set(txValues.stateGasReservoir().get() + amount); + } + + /** + * Consumes state gas: draws from the reservoir first, then from gasRemaining. Also increments + * stateGasUsed. Returns false if insufficient total gas (reservoir + gasRemaining). + * + * @param amount the amount of state gas to consume + * @return true if the gas was successfully consumed, false if insufficient gas + */ + public boolean consumeStateGas(final long amount) { + final long reservoir = txValues.stateGasReservoir().get(); + if (reservoir >= amount) { + txValues.stateGasReservoir().set(reservoir - amount); + } else { + // Overflow goes to gasRemaining + final long overflow = amount - reservoir; + if (gasRemaining < overflow) { + return false; + } + txValues.stateGasReservoir().set(0L); + gasRemaining -= overflow; + } + txValues.stateGasUsed().set(txValues.stateGasUsed().get() + amount); + return true; + } + + /** + * Consumes state gas, draining all available gas even when the full amount cannot be covered. + * Always increments stateGasUsed by the full amount regardless of gas availability. Used when a + * transaction-level (depth-0) contract creation fails after state gas has been partially + * committed: we must record the charge for block accounting even though execution fails. + * + * @param amount the amount of state gas to consume + * @return true if sufficient gas was available, false if gas was insufficient (but drained + * anyway) + */ + public boolean consumeStateGasForced(final long amount) { + final long reservoir = txValues.stateGasReservoir().get(); + if (reservoir >= amount) { + txValues.stateGasReservoir().set(reservoir - amount); + txValues.stateGasUsed().set(txValues.stateGasUsed().get() + amount); + return true; + } else { + final long overflow = amount - reservoir; + txValues.stateGasReservoir().set(0L); + final boolean sufficient = gasRemaining >= overflow; + gasRemaining = Math.max(0L, gasRemaining - overflow); + txValues.stateGasUsed().set(txValues.stateGasUsed().get() + amount); + return sufficient; + } + } + + /** + * Accumulates state gas that spilled into gasRemaining in a reverted child frame (EIP-8037). This + * counter is NOT undone on revert — it tracks permanently burned spill gas for block accounting. + * + * @param amount the spill amount to accumulate + */ + public void accumulateStateGasSpillBurned(final long amount) { + txValues.stateGasSpillBurned()[0] += amount; + } + + /** + * Returns the total state gas spill burned by reverted child frames (EIP-8037). + * + * @return accumulated spill burned + */ + public long getStateGasSpillBurned() { + return txValues.stateGasSpillBurned()[0]; + } + + /** + * Accumulates gas that was sitting unused in the initial frame's gasRemaining at the moment of an + * exceptional halt (EIP-7778/EIP-8037). The sender still pays for this gas via receipts, but it + * did not correspond to any executed regular or state gas, so it must be excluded from the block + * regular gas total. Not undone on revert. + * + * @param amount the gasRemaining snapshot taken immediately before clearGasRemaining on the + * initial frame's exceptional halt + */ + public void accumulateInitialFrameRegularHaltBurn(final long amount) { + txValues.initialFrameRegularHaltBurn()[0] += amount; + } + + /** + * Returns the gas burned on the initial frame's exceptional halt. + * + * @return accumulated halt-burned gas + */ + public long getInitialFrameRegularHaltBurn() { + return txValues.initialFrameRegularHaltBurn()[0]; + } + + /** + * Marks that opcode execution has started on this frame. Once set, an exceptional halt is + * classified as "during code execution" (halt-burned gas counts toward block regular gas) rather + * than pre-execution (halt-burned gas is excluded). + */ + public void markCodeExecuted() { + this.codeExecuted = true; + } + + /** + * Returns whether opcode execution has started on this frame. + * + * @return true if {@link #markCodeExecuted()} was invoked + */ + public boolean isCodeExecuted() { + return codeExecuted; + } + + /** + * Add recipient to the self-destruct set if not already present. + * + * @param address The recipient to self-destruct + */ + public void addSelfDestruct(final Address address) { + txValues.selfDestructs().add(address); + } + + /** + * Add addresses to the self-destruct set if they are not already present. + * + * @param addresses The addresses to self-destruct + */ + public void addSelfDestructs(final Set

addresses) { + txValues.selfDestructs().addAll(addresses); + } + + /** + * Returns the self-destruct set. + * + * @return the self-destruct set + */ + public Set
getSelfDestructs() { + return txValues.selfDestructs(); + } + + /** + * Add recipient to the create set if not already present. + * + * @param address The recipient to create + */ + public void addCreate(final Address address) { + txValues.creates().add(address); + } + + /** + * Add addresses to the create set if they are not already present. + * + * @param addresses The addresses to create + */ + public void addCreates(final Set
addresses) { + txValues.creates().addAll(addresses); + } + + /** + * Returns the create set. + * + * @return the create set + */ + public Set
getCreates() { + return txValues.creates(); + } + + /** + * Was the account at this address created in this transaction? (in any of the previously executed + * message frames in this transaction). + * + * @param address the address to check + * @return true if the account was created in any parent or prior message frame in this + * transaction. False if the account existed in the world state at the beginning of the + * transaction. + */ + public boolean wasCreatedInTransaction(final Address address) { + return txValues.creates().contains((address)); + } + + /** + * Add refund to the refunds map if not already present. + * + * @param beneficiary the beneficiary of the refund. + * @param amount the amount of the refund. + */ + public void addRefund(final Address beneficiary, final Wei amount) { + refunds.put(beneficiary, amount); + } + + /** + * Returns the refunds map. + * + * @return the refunds map + */ + public Map getRefunds() { + return refunds; + } + + /** + * "Warms up" the address as per EIP-2929 + * + * @param address the address to warm up + * @return true if the address was already warmed up + */ + public boolean warmUpAddress(final Address address) { + return !txValues.warmedUpAddresses().add(address); + } + + /** + * Returns whether an address has been warmed up. Is deliberately publicly exposed for access from + * tracers. + * + * @param address the address context + * @return whether the address has been warmed up + */ + public boolean isAddressWarm(final Address address) { + return txValues.warmedUpAddresses().contains(address); + } + + /** + * "Warms up" the storage slot as per EIP-2929 + * + * @param address the address whose storage is being warmed up + * @param slot the slot being warmed up + * @return true if the storage slot was already warmed up + */ + public boolean warmUpStorage(final Address address, final Bytes32 slot) { + return txValues.warmedUpStorage().put(address, slot, Boolean.TRUE) != null; + } + + /** + * Return the world state. + * + * @return the world state + */ + public WorldUpdater getWorldUpdater() { + return worldUpdater; + } + + /** + * Returns the message frame type. + * + * @return the message frame type + */ + public Type getType() { + return type; + } + + /** + * Returns the current execution state. + * + * @return the current execution state + */ + public State getState() { + return state; + } + + /** + * Sets the current execution state. + * + * @param state The new execution state + */ + public void setState(final State state) { + this.state = state; + } + + /** + * Returns the code currently being executed. + * + * @return the code currently being executed + */ + public Code getCode() { + return code; + } + + /** + * Returns the current input data. + * + * @return the current input data + */ + public Bytes getInputData() { + return inputData; + } + + /** + * Returns the recipient account recipient + * + * @return the callee account recipient + */ + public Address getRecipientAddress() { + return recipient; + } + + /** + * Returns the message stack size. + * + * @return the message stack size + */ + public int getMessageStackSize() { + return txValues.messageFrameStack().size(); + } + + /** + * Returns the Call Depth, where the rootmost call is depth 0 + * + * @return the call depth + */ + public int getDepth() { + return getMessageStackSize() - 1; + } + + /** + * Returns the recipient that originated the message. + * + * @return the recipient that originated the message + */ + public Address getOriginatorAddress() { + return txValues.originator(); + } + + /** + * Returns the recipient of the code currently executing. + * + * @return the recipient of the code currently executing + */ + public Address getContractAddress() { + return contract; + } + + /** + * Returns the current gas price. + * + * @return the current gas price + */ + public Wei getGasPrice() { + return txValues.gasPrice(); + } + + /** + * Returns the current blob gas price. + * + * @return the current blob gas price + */ + public Wei getBlobGasPrice() { + return txValues.blobGasPrice(); + } + + /** + * Returns the recipient of the sender. + * + * @return the recipient of the sender + */ + public Address getSenderAddress() { + return sender; + } + + /** + * Returns the value being transferred. + * + * @return the value being transferred + */ + public Wei getValue() { + return value; + } + + /** + * Returns the apparent value being transferred. + * + * @return the apparent value being transferred + */ + public Wei getApparentValue() { + return apparentValue; + } + + /** + * Returns the current block header. + * + * @return the current block header + */ + public BlockValues getBlockValues() { + return txValues.blockValues(); + } + + /** Performs updates based on the message frame's execution. */ + public void notifyCompletion() { + completer.accept(this); + } + + /** + * Returns the current message frame stack. + * + * @return the current message frame stack + */ + public Deque getMessageFrameStack() { + return txValues.messageFrameStack(); + } + + /** + * Sets exceptional halt reason. + * + * @param exceptionalHaltReason the exceptional halt reason + */ + public void setExceptionalHaltReason( + final Optional exceptionalHaltReason) { + this.exceptionalHaltReason = exceptionalHaltReason; + } + + /** + * Gets exceptional halt reason. + * + * @return the exceptional halt reason + */ + public Optional getExceptionalHaltReason() { + return exceptionalHaltReason; + } + + /** + * Returns the current miningBeneficiary (aka coinbase) + * + * @return the current mining beneficiary + */ + public Address getMiningBeneficiary() { + return txValues.miningBeneficiary(); + } + + /** + * Gets block hash lookup. + * + * @return the block hash lookup + */ + public BlockHashLookup getBlockHashLookup() { + return txValues.blockHashLookup(); + } + + /** + * Gets current operation. + * + * @return the current operation + */ + public Operation getCurrentOperation() { + return currentOperation; + } + + /** + * Gets max stack size. + * + * @return the max stack size + */ + public int getMaxStackSize() { + return txValues.maxStackSize(); + } + + /** + * Gets context variable. + * + * @param the type parameter + * @param name the name + * @return the context variable + */ + @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) + public T getContextVariable(final String name) { + return (T) contextVariables.get(name); + } + + /** + * Gets context variable. + * + * @param the type parameter + * @param name the name + * @param defaultValue the default value + * @return the context variable + */ + @SuppressWarnings("unchecked") + public T getContextVariable(final String name, final T defaultValue) { + return (T) contextVariables.getOrDefault(name, defaultValue); + } + + /** + * Has context variable. + * + * @param name the name + * @return the boolean + */ + public boolean hasContextVariable(final String name) { + return contextVariables.containsKey(name); + } + + /** + * Sets current operation. + * + * @param currentOperation the current operation + */ + public void setCurrentOperation(final Operation currentOperation) { + this.currentOperation = currentOperation; + } + + /** + * Gets warmedUp Storage. + * + * @return the warmed up storage + */ + public Table getWarmedUpStorage() { + return txValues.warmedUpStorage(); + } + + /** + * Gets maybe updated memory. + * + * @return the maybe updated memory + */ + public Optional getMaybeUpdatedMemory() { + return maybeUpdatedMemory; + } + + /** + * Gets maybe updated storage. + * + * @return the maybe updated storage + */ + public Optional getMaybeUpdatedStorage() { + return maybeUpdatedStorage; + } + + /** + * Gets the transient storage value, including values from parent frames if not set + * + * @param accountAddress The address of the executing context + * @param slot the slot to retrieve + * @return the data value read + */ + public Bytes32 getTransientStorageValue(final Address accountAddress, final Bytes32 slot) { + Bytes32 v = txValues.transientStorage().get(accountAddress, slot); + return v == null ? Bytes32.ZERO : v; + } + + /** + * Gets the transient storage value, including values from parent frames if not set + * + * @param accountAddress The address of the executing context + * @param slot the slot to set + * @param value the value to set in the transient store + */ + public void setTransientStorageValue( + final Address accountAddress, final Bytes32 slot, final Bytes32 value) { + txValues.transientStorage().put(accountAddress, slot, value); + } + + /** Undo all the changes done by this message frame, such as when a revert is called for. */ + public void rollback() { + txValues.undoChanges(undoMark); + } + + /** + * Advances the undo mark to the current point, so that subsequent rollback() calls will not undo + * changes made before this point. Used by the transaction processor to protect intrinsic state + * gas charges (EIP-8037 auth delegation and contract creation) from being rolled back when the + * initial frame's execution reverts. + */ + public void advanceUndoMark() { + this.undoMark = txValues.transientStorage().mark(); + } + + /** + * Accessor for versionedHashes, if present. + * + * @return optional list of hashes + */ + public Optional> getVersionedHashes() { + return txValues.versionedHashes(); + } + + /** + * Accessor for Eip7928AccessList, if present. + * + * @return optional Eip7928AccessList + */ + public Optional getEip7928AccessList() { + return eip7928AccessList; + } + + /** Reset. */ + public void reset() { + maybeUpdatedMemory = Optional.empty(); + maybeUpdatedStorage = Optional.empty(); + } + + /** The MessageFrame Builder. */ + public static class Builder { + + private MessageFrame parentMessageFrame; + private Type type; + private WorldUpdater worldUpdater; + private Long initialGas; + private Address address; + private Address originator; + private Address contract; + private Wei gasPrice; + private Wei blobGasPrice = Wei.ZERO; + private Bytes inputData; + private Address sender; + private Wei value; + private Wei apparentValue; + private Code code; + private BlockValues blockValues; + private int maxStackSize = DEFAULT_MAX_STACK_SIZE; + private boolean isStatic = false; + private Consumer completer; + private Address miningBeneficiary; + private BlockHashLookup blockHashLookup; + private Map contextVariables; + private Optional reason = Optional.empty(); + private Set
eip2930AccessListWarmAddresses = emptySet(); + private Multimap eip2930AccessListWarmStorage = HashMultimap.create(); + private Optional eip7928AccessList = Optional.empty(); + + private Optional> versionedHashes = Optional.empty(); + + private boolean enableEvmV2 = false; + + /** Instantiates a new Builder. */ + public Builder() { + // constructor added to deal with JavaDoc linting rules. + } + + /** + * The "parent" message frame. When present some fields will be populated from the parent and + * ignored if passed in via builder + * + * @param parentMessageFrame the parent message frame + * @return the builder + */ + public Builder parentMessageFrame(final MessageFrame parentMessageFrame) { + this.parentMessageFrame = parentMessageFrame; + return this; + } + + /** + * Sets Type. + * + * @param type the type + * @return the builder + */ + public Builder type(final Type type) { + this.type = type; + return this; + } + + /** + * Sets World updater. + * + * @param worldUpdater the world updater + * @return the builder + */ + public Builder worldUpdater(final WorldUpdater worldUpdater) { + this.worldUpdater = worldUpdater; + return this; + } + + /** + * Sets Initial gas. + * + * @param initialGas the initial gas + * @return the builder + */ + public Builder initialGas(final long initialGas) { + this.initialGas = initialGas; + return this; + } + + /** + * Sets Address. + * + * @param address the address + * @return the builder + */ + public Builder address(final Address address) { + this.address = address; + return this; + } + + /** + * Sets Originator. + * + * @param originator the originator + * @return the builder + */ + public Builder originator(final Address originator) { + this.originator = originator; + return this; + } + + /** + * Sets Contract. + * + * @param contract the contract + * @return the builder + */ + public Builder contract(final Address contract) { + this.contract = contract; + return this; + } + + /** + * Sets Gas price. + * + * @param gasPrice the gas price + * @return the builder + */ + public Builder gasPrice(final Wei gasPrice) { + this.gasPrice = gasPrice; + return this; + } + + /** + * Sets Blob Gas price. + * + * @param blobGasPrice the blob gas price + * @return the builder + */ + public Builder blobGasPrice(final Wei blobGasPrice) { + this.blobGasPrice = blobGasPrice; + return this; + } + + /** + * Sets Input data. + * + * @param inputData the input data + * @return the builder + */ + public Builder inputData(final Bytes inputData) { + this.inputData = inputData; + return this; + } + + /** + * Sets Sender address. + * + * @param sender the sender + * @return the builder + */ + public Builder sender(final Address sender) { + this.sender = sender; + return this; + } + + /** + * Sets Value. + * + * @param value the value + * @return the builder + */ + public Builder value(final Wei value) { + this.value = value; + return this; + } + + /** + * Sets Apparent value. + * + * @param apparentValue the apparent value + * @return the builder + */ + public Builder apparentValue(final Wei apparentValue) { + this.apparentValue = apparentValue; + return this; + } + + /** + * Sets Code. + * + * @param code the code + * @return the builder + */ + public Builder code(final Code code) { + this.code = code; + return this; + } + + /** + * Sets Block values. + * + * @param blockValues the block values + * @return the builder + */ + public Builder blockValues(final BlockValues blockValues) { + this.blockValues = blockValues; + return this; + } + + /** + * Sets Is static. + * + * @param isStatic the is static + * @return the builder + */ + public Builder isStatic(final boolean isStatic) { + this.isStatic = isStatic; + return this; + } + + /** + * Sets Max stack size. + * + * @param maxStackSize the max stack size + * @return the builder + */ + public Builder maxStackSize(final int maxStackSize) { + this.maxStackSize = maxStackSize; + return this; + } + + /** + * Sets Completer. + * + * @param completer the completer + * @return the builder + */ + public Builder completer(final Consumer completer) { + this.completer = completer; + return this; + } + + /** + * Sets Mining beneficiary. + * + * @param miningBeneficiary the mining beneficiary + * @return the builder + */ + public Builder miningBeneficiary(final Address miningBeneficiary) { + this.miningBeneficiary = miningBeneficiary; + return this; + } + + /** + * Sets Block hash lookup. + * + * @param blockHashLookup the block hash lookup + * @return the builder + */ + public Builder blockHashLookup(final BlockHashLookup blockHashLookup) { + this.blockHashLookup = blockHashLookup; + return this; + } + + /** + * Sets Context variables. + * + * @param contextVariables the context variables + * @return the builder + */ + public Builder contextVariables(final Map contextVariables) { + this.contextVariables = contextVariables; + return this; + } + + /** + * Sets Reason. + * + * @param reason the reason + * @return the builder + */ + public Builder reason(final Bytes reason) { + this.reason = Optional.ofNullable(reason); + return this; + } + + /** + * Sets EIP 2930 Access list warm addresses. + * + * @param accessListWarmAddresses the access list warm addresses + * @return the builder + */ + public Builder eip2930AccessListWarmAddresses(final Set
accessListWarmAddresses) { + this.eip2930AccessListWarmAddresses = accessListWarmAddresses; + return this; + } + + /** + * Sets EIP 2930 Access list warm storage. + * + * @param accessListWarmStorage the access list warm storage + * @return the builder + */ + public Builder eip2930AccessListWarmStorage( + final Multimap accessListWarmStorage) { + this.eip2930AccessListWarmStorage = accessListWarmStorage; + return this; + } + + /** + * Sets EIP 7928 access list to record account and storage accesses. + * + * @param eip7928AccessList access list to record account and storage accesses + * @return the builder + */ + public Builder eip7928AccessList(final Eip7928AccessList eip7928AccessList) { + this.eip7928AccessList = Optional.of(eip7928AccessList); + return this; + } + + /** + * Sets versioned hashes list. + * + * @param versionedHashes the Optional list of versioned hashes + * @return the builder + */ + public Builder versionedHashes(final Optional> versionedHashes) { + this.versionedHashes = versionedHashes; + return this; + } + + /** + * Sets whether the experimental EVM v2 (long[] stack) is enabled. + * + * @param enableEvmV2 true to enable EVM v2 + * @return the builder + */ + public Builder enableEvmV2(final boolean enableEvmV2) { + this.enableEvmV2 = enableEvmV2; + return this; + } + + private void validate() { + if (parentMessageFrame == null) { + checkState(worldUpdater != null, "Missing message frame world updater"); + checkState(originator != null, "Missing message frame originator"); + checkState(gasPrice != null, "Missing message frame getGasRemaining price"); + checkState(blobGasPrice != null, "Missing message frame blob gas price"); + checkState(blockValues != null, "Missing message frame block header"); + checkState(miningBeneficiary != null, "Missing mining beneficiary"); + checkState(blockHashLookup != null, "Missing block hash lookup"); + } + checkState(type != null, "Missing message frame type"); + checkState(initialGas != null, "Missing message frame initial getGasRemaining"); + checkState(address != null, "Missing message frame recipient"); + checkState(contract != null, "Missing message frame contract"); + checkState(inputData != null, "Missing message frame input data"); + checkState(sender != null, "Missing message frame sender"); + checkState(value != null, "Missing message frame value"); + checkState(apparentValue != null, "Missing message frame apparent value"); + checkState(code != null, "Missing message frame code"); + checkState(completer != null, "Missing message frame completer"); + } + + /** + * Build MessageFrame. + * + * @return instance of MessageFrame + */ + public MessageFrame build() { + validate(); + + WorldUpdater updater; + boolean newStatic; + TxValues newTxValues; + + if (parentMessageFrame == null) { + TreeSet
warmedUpAddresses = new TreeSet<>(); // AERE D-434 (adfa98d013): TreeSet defeats the grindable base-31 HashDoS + warmedUpAddresses.add(contract); + newTxValues = + TxValues.forTransaction( + blockHashLookup, + maxStackSize, + UndoSet.of(warmedUpAddresses), + originator, + gasPrice, + blobGasPrice, + blockValues, + miningBeneficiary, + versionedHashes); + updater = worldUpdater; + newStatic = isStatic; + } else { + newTxValues = parentMessageFrame.txValues; + updater = parentMessageFrame.getWorldUpdater().updater(); + newStatic = isStatic || parentMessageFrame.isStatic; + parentMessageFrame.warmUpAddress(contract); + } + + MessageFrame messageFrame = + new MessageFrame( + enableEvmV2, + type, + updater, + initialGas, + address, + contract, + inputData, + sender, + value, + apparentValue, + code, + newStatic, + completer, + contextVariables == null ? Map.of() : contextVariables, + reason, + newTxValues, + eip7928AccessList); + newTxValues.messageFrameStack().addFirst(messageFrame); + messageFrame.warmUpAddress(sender); + for (Address a : eip2930AccessListWarmAddresses) { + messageFrame.warmUpAddress(a); + } + for (var e : eip2930AccessListWarmStorage.entries()) { + messageFrame.warmUpStorage(e.getKey(), e.getValue()); + } + return messageFrame; + } + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java new file mode 100644 index 0000000..b13a96f --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java @@ -0,0 +1,152 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-warm-access-hashdos-treeset.patch. + */ +package org.hyperledger.besu.evm.frame; + +import org.hyperledger.besu.collections.undo.UndoScalar; +import org.hyperledger.besu.collections.undo.UndoSet; +import org.hyperledger.besu.collections.undo.UndoTable; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.VersionedHash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.TreeSet; +import java.util.List; +import java.util.Optional; + +import com.google.common.collect.TreeBasedTable; +import org.apache.tuweni.bytes.Bytes32; + +/** + * Transaction Values used by various EVM Opcodes. These are the values that either do not change or + * the backing stores whose changes transcend message frames and are not part of state, such as + * transient storage and address warming. + * + * @param blockHashLookup The block hash lookup function + * @param maxStackSize The maximum stack size + * @param warmedUpAddresses The warmed-up addresses + * @param warmedUpStorage The warmed-up storage + * @param originator The originator address + * @param gasPrice The gas price + * @param blobGasPrice The blob gas price + * @param blockValues The block values + * @param messageFrameStack The message frame stack + * @param miningBeneficiary The mining beneficiary address + * @param versionedHashes The optional list of versioned hashes + * @param transientStorage The transient storage + * @param creates The set of addresses that creates + * @param selfDestructs The set of addresses that self-destructs + * @param gasRefunds The gas refunds + * @param stateGasUsed The cumulative state gas used (EIP-8037), undone on revert + * @param stateGasReservoir The EIP-8037 state gas reservoir (overflow from regular gas budget), + * undone on revert + * @param stateGasSpillBurned EIP-8037 accumulated state gas that spilled from reverted child + * frames; NOT undone on revert (permanent burn counter for block accounting) + * @param initialFrameRegularHaltBurn EIP-7778/EIP-8037 gas burned when the initial frame halts + * exceptionally (gasRemaining at halt time). Paid by the sender via receipts, but must be + * excluded from block regular gas. Single-element long[] so it is NOT undone on revert. + */ +public record TxValues( + BlockHashLookup blockHashLookup, + int maxStackSize, + UndoSet
warmedUpAddresses, + UndoTable warmedUpStorage, + Address originator, + Wei gasPrice, + Wei blobGasPrice, + BlockValues blockValues, + Deque messageFrameStack, + Address miningBeneficiary, + Optional> versionedHashes, + UndoTable transientStorage, + UndoSet
creates, + UndoSet
selfDestructs, + UndoScalar gasRefunds, + UndoScalar stateGasUsed, + UndoScalar stateGasReservoir, + long[] stateGasSpillBurned, + long[] initialFrameRegularHaltBurn) { + + /** + * Creates a new TxValues for the initial (depth-0) frame of a transaction. EIP-8037 gas tracking + * fields are initialized to zero. + * + * @param blockHashLookup block hash lookup function + * @param maxStackSize maximum stack size + * @param warmedUpAddresses pre-warmed addresses + * @param originator the transaction originator + * @param gasPrice the gas price + * @param blobGasPrice the blob gas price + * @param blockValues the block values + * @param miningBeneficiary the mining beneficiary + * @param versionedHashes optional versioned hashes + * @return a new TxValues instance + */ + public static TxValues forTransaction( + final BlockHashLookup blockHashLookup, + final int maxStackSize, + final UndoSet
warmedUpAddresses, + final Address originator, + final Wei gasPrice, + final Wei blobGasPrice, + final BlockValues blockValues, + final Address miningBeneficiary, + final Optional> versionedHashes) { + return new TxValues( + blockHashLookup, + maxStackSize, + warmedUpAddresses, + UndoTable.of(TreeBasedTable.create()), + originator, + gasPrice, + blobGasPrice, + blockValues, + new ArrayDeque<>(), + miningBeneficiary, + versionedHashes, + UndoTable.of(TreeBasedTable.create()), + UndoSet.of(new TreeSet<>()), + UndoSet.of(new TreeSet<>()), + new UndoScalar<>(0L), + new UndoScalar<>(0L), + new UndoScalar<>(0L), + new long[] {0L}, + new long[] {0L}); + } + + /** + * For all data stored in this record, undo the changes since the mark. + * + * @param mark the mark to which it should be rolled back to + */ + public void undoChanges(final long mark) { + warmedUpAddresses.undo(mark); + warmedUpStorage.undo(mark); + transientStorage.undo(mark); + creates.undo(mark); + selfDestructs.undo(mark); + gasRefunds.undo(mark); + stateGasUsed.undo(mark); + stateGasReservoir.undo(mark); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/FrontierGasCalculator.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/FrontierGasCalculator.java new file mode 100644 index 0000000..4dd3f52 --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/FrontierGasCalculator.java @@ -0,0 +1,611 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0006-aere-pq-transactions.patch. + */ +package org.hyperledger.besu.evm.gascalculator; + +import static org.hyperledger.besu.evm.internal.Words.clampedAdd; +import static org.hyperledger.besu.evm.internal.Words.clampedMultiply; +import static org.hyperledger.besu.evm.internal.Words.clampedToInt; +import static org.hyperledger.besu.evm.internal.Words.numWords; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.operation.ExpOperation; + +import java.util.function.Supplier; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** The Frontier gas calculator. */ +public class FrontierGasCalculator implements GasCalculator { + + private static final long TX_DATA_ZERO_COST = 4L; + + private static final long TX_DATA_NON_ZERO_COST = 68L; + + /** Minimum base cost that every transaction needs to pay */ + protected static final long TX_BASE_COST = 21_000L; + + private static final long TX_CREATE_EXTRA_COST = 0L; + + private static final long CODE_DEPOSIT_BYTE_COST = 200L; + + private static final long ID_PRECOMPILED_BASE_GAS_COST = 15L; + + private static final long ID_PRECOMPILED_WORD_GAS_COST = 3L; + + private static final long ECREC_PRECOMPILED_GAS_COST = 3_000L; + + private static final long P256VERIFY_PRECOMPILED_GAS_COST = 6_900L; + + private static final long SHA256_PRECOMPILED_BASE_GAS_COST = 60L; + + private static final long SHA256_PRECOMPILED_WORD_GAS_COST = 12L; + + private static final long RIPEMD160_PRECOMPILED_WORD_GAS_COST = 120L; + + private static final long RIPEMD160_PRECOMPILED_BASE_GAS_COST = 600L; + + private static final long VERY_LOW_TIER_GAS_COST = 3L; + + private static final long LOW_TIER_GAS_COST = 5L; + + private static final long BASE_TIER_GAS_COST = 2L; + + private static final long MID_TIER_GAS_COST = 8L; + + private static final long HIGH_TIER_GAS_COST = 10L; + + private static final long CALL_OPERATION_BASE_GAS_COST = 40L; + + private static final long CALL_VALUE_TRANSFER_GAS_COST = 9_000L; + + /** additional call stipend to be used for gas estimate calculations */ + public static final long ADDITIONAL_CALL_STIPEND = 2_300L; + + private static final long NEW_ACCOUNT_GAS_COST = 25_000L; + + /** Yellow paper constant for the cost of creating a new contract on-chain */ + protected static final long CREATE_OPERATION_GAS_COST = 32_000L; + + private static final long COPY_WORD_GAS_COST = 3L; + + private static final long MEMORY_WORD_GAS_COST = 3L; + + private static final long BALANCE_OPERATION_GAS_COST = 20L; + + /** The constant BLOCKHASH_OPERATION_GAS_COST = 20gwei. */ + public static final long BLOCKHASH_OPERATION_GAS_COST = 20L; + + private static final long EXP_OPERATION_BASE_GAS_COST = 10L; + + private static final long EXP_OPERATION_BYTE_GAS_COST = 10L; + + private static final long EXT_CODE_BASE_GAS_COST = 20L; + + private static final long JUMPDEST_OPERATION_GAS_COST = 1L; + + private static final long LOG_OPERATION_BASE_GAS_COST = 375L; + + private static final long LOG_OPERATION_DATA_BYTE_GAS_COST = 8L; + + private static final long LOG_OPERATION_TOPIC_GAS_COST = 375L; + + private static final long KECCAK256_OPERATION_BASE_GAS_COST = 30L; + + /** The constant KECCAK256_OPERATION_WORD_GAS_COST. */ + static final long KECCAK256_OPERATION_WORD_GAS_COST = 6L; + + private static final long SLOAD_OPERATION_GAS_COST = 50L; + + /** The constant STORAGE_SET_GAS_COST. */ + public static final long STORAGE_SET_GAS_COST = 20_000L; + + /** The constant STORAGE_RESET_GAS_COST. */ + public static final long STORAGE_RESET_GAS_COST = 5_000L; + + /** The constant STORAGE_RESET_REFUND_AMOUNT. */ + public static final long STORAGE_RESET_REFUND_AMOUNT = 15_000L; + + private static final long SELF_DESTRUCT_REFUND_AMOUNT = 24_000L; + + /** Default constructor. */ + public FrontierGasCalculator() { + // Default Constructor, for JavaDoc lint + } + + @Override + public long transactionIntrinsicGasCost(final Transaction transaction, final long baselineGas) { + final long dynamicIntrinsicGasCost = dynamicIntrinsicGasCost(transaction, baselineGas); + + if (dynamicIntrinsicGasCost == Long.MIN_VALUE || dynamicIntrinsicGasCost == Long.MAX_VALUE) { + return dynamicIntrinsicGasCost; + } + return clampedAdd(getMinimumTransactionCost(), dynamicIntrinsicGasCost); + } + + /** + * Calculates the dynamic part of the intrinsic gas cost + * + * @param transaction the transaction + * @param baselineGas how much gas is used by access lists and code delegations + * @return the dynamic part of the intrinsic gas cost + */ + protected long dynamicIntrinsicGasCost(final Transaction transaction, final long baselineGas) { + final int payloadSize = transaction.getPayload().size(); + + long cost = + clampedAdd(callDataCost(payloadSize, transaction.getPayloadZeroBytes()), baselineGas); + + if (cost == Long.MIN_VALUE || cost == Long.MAX_VALUE) { + return cost; + } + + // AIP-20: the post-quantum authorization bytes are charged like non-zero calldata (16 per byte) + // and the verification itself at the algorithm's flat cost. Zero for every other type. + final long pqCost = + transaction + .getPqAuthorization() + .map(a -> clampedAdd(16L * a.authorizationByteCount(), a.algorithm().verifyGas())) + .orElse(0L); + cost = clampedAdd(cost, pqCost); + + if (cost == Long.MIN_VALUE || cost == Long.MAX_VALUE) { + return cost; + } + + if (transaction.isContractCreation()) { + cost = clampedAdd(cost, contractCreationCost(payloadSize)); + + if (cost == Long.MIN_VALUE || cost == Long.MAX_VALUE) { + return cost; + } + } + + return cost; + } + + /** + * Calculates the cost of the call data + * + * @param payloadSize the total size of the payload + * @param zeroBytes the number of zero bytes in the payload + * @return the cost of the call data + */ + protected long callDataCost(final long payloadSize, final long zeroBytes) { + return clampedAdd( + TX_DATA_NON_ZERO_COST * (payloadSize - zeroBytes), TX_DATA_ZERO_COST * zeroBytes); + } + + /** + * Returns the gas cost for contract creation transactions + * + * @param ignored the size of the contract creation code (ignored in Frontier) + * @return the gas cost for contract creation transactions + */ + protected long contractCreationCost(final int ignored) { + return txCreateExtraGasCost(); + } + + /** + * Returns the additional gas cost for contract creation transactions + * + * @return the additional gas cost for contract creation transactions + */ + protected long txCreateExtraGasCost() { + return TX_CREATE_EXTRA_COST; + } + + @Override + public long transactionFloorCost(final Bytes payload, final long payloadZeroBytes) { + return 0L; + } + + @Override + public long codeDepositGasCost(final int codeSize) { + return CODE_DEPOSIT_BYTE_COST * codeSize; + } + + @Override + public long idPrecompiledContractGasCost(final Bytes input) { + return ID_PRECOMPILED_WORD_GAS_COST * numWords(input) + ID_PRECOMPILED_BASE_GAS_COST; + } + + @Override + public long getEcrecPrecompiledContractGasCost() { + return ECREC_PRECOMPILED_GAS_COST; + } + + @Override + public long getP256VerifyPrecompiledContractGasCost() { + return P256VERIFY_PRECOMPILED_GAS_COST; + } + + @Override + public long sha256PrecompiledContractGasCost(final Bytes input) { + return SHA256_PRECOMPILED_WORD_GAS_COST * numWords(input) + SHA256_PRECOMPILED_BASE_GAS_COST; + } + + @Override + public long ripemd160PrecompiledContractGasCost(final Bytes input) { + return RIPEMD160_PRECOMPILED_WORD_GAS_COST * numWords(input) + + RIPEMD160_PRECOMPILED_BASE_GAS_COST; + } + + @Override + public long getZeroTierGasCost() { + return 0L; + } + + @Override + public long getVeryLowTierGasCost() { + return VERY_LOW_TIER_GAS_COST; + } + + @Override + public long getLowTierGasCost() { + return LOW_TIER_GAS_COST; + } + + @Override + public long getBaseTierGasCost() { + return BASE_TIER_GAS_COST; + } + + @Override + public long getMidTierGasCost() { + return MID_TIER_GAS_COST; + } + + @Override + public long getHighTierGasCost() { + return HIGH_TIER_GAS_COST; + } + + /** + * Returns the base gas cost to execute a call operation. + * + * @return the base gas cost to execute a call operation + */ + @Override + public long callOperationBaseGasCost() { + return CALL_OPERATION_BASE_GAS_COST; + } + + @Override + public long callValueTransferGasCost() { + return CALL_VALUE_TRANSFER_GAS_COST; + } + + @Override + public long newAccountGasCost() { + return NEW_ACCOUNT_GAS_COST; + } + + @Override + public long callOperationStaticGasCost( + final MessageFrame frame, + final long stipend, + final long inputDataOffset, + final long inputDataLength, + final long outputDataOffset, + final long outputDataLength, + final Wei transferValue, + final Address recipientAddress, + final boolean accountIsWarm) { + final long inputDataMemoryExpansionCost = + memoryExpansionGasCost(frame, inputDataOffset, inputDataLength); + final long outputDataMemoryExpansionCost = + memoryExpansionGasCost(frame, outputDataOffset, outputDataLength); + final long memoryExpansionCost = + Math.max(inputDataMemoryExpansionCost, outputDataMemoryExpansionCost); + + long cost = clampedAdd(clampedAdd(callOperationBaseGasCost(), stipend), memoryExpansionCost); + + if (!transferValue.isZero()) { + cost = clampedAdd(cost, callValueTransferGasCost()); + } + + return cost; + } + + @Override + public long callOperationGasCost( + final MessageFrame frame, + final long staticCallCost, + final long stipend, + final long inputDataOffset, + final long inputDataLength, + final long outputDataOffset, + final long outputDataLength, + final Wei transferValue, + final Address recipientAddress, + final boolean accountIsWarm) { + long cost = staticCallCost; + + final Account recipient = frame.getWorldUpdater().get(recipientAddress); + if (recipient == null) { + cost = clampedAdd(cost, newAccountGasCost()); + } + + return cost; + } + + /** + * Returns the additional call stipend for calls with value transfers. + * + * @return the additional call stipend for calls with value transfers + */ + @Override + public long getAdditionalCallStipend() { + return ADDITIONAL_CALL_STIPEND; + } + + @Override + public long gasAvailableForChildCall( + final MessageFrame frame, final long stipend, final boolean transfersValue) { + if (transfersValue) { + return stipend + getAdditionalCallStipend(); + } else { + return stipend; + } + } + + @Override + public long getMinRetainedGas() { + return 0; + } + + @Override + public long getMinCalleeGas() { + return 0; + } + + @Override + public long txCreateCost() { + return CREATE_OPERATION_GAS_COST; + } + + @Override + public long createKeccakCost(final int initCodeLength) { + return clampedMultiply(KECCAK256_OPERATION_WORD_GAS_COST, numWords(initCodeLength)); + } + + @Override + public long initcodeCost(final int initCodeLength) { + return 0; + } + + @Override + public long gasAvailableForChildCreate(final long stipend) { + return stipend; + } + + @Override + public long dataCopyOperationGasCost( + final MessageFrame frame, final long offset, final long length) { + return copyWordsToMemoryGasCost( + frame, VERY_LOW_TIER_GAS_COST, COPY_WORD_GAS_COST, offset, length); + } + + @Override + public long memoryExpansionGasCost( + final MessageFrame frame, final long offset, final long length) { + + final long pre = memoryCost(frame.memoryWordSize()); + final long post = memoryCost(frame.calculateMemoryExpansion(offset, length)); + if (post == Long.MAX_VALUE) { + return Long.MAX_VALUE; + } + return post - pre; + } + + @Override + public long getBalanceOperationGasCost() { + return BALANCE_OPERATION_GAS_COST; + } + + @Override + public long getBlockHashOperationGasCost() { + return BLOCKHASH_OPERATION_GAS_COST; + } + + /** + * Returns the gas cost for a byte in the {@link ExpOperation}. + * + * @return the gas cost for a byte in the exponent operation + */ + protected long expOperationByteGasCost() { + return EXP_OPERATION_BYTE_GAS_COST; + } + + @Override + public long expOperationGasCost(final int numBytes) { + return expOperationByteGasCost() * numBytes + EXP_OPERATION_BASE_GAS_COST; + } + + /** + * Returns the base gas cost for external code accesses. + * + * @return the base gas cost for external code accesses + */ + protected long extCodeBaseGasCost() { + return EXT_CODE_BASE_GAS_COST; + } + + @Override + public long extCodeCopyOperationGasCost( + final MessageFrame frame, final long offset, final long length) { + return copyWordsToMemoryGasCost( + frame, extCodeBaseGasCost(), COPY_WORD_GAS_COST, offset, length); + } + + @Override + public long extCodeHashOperationGasCost() { + throw new UnsupportedOperationException( + "EXTCODEHASH not supported by " + getClass().getSimpleName()); + } + + @Override + public long getExtCodeSizeOperationGasCost() { + return extCodeBaseGasCost(); + } + + @Override + public long getJumpDestOperationGasCost() { + return JUMPDEST_OPERATION_GAS_COST; + } + + @Override + public long logOperationGasCost( + final MessageFrame frame, final long dataOffset, final long dataLength, final int numTopics) { + return clampedAdd( + LOG_OPERATION_BASE_GAS_COST, + clampedAdd( + clampedMultiply(LOG_OPERATION_DATA_BYTE_GAS_COST, dataLength), + clampedAdd( + clampedMultiply(LOG_OPERATION_TOPIC_GAS_COST, numTopics), + memoryExpansionGasCost(frame, dataOffset, dataLength)))); + } + + @Override + public long mLoadOperationGasCost(final MessageFrame frame, final long offset) { + return clampedAdd(VERY_LOW_TIER_GAS_COST, memoryExpansionGasCost(frame, offset, 32)); + } + + @Override + public long mStoreOperationGasCost(final MessageFrame frame, final long offset) { + return clampedAdd(VERY_LOW_TIER_GAS_COST, memoryExpansionGasCost(frame, offset, 32)); + } + + @Override + public long mStore8OperationGasCost(final MessageFrame frame, final long offset) { + return clampedAdd(VERY_LOW_TIER_GAS_COST, memoryExpansionGasCost(frame, offset, 1)); + } + + @Override + public long keccak256OperationGasCost( + final MessageFrame frame, final long offset, final long length) { + return copyWordsToMemoryGasCost( + frame, + KECCAK256_OPERATION_BASE_GAS_COST, + KECCAK256_OPERATION_WORD_GAS_COST, + offset, + length); + } + + @Override + public long getSloadOperationGasCost() { + return SLOAD_OPERATION_GAS_COST; + } + + @Override + public long calculateStorageCost( + final UInt256 newValue, + final Supplier currentValue, + final Supplier originalValue) { + return !newValue.isZero() && currentValue.get().isZero() + ? STORAGE_SET_GAS_COST + : STORAGE_RESET_GAS_COST; + } + + @Override + public long calculateStorageRefundAmount( + final UInt256 newValue, + final Supplier currentValue, + final Supplier originalValue) { + return newValue.isZero() && !currentValue.get().isZero() ? STORAGE_RESET_REFUND_AMOUNT : 0L; + } + + @Override + public long getSelfDestructRefundAmount() { + return SELF_DESTRUCT_REFUND_AMOUNT; + } + + /** + * Copy words to memory gas cost. + * + * @param frame the frame + * @param baseGasCost the base gas cost + * @param wordGasCost the word gas cost + * @param offset the offset + * @param length the length + * @return the cost + */ + protected long copyWordsToMemoryGasCost( + final MessageFrame frame, + final long baseGasCost, + final long wordGasCost, + final long offset, + final long length) { + final long copyCost = + clampedAdd(clampedMultiply(wordGasCost, numWords(clampedToInt(length))), baseGasCost); + final long memoryCost = memoryExpansionGasCost(frame, offset, length); + + return clampedAdd(copyCost, memoryCost); + } + + /** + * Memory cost. + * + * @param length the length + * @return the cost + */ + static long memoryCost(final long length) { + final long lengthSquare = clampedMultiply(length, length); + final long base = + (lengthSquare == Long.MAX_VALUE) + ? clampedMultiply(length / 512, length) + : lengthSquare / 512; + + return clampedAdd(clampedMultiply(MEMORY_WORD_GAS_COST, length), base); + } + + @Override + public long getMinimumTransactionCost() { + return TX_BASE_COST; + } + + @Override + public long calculateGasRefund( + final Transaction transaction, + final MessageFrame initialFrame, + final long ignoredCodeDelegationRefund) { + + final long refundAllowance = calculateRefundAllowance(transaction, initialFrame); + + return initialFrame.getRemainingGas() + refundAllowance; + } + + private long calculateRefundAllowance( + final Transaction transaction, final MessageFrame initialFrame) { + final long selfDestructRefund = + getSelfDestructRefundAmount() * initialFrame.getSelfDestructs().size(); + final long executionRefund = initialFrame.getGasRefund() + selfDestructRefund; + // Integer truncation takes care of the floor calculation needed after the divide. + final long maxRefundAllowance = + (transaction.getGasLimit() - initialFrame.getRemainingGas()) / getMaxRefundQuotient(); + return Math.min(executionRefund, maxRefundAllowance); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/AereFalconSupport.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/AereFalconSupport.java new file mode 100644 index 0000000..8f15430 --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/AereFalconSupport.java @@ -0,0 +1,94 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.evm.precompile; + +import java.util.Arrays; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.pqc.crypto.falcon.FalconParameters; +import org.bouncycastle.pqc.crypto.falcon.FalconPublicKeyParameters; +import org.bouncycastle.pqc.crypto.falcon.FalconSigner; + +/** + * Shared logic for the AERE Falcon-512 / Falcon-1024 verification precompiles. + * + *

Transcodes the NIST round-3 reference "signed message" (sm) blob into the encoding Bouncy + * Castle's {@link FalconSigner#verifySignature(byte[], byte[])} expects and drives that + * verifier. No cryptography is reimplemented here. + * + *

NIST sm layout: {@code sigLen(2, big-endian) || nonce(40) || message || esig} where + * {@code esig = (0x20+logn) || compressedSig} and {@code sigLen == esig.length}. The Falcon public + * key is {@code (0x00+logn) || packed_h}; Bouncy Castle wants only {@code packed_h} (header + * stripped), and the signature it wants is {@code (0x30+logn) || nonce(40) || compressedSig}. + */ +final class AereFalconSupport { + + static final int NONCE_LEN = 40; + + private AereFalconSupport() {} + + static Bytes resultWord(final boolean valid) { + return valid ? Bytes32.leftPad(Bytes.of((byte) 1)) : Bytes32.ZERO; + } + + /** + * Verify a NIST signed-message blob against a Falcon public key. + * + * @param params Bouncy Castle Falcon parameter set (falcon_512 / falcon_1024) + * @param logn 9 for Falcon-512, 10 for Falcon-1024 + * @param pkFull the 897- (512) or 1793-byte (1024) public key including the leading header byte + * @param sm the NIST signed-message blob + * @return true iff the signature is valid for the embedded message under pkFull + */ + static boolean verify( + final FalconParameters params, final int logn, final byte[] pkFull, final byte[] sm) { + try { + if (pkFull.length < 2 || sm.length < 2 + NONCE_LEN + 2) { + return false; + } + if ((pkFull[0] & 0xff) != logn) { + return false; + } + final int sigLen = ((sm[0] & 0xff) << 8) | (sm[1] & 0xff); + if (sigLen < 2 || 2 + NONCE_LEN + sigLen > sm.length) { + return false; + } + final int msgLen = sm.length - 2 - NONCE_LEN - sigLen; + if (msgLen < 0) { + return false; + } + final byte[] esig = Arrays.copyOfRange(sm, sm.length - sigLen, sm.length); + if ((esig[0] & 0xff) != (0x20 + logn)) { + return false; + } + final byte[] H = Arrays.copyOfRange(pkFull, 1, pkFull.length); + final byte[] nonce = Arrays.copyOfRange(sm, 2, 2 + NONCE_LEN); + final byte[] message = Arrays.copyOfRange(sm, 2 + NONCE_LEN, 2 + NONCE_LEN + msgLen); + + final byte[] bcSig = new byte[1 + NONCE_LEN + (esig.length - 1)]; + bcSig[0] = (byte) (0x30 + logn); + System.arraycopy(nonce, 0, bcSig, 1, NONCE_LEN); + System.arraycopy(esig, 1, bcSig, 1 + NONCE_LEN, esig.length - 1); + + final FalconPublicKeyParameters pub = new FalconPublicKeyParameters(params, H); + final FalconSigner signer = new FalconSigner(); + signer.init(false, pub); + return signer.verifySignature(message, bcSig); + } catch (final Throwable t) { + return false; + } + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon1024PrecompiledContract.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon1024PrecompiledContract.java new file mode 100644 index 0000000..80aca57 --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon1024PrecompiledContract.java @@ -0,0 +1,62 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.evm.precompile; + +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.bouncycastle.pqc.crypto.falcon.FalconParameters; + +/** + * AERE PQC precompile: Falcon-1024 signature verification (NIST round-3 reference encoding). + * + *

Input: {@code pk(1793) || sm(rest)} where pk is the reference public key (header 0x0A) and sm + * is the reference signed-message blob. Output: 32-byte word, {@code ...01} valid else {@code ...00}. + */ +public class Falcon1024PrecompiledContract extends AbstractPrecompiledContract { + + static final int PK_LEN = 1793; + private static final int LOGN = 10; + private static final long GAS = 75_000L; + + /** + * Instantiates a new Falcon-1024 precompiled contract. + * + * @param gasCalculator the gas calculator + */ + Falcon1024PrecompiledContract(final GasCalculator gasCalculator) { + super("AereFalcon1024", gasCalculator); + } + + @Override + public long gasRequirement(final Bytes input) { + return GAS; + } + + @NotNull + @Override + public PrecompileContractResult computePrecompile( + final Bytes input, @NotNull final MessageFrame messageFrame) { + boolean valid = false; + if (input.size() > PK_LEN) { + final byte[] pk = input.slice(0, PK_LEN).toArrayUnsafe(); + final byte[] sm = input.slice(PK_LEN).toArrayUnsafe(); + valid = AereFalconSupport.verify(FalconParameters.falcon_1024, LOGN, pk, sm); + } + return PrecompileContractResult.success(AereFalconSupport.resultWord(valid)); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon512PrecompiledContract.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon512PrecompiledContract.java new file mode 100644 index 0000000..5df074c --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/Falcon512PrecompiledContract.java @@ -0,0 +1,62 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.evm.precompile; + +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.bouncycastle.pqc.crypto.falcon.FalconParameters; + +/** + * AERE PQC precompile: Falcon-512 signature verification (NIST round-3 reference encoding). + * + *

Input: {@code pk(897) || sm(rest)} where pk is the reference public key (header 0x09) and sm is + * the reference signed-message blob. Output: 32-byte word, {@code ...01} if valid else {@code ...00}. + */ +public class Falcon512PrecompiledContract extends AbstractPrecompiledContract { + + static final int PK_LEN = 897; + private static final int LOGN = 9; + private static final long GAS = 40_000L; + + /** + * Instantiates a new Falcon-512 precompiled contract. + * + * @param gasCalculator the gas calculator + */ + Falcon512PrecompiledContract(final GasCalculator gasCalculator) { + super("AereFalcon512", gasCalculator); + } + + @Override + public long gasRequirement(final Bytes input) { + return GAS; + } + + @NotNull + @Override + public PrecompileContractResult computePrecompile( + final Bytes input, @NotNull final MessageFrame messageFrame) { + boolean valid = false; + if (input.size() > PK_LEN) { + final byte[] pk = input.slice(0, PK_LEN).toArrayUnsafe(); + final byte[] sm = input.slice(PK_LEN).toArrayUnsafe(); + valid = AereFalconSupport.verify(FalconParameters.falcon_512, LOGN, pk, sm); + } + return PrecompileContractResult.success(AereFalconSupport.resultWord(valid)); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/MLDSA44PrecompiledContract.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/MLDSA44PrecompiledContract.java new file mode 100644 index 0000000..f25be64 --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/MLDSA44PrecompiledContract.java @@ -0,0 +1,84 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.evm.precompile; + +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.pqc.crypto.mldsa.MLDSAParameters; +import org.bouncycastle.pqc.crypto.mldsa.MLDSAPublicKeyParameters; +import org.bouncycastle.pqc.crypto.mldsa.MLDSASigner; + +/** + * AERE PQC precompile: ML-DSA-44 (FIPS 204) signature verification via the INTERNAL interface + * (ML-DSA.Verify_internal, Algorithm 8 — no context/domain-separation prefix). + * + *

Input: {@code pk(1312) || sig(2420) || message(rest)}. Output: 32-byte word, {@code ...01} + * valid else {@code ...00}. The internal (rather than pure) verifier is reached by subclassing + * Bouncy Castle's {@link MLDSASigner} and calling its {@code protected internalVerifySignature}. + */ +public class MLDSA44PrecompiledContract extends AbstractPrecompiledContract { + + static final int PK_LEN = 1312; + static final int SIG_LEN = 2420; + private static final long GAS = 55_000L; + + /** Subclass exposing Bouncy Castle's protected internal (Verify_internal) verifier. */ + private static final class InternalVerifier extends MLDSASigner { + boolean verifyInternal(final byte[] message, final byte[] signature) { + return internalVerifySignature(message, signature); + } + } + + /** + * Instantiates a new ML-DSA-44 precompiled contract. + * + * @param gasCalculator the gas calculator + */ + MLDSA44PrecompiledContract(final GasCalculator gasCalculator) { + super("AereMLDSA44", gasCalculator); + } + + @Override + public long gasRequirement(final Bytes input) { + return GAS; + } + + @NotNull + @Override + public PrecompileContractResult computePrecompile( + final Bytes input, @NotNull final MessageFrame messageFrame) { + boolean valid = false; + if (input.size() >= PK_LEN + SIG_LEN) { + try { + final byte[] pk = input.slice(0, PK_LEN).toArrayUnsafe(); + final byte[] sig = input.slice(PK_LEN, SIG_LEN).toArrayUnsafe(); + final byte[] message = input.slice(PK_LEN + SIG_LEN).toArrayUnsafe(); + final MLDSAPublicKeyParameters pub = + new MLDSAPublicKeyParameters(MLDSAParameters.ml_dsa_44, pk); + final InternalVerifier verifier = new InternalVerifier(); + verifier.init(false, pub); + valid = verifier.verifyInternal(message, sig); + } catch (final Throwable t) { + valid = false; + } + } + return PrecompileContractResult.success( + valid ? Bytes32.leftPad(Bytes.of((byte) 1)) : Bytes32.ZERO); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/MainnetPrecompiledContracts.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/MainnetPrecompiledContracts.java new file mode 100644 index 0000000..b4ae37b --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/MainnetPrecompiledContracts.java @@ -0,0 +1,246 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0001-aere-pqc-precompiles-mainnet.patch.patch. + */ +package org.hyperledger.besu.evm.precompile; + +import static org.hyperledger.besu.datatypes.Address.P256_VERIFY; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +/** Provides the various precompiled contracts used on mainnet hard forks. */ +public interface MainnetPrecompiledContracts { + + /** + * Frontier precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry frontier(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForFrontier(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for frontier. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForFrontier( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + registry.put(Address.ECREC, new ECRECPrecompiledContract(gasCalculator)); + registry.put(Address.SHA256, new SHA256PrecompiledContract(gasCalculator)); + registry.put(Address.RIPEMD160, new RIPEMD160PrecompiledContract(gasCalculator)); + registry.put(Address.ID, new IDPrecompiledContract(gasCalculator)); + } + + /** + * Homestead precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry homestead(final GasCalculator gasCalculator) { + return frontier(gasCalculator); + } + + /** + * Byzantium precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry byzantium(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForByzantium(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for byzantium. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForByzantium( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + populateForFrontier(registry, gasCalculator); + registry.put( + Address.MODEXP, + new BigIntegerModularExponentiationPrecompiledContract(gasCalculator, Long.MAX_VALUE)); + registry.put(Address.ALTBN128_ADD, new AltBN128AddPrecompiledContract(gasCalculator, 500L)); + registry.put(Address.ALTBN128_MUL, new AltBN128MulPrecompiledContract(gasCalculator, 40_000L)); + registry.put( + Address.ALTBN128_PAIRING, + new AltBN128PairingPrecompiledContract(gasCalculator, 80_000L, 100_000L)); + } + + /** + * Istanbul precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry istanbul(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForIstanbul(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for istanbul. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForIstanbul( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + populateForByzantium(registry, gasCalculator); + registry.put(Address.ALTBN128_ADD, new AltBN128AddPrecompiledContract(gasCalculator, 150L)); + registry.put(Address.ALTBN128_MUL, new AltBN128MulPrecompiledContract(gasCalculator, 6_000L)); + registry.put( + Address.ALTBN128_PAIRING, + new AltBN128PairingPrecompiledContract(gasCalculator, 34_000L, 45_000L)); + registry.put(Address.BLAKE2B_F_COMPRESSION, new BLAKE2BFPrecompileContract(gasCalculator)); + } + + /** + * Cancun precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry cancun(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForCancun(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for Cancun. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForCancun( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + populateForIstanbul(registry, gasCalculator); + + // EIP-4844 - shard blob transactions + registry.put(Address.KZG_POINT_EVAL, new KZGPointEvalPrecompiledContract()); + } + + /** + * Prague precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry prague(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForPrague(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for Prague. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForPrague( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + populateForCancun(registry, gasCalculator); + + // EIP-2537 - BLS12-381 curve operations + registry.put(Address.BLS12_G1ADD, new BLS12G1AddPrecompiledContract()); + registry.put(Address.BLS12_G1MULTIEXP, new BLS12G1MultiExpPrecompiledContract()); + registry.put(Address.BLS12_G2ADD, new BLS12G2AddPrecompiledContract()); + registry.put(Address.BLS12_G2MULTIEXP, new BLS12G2MultiExpPrecompiledContract()); + registry.put(Address.BLS12_PAIRING, new BLS12PairingPrecompiledContract()); + registry.put(Address.BLS12_MAP_FP_TO_G1, new BLS12MapFpToG1PrecompiledContract()); + registry.put(Address.BLS12_MAP_FP2_TO_G2, new BLS12MapFp2ToG2PrecompiledContract()); + } + + /** + * Osaka precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry osaka(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForOsaka(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for Osaka. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForOsaka( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + populateForPrague(registry, gasCalculator); + + // EIP-7823 - Set upper bounds for MODEXP + registry.put( + Address.MODEXP, + new BigIntegerModularExponentiationPrecompiledContract(gasCalculator, 1024L)); + // EIP-7951 - secp256r1 P256VERIFY + registry.put(P256_VERIFY, new P256VerifyPrecompiledContract(gasCalculator)); + } + + /** + * FutureEIPs precompile contract registry. + * + * @param gasCalculator the gas calculator + * @return the precompile contract registry + */ + static PrecompileContractRegistry futureEIPs(final GasCalculator gasCalculator) { + PrecompileContractRegistry precompileContractRegistry = new PrecompileContractRegistry(); + populateForFutureEIPs(precompileContractRegistry, gasCalculator); + return precompileContractRegistry; + } + + /** + * Populate registry for Future EIPs. + * + * @param registry the registry + * @param gasCalculator the gas calculator + */ + static void populateForFutureEIPs( + final PrecompileContractRegistry registry, final GasCalculator gasCalculator) { + // AERE "AerePQC" fork: full Osaka precompile set plus native post-quantum verifiers. + // Activated on a running chain via genesis config "futureEipsTime"; no re-genesis needed. + populateForOsaka(registry, gasCalculator); + + // Native post-quantum precompiles (Bouncy Castle BCPQC verifiers). + registry.put(Address.AERE_FALCON512, new Falcon512PrecompiledContract(gasCalculator)); + registry.put(Address.AERE_FALCON1024, new Falcon1024PrecompiledContract(gasCalculator)); + registry.put(Address.AERE_MLDSA44, new MLDSA44PrecompiledContract(gasCalculator)); + registry.put(Address.AERE_SLHDSA128S, new SLHDSA128sPrecompiledContract(gasCalculator)); + registry.put(Address.AERE_SHAKE256, new SHAKE256PrecompiledContract(gasCalculator)); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/P256VerifyPrecompiledContract.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/P256VerifyPrecompiledContract.java new file mode 100644 index 0000000..220194b --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/P256VerifyPrecompiledContract.java @@ -0,0 +1,251 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/aere-drop-per-request-console-writes.patch. + */ +package org.hyperledger.besu.evm.precompile; + +import org.hyperledger.besu.crypto.SECP256R1; +import org.hyperledger.besu.crypto.SECPPublicKey; +import org.hyperledger.besu.crypto.SECPSignature; +import org.hyperledger.besu.crypto.SignatureAlgorithm; +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; +import org.hyperledger.besu.nativelib.boringssl.BoringSSLPrecompiles; + +import java.math.BigInteger; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.asn1.sec.SECNamedCurves; +import org.bouncycastle.asn1.x9.X9ECParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Implementation of EIP-7951. */ +public class P256VerifyPrecompiledContract extends AbstractPrecompiledContract { + private static final Logger LOG = LoggerFactory.getLogger(P256VerifyPrecompiledContract.class); + private static final String PRECOMPILE_NAME = "P256VERIFY"; + private static final Bytes32 VALID = Bytes32.leftPad(Bytes.of(1), (byte) 0); + private static final Bytes INVALID = Bytes.EMPTY; + private static final int SECP256R1_INPUT_LENGTH = 160; + private static final SignatureAlgorithm SECP256R1_SIGNATURE_ALGORITHM_INSTANCE; + + static final X9ECParameters R1_PARAMS = SECNamedCurves.getByName("secp256r1"); + static final BigInteger N = R1_PARAMS.getN(); + static final BigInteger P = R1_PARAMS.getCurve().getField().getCharacteristic(); + + /** The constant useNative. */ + // use the BoringSSL native library implementation, if it is available + private static boolean useNativeBoringSSL; + + static { + maybeEnableNativeBoringSSL(); + SECP256R1_SIGNATURE_ALGORITHM_INSTANCE = new SECP256R1(); + } + + /** + * Attempt to enable the BoringSSL native library for P256Verify contract Note, if BoringSSL is + * disabled, then native SECP256R1 may still be enabled + * + * @return true if the native library was enabled. + */ + public static boolean maybeEnableNativeBoringSSL() { + try { + useNativeBoringSSL = BoringSSLPrecompiles.ENABLED; + } catch (UnsatisfiedLinkError | NoClassDefFoundError ule) { + LOG.info( + "BoringSSL secp256r1 p256verify native precompile not available: {}", ule.getMessage()); + useNativeBoringSSL = false; + } + return useNativeBoringSSL; + } + + /** Disable native. Note SECP256R1 must additionally be disabled to fully disable native */ + public static void disableNativeBoringSSL() { + useNativeBoringSSL = false; + } + + /** + * Is native BoringSSL boolean. + * + * @return the boolean indicating whether to use the BoringSSL native library implementation, if + * it is available + */ + public static boolean isNativeBoringSSL() { + return useNativeBoringSSL; + } + + private final GasCalculator gasCalculator; + private final SignatureAlgorithm signatureAlgorithm; + + private static final Cache p256VerifyCache = + Caffeine.newBuilder().maximumSize(1000).build(); + + /** + * Instantiates a new Abstract precompiled contract. + * + * @param gasCalculator the gas calculator + */ + public P256VerifyPrecompiledContract(final GasCalculator gasCalculator) { + this(gasCalculator, SECP256R1_SIGNATURE_ALGORITHM_INSTANCE); + } + + /** + * Instantiates a new Abstract precompiled contract. + * + * @param gasCalculator the gas calculator + * @param signatureAlgorithm the signature algorithm to use + */ + public P256VerifyPrecompiledContract( + final GasCalculator gasCalculator, final SignatureAlgorithm signatureAlgorithm) { + super(PRECOMPILE_NAME, gasCalculator); + this.gasCalculator = gasCalculator; + this.signatureAlgorithm = signatureAlgorithm; + } + + @Override + public long gasRequirement(final Bytes input) { + return gasCalculator.getP256VerifyPrecompiledContractGasCost(); + } + + @Override + public PrecompileContractResult computePrecompile( + final Bytes input, final MessageFrame messageFrame) { + if (input.size() != SECP256R1_INPUT_LENGTH) { + LOG.debug( + "Invalid input length for P256VERIFY precompile: expected {} bytes but got {}", + SECP256R1_INPUT_LENGTH, + input.size()); + return PrecompileContractResult.success(INVALID); + } + PrecompileInputResultTuple res = null; + Integer cacheKey = null; + if (enableResultCaching) { + cacheKey = getCacheKey(input); + res = p256VerifyCache.getIfPresent(cacheKey); + + if (res != null) { + if (res.cachedInput().equals(input)) { + cacheEventConsumer.accept(new CacheEvent(PRECOMPILE_NAME, CacheMetric.HIT)); + return res.cachedResult(); + } else { + LOG.debug( + "false positive p256verify {}, cache key {}, cached input: {}, input: {}", + input.getClass().getSimpleName(), + cacheKey, + res.cachedInput().toHexString(), + input.toHexString()); + cacheEventConsumer.accept(new CacheEvent(PRECOMPILE_NAME, CacheMetric.FALSE_POSITIVE)); + } + } else { + cacheEventConsumer.accept(new CacheEvent(PRECOMPILE_NAME, CacheMetric.MISS)); + } + } + + try { + if (useNativeBoringSSL) { + res = computeNative(input); + } else { + res = computeDefault(input); + } + + if (enableResultCaching) { + p256VerifyCache.put(cacheKey, res); + } + return res.cachedResult(); + + } catch (Exception e) { + LOG.warn("P256VERIFY verification failed: {}", e.getMessage()); + // AERE D-434 (fbfb5f9ecd): removed System.err print on the P256VERIFY failure path; a cheap invalid signature could otherwise spam the process console, bypassing log4j. LOG.warn above records it. + return PrecompileContractResult.success(INVALID); + } + } + + @NotNull + private static PrecompileInputResultTuple computeNative(final Bytes input) { + PrecompileInputResultTuple res; + BoringSSLPrecompiles.P256VerifyResult result = + BoringSSLPrecompiles.p256Verify(input.toArrayUnsafe(), input.size()); + + if (result.status != 0) { + LOG.atTrace().setMessage("Verify failed: {}").addArgument(result.error).log(); + } + res = + new PrecompileInputResultTuple( + enableResultCaching ? input.copy() : input, + PrecompileContractResult.success(result.status == 0 ? VALID : INVALID)); + return res; + } + + // This may still use native SECP256R1 depending on how SignatureAlgorithm is configured + @NotNull + PrecompileInputResultTuple computeDefault(final Bytes input) { + final Bytes messageHash = input.slice(0, 32); + final Bytes rBytes = input.slice(32, 32); + final Bytes sBytes = input.slice(64, 32); + final Bytes pubKeyBytes = input.slice(96, 64); + final BigInteger qx = pubKeyBytes.slice(0, 32).toUnsignedBigInteger(); + final BigInteger qy = pubKeyBytes.slice(32, 32).toUnsignedBigInteger(); + + // Convert r and s to BigIntegers (unsigned) + final BigInteger r = rBytes.toUnsignedBigInteger(); + final BigInteger s = sBytes.toUnsignedBigInteger(); + + // Check r, s in (0, n) + if (r.signum() <= 0 || r.compareTo(N) >= 0 || s.signum() <= 0 || s.compareTo(N) >= 0) { + LOG.trace("Invalid r or s: must satisfy 0 < r,s < n"); + return new PrecompileInputResultTuple( + enableResultCaching ? input.copy() : input, PrecompileContractResult.success(INVALID)); + } + + // Check qx, qy in [0, p) + if (qx.signum() < 0 || qx.compareTo(P) >= 0 || qy.signum() < 0 || qy.compareTo(P) >= 0) { + LOG.trace("Invalid qx or qy: must satisfy 0 <= qx,qy < p"); + return new PrecompileInputResultTuple( + enableResultCaching ? input.copy() : input, PrecompileContractResult.success(INVALID)); + } + + // Check point not at infinity (qx, qy ≠ 0,0), and non-trivial infinity encoding + if ((qx.signum() == 0 && qy.signum() == 0)) { + LOG.trace("Invalid public key: point at infinity"); + return new PrecompileInputResultTuple( + enableResultCaching ? input.copy() : input, PrecompileContractResult.success(INVALID)); + } + try { + final org.bouncycastle.math.ec.ECPoint ecPoint = R1_PARAMS.getCurve().createPoint(qx, qy); + signatureAlgorithm.getCurve().validatePublicPoint(ecPoint); + } catch (IllegalArgumentException e) { + LOG.trace("Public key not on curve: {}", e.getMessage()); + return new PrecompileInputResultTuple( + enableResultCaching ? input.copy() : input, PrecompileContractResult.success(INVALID)); + } + // Create the signature; recID is not used in verification - use 0 + final SECPSignature signature = signatureAlgorithm.createSignature(r, s, (byte) 0); + final SECPPublicKey publicKey = signatureAlgorithm.createPublicKey(pubKeyBytes); + + final boolean isValid = signatureAlgorithm.verifyMalleable(messageHash, signature, publicKey); + + return new PrecompileInputResultTuple( + enableResultCaching ? input.copy() : input, + PrecompileContractResult.success(isValid ? VALID : INVALID)); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/SHAKE256PrecompiledContract.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/SHAKE256PrecompiledContract.java new file mode 100644 index 0000000..b108b7e --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/SHAKE256PrecompiledContract.java @@ -0,0 +1,96 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.evm.precompile; + +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.bouncycastle.crypto.digests.SHAKEDigest; + +/** + * AERE PQC precompile: SHAKE256 extendable-output function (FIPS 202). + * + *

Input layout: outLen (32 bytes, big-endian, capped at MAX_OUTPUT) || data (arbitrary length). + * Output: exactly {@code outLen} bytes of SHAKE256(data). + * + *

SHAKE256 is the hashing bottleneck inside Falcon, ML-DSA and SLH-DSA; exposing it natively lets + * on-chain PQC flows offload the hot path to Bouncy Castle rather than hand-rolled Solidity. The + * provider we ship, bcprov-jdk18on, is not a FIPS validated module and we know of no published + * third-party audit of it; what is citable is byte-for-byte agreement with the NIST KAT and ACVP + * vectors, in both directions. See AERE-NIST-VALIDATION-STATUS.md section 3.4. + */ +public class SHAKE256PrecompiledContract extends AbstractPrecompiledContract { + + /** Upper bound on requested output length to keep gas/allocation bounded. */ + static final int MAX_OUTPUT = 1 << 16; // 65536 bytes + + private static final int BASE_GAS = 60; + private static final int GAS_PER_WORD = 12; + + /** + * Instantiates a new SHAKE256 precompiled contract. + * + * @param gasCalculator the gas calculator + */ + SHAKE256PrecompiledContract(final GasCalculator gasCalculator) { + super("AereSHAKE256", gasCalculator); + } + + private static int outputLength(final Bytes input) { + if (input.size() < 32) { + return 0; + } + // Big-endian 32-byte length; only the low 4 bytes are honoured, then capped. + long v = input.slice(28, 4).toLong() & 0xFFFFFFFFL; + // If any of the high 28 bytes are non-zero the value is enormous; cap regardless. + if (!input.slice(0, 28).isZero()) { + return MAX_OUTPUT; + } + if (v > MAX_OUTPUT) { + return MAX_OUTPUT; + } + return (int) v; + } + + @Override + public long gasRequirement(final Bytes input) { + int outLen = outputLength(input); + int dataLen = input.size() < 32 ? 0 : input.size() - 32; + long words = ((long) dataLen + 31) / 32 + ((long) outLen + 31) / 32; + return BASE_GAS + GAS_PER_WORD * words; + } + + @NotNull + @Override + public PrecompileContractResult computePrecompile( + final Bytes input, @NotNull final MessageFrame messageFrame) { + if (input.size() < 32) { + return PrecompileContractResult.success(Bytes.EMPTY); + } + final int outLen = outputLength(input); + final byte[] data = input.slice(32).toArrayUnsafe(); + final SHAKEDigest digest = new SHAKEDigest(256); + if (data.length > 0) { + digest.update(data, 0, data.length); + } + final byte[] out = new byte[outLen]; + if (outLen > 0) { + digest.doFinal(out, 0, outLen); + } + return PrecompileContractResult.success(Bytes.wrap(out)); + } +} diff --git a/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/SLHDSA128sPrecompiledContract.java b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/SLHDSA128sPrecompiledContract.java new file mode 100644 index 0000000..2666602 --- /dev/null +++ b/anchor/evm/src/main/java/org/hyperledger/besu/evm/precompile/SLHDSA128sPrecompiledContract.java @@ -0,0 +1,84 @@ +/* + * Copyright contributors to the AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.evm.precompile; + +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; + +import jakarta.validation.constraints.NotNull; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.pqc.crypto.slhdsa.SLHDSAParameters; +import org.bouncycastle.pqc.crypto.slhdsa.SLHDSAPublicKeyParameters; +import org.bouncycastle.pqc.crypto.slhdsa.SLHDSASigner; + +/** + * AERE PQC precompile: SLH-DSA-SHA2-128s (SPHINCS+, FIPS 205) signature verification via the + * INTERNAL interface (slh_verify_internal, Algorithm 20 — message hashed directly, no prefix). + * + *

Input: {@code pk(32) || sig(7856) || message(rest)}. Output: 32-byte word, {@code ...01} valid + * else {@code ...00}. The internal verifier is reached by subclassing Bouncy Castle's + * {@link SLHDSASigner} and calling its {@code protected internalVerifySignature}. + */ +public class SLHDSA128sPrecompiledContract extends AbstractPrecompiledContract { + + static final int PK_LEN = 32; + static final int SIG_LEN = 7856; + private static final long GAS = 350_000L; + + /** Subclass exposing Bouncy Castle's protected internal (slh_verify_internal) verifier. */ + private static final class InternalVerifier extends SLHDSASigner { + boolean verifyInternal(final byte[] message, final byte[] signature) { + return internalVerifySignature(message, signature); + } + } + + /** + * Instantiates a new SLH-DSA-128s precompiled contract. + * + * @param gasCalculator the gas calculator + */ + SLHDSA128sPrecompiledContract(final GasCalculator gasCalculator) { + super("AereSLHDSA128s", gasCalculator); + } + + @Override + public long gasRequirement(final Bytes input) { + return GAS; + } + + @NotNull + @Override + public PrecompileContractResult computePrecompile( + final Bytes input, @NotNull final MessageFrame messageFrame) { + boolean valid = false; + if (input.size() >= PK_LEN + SIG_LEN) { + try { + final byte[] pk = input.slice(0, PK_LEN).toArrayUnsafe(); + final byte[] sig = input.slice(PK_LEN, SIG_LEN).toArrayUnsafe(); + final byte[] message = input.slice(PK_LEN + SIG_LEN).toArrayUnsafe(); + final SLHDSAPublicKeyParameters pub = + new SLHDSAPublicKeyParameters(SLHDSAParameters.sha2_128s, pk); + final InternalVerifier verifier = new InternalVerifier(); + verifier.init(false, pub); + valid = verifier.verifyInternal(message, sig); + } catch (final Throwable t) { + valid = false; + } + } + return PrecompileContractResult.success( + valid ? Bytes32.leftPad(Bytes.of((byte) 1)) : Bytes32.ZERO); + } +}