From 0fe71db4909af5dc9098359165fe6d6f713090bd Mon Sep 17 00:00:00 2001 From: circle-github-action-bot Date: Wed, 22 Apr 2026 12:28:24 +0000 Subject: [PATCH] chore: sync from arc-node - b49d2836bffb1fbc0e176cd6088d852d461c4a04 chore: sync from main@b6afef6a16850fdb1b3174403fb4922cf00... by circle-github-action-bot <[internal]> - 46f078d083054d0d05e262bb0ee816b62ad323cd chore: sync from main@befb3d5970ae11236c727652dfadadf9844... by circle-github-action-bot <[internal]> - 0bb166c8c321ed587bcd80e72a4f8851a3239c74 chore: sync from main@4fd7ba1865e74b1042292bde0942c24d980... by circle-github-action-bot <[internal]> - 57d25b56e90a5baf5ed2b741d104aea95ce2e98a chore: sync from main@11a118226783f0d2aa44b9ac4b45bc5f52c... by circle-github-action-bot <[internal]> - c4c7733e9d74720ccf10fad2fe1c64e5ec9ebdd6 chore: sync from main@11a8e3a25166b3534c344c00d4b07899be3... by circle-github-action-bot <[internal]> - 69d5e7008c70a879156532c6055eb6018547feaf chore: sync from main@50417d277768d584f7994afb93d05b09b98... by circle-github-action-bot <[internal]> - 591b7c7e2cbfa9ebee819fc2372a17abcf6763b9 chore: sync from main@bc29861d48aa768a90d959dc70c795d7449... by circle-github-action-bot <[internal]> - 9b4ec55e6c06b73150f6263bd821473d725007cc chore: sync from main@4409339661ed58f301248e81352aa1d579d... by circle-github-action-bot <[internal]> - 229ae29a5c7f9ec663bee4fb820dddcc0b7627b2 chore: sync from main@2a19fe7385a429c94b13fd6c02e167e12e0... by circle-github-action-bot <[internal]> - ccd874654903963358ce8127b1f07eea2eda2a80 chore: sync from main@ecac30fce4934294b587011fa6ad6bdc014... by circle-github-action-bot <[internal]> - 1b0a9cd598a56ab66035722a23f5f8aeaff902f0 chore: sync from main@4393dc14973a7981852f13a0f9a3ac2d318... by circle-github-action-bot <[internal]> - 1a6c0875b7a1dc8a4f061afe381e5f0057833ef6 chore: sync from main@81a5e45c21fab22037119846b0b312ee79f... by circle-github-action-bot <[internal]> - af0edf12fd52d4cd9a32a3b76fc15223e8b60b48 chore: sync from main@2b21154e52fae300e1ab0742949bc2aae88... by circle-github-action-bot <[internal]> - b1129404e39ef7af97d971902e8f56f219e35d19 chore: sync from main@c04b2457af8972b3fea565b7552affa02a7... by circle-github-action-bot <[internal]> - 117f0c839963b4bd775a2f23da80d207502c8468 chore: sync from main@87570120a529bbb3bb1a2e24ee21039fcb4... by circle-github-action-bot <[internal]> - 865daccd6d7f5e286718a7c166afe50b1c023117 chore: sync from main@d72493dd7a5e6c000649a2ece7b8c7b8991... by circle-github-action-bot <[internal]> - 90a44d1c4249bde2f6e664ab5e8320ceb34f3628 chore: sync from main@637df919468dcb4013805c4a7925de2f80b... by circle-github-action-bot <[internal]> - 034a1ce89af83facb85f29710f0fb078e79099df chore: sync from main@e6ccd9029877bb939e3f7a7066bd558473b... by circle-github-action-bot <[internal]> - bc231679022f6c14a5b8e676e51553d39063a46d chore: sync from main@7da9d6476385ecc70f64a033350b696c05f... by circle-github-action-bot <[internal]> - c9d44aad7708126dbb357084285ea67ee704a491 chore: sync from main@75725fb4652a28273ea1e3f2a7895d07bdb... by circle-github-action-bot <[internal]> - 2858d92b4d4ab0ec403b5201038211d06fa8cb54 chore: sync from main@9e452fb5d47c711de29c4d83037546fa299... by circle-github-action-bot <[internal]> - 9acf38af34fb6d40eeec91b3096125419bd1846d chore: sync from main@46dcd3335c64f5a25b127e549c49b040178... by circle-github-action-bot <[internal]> - c9b0f18eb0a622f4516f3386c7f3c5ec224d9da1 chore: sync from main@5b87cef9a731800fe9c41b419baa8679ce6... by circle-github-action-bot <[internal]> - fd564d7ddc9d8bc223c8af23a74d0da6445d3e05 chore: sync from main@98e854bde5e4d1b8bb08eed006763d633d2... by circle-github-action-bot <[internal]> - 904fd879f08432010fef6184e982105cc666eaf7 chore: sync from main@3a11a967395eee5eaaecdde8379fbaf7c4f... by circle-github-action-bot <[internal]> - ca79402de70abbd905c75b625b4c0fc8636fc28b chore: sync from main@1b3a6c365e4cbba9690056b11c5364d8a45... by circle-github-action-bot <[internal]> - ea872c09101e4b108563e9a164b1b08d2b0822c4 chore: sync from main@bf05dba1d8a8b84f2295711d91df7c5bbb0... by circle-github-action-bot <[internal]> - f6a572f29ae38a03a9a8888d469977c2559911c0 chore: sync from main@cfac75da87f434abb953f4fc9d59fc41ff5... by circle-github-action-bot <[internal]> - 6cc13b695710992b8fc70d9dcad9dccd1057273b chore: sync from main@ecf511b97786a34fb095f7646b708136f8a... by circle-github-action-bot <[internal]> - 6d971886a5aa9d1158ef0b28d7a9687378ed09ba chore: sync from main@98583140afd2e5a13f4217b00704d041b28... by circle-github-action-bot <[internal]> - 31cc7654c6dcd06d2f75c6188f5541d26afae37a chore: sync from main@58f6983f90ac8375c08895c08cc6d4b2aed... by circle-github-action-bot <[internal]> - c898e3708d20f7005f9b3fdbfe481d4fa7173ac1 chore: sync from main@34533e716488b82c4ed4fa5e83368607c8d... by circle-github-action-bot <[internal]> - 6a5ca8c6776c77d56f79a32d459089f71ffd1daf chore: sync from main@8a66cfa4cc18b4dcf17d16746d1a5cf1eea... by circle-github-action-bot <[internal]> - 75ec9895925972a23c58123c8dc0185745bf5ee2 chore: sync from master@2c81c103436c9687291194f5228c6a10f... by circle-github-action-bot <[internal]> - d3a73437380446cfe63d4839ffad9f22b72f37ef chore: sync from main@41ca7fc3b80bfa2b481da75fb313f6fa2bf... by circle-github-action-bot <[internal]> - 82e53174ade8d9f2986272edaa7bf411d8500fe3 chore: sync from main@f29a838032273676a6ea194c0360de5b438... by circle-github-action-bot <[internal]> - b75ec864aa837907b93dab0de4f5e941f91355ff chore: sync from main@60429a10d48c552425e50de39878b24b841... by circle-github-action-bot <[internal]> - 1d8b3dfecb4ae043ad4df542d146ebb2efe702e7 chore: sync from main@cd874feeb782990b397fc5d0dce3792c2c5... by circle-github-action-bot <[internal]> - 6703f250d521a8017459ae6f6db9aca5cb1888e0 chore: sync from main@2a66ec4604d0c0769720259cacabb6ac2cd... by circle-github-action-bot <[internal]> - bc9bae2fa7c6921a0bec7d63a2643219abfd34ee chore: sync from main@cc4738ac5940143065cd96643ac6a37b581... by circle-github-action-bot <[internal]> - 1816d53d235d2fa124fef4b76ce62149f48a9c51 chore: sync from main@78b3ff687fd9cffc7b930c93ba23bcc5830... by circle-github-action-bot <[internal]> - 9b51a009ccf4e18c1ddae78c6bdfc989ccef4ba6 chore: sync from main@457c5b29e510d8d4a2bd119c83ae1b6bb87... by circle-github-action-bot <[internal]> - 3f6cd5947a87659a558c1f8ac01a604238a1f8aa chore: sync from main@65df33917e65e211610a0db7deb8064eb1c... by circle-github-action-bot <[internal]> - facc06a4d225b9b85f8bb35fab9a94683ba9197e chore: sync from main@f621f21c4656064a0f77b5247d1bb00ca31... by circle-github-action-bot <[internal]> - f4f3e109d61b7489c98ffd2d551b1ada96ac67c5 chore: sync from main@3a21786b5985c201f861a68c7b362f5fd8d... by circle-github-action-bot <[internal]> - 81da1a6740b4e2476035406eb3c4685c7b606cfc chore: sync from master@44b59535d4c6ae8ad107fbbd05c192c9a... by circle-github-action-bot <[internal]> - b474b00f325794c7b3572391f7f78d3e47ff34d1 chore: sync from main@08cd975bb34338bf943eff1e3561a3a6814... by circle-github-action-bot <[internal]> - 333368bed853c9638f8ac4a9617034708733eee3 fix: add frontend.env to deployments (#1639) by circle-github-action-bot <[internal]> - c4ce096cc2ac9b7e4cc77c2d69de2a6e39d5ba7d chore: sync from master@13b953786f0b37e5dc660b0894c0415eb... by circle-github-action-bot <[internal]> - 3b47ca138b3a50f3705c3e30b0433e8c97ca5d39 fix(NoStory): add missing blocksount env file (#1634) by circle-github-action-bot <[internal]> - a4f9abf8e67ac70c4a955763e612616a73e28056 chore: sync from master@470d239ef9df1b55e8881182b314d8fd6... by circle-github-action-bot <[internal]> - b53911052ba786a539aa33fd22f46d7b7ade5d9f chore: sync from master@0c74523fbd119e5215c59e1b6a56c5d45... by circle-github-action-bot <[internal]> - 7e92113b4eaa0d173b407e06691b101677109bd2 ci: sync arc-node CI target main commit (#1628) by circle-github-action-bot <[internal]> - 379d5668e6de35b44a67a1ad99f127eedc53d560 chore: sync from master@a204a066fa3cfee4376daad374f0639a5... by circle-github-action-bot <[internal]> - 35bd0ad25db2653bc6d4b029b71f67017b2d7d6d chore: sync from master@745ed73f46261352159ad152d789b6465... by circle-github-action-bot <[internal]> - 887b08aad23a28ff83289bb35230ca0b05075ee6 chore: sync from master@3453512d8739cafeb7d9187b9cd479d28... by circle-github-action-bot <[internal]> - 6ef6b0927f807f70de6f6131789c6b14f9f287ac chore: sync from master@a21f05a83919bfe83a1273051817d80f5... by circle-github-action-bot <[internal]> - 7b0b8215870ce59f852c5cae7b4dba12a0b37c75 chore: sync from master@68f79671b65d071dbfe7e55b6eacb9b8f... by circle-github-action-bot <[internal]> - 74a80c0c271c9e0e257a041ce3c9764c47ac706d chore: sync from master@2b7bd5e6c8f9f2cb9609e6b3f4e657937... by circle-github-action-bot <[internal]> - 8ef157b12f57beba02ff4d89bb9cebfe158538df chore: sync from master@34892f11b767259b38c87d240e02791a4... by circle-github-action-bot <[internal]> - d7ccb1ba07fc7a0467ceb82666e8ae41f04e32f3 chore: sync from master@3467508bf0279c277e668cf844d8edc86... by circle-github-action-bot <[internal]> - b2ccb4dfb7481baa85550c831d052cf3a82ac8e9 chore: sync from master@375b2837007aeaf599b4d2cc43f6d912a... by circle-github-action-bot <[internal]> - c4b21d8fb45da4e6fd98a6a44dae3a042e88918f chore: sync from master@b62158c9f0204b8a604268b3e420f9ee5... by circle-github-action-bot <[internal]> - a712c669351b0e65dc99daaf3a3c7c0c86d1409a chore: sync from master@b3100bd066aff8ea1ee720f07a98ba4b9... by circle-github-action-bot <[internal]> - 7114a8afed887d0c25522c0e452766a3ad4bb6f8 chore: sync from master@5a2d82d2572489bec0fcdeea8a2510160... by circle-github-action-bot <[internal]> - 338fc862c967b440378070f81ecc33c1606fea0c chore: sync from master@b168d6857d781b344788b5f60b00ed457... by circle-github-action-bot <[internal]> - d32f0c1740f07d86a7e27d9bcf29a794533d4280 chore: sync from master@69fec733d8a9a42d9df7ba81ae700fa33... by circle-github-action-bot <[internal]> - e6c6255af2d8df6e3e18fa85b0f66aeafed03bc2 chore: sync from master@7eab078b4e1009c88ca41b3c3f0f77f2b... by circle-github-action-bot <[internal]> - dcc4026c541b60ab294a10e66e9a2a04c29701a9 chore: sync from master@c4b3962a39bbcbd25b633a9d8f7c688ef... by circle-github-action-bot <[internal]> - 6771c878fc0d55d6ad70a87c9eb5e1a94b8fbdb3 chore: sync from master@549b2047c1337338cec96ca7660a3132c... by circle-github-action-bot <[internal]> - c424137792989883638a37bd1de75cca1b8a0796 chore: sync from master@3a30370b64709ccaef6642f84e8563719... by circle-github-action-bot <[internal]> - b59d239b95661eca15fd1003b801ca1e438d9a66 chore: sync from master@eb4719e3ec36a23cbed865492ca7eb7e0... by circle-github-action-bot <[internal]> - 8f06cff704d5cf325f57b5c978ab1b43e3a2a27e fix(NoStory): bump trivy-action to v0.35.0 (compromised v... by circle-github-action-bot <[internal]> - 099124672c571777333a004741ab026bb076b5b1 chore(NoStory): update cloudsmith namespace to circle/arc... by circle-github-action-bot <[internal]> - 8a3e359dd649f83c1363c77cdecd4f3f2b7970dc chore: sync from master@6f811d880e7ca43f6a336738f202b7123... by circle-github-action-bot <[internal]> - af4ca214db5409e5e49e9fbb7045216b17ca86ed chore: sync from master@f9009ec3d5a56472bb63b779e1365560b... by circle-github-action-bot <[internal]> - 0e16555bd9e98358bb46d7c11b8f245e2bc4bd41 chore: sync from master@e51a19d1c8f692bf492c6f71e5205d59e... by circle-github-action-bot <[internal]> - fb43f1829a2355cd4b0492ada5df9e544c8595a6 chore: sync from master@b8011ade4b7961ede147c7cbcb7301f2e... by circle-github-action-bot <[internal]> - a17edf77039578a13f68ffcf9b9e0af52975f408 chore: sync from master@46c4ed22aab81ee7fd2af7c326bf7086b... by circle-github-action-bot <[internal]> - 71f9ed7dc5e31305a6568cb3641ed12bb1533e31 chore: sync from master@67bbc592ef809f73cb11b32e6549cf8c4... by circle-github-action-bot <[internal]> - edd03bf2e830bedd036b903031444a7061b7df2b chore: sync from master@1ecf507bb272d874d421d719ee08e4898... by circle-github-action-bot <[internal]> - 99235d6b6e15feaeb03354b86c9d276768bf2ec8 chore: sync from master@9d9c04db48e26d6ae98f08a9d2cedea6a... by circle-github-action-bot <[internal]> - fdbc81a4fc12ffb061be59be675d876fe4449f9a chore: sync from master@43e89a7d8a4eb4c3c38690ee20118cd16... by circle-github-action-bot <[internal]> - 734adf4cbf23989a508940320b9834760ec37478 chore: sync from master@96f6ac140dc73fc6807909d16cdb66a85... by circle-github-action-bot <[internal]> - fec9b3508adbf1dff2220ca77fbe0d44b8436dda chore: sync from master@92c9ce5116e5a7ad42e04081c28bea743... by circle-github-action-bot <[internal]> - cb03ff808ab4a2e7ae1b651c67f7ec49551f70ce chore: sync from master@bf150f424e3d8459ddf8ef0f4c03848a4... by circle-github-action-bot <[internal]> - 79fb89822700a8a5d24d2616941edb4b953cd70a chore: sync from master@fef84266c6249254037d86ff35829ec0a... by circle-github-action-bot <[internal]> - 96282d3d36132c330c4f24e7febeb65a3a656df6 chore: remove internal-only CLAUDE.md and atlantis.yaml f... by circle-github-action-bot <[internal]> - c706c92646a30f59401333e47a61e3332fd0c2ed chore: sync from master@8fb688322fc845e6118bb00a91e1c9065... by circle-github-action-bot <[internal]> - 3b458113e59f6bbe2c9b348f79b42266402a4ea0 chore: sync from master@73e4c85d5e20ac29f3a30d58450660a51... by circle-github-action-bot <[internal]> - dea1e5231573b0bd4a7b4bc83e8183d62a984299 chore: sync from master@a0046d36c58a3df93da53dba35ff48771... by circle-github-action-bot <[internal]> - e8faf66a325f6f815d23d942e2dce3f6951d8191 chore: sync from master@51268f8a6432c247268cef2f88606a17f... by circle-github-action-bot <[internal]> - 7eb56e431e70e33b15eddd3574bdd65240f8fe8c chore: sync from master@c53b90af5d96c9e12cb51b747650985db... by circle-github-action-bot <[internal]> - a4361f4a1439359dd5b901d5443b81dbcd99384c chore: sync from master@9fc868ffd4e424bc34468a1dc7f7eeb3d... by circle-github-action-bot <[internal]> - 4823209bc88e6e965c7d64c76e0bb8097cab136f chore: sync from master@8f7a9c007ccee88d344bd83149e51e4cf... by circle-github-action-bot <[internal]> - 5a9100d74c235be7237e9962d202e24b5131b4b7 chore: sync from master@8de70cfbea02b3012b68cf6c8b10bd9ff... by circle-github-action-bot <[internal]> - 3a1af58a45c89ae3449cdca89cc085137fb8b70f chore: sync from master@651b0f71ab5afa18bf227e16c3bf43165... by circle-github-action-bot <[internal]> - 309c88baf1ab6b7a731144ed200efed2d750de81 chore: sync from master@545151f25f4494ce819abc49ca3ac76b5... by circle-github-action-bot <[internal]> - 404fe049aace2689f1d99d8a7ae095eff88cb65f chore: remove atlantis.yaml and CLAUDE.md from public-rep... by circle-github-action-bot <[internal]> - 31555ff956e620442468659800d721d30748e2c9 chore: sync from master@6057669f08857e87e67ca3b48f7fd90c0... by circle-github-action-bot <[internal]> (And 718 more changes) GitOrigin-RevId: b49d2836bffb1fbc0e176cd6088d852d461c4a04 --- .cargo/audit.toml | 5 + Cargo.lock | 163 +- Cargo.toml | 8 + Makefile | 7 +- README.md | 7 +- assets/devnet/config.json | 1 + assets/devnet/genesis.config.ts | 1 + assets/localdev/genesis.config.ts | 6 +- assets/localdev/genesis.json | 2 +- assets/testnet/config.json | 1 + assets/testnet/genesis.config.ts | 1 + crates/consensus-db/src/decoder.rs | 6 +- crates/consensus-db/src/encoder.rs | 21 +- crates/consensus-db/src/migrations.rs | 267 +- .../consensus-db/src/migrations/migrators.rs | 4 + crates/consensus-db/src/store.rs | 119 +- crates/consensus-db/src/versions.rs | 2 +- crates/engine-bench/Cargo.toml | 38 + crates/engine-bench/README.md | 201 ++ crates/engine-bench/src/bench/context.rs | 250 ++ crates/engine-bench/src/bench/fixture.rs | 542 ++++ crates/engine-bench/src/bench/helpers.rs | 27 + crates/engine-bench/src/bench/mod.rs | 31 + .../engine-bench/src/bench/new_payload_fcu.rs | 212 ++ crates/engine-bench/src/bench/output.rs | 313 +++ .../engine-bench/src/bench/prepare_payload.rs | 164 ++ crates/engine-bench/src/cli.rs | 94 + crates/engine-bench/src/lib.rs | 27 + crates/engine-bench/src/main.rs | 45 + crates/eth-engine/src/constants.rs | 7 + crates/eth-engine/src/engine.rs | 162 +- crates/eth-engine/src/ipc/engine_ipc.rs | 17 +- crates/eth-engine/src/ipc/ethereum_ipc.rs | 8 + crates/eth-engine/src/ipc/ipc_builder.rs | 13 +- crates/eth-engine/src/persistence_meter.rs | 33 +- crates/eth-engine/src/rpc/ethereum_rpc.rs | 72 +- crates/eth-engine/tests/integration.rs | 6 +- crates/evm-node/Cargo.toml | 1 - crates/evm-node/src/lib.rs | 1 + crates/evm-node/src/node.rs | 4 +- crates/evm-node/src/payload.rs | 138 ++ crates/evm-specs-tests/Cargo.toml | 39 + crates/evm-specs-tests/README.md | 179 ++ crates/evm-specs-tests/src/adapter.rs | 305 +++ crates/evm-specs-tests/src/cmd/mod.rs | 17 + crates/evm-specs-tests/src/cmd/statetest.rs | 58 + crates/evm-specs-tests/src/error.rs | 147 ++ crates/evm-specs-tests/src/exception_match.rs | 457 ++++ .../evm-specs-tests/src/fixture_sanitizer.rs | 154 ++ crates/evm-specs-tests/src/lib.rs | 24 + crates/evm-specs-tests/src/main.rs | 137 ++ crates/evm-specs-tests/src/result.rs | 328 +++ crates/evm-specs-tests/src/roots.rs | 295 +++ crates/evm-specs-tests/src/runner.rs | 1854 ++++++++++++++ crates/evm/src/evm.rs | 2191 +++++++++++++---- crates/evm/src/executor.rs | 43 +- crates/evm/src/frame_result.rs | 14 +- crates/evm/src/handler.rs | 8 +- crates/evm/src/lib.rs | 5 + crates/evm/src/subcall.rs | 2 +- crates/execution-config/src/follow.rs | 24 +- crates/execution-config/src/gas_fee.rs | 44 +- crates/execution-e2e/src/lib.rs | 6 +- crates/execution-e2e/tests/base_fee.rs | 2 + .../tests/beneficiary_mismatch.rs | 7 +- crates/execution-e2e/tests/denylist.rs | 2 + .../tests/native_transfer_balance.rs | 2 + crates/execution-payload/src/payload.rs | 33 +- crates/execution-txpool/src/error.rs | 9 +- crates/execution-txpool/src/validator.rs | 6 +- crates/execution-validation/src/consensus.rs | 2 +- crates/malachite-app/README.md | 12 +- crates/malachite-app/src/app.rs | 101 +- crates/malachite-app/src/config.rs | 176 +- .../src/handlers/consensus_ready.rs | 67 +- .../src/handlers/get_decided_values.rs | 27 +- .../src/handlers/get_history_min_height.rs | 124 +- .../src/handlers/received_proposal_part.rs | 11 +- .../src/handlers/started_round.rs | 38 +- crates/malachite-app/src/lib.rs | 7 +- crates/malachite-app/src/main.rs | 26 +- crates/malachite-app/src/metrics/app.rs | 42 +- crates/malachite-app/src/metrics/process.rs | 6 +- crates/malachite-app/src/node.rs | 167 +- crates/malachite-app/src/payload.rs | 12 +- crates/malachite-app/src/proposal_parts.rs | 277 ++- crates/malachite-app/src/request.rs | 26 +- crates/malachite-app/src/rpc/handlers.rs | 19 + crates/malachite-app/src/rpc/routes.rs | 40 +- crates/malachite-app/src/rpc/types.rs | 11 + crates/malachite-app/src/rpc_sync/network.rs | 8 +- .../src/rpc_sync/ws_subscription.rs | 10 +- crates/malachite-app/src/spec.rs | 4 +- crates/malachite-app/src/state.rs | 20 +- crates/malachite-app/src/streaming.rs | 524 +++- crates/malachite-app/src/utils/sync_state.rs | 2 +- crates/malachite-app/tests/cli_db_migrate.rs | 3 +- crates/malachite-app/tests/rpc_integration.rs | 11 +- crates/malachite-cli/src/args.rs | 24 +- crates/malachite-cli/src/cmd/start.rs | 107 +- crates/malachite-cli/src/logging.rs | 2 +- crates/malachite-cli/src/new.rs | 4 + crates/mesh-analysis/src/lib.rs | 7 +- crates/mesh-analysis/src/main.rs | 8 + crates/mesh-analysis/src/parse.rs | 26 +- crates/mesh-analysis/src/report.rs | 68 + crates/mesh-analysis/src/types.rs | 27 + crates/node/src/main.rs | 2 +- crates/node/tests/common.rs | 7 +- crates/node/tests/native_transfer.rs | 2 + crates/precompiles/src/call_from.rs | 6 + crates/precompiles/src/helpers.rs | 26 +- crates/precompiles/src/pq.rs | 2 + crates/precompiles/src/subcall.rs | 7 + crates/quake/README.md | 25 + crates/quake/scenarios/localdev.toml | 11 + crates/quake/src/infra/export.rs | 3 +- crates/quake/src/infra/mod.rs | 26 +- crates/quake/src/infra/remote.rs | 167 ++ crates/quake/src/infra/ssm.rs | 986 ++++++-- crates/quake/src/main.rs | 117 +- crates/quake/src/manifest.rs | 5 + crates/quake/src/manifest/raw.rs | 7 + crates/quake/src/mcp.rs | 42 +- crates/quake/src/setup.rs | 302 ++- crates/quake/src/shell.rs | 23 + crates/quake/src/testnet.rs | 163 +- crates/quake/src/tests/mesh.rs | 29 + crates/quake/src/tests/sanity.rs | 45 +- crates/quake/src/tests/snapshot.rs | 4 +- crates/quake/src/tests/sync.rs | 2 +- crates/quake/src/util.rs | 30 + crates/quake/terraform/cc-data.yaml | 129 + crates/remote-signer/Cargo.toml | 5 +- crates/remote-signer/src/client.rs | 8 +- crates/remote-signer/src/provider.rs | 258 +- crates/shared/src/chain_ids.rs | 2 +- crates/signer/Cargo.toml | 1 + crates/signer/src/local.rs | 6 +- crates/snapshots/src/download.rs | 5 +- crates/spammer/src/lib.rs | 6 +- crates/spammer/src/main.rs | 6 +- crates/test/checks/src/lib.rs | 13 +- crates/test/checks/src/perf.rs | 292 ++- crates/test/checks/src/types.rs | 12 +- crates/test/framework/src/lib.rs | 6 +- crates/types/src/address.rs | 26 +- crates/types/src/codec/mod.rs | 44 +- crates/types/src/codec/network.rs | 8 +- crates/types/src/codec/proto.rs | 247 ++ crates/types/src/height.rs | 4 +- crates/types/src/proposal_part.rs | 1 + crates/types/src/proposer.rs | 8 +- crates/types/src/rpc_sync.rs | 4 +- crates/types/src/validator_set.rs | 20 +- crates/types/tests/unit/main.rs | 6 +- crates/version/build.rs | 1 + deploy/helm/circle-chain-reth/Chart.yaml | 9 + deploy/helm/circle-chain-reth/values.yaml | 14 + deployments/Dockerfile.engine-bench | 81 + .../config-prometheus/prometheus.yml | 8 + docker-bake.hcl | 1 + docs/installation.md | 43 +- docs/running-an-arc-node.md | 26 +- scripts/genesis/genesis.ts | 8 +- scripts/localdev.mjs | 36 +- tests/helpers/networks/index.ts | 2 + tests/helpers/networks/localdev.ts | 5 + tests/localdev/NativeFiatToken.test.ts | 20 +- tests/localdev/ProtocolConfig.test.ts | 13 +- tests/localdev/genesis.test.ts | 15 +- tests/localdev/subcall.test.ts | 205 +- tests/simulation/ProtocolConfig.test.ts | 21 +- tests/simulation/native_transfer.test.ts | 30 +- 174 files changed, 13291 insertions(+), 1868 deletions(-) create mode 100644 crates/engine-bench/Cargo.toml create mode 100644 crates/engine-bench/README.md create mode 100644 crates/engine-bench/src/bench/context.rs create mode 100644 crates/engine-bench/src/bench/fixture.rs create mode 100644 crates/engine-bench/src/bench/helpers.rs create mode 100644 crates/engine-bench/src/bench/mod.rs create mode 100644 crates/engine-bench/src/bench/new_payload_fcu.rs create mode 100644 crates/engine-bench/src/bench/output.rs create mode 100644 crates/engine-bench/src/bench/prepare_payload.rs create mode 100644 crates/engine-bench/src/cli.rs create mode 100644 crates/engine-bench/src/lib.rs create mode 100644 crates/engine-bench/src/main.rs create mode 100644 crates/evm-node/src/payload.rs create mode 100644 crates/evm-specs-tests/Cargo.toml create mode 100644 crates/evm-specs-tests/README.md create mode 100644 crates/evm-specs-tests/src/adapter.rs create mode 100644 crates/evm-specs-tests/src/cmd/mod.rs create mode 100644 crates/evm-specs-tests/src/cmd/statetest.rs create mode 100644 crates/evm-specs-tests/src/error.rs create mode 100644 crates/evm-specs-tests/src/exception_match.rs create mode 100644 crates/evm-specs-tests/src/fixture_sanitizer.rs create mode 100644 crates/evm-specs-tests/src/lib.rs create mode 100644 crates/evm-specs-tests/src/main.rs create mode 100644 crates/evm-specs-tests/src/result.rs create mode 100644 crates/evm-specs-tests/src/roots.rs create mode 100644 crates/evm-specs-tests/src/runner.rs create mode 100644 deploy/helm/circle-chain-reth/Chart.yaml create mode 100644 deploy/helm/circle-chain-reth/values.yaml create mode 100644 deployments/Dockerfile.engine-bench diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 32636a0..f1c4c40 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -19,4 +19,9 @@ ignore = [ "RUSTSEC-2025-0134", # lru IterMut violates Stacked Borrows - used by discv5, reth-network "RUSTSEC-2026-0002", + + # rand 0.8.5 is unsound with a custom logger - no 0.8.x patch available; upgrading + # to 0.9+ requires a major-version API migration and transitive deps (yamux, libp2p) + # still require rand 0.8.x + "RUSTSEC-2026-0097", ] \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 84226e6..b8162db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -413,7 +413,7 @@ dependencies = [ "paste", "proptest", "proptest-derive", - "rand 0.9.2", + "rand 0.9.4", "rapidhash", "ruint", "rustc-hash", @@ -1094,6 +1094,31 @@ dependencies = [ "url", ] +[[package]] +name = "arc-engine-bench" +version = "0.0.1" +dependencies = [ + "alloy-genesis", + "alloy-primitives", + "alloy-rpc-types-engine", + "arc-eth-engine", + "arc-execution-config", + "arc-version", + "chrono", + "clap", + "color-eyre", + "csv", + "eyre", + "reqwest", + "reth-cli", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "arc-eth-engine" version = "0.0.1" @@ -1191,7 +1216,6 @@ dependencies = [ "jsonrpsee", "reqwest", "reth-chainspec", - "reth-engine-local", "reth-engine-primitives", "reth-ethereum", "reth-ethereum-engine-primitives", @@ -1223,6 +1247,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "arc-evm-specs-tests" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "alloy-trie", + "arc-evm", + "arc-execution-config", + "clap", + "hash-db", + "plain_hasher", + "reth-evm", + "revm", + "revm-context-interface", + "revm-database", + "revm-inspector", + "revm-primitives", + "revm-statetest-types", + "serde", + "serde_json", + "thiserror 2.0.18", + "triehash", +] + [[package]] name = "arc-execution-config" version = "0.0.1" @@ -1867,12 +1916,13 @@ dependencies = [ "arc-malachitebft-signing", "async-trait", "backon", - "ed25519-dalek", + "base64 0.22.1", "eyre", "hex", "prometheus-client", "prost 0.13.5", "protox", + "rand 0.8.5", "thiserror 2.0.18", "tokio", "tonic 0.12.3", @@ -1896,6 +1946,7 @@ dependencies = [ "arc-malachitebft-signing-ed25519", "arc-remote-signer", "async-trait", + "base64 0.22.1", "bytes", "hex", "rand 0.8.5", @@ -2926,7 +2977,7 @@ dependencies = [ "num_enum", "paste", "portable-atomic", - "rand 0.9.2", + "rand 0.9.4", "regress", "rustc-hash", "ryu-js", @@ -3828,6 +3879,27 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -4030,7 +4102,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -5273,7 +5345,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.2", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -5454,7 +5526,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.4", "ring", "serde", "socket2 0.5.10", @@ -5478,7 +5550,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "resolv-conf", "serde", "smallvec", @@ -5894,7 +5966,7 @@ dependencies = [ "hyper", "hyper-util", "log", - "rand 0.9.2", + "rand 0.9.4", "tokio", "url", "xmltree", @@ -6248,7 +6320,7 @@ dependencies = [ "jsonrpsee-types", "parking_lot", "pin-project", - "rand 0.9.2", + "rand 0.9.4", "rustc-hash", "serde", "serde_json", @@ -7282,7 +7354,7 @@ dependencies = [ "hashbrown 0.16.1", "metrics", "quanta", - "rand 0.9.2", + "rand 0.9.4", "rand_xoshiro", "sketches-ddsketch", ] @@ -8107,7 +8179,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "thiserror 2.0.18", ] @@ -8770,7 +8842,7 @@ dependencies = [ "bit-vec", "bitflags 2.11.0", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -9113,7 +9185,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -9195,9 +9267,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -9206,9 +9278,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", "getrandom 0.4.1", @@ -9284,7 +9356,7 @@ version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", "rustversion", ] @@ -9603,7 +9675,7 @@ dependencies = [ "metrics", "parking_lot", "pin-project", - "rand 0.9.2", + "rand 0.9.4", "rayon", "reth-chainspec", "reth-errors", @@ -9998,7 +10070,7 @@ dependencies = [ "futures", "itertools 0.14.0", "metrics", - "rand 0.9.2", + "rand 0.9.4", "reth-chainspec", "reth-ethereum-forks", "reth-metrics", @@ -10863,7 +10935,7 @@ dependencies = [ "parking_lot", "pin-project", "rand 0.8.5", - "rand 0.9.2", + "rand 0.9.4", "rayon", "reth-chainspec", "reth-consensus", @@ -11103,7 +11175,7 @@ dependencies = [ "futures", "humantime", "ipnet", - "rand 0.9.2", + "rand 0.9.4", "reth-chainspec", "reth-cli-util", "reth-config", @@ -11742,7 +11814,7 @@ dependencies = [ "jsonrpsee-core", "jsonrpsee-types", "metrics", - "rand 0.9.2", + "rand 0.9.4", "reqwest", "reth-chain-state", "reth-chainspec", @@ -11995,7 +12067,7 @@ dependencies = [ "alloy-genesis", "alloy-primitives", "rand 0.8.5", - "rand 0.9.2", + "rand 0.9.4", "reth-ethereum-primitives", "reth-primitives-traits", "secp256k1 0.30.0", @@ -12064,7 +12136,7 @@ dependencies = [ "pin-project", "proptest", "proptest-arbitrary-interop", - "rand 0.9.2", + "rand 0.9.4", "reth-chain-state", "reth-chainspec", "reth-eth-wire-types", @@ -12427,6 +12499,25 @@ dependencies = [ "serde", ] +[[package]] +name = "revm-statetest-types" +version = "14.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab40862cebf520cf8f67dcd6a81dc8207140e57b36a9f15712e0dc7974a036d" +dependencies = [ + "alloy-eip7928", + "k256", + "revm-bytecode", + "revm-context", + "revm-context-interface", + "revm-database", + "revm-primitives", + "revm-state", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -12501,7 +12592,7 @@ dependencies = [ "http-body-util", "pastey", "pin-project-lite", - "rand 0.10.0", + "rand 0.10.1", "rmcp-macros", "schemars 1.2.1", "serde", @@ -12642,7 +12733,7 @@ dependencies = [ "primitive-types", "proptest", "rand 0.8.5", - "rand 0.9.2", + "rand 0.9.4", "rlp", "ruint-macro", "serde_core", @@ -12814,9 +12905,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -13001,7 +13092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" dependencies = [ "bitcoin_hashes", - "rand 0.9.2", + "rand 0.9.4", "secp256k1-sys 0.11.0", ] @@ -13887,9 +13978,9 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "thin-vec" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144f754d318415ac792f9d69fc87abbbfc043ce2ef041c60f16ad828f638717d" +checksum = "259cdf8ed4e4aca6f1e9d011e10bd53f524a2d0637d7b28450f6c64ac298c4c6" [[package]] name = "thiserror" @@ -14663,7 +14754,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1", @@ -14682,7 +14773,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "sha1", "thiserror 2.0.18", "utf-8", @@ -15900,7 +15991,7 @@ dependencies = [ "nohash-hasher", "parking_lot", "pin-project", - "rand 0.9.2", + "rand 0.9.4", "static_assertions", "web-time", ] diff --git a/Cargo.toml b/Cargo.toml index 007b906..49a8777 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ repository = "https://github.com/circlefin/arc-node" unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage_nightly)'] } [workspace.lints.clippy] +arithmetic_side_effects = "deny" +cast_possible_truncation = "deny" collapsible_if = "allow" unwrap_used = "deny" @@ -47,6 +49,7 @@ alloy-signer-local = { version = "1.6.3", default-features = false, features = [ alloy-sol-macro = "1.5.6" alloy-sol-types = { version = "1.5.6", default-features = false } alloy-transport-ws = { version = "1.6.3", default-features = false } +alloy-trie = { version = "0.9.4", default-features = false } arbitrary = "1.3" @@ -99,6 +102,7 @@ config = { version = "0.14", features = ["toml"], default-features = # # See: https://github.com/eira-fransham/crunchy/issues/13 crunchy = "=0.2.2" +csv = "1.4" deranged = "0.5.5" directories = "5.0.1" @@ -139,6 +143,7 @@ oneline-eyre = "0.1" # reth parking_lot = "0.12" +plain_hasher = "0.2" # proc-macros proc-macro2 = "1.0" @@ -202,9 +207,11 @@ reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", tag = "v1 revm = { version = "34.0.0", default-features = false } revm-context-interface = { version = "14.0.0", default-features = false } revm-database = { version = "10.0.0", default-features = false } +revm-inspector = { version = "15.0.0", default-features = false } revm-inspectors = "0.34.2" revm-interpreter = { version = "32.0.0", default-features = false } revm-primitives = { version = "22.0.0", default-features = false } +revm-statetest-types = { version = "14.0.0", default-features = false } rmcp = { version = "0.16", features = ["server", "macros", "transport-io", "transport-streamable-http-server"] } rstest = "0.24.0" @@ -243,6 +250,7 @@ tower = "0.5" tracing = { version = "0.1.0", default-features = false } tracing-appender = "0.2" tracing-subscriber = { version = "0.3", default-features = false, features = ["ansi"] } +triehash = "0.8" url = { version = "2.3", default-features = false } uuid = { version = "1", default-features = false, features = ["v4"] } vergen-git2 = { version = "9.1.0", default-features = false } diff --git a/Makefile b/Makefile index 7adb3e7..e9942c4 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,7 @@ test-unit-hardhat: ## Run hardhat unit tests .PHONY: test-localdev test-localdev: ## Run hardhat localdev tests $(HARDHAT) test ./tests/localdev/*.test.ts --network localdev + $(MAKE) test-simulation .PHONY: test-simulation test-simulation: ## Run hardhat simulation tests @@ -192,12 +193,12 @@ smoke: genesis ## Run smoke tests (both reth and malachite) .PHONY: smoke-reth smoke-reth: genesis ## Run Reth smoke tests @echo "Running smoke tests on local reth(mock CL)..." + cargo build --release --bin arc-node-execution @bash -c '\ set -ex; \ trap "./scripts/localdev.mjs stop --network=localdev" EXIT; \ - ./scripts/localdev.mjs stop clean daemon --network=localdev $(LAUNCH_ARGS); \ + ./scripts/localdev.mjs stop clean daemon --network=localdev --bin=target/release/arc-node-execution $(LAUNCH_ARGS); \ $(MAKE) test-localdev; \ - $(MAKE) test-simulation; \ ' .PHONY: smoke-malachite @@ -218,7 +219,7 @@ smoke-quake: testnet .PHONY: testnet testnet: genesis build-docker ## Start testnet as defined in QUAKE_MANIFEST file @echo "Setting up and starting Quake testnet..." - $(QUAKE) -f $(QUAKE_MANIFEST) start + $(QUAKE) -f $(QUAKE_MANIFEST) start $(QUAKE_START_ARGS) .PHONY: testnet-test testnet-test: ## Run tests against running testnet diff --git a/README.md b/README.md index 84014da..b08ad1e 100644 --- a/README.md +++ b/README.md @@ -186,18 +186,17 @@ For more details, see our [Contributing Guide](CONTRIBUTING.md). - [Local Documentation](docs/) - Implementation guides and references ## Acknowledgements - arc-node is open-source software, licensed under Apache 2.0, built from a number of open source libraries, and inspired by others. We would like to highlight several of them in particular and credit the teams that develop and maintain them. [Malachite](https://github.com/circlefin/malachite) - Malachite, a flexible BFT consensus engine written in Rust, was originally developed at [Informal Systems](https://github.com/informalsystems) and [is now maintained by Circle](https://www.circle.com/blog/introducing-arc-an-open-layer-1-blockchain-purpose-built-for-stablecoin-finance) as part of Arc. We thank Informal Systems for originating and stewarding Malachite, and their continued contributions to the project. -[Reth / Paradigm](https://github.com/paradigmxyz/reth) - Reth is an EVM execution client that is used in Arc's execution layer via Reth SDK. We thank the Paradigm team for continuing to push the envelope with Reth and their continued emphasis on performance, extensibility, and customization, as well as their commitment to open source. Additionally, we're big fans of the Foundry toolchain as well! +[Reth / Paradigm](https://github.com/paradigmxyz/reth) - Reth is an EVM execution client that is used in Arc’s execution layer via Reth SDK. We thank the Paradigm team for continuing to push the envelope with Reth and their continued emphasis on performance, extensibility, and customization, as well as their commitment to open source. Additionally, we’re big fans of the Foundry toolchain as well! [libp2p](https://github.com/libp2p/rust-libp2p) - libp2p is used extensively through the arc-node consensus layer, and we thank the team for their development of it. [Tokio](https://github.com/tokio-rs/tokio) - Tokio is used extensively throughout the consensus and execution layers, and we are grateful to the team for their continued development and maintenance of it. -[Celo](https://celo.org/) - USDC is the native token on Arc and supports interacting with it through an ERC-20 interface; this "linked interface" design was first (as far as we know) pioneered on Celo, and we'd like to credit the team for devising it. +[Celo](https://celo.org/) - USDC is the native token on Arc and supports interacting with it through an ERC-20 interface; this “linked interface” design was first (as far as we know) pioneered on Celo, and we’d like to credit the team for devising it. [Alloy-rs](https://github.com/alloy-rs/alloy) - Alloy is used throughout the consensus and execution layers, and we are very thankful to the team for this excellent library. @@ -205,4 +204,4 @@ arc-node is open-source software, licensed under Apache 2.0, built from a number [Hardhat / Nomic Foundation](https://github.com/NomicFoundation/hardhat) - we thank the team for their continued development of the Hardhat toolchain. -[Viem](https://github.com/wevm/viem) - we thank the team for their continued development of Viem and other libraries. +[Viem](https://github.com/wevm/viem) - we thank the team for their continued development of Viem and other libraries. \ No newline at end of file diff --git a/assets/devnet/config.json b/assets/devnet/config.json index f4c91b4..1400a18 100644 --- a/assets/devnet/config.json +++ b/assets/devnet/config.json @@ -1,5 +1,6 @@ { "timestamp": "1758617247", + "coinbase": "0x65E0a200006D4FF91bD59F9694220dafc49dbBC1", "NativeFiatToken": { "proxy": { "admin": "0x2454fd9B923cD74d690bf500b4120E2BAEe3781a" diff --git a/assets/devnet/genesis.config.ts b/assets/devnet/genesis.config.ts index d17e2ea..d2cdef6 100644 --- a/assets/devnet/genesis.config.ts +++ b/assets/devnet/genesis.config.ts @@ -46,6 +46,7 @@ const build = async () => { const config: GenesisConfig = { timestamp: currentTimestamp(), + coinbase: '0x65E0a200006D4FF91bD59F9694220dafc49dbBC1', NativeFiatToken: { proxy: { admin: creator.nextAccount('NativeFiatToken.proxyAdmin', adminPrefund) }, diff --git a/assets/localdev/genesis.config.ts b/assets/localdev/genesis.config.ts index e84ec77..dc08927 100644 --- a/assets/localdev/genesis.config.ts +++ b/assets/localdev/genesis.config.ts @@ -16,7 +16,7 @@ import fs from 'fs' import { z } from 'zod' -import { parseEther, parseGwei, toHex } from 'viem' +import { parseEther, parseGwei, toHex, zeroAddress } from 'viem' import { privateKeyToAccount } from 'viem/accounts' import { createBuilderContext, buildGenesis, GenesisConfig, schemaGenesisConfig } from '../../scripts/genesis' import { bigintReplacer } from '../../scripts/genesis/types' @@ -73,6 +73,7 @@ const build = async (options: z.infer) => { const config: GenesisConfig = { timestamp: 1763620028n, + coinbase: proxyAdmin.address, hardforks: { zero3Block: 0, ...hardforks, @@ -103,7 +104,8 @@ const build = async (options: z.infer) => { owner: admin.address, controller: admin.address, pauser: admin.address, - beneficiary: proxyAdmin.address, + // Zero = unset; EL honors CL-provided --suggested-fee-recipient per validator. + beneficiary: zeroAddress, feeParams: { alpha: 20n, // 20% kRate: 200n, // 2% diff --git a/assets/localdev/genesis.json b/assets/localdev/genesis.json index 2f147a8..f705ab6 100644 --- a/assets/localdev/genesis.json +++ b/assets/localdev/genesis.json @@ -87,7 +87,7 @@ "0x668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385201": "0x0000000000000000000000000000000000000000000000000000000000000001", "0x668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385202": "0x000000000000000000000000000000000000000000000000000000e8d4a51000", "0x668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385203": "0x0000000000000000000000000000000000000000000000000000000001c9c380", - "0x668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385204": "0x000000000000000000000000a0ee7a142d267c1f36714e4a8f75612f20a79720", + "0x668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385204": "0x0000000000000000000000000000000000000000000000000000000000000000", "0x668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385205": "0x0000000000000000000000000000000001f403e801f403e801f403e801f40bb8" } }, diff --git a/assets/testnet/config.json b/assets/testnet/config.json index db0d315..c019427 100644 --- a/assets/testnet/config.json +++ b/assets/testnet/config.json @@ -1,5 +1,6 @@ { "timestamp": "1757515087", + "coinbase": "0xa693CC18Aa09d33dD388013B7A02E5Ff863b8760", "NativeFiatToken": { "proxy": { "admin": "0x49f78af090F1f98e7184B7f61f1F1a8a8064b40d" diff --git a/assets/testnet/genesis.config.ts b/assets/testnet/genesis.config.ts index 03faf97..468a2c1 100644 --- a/assets/testnet/genesis.config.ts +++ b/assets/testnet/genesis.config.ts @@ -46,6 +46,7 @@ const build = async () => { const config: GenesisConfig = { timestamp: currentTimestamp(), + coinbase: '0xa693CC18Aa09d33dD388013B7A02E5Ff863b8760', NativeFiatToken: { proxy: { admin: creator.nextAccount('FiatTokenCircleChain.proxyAdmin', adminPrefund) }, diff --git a/crates/consensus-db/src/decoder.rs b/crates/consensus-db/src/decoder.rs index 540ac78..5984b26 100644 --- a/crates/consensus-db/src/decoder.rs +++ b/crates/consensus-db/src/decoder.rs @@ -152,11 +152,13 @@ pub fn decode_proposal_monitor_data(bytes: &[u8]) -> Result 0 { + #[allow(clippy::arithmetic_side_effects)] Some(UNIX_EPOCH + Duration::from_millis(proto_data.receive_time_ms)) } else { None @@ -515,12 +517,14 @@ mod tests { successful: ProposalSuccessState, synced: bool, ) -> ProposalMonitor { + #[allow(clippy::arithmetic_side_effects)] let start_time = UNIX_EPOCH + Duration::from_secs(1000000); let proposer = Address::new([0x42; 20]); let mut monitor = ProposalMonitor::new(Height::new(height), proposer, start_time); if with_proposal { + #[allow(clippy::arithmetic_side_effects)] let receive_time = start_time + Duration::from_millis(150); let value_id = ValueId::new(B256::repeat_byte(0xAB)); monitor.proposal_receive_time = Some(receive_time); diff --git a/crates/consensus-db/src/encoder.rs b/crates/consensus-db/src/encoder.rs index a7c7845..20ac7c4 100644 --- a/crates/consensus-db/src/encoder.rs +++ b/crates/consensus-db/src/encoder.rs @@ -34,8 +34,12 @@ use crate::versions::{ ProposalPartsVersion, }; +// All encode_* functions prepend a 1-byte version tag to the serialized payload. +// The `1 + len` capacity calculation cannot overflow because any valid allocation +// already fits in usize and adding 1 byte stays within bounds. pub fn encode_execution_payload(payload: &ExecutionPayloadV3) -> Vec { let ssz_bytes = payload.as_ssz_bytes(); + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + ssz_bytes.len()); bytes.push(ExecutionPayloadVersion::V3 as u8); bytes.extend_from_slice(&ssz_bytes); @@ -46,6 +50,7 @@ pub fn encode_proposal_parts(parts: &ProposalParts) -> Result, malachite let proto = proto_funcs::encode_proposal_parts(parts)?; let proto_bytes = proto.encode_to_vec(); // version byte + encoded protobuf + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + proto_bytes.len()); bytes.push(ProposalPartsVersion::V1 as u8); bytes.extend_from_slice(&proto_bytes); @@ -59,6 +64,7 @@ pub fn encode_certificate( let proto = proto_funcs::encode_store_commit_certificate(certificate)?; let proto_bytes = proto.encode_to_vec(); // version byte + encoded protobuf + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + proto_bytes.len()); bytes.push(CommitCertificateVersion::V1 as u8); bytes.extend_from_slice(&proto_bytes); @@ -70,6 +76,7 @@ pub fn encode_block(block: &ConsensusBlock) -> Bytes { let data = block_as_ssz_data(block); let ssz_bytes = data.as_ssz_bytes(); // version byte + encoded SSZ + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + ssz_bytes.len()); bytes.push(ConsensusBlockVersion::V1 as u8); bytes.extend_from_slice(&ssz_bytes); @@ -80,20 +87,25 @@ pub fn encode_block(block: &ConsensusBlock) -> Bytes { pub fn encode_proposal_monitor_data( data: &ProposalMonitor, ) -> Result, malachitebft_proto::Error> { - // SystemTime is converted to milliseconds since UNIX_EPOCH + // SystemTime is converted to milliseconds since UNIX_EPOCH. + // u64 millis covers ~584 million years from epoch — truncation is unreachable. let start_time_ms = { let duration = data .start_time .duration_since(UNIX_EPOCH) .unwrap_or_default(); - duration.as_millis() as u64 + #[allow(clippy::cast_possible_truncation)] + let ms = duration.as_millis() as u64; + ms }; let receive_time_ms = data .proposal_receive_time .map(|t| { let duration = t.duration_since(UNIX_EPOCH).unwrap_or_default(); - duration.as_millis() as u64 + #[allow(clippy::cast_possible_truncation)] + let ms = duration.as_millis() as u64; + ms }) .unwrap_or(0); @@ -117,6 +129,7 @@ pub fn encode_proposal_monitor_data( let proto_bytes = proto_data.encode_to_vec(); // version byte + encoded protobuf + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + proto_bytes.len()); bytes.push(ProposalMonitorDataVersion::V1 as u8); bytes.extend_from_slice(&proto_bytes); @@ -168,6 +181,7 @@ pub fn encode_misbehavior_evidence( let proto_bytes = proto_evidence.encode_to_vec(); // version byte + encoded protobuf + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + proto_bytes.len()); bytes.push(MisbehaviorEvidenceVersion::V1 as u8); bytes.extend_from_slice(&proto_bytes); @@ -203,6 +217,7 @@ pub fn encode_invalid_payloads( let proto_bytes = proto_payloads.encode_to_vec(); // version byte + encoded protobuf + #[allow(clippy::arithmetic_side_effects)] let mut bytes = Vec::with_capacity(1 + proto_bytes.len()); bytes.push(InvalidPayloadsVersion::V1 as u8); bytes.extend_from_slice(&proto_bytes); diff --git a/crates/consensus-db/src/migrations.rs b/crates/consensus-db/src/migrations.rs index b582076..a813145 100644 --- a/crates/consensus-db/src/migrations.rs +++ b/crates/consensus-db/src/migrations.rs @@ -47,6 +47,8 @@ pub struct MigrationStats { } impl MigrationStats { + // Migration counters are bounded by total DB records — overflow is not reachable. + #[allow(clippy::arithmetic_side_effects)] fn merge(&mut self, other: MigrationStats) { self.tables_migrated += other.tables_migrated; self.records_scanned += other.records_scanned; @@ -184,10 +186,10 @@ impl MigrationCoordinator { for (schema_version, migrators) in migration_chain { info!("Migrating to schema version {}", schema_version); - stats.merge(self.migrate_certificates(migrators.certificate.as_ref())?); - stats.merge(self.migrate_decided_blocks(migrators.decided_block.as_ref())?); - stats.merge(self.migrate_undecided_blocks(migrators.undecided_block.as_ref())?); - stats.merge(self.migrate_pending_parts(migrators.pending_parts.as_ref())?); + stats.merge(self.migrate_certificates(migrators.certificate.as_ref(), false)?); + stats.merge(self.migrate_decided_blocks(migrators.decided_block.as_ref(), false)?); + stats.merge(self.migrate_undecided_blocks(migrators.undecided_block.as_ref(), false)?); + stats.merge(self.migrate_pending_parts(migrators.pending_parts.as_ref(), false)?); // Update schema version after successful migration self.set_schema_version(schema_version)?; @@ -199,6 +201,55 @@ impl MigrationCoordinator { Ok(stats) } + /// Scan what [`Self::migrate`] would apply without persisting: schema steps, per-table + /// migrator names, and record counts. Uses the same code path as migration (including + /// write transactions) but aborts each transaction instead of committing. + pub fn preview_migrate(&self) -> Result { + let start = Instant::now(); + let mut stats = MigrationStats::default(); + + let from_version = self.current_schema_version()?.ok_or_else(|| { + StoreError::Migration( + "database schema version is not set; cannot preview migration".to_owned(), + ) + })?; + let to_version = DB_SCHEMA_VERSION; + + info!( + from = %from_version, + to = %to_version, + "Dry-run: scanning pending database migrations (no commits)" + ); + + let migration_chain = self.build_migration_chain(from_version, to_version)?; + + if migration_chain.is_empty() { + info!("Dry-run: no schema steps in migration chain"); + stats.duration = start.elapsed(); + return Ok(stats); + } + + for (schema_version, migrators) in &migration_chain { + info!(schema_version = %schema_version, "Would migrate to schema version (dry-run)"); + info!( + certificates = migrators.certificate.name(), + decided_blocks = migrators.decided_block.name(), + undecided_blocks = migrators.undecided_block.name(), + pending_parts = migrators.pending_parts.name(), + "Pending migrators for this step" + ); + + stats.merge(self.migrate_certificates(migrators.certificate.as_ref(), true)?); + stats.merge(self.migrate_decided_blocks(migrators.decided_block.as_ref(), true)?); + stats.merge(self.migrate_undecided_blocks(migrators.undecided_block.as_ref(), true)?); + stats.merge(self.migrate_pending_parts(migrators.pending_parts.as_ref(), true)?); + } + + stats.duration = start.elapsed(); + + Ok(stats) + } + /// Ensure the metadata table exists by creating it if necessary fn ensure_metadata_table_exists(&self) -> Result<(), StoreError> { let tx = self.db.begin_write()?; @@ -239,8 +290,16 @@ impl MigrationCoordinator { } /// Migrate certificates table. - fn migrate_certificates(&self, migrator: &dyn Migrator) -> Result { - info!("Migrating certificates table"); + /// + /// When `dry_run` is true, uses the same write transaction and iteration path as a real + /// migration but aborts each batch without persisting changes. + #[allow(clippy::arithmetic_side_effects)] // counter arithmetic bounded by DB record counts + fn migrate_certificates( + &self, + migrator: &dyn Migrator, + dry_run: bool, + ) -> Result { + info!(dry_run, "Processing certificates table"); let mut stats = MigrationStats::default(); @@ -253,7 +312,7 @@ impl MigrationCoordinator { { min_height.value() } else { - // no records to migrate, return early + stats.tables_migrated += 1; return Ok(stats); }; @@ -287,14 +346,19 @@ impl MigrationCoordinator { next_height = key.value().increment(); } - // migrate the records in this batch records_upgraded = batch.len(); - for (key, value) in batch { - table.insert(key, value)?; + if !dry_run { + for (key, value) in batch { + table.insert(key, value)?; + } } } - tx.commit()?; - debug!("migrated batch of {} records", records_upgraded); + if dry_run { + tx.abort()?; + } else { + tx.commit()?; + } + debug!("{} records in batch", records_upgraded); stats.records_scanned += records_scanned; stats.records_upgraded += records_upgraded; @@ -309,20 +373,25 @@ impl MigrationCoordinator { stats.tables_migrated += 1; info!( + dry_run, scanned = stats.records_scanned, upgraded = stats.records_upgraded, - "Completed certificates migration" + "Finished certificates table" ); Ok(stats) } /// Migrate decided blocks table. + /// + /// When `dry_run` is true, uses write transactions but aborts each batch without persisting. + #[allow(clippy::arithmetic_side_effects)] // counter arithmetic bounded by DB record counts fn migrate_decided_blocks( &self, migrator: &dyn Migrator, + dry_run: bool, ) -> Result { - info!("Migrating decided blocks table"); + info!(dry_run, "Processing decided blocks table"); let mut stats = MigrationStats::default(); @@ -335,7 +404,7 @@ impl MigrationCoordinator { { min_height.value() } else { - // no records to migrate, return early + stats.tables_migrated += 1; return Ok(stats); }; @@ -369,14 +438,19 @@ impl MigrationCoordinator { next_height = key.value().increment(); } - // migrate the records in this batch records_upgraded = batch.len(); - for (key, value) in batch { - table.insert(key, value)?; + if !dry_run { + for (key, value) in batch { + table.insert(key, value)?; + } } } - tx.commit()?; - debug!("migrated batch of {} records", records_upgraded); + if dry_run { + tx.abort()?; + } else { + tx.commit()?; + } + debug!("{} records in batch", records_upgraded); stats.records_scanned += records_scanned; stats.records_upgraded += records_upgraded; @@ -391,20 +465,26 @@ impl MigrationCoordinator { stats.tables_migrated += 1; info!( + dry_run, scanned = stats.records_scanned, upgraded = stats.records_upgraded, - "Completed decided blocks migration" + "Finished decided blocks table" ); Ok(stats) } /// Migrate undecided blocks table. + /// + /// When `dry_run` is true, opens a write transaction (creating the table if needed) but + /// aborts without applying updates. + #[allow(clippy::arithmetic_side_effects)] // counter arithmetic bounded by DB record counts fn migrate_undecided_blocks( &self, migrator: &dyn Migrator, + dry_run: bool, ) -> Result { - info!("Migrating undecided blocks table"); + info!(dry_run, "Processing undecided blocks table"); let mut stats = MigrationStats::default(); @@ -433,29 +513,43 @@ impl MigrationCoordinator { } } - // Apply all migrations - for (key, value) in to_migrate { - table.insert(key, value)?; - stats.records_upgraded += 1; + stats.records_upgraded = to_migrate.len(); + if !dry_run { + for (key, value) in to_migrate { + table.insert(key, value)?; + } } } - tx.commit()?; + if dry_run { + tx.abort()?; + } else { + tx.commit()?; + } stats.records_skipped = stats.records_scanned - stats.records_upgraded; stats.tables_migrated += 1; info!( + dry_run, scanned = stats.records_scanned, upgraded = stats.records_upgraded, - "Completed undecided blocks migration" + "Finished undecided blocks table" ); Ok(stats) } /// Migrate pending proposal parts table. - fn migrate_pending_parts(&self, migrator: &dyn Migrator) -> Result { - info!("Migrating pending proposal parts table"); + /// + /// When `dry_run` is true, opens a write transaction (creating the table if needed) but + /// aborts without applying updates. + #[allow(clippy::arithmetic_side_effects)] // counter arithmetic bounded by DB record counts + fn migrate_pending_parts( + &self, + migrator: &dyn Migrator, + dry_run: bool, + ) -> Result { + info!(dry_run, "Processing pending proposal parts table"); let mut stats = MigrationStats::default(); @@ -484,21 +578,27 @@ impl MigrationCoordinator { } } - // Apply all migrations - for (key, value) in to_migrate { - table.insert(key, value)?; - stats.records_upgraded += 1; + stats.records_upgraded = to_migrate.len(); + if !dry_run { + for (key, value) in to_migrate { + table.insert(key, value)?; + } } } - tx.commit()?; + if dry_run { + tx.abort()?; + } else { + tx.commit()?; + } stats.records_skipped = stats.records_scanned - stats.records_upgraded; stats.tables_migrated += 1; info!( + dry_run, scanned = stats.records_scanned, upgraded = stats.records_upgraded, - "Completed pending parts migration" + "Finished pending proposal parts table" ); Ok(stats) @@ -739,6 +839,70 @@ mod tests { assert_eq!(chain.len(), 0); } + #[test] + fn test_preview_migrate_stats_match_migrate() { + let (db, _path) = create_test_db(); + let tx = db.begin_write().unwrap(); + { + let mut meta = tx.open_table(METADATA_TABLE).unwrap(); + meta.insert("schema_version", SchemaVersion::V0).unwrap(); + // Use V0 version byte (0x00) so records actually need migration. + let mut cert = tx.open_table(CERTIFICATES_TABLE).unwrap(); + cert.insert(Height::new(1), vec![0u8, 1, 2, 3]).unwrap(); + let mut dec = tx.open_table(DECIDED_BLOCKS_TABLE).unwrap(); + dec.insert(Height::new(1), vec![0u8, 4, 5, 6]).unwrap(); + } + tx.commit().unwrap(); + + let coordinator = MigrationCoordinator::new(db); + let preview = coordinator.preview_migrate().unwrap(); + + // Dry-run must not commit: schema version and record data must be unchanged. + assert_eq!( + coordinator.current_schema_version().unwrap(), + Some(SchemaVersion::V0), + "preview_migrate must not alter schema version" + ); + let tx = coordinator.db.begin_read().unwrap(); + let cert_table = tx.open_table(CERTIFICATES_TABLE).unwrap(); + assert_eq!( + cert_table.get(Height::new(1)).unwrap().unwrap().value()[0], + 0, + "preview_migrate must not alter record data" + ); + drop(cert_table); + drop(tx); + + let migrated = coordinator.migrate().unwrap(); + + assert_eq!(preview.records_scanned, migrated.records_scanned); + assert_eq!(preview.records_upgraded, migrated.records_upgraded); + assert_eq!(preview.records_skipped, migrated.records_skipped); + assert_eq!(preview.tables_migrated, migrated.tables_migrated); + } + + #[test] + fn test_preview_migrate_already_current() { + let (db, _path) = create_test_db(); + let coordinator = MigrationCoordinator::new(db); + + // Initialize DB at current schema version (simulates a node that is already up to date). + coordinator.needs_migration(false).unwrap(); + assert_eq!( + coordinator.current_schema_version().unwrap(), + Some(DB_SCHEMA_VERSION) + ); + + let stats = coordinator + .preview_migrate() + .expect("preview_migrate must not error when already at current version"); + + assert_eq!(stats.tables_migrated, 0); + assert_eq!(stats.records_scanned, 0); + assert_eq!(stats.records_upgraded, 0); + assert_eq!(stats.records_skipped, 0); + } + #[test] fn test_certificate_table_migration() { let (db, _path) = create_test_db(); @@ -764,7 +928,7 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_certificates(&migrator).unwrap(); + let stats = coordinator.migrate_certificates(&migrator, false).unwrap(); assert_eq!(stats.records_scanned, 2); assert_eq!(stats.records_upgraded, 2); @@ -805,7 +969,9 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_decided_blocks(&migrator).unwrap(); + let stats = coordinator + .migrate_decided_blocks(&migrator, false) + .unwrap(); assert_eq!(stats.records_scanned, 2); assert_eq!(stats.records_upgraded, 0); @@ -838,7 +1004,9 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_decided_blocks(&migrator).unwrap(); + let stats = coordinator + .migrate_decided_blocks(&migrator, false) + .unwrap(); assert_eq!(stats.records_scanned, 3); assert_eq!(stats.records_upgraded, 2); @@ -943,6 +1111,7 @@ mod tests { } #[test] + #[allow(clippy::cast_possible_truncation)] fn test_certificate_migration_batch_size_plus_one() { let (db, _path) = create_test_db(); @@ -985,7 +1154,7 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_certificates(&migrator).unwrap(); + let stats = coordinator.migrate_certificates(&migrator, false).unwrap(); // Verify all records were scanned and upgraded assert_eq!( @@ -1080,7 +1249,9 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_undecided_blocks(&migrator).unwrap(); + let stats = coordinator + .migrate_undecided_blocks(&migrator, false) + .unwrap(); assert_eq!(stats.records_scanned, 3); assert_eq!(stats.records_upgraded, 3); @@ -1127,7 +1298,9 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_undecided_blocks(&migrator).unwrap(); + let stats = coordinator + .migrate_undecided_blocks(&migrator, false) + .unwrap(); assert_eq!(stats.records_scanned, 0); assert_eq!(stats.records_upgraded, 0); @@ -1172,7 +1345,9 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_undecided_blocks(&migrator).unwrap(); + let stats = coordinator + .migrate_undecided_blocks(&migrator, false) + .unwrap(); assert_eq!(stats.records_scanned, 3); assert_eq!(stats.records_upgraded, 2); @@ -1251,7 +1426,7 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_pending_parts(&migrator).unwrap(); + let stats = coordinator.migrate_pending_parts(&migrator, false).unwrap(); assert_eq!(stats.records_scanned, 4); assert_eq!(stats.records_upgraded, 4); @@ -1305,7 +1480,7 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_pending_parts(&migrator).unwrap(); + let stats = coordinator.migrate_pending_parts(&migrator, false).unwrap(); assert_eq!(stats.records_scanned, 0); assert_eq!(stats.records_upgraded, 0); @@ -1354,7 +1529,7 @@ mod tests { from: SchemaVersion::V0, to: SchemaVersion::V1, }; - let stats = coordinator.migrate_pending_parts(&migrator).unwrap(); + let stats = coordinator.migrate_pending_parts(&migrator, false).unwrap(); assert_eq!(stats.records_scanned, 4); assert_eq!(stats.records_upgraded, 2); diff --git a/crates/consensus-db/src/migrations/migrators.rs b/crates/consensus-db/src/migrations/migrators.rs index 316239e..959e530 100644 --- a/crates/consensus-db/src/migrations/migrators.rs +++ b/crates/consensus-db/src/migrations/migrators.rs @@ -71,6 +71,7 @@ impl Migrator for CommitCertificateMigrator0To1 { // Unconditionally prepend version byte. Otherwise, we might have some // false positives when going from unversioned to versioned // data. + #[allow(clippy::arithmetic_side_effects)] // 1 + slice.len() cannot overflow usize let mut result = Vec::with_capacity(1 + old_bytes.len()); result.push(self.target_version().as_u8()); result.extend_from_slice(old_bytes); @@ -107,6 +108,7 @@ impl Migrator for ExecutionPayloadMigrator0To3 { // Unconditionally prepend version byte. Otherwise, we might have some // false positives when going from unversioned to versioned // data. + #[allow(clippy::arithmetic_side_effects)] // 1 + slice.len() cannot overflow usize let mut result = Vec::with_capacity(1 + old_bytes.len()); result.push(self.target_version().as_u8()); result.extend_from_slice(old_bytes); @@ -142,6 +144,7 @@ impl Migrator for ConsensusBlockMigrator0To1 { // Unconditionally prepend version byte. Otherwise, we might have some // false positives when going from unversioned to versioned // data. + #[allow(clippy::arithmetic_side_effects)] // 1 + slice.len() cannot overflow usize let mut result = Vec::with_capacity(1 + old_bytes.len()); result.push(self.target_version().as_u8()); result.extend_from_slice(old_bytes); @@ -177,6 +180,7 @@ impl Migrator for ProposalPartsMigrator0To1 { // Unconditionally prepend version byte. Otherwise, we might have some // false positives when going from unversioned to versioned // data. + #[allow(clippy::arithmetic_side_effects)] // 1 + slice.len() cannot overflow usize let mut result = Vec::with_capacity(1 + old_bytes.len()); result.push(self.target_version().as_u8()); result.extend_from_slice(old_bytes); diff --git a/crates/consensus-db/src/store.rs b/crates/consensus-db/src/store.rs index 06acdd8..5dc4c43 100644 --- a/crates/consensus-db/src/store.rs +++ b/crates/consensus-db/src/store.rs @@ -164,8 +164,10 @@ impl Db { let path = path.as_ref().to_owned(); let db_exists = path.exists(); + #[allow(clippy::cast_possible_truncation)] // 32-bit targets won't have multi-GB caches + let cache_size_bytes = cache_size.as_u64() as usize; let mut db = redb::Database::builder() - .set_cache_size(cache_size.as_u64() as usize) + .set_cache_size(cache_size_bytes) .set_repair_callback(|session| { let status = session.progress() * 100.0; info!("Database repair in progress: {status:.2}%"); @@ -216,6 +218,7 @@ impl Db { )) } + // Metric byte counters accumulate bounded DB record sizes — overflow is not reachable. fn get_payload(&self, height: Height) -> Result, StoreError> { let start = Instant::now(); let mut read_bytes = 0; @@ -227,7 +230,10 @@ impl Db { let payload = payload .map(|value| { let bytes = value.value(); - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_execution_payload(&bytes) }) .transpose() @@ -257,7 +263,10 @@ impl Db { }; let result = bytes.and_then(|bytes| { - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_certificate(&bytes).ok() }); @@ -278,7 +287,10 @@ impl Db { let payload = table.get(&height)?.map(|value| value.value()); payload .map(|bytes| { - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_execution_payload(&bytes) }) .transpose() @@ -290,7 +302,10 @@ impl Db { let value = table.get(&height)?; value.and_then(|value| { let bytes = value.value(); - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_certificate(&bytes).ok() }) }; @@ -319,7 +334,10 @@ impl Db { { let mut blocks = tx.open_table(DECIDED_BLOCKS_TABLE)?; let block_bytes = encode_execution_payload(&decided_block.execution_payload); - write_bytes += block_bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + write_bytes += block_bytes.len(); + } blocks.insert(height, block_bytes)?; } @@ -502,7 +520,10 @@ impl Db { let data = bytes .map(|bytes| { - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_proposal_monitor_data(&bytes) }) .transpose() @@ -541,7 +562,10 @@ impl Db { let evidence = bytes .map(|bytes| { - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_misbehavior_evidence(&bytes) }) .transpose() @@ -584,7 +608,10 @@ impl Db { let payloads = bytes .map(|bytes| { - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } decode_invalid_payloads(&bytes) }) .transpose() @@ -649,7 +676,10 @@ impl Db { let value = if let Ok(Some(value)) = table.get(&(height, round, block_hash)) { let bytes = value.value(); - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } let block = decode_block(&bytes)?; Some(block) @@ -685,7 +715,10 @@ impl Db { if key_height == height && key_block_hash == block_hash { let bytes = value.value(); - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } let block = decode_block(&bytes)?; @@ -724,6 +757,7 @@ impl Db { // Iterate through all entries that start with (height, round, *) let range_start = (height, round, BlockHash::new([0; 32])); + #[allow(clippy::arithmetic_side_effects)] // round + 1 for range upper bound let range_end = ( height, Round::from(round.as_i64() + 1), @@ -737,18 +771,19 @@ impl Db { // Only include entries that match exactly height and round if key_tuple.0 == height && key_tuple.1 == round { let bytes = value.value(); - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } let proposal = decode_block(&bytes)?; blocks.push(proposal); } } - self.update_read_metrics( - read_bytes, - size_of::<(Height, Round, BlockHash)>() * blocks.len(), - start.elapsed(), - ); + #[allow(clippy::arithmetic_side_effects)] + let key_bytes = size_of::<(Height, Round, BlockHash)>() * blocks.len(); + self.update_read_metrics(read_bytes, key_bytes, start.elapsed()); Ok(blocks) } @@ -827,18 +862,19 @@ impl Db { if h == height && r == round { let bytes = value.value(); - read_bytes += bytes.len(); + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + } let parts = decode_proposal_parts(&bytes)?; proposals.push(parts); } } - self.update_read_metrics( - read_bytes, - size_of::<(Height, Round, BlockHash)>() * proposals.len(), - start.elapsed(), - ); + #[allow(clippy::arithmetic_side_effects)] + let key_bytes = size_of::<(Height, Round, BlockHash)>() * proposals.len(); + self.update_read_metrics(read_bytes, key_bytes, start.elapsed()); Ok(proposals) } @@ -849,6 +885,8 @@ impl Db { let tx = self.db.begin_read()?; let table = tx.open_table(PENDING_PROPOSAL_PARTS_TABLE)?; + // redb returns u64; table won't exceed usize on any supported target + #[allow(clippy::cast_possible_truncation)] let count = table.len()? as usize; self.update_read_metrics(0, 0, start.elapsed()); @@ -874,19 +912,17 @@ impl Db { let (height, _, _) = key.value(); let bytes = value.value(); - read_bytes += bytes.len(); - - let count = counts.entry(height).or_insert(0); - *count += 1; - - total_keys += 1; + #[allow(clippy::arithmetic_side_effects)] + { + read_bytes += bytes.len(); + *counts.entry(height).or_insert(0) += 1; + total_keys += 1; + } } - self.update_read_metrics( - read_bytes, - size_of::<(Height, Round, BlockHash)>() * total_keys, - start.elapsed(), - ); + #[allow(clippy::arithmetic_side_effects)] + let key_bytes = size_of::<(Height, Round, BlockHash)>() * total_keys; + self.update_read_metrics(read_bytes, key_bytes, start.elapsed()); Ok(counts.into_iter().collect()) } @@ -901,7 +937,11 @@ impl Db { // Calculate max allowed height (inclusive). // For current_height=10 and max_pending_parts=4, we allow heights 10, 11, 12, 13. - let max_allowed_height = current_height.increment_by(max_pending_parts as u64 - 1); + let max_allowed_height = current_height.increment_by( + max_pending_parts + .checked_sub(1) + .expect("max_pending_parts must be > 0") as u64, + ); // Do not insert if proposal is outside the allowed range (too far in the future) if parts.height() > max_allowed_height { @@ -918,6 +958,7 @@ impl Db { { let mut table = tx.open_table(PENDING_PROPOSAL_PARTS_TABLE)?; + #[allow(clippy::cast_possible_truncation)] let count = table.len()? as usize; if count < max_pending_parts { @@ -949,7 +990,11 @@ impl Db { // Calculate max allowed height (inclusive). // For current_height=10 and max_pending_parts=4, we allow heights 10, 11, 12, 13. - let max_allowed_height = current_height.increment_by(max_pending_proposals as u64 - 1); + let max_allowed_height = current_height.increment_by( + max_pending_proposals + .checked_sub(1) + .expect("max_pending_proposals must be > 0") as u64, + ); // Collect all keys, categorize as: stale, within_range, or too_far // Keys are sorted by (height, round, hash) ascending @@ -1880,6 +1925,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] async fn test_pending_proposal_parts_counts() { // no proposal parts let store = create_store().await; @@ -2560,6 +2606,7 @@ mod tests { .await; } + #[allow(clippy::arithmetic_side_effects)] async fn test_enforce_pending_limit_on_startup( initial_proposals: Vec<(u64, u32)>, current_height: u64, diff --git a/crates/consensus-db/src/versions.rs b/crates/consensus-db/src/versions.rs index 8e9d1e8..8350174 100644 --- a/crates/consensus-db/src/versions.rs +++ b/crates/consensus-db/src/versions.rs @@ -39,7 +39,7 @@ impl SchemaVersion { } pub const fn next(&self) -> Self { - Self(self.0 + 1) + Self(self.0.checked_add(1).expect("schema version overflow")) } pub fn previous(&self) -> Option { diff --git a/crates/engine-bench/Cargo.toml b/crates/engine-bench/Cargo.toml new file mode 100644 index 0000000..5d8b06d --- /dev/null +++ b/crates/engine-bench/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "arc-engine-bench" +version.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true +rust-version.workspace = true +publish.workspace = true + +[[bin]] +name = "arc-engine-bench" +path = "src/main.rs" + +[dependencies] +alloy-genesis = { workspace = true } +alloy-primitives = { workspace = true } +alloy-rpc-types-engine = { workspace = true, features = ["jwt", "serde"] } +arc-eth-engine = { workspace = true } +arc-execution-config = { workspace = true } +arc-version = { workspace = true } +chrono = { workspace = true } +clap = { workspace = true, features = ["derive", "env"] } +color-eyre = { workspace = true } +csv = { workspace = true } +eyre = { workspace = true } +reqwest = { workspace = true, features = ["json", "rustls-tls", "native-tls-vendored"] } +reth-cli = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +[dev-dependencies] +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/crates/engine-bench/README.md b/crates/engine-bench/README.md new file mode 100644 index 0000000..e6aa6f0 --- /dev/null +++ b/crates/engine-bench/README.md @@ -0,0 +1,201 @@ +# Arc Engine Bench + +Inspired by [`reth-bench`](https://github.com/paradigmxyz/reth/tree/main/bin/reth-bench) from the +upstream Reth project. + +`arc-engine-bench` replays historical blocks into an Arc execution node and measures Engine API import +latency. The current benchmark mode, `new-payload-fcu`, submits each block with +`engine_newPayloadV4`, follows it with `engine_forkchoiceUpdatedV3`, and writes CSV artifacts for +per-block latency and aggregate throughput. + +## CLI + +| Command | Description | +| --- | --- | +| `arc-engine-bench prepare-payload` | Fetches a contiguous source block range and writes a local payload fixture directory with `genesis.json`, `metadata.json`, and `payloads.jsonl`. | +| `arc-engine-bench new-payload-fcu` | Replays a prepared payload fixture into a target execution node with `engine_newPayloadV4` followed by `engine_forkchoiceUpdatedV3`. | + +## What You Need + +- A running `arc-node-execution` instance to benchmark. This is the **target** node. +- A payload fixture directory containing `genesis.json`, `metadata.json`, and `payloads.jsonl` for + the block range you want to replay. +- A source RPC endpoint with the historical blocks you want to replay. This is only needed when you + run `prepare-payload` to create or refresh a fixture. +- The target must already be at block `FROM_BLOCK - 1`. The benchmark verifies that the target head + matches the fixture metadata before replay starts and exits if it does not. +- The target Engine API must be reachable via **IPC** or **authenticated RPC**. Pass either + `--engine-ipc ` or `--engine-rpc-url --jwt-secret ` (mutually exclusive). + +## Example Environment + +Copy this block, adjust it for your setup, then `source` it before running the commands below: + +`BENCH_DATADIR` is the directory where the target node snapshot lives. + +```bash +BENCH_DATADIR=datadir/bench-target +TARGET_ETH_RPC_URL=http://127.0.0.1:7545 +SOURCE_RPC_URL=http://127.0.0.1:8545 +HTTP_PORT=7545 +METRICS_PORT=19001 +FROM_BLOCK=1 +TO_BLOCK=3000 +PAYLOAD_DIR=target/engine-bench/payload-fixture +CHAIN=arc-localdev +# ipc +ENGINE_IPC="$BENCH_DATADIR/reth.ipc" +# rpc +ENGINE_RPC_URL=http://127.0.0.1:7551 +AUTHRPC_PORT=7551 +``` + +## Prepare the Target Node + +The target node must start at the parent of the first replayed block. + +- If `FROM_BLOCK=1`, you can start from a fresh datadir. +- If `FROM_BLOCK>1`, you need the target node at block `FROM_BLOCK - 1` before replay. In + practice, that means either: + - prepare a snapshot at the desired height, or + - sync the node past that height and unwind it back to `FROM_BLOCK - 1`. + +### 1. Create a datadir and JWT secret + +```bash +mkdir -p "$BENCH_DATADIR" +# Only needed for RPC transport: +openssl rand -hex 32 | tr -d '\n' > "$BENCH_DATADIR/jwt.hex" +chmod 600 "$BENCH_DATADIR/jwt.hex" +``` + +### 2. Unwind the target to the replay parent block + +Skip this step when `FROM_BLOCK=1`. If you synced the node past the replay start, stop it before running the unwind command: + +```bash +arc-node-execution stage unwind \ + --chain "$CHAIN" \ + --datadir "$BENCH_DATADIR" \ + to-block "$((FROM_BLOCK - 1))" +``` + +### 3. Start the target node + +**IPC transport:** + +```bash +arc-node-execution node \ + --chain "$CHAIN" \ + --datadir "$BENCH_DATADIR" \ + --dev \ + --disable-discovery \ + --http \ + --http.api=eth \ + --http.port "$HTTP_PORT" \ + --metrics 127.0.0.1:"$METRICS_PORT" \ + --auth-ipc \ + --auth-ipc.path "$ENGINE_IPC" \ + --arc.denylist.enabled +``` + +**RPC transport:** + +```bash +arc-node-execution node \ + --chain "$CHAIN" \ + --datadir "$BENCH_DATADIR" \ + --dev \ + --disable-discovery \ + --http \ + --http.api=eth \ + --http.port "$HTTP_PORT" \ + --metrics 127.0.0.1:"$METRICS_PORT" \ + --authrpc.addr=127.0.0.1 \ + --authrpc.port="$AUTHRPC_PORT" \ + --authrpc.jwtsecret="$BENCH_DATADIR/jwt.hex" \ + --arc.denylist.enabled +``` + +## Prepare the Payload Fixture + +Fetch source blocks `FROM_BLOCK..=TO_BLOCK` once and write them to a local fixture directory: + +```bash +arc-engine-bench prepare-payload \ + --chain "$CHAIN" \ + --source-rpc-url "$SOURCE_RPC_URL" \ + --from "$FROM_BLOCK" \ + --to "$TO_BLOCK" \ + --output-dir "$PAYLOAD_DIR" +``` + +Other flags: + +- `--chain ` sets the chain spec used to record genesis config. Accepts built-in + names (`arc-localdev`, `arc-devnet`, `arc-testnet`) or a path to a genesis JSON file. The default + is `arc-localdev`. +- `--eth-rpc-timeout-ms ` sets the timeout for source Ethereum RPC requests. The + default is `10000` ms. Batch requests use the larger of this value or 30 seconds. +- `--batch-size ` controls source RPC fetch batching. The default is `20`. + +The fixture directory contains: + +| File | Content | +| --- | --- | +| `genesis.json` | Chain genesis configuration (chain ID, hardfork activations, initial state). | +| `metadata.json` | Replay metadata including `from_block`, `to_block`, `payload_count`, and the expected parent block. | +| `payloads.jsonl` | One `ExecutionPayloadV3` JSON document per line, ordered by block number. | + +## Run `new-payload-fcu` + +Replay the prepared fixture into the target node: + +**IPC transport:** + +```bash +arc-engine-bench new-payload-fcu \ + --engine-ipc "$ENGINE_IPC" \ + --target-eth-rpc-url "$TARGET_ETH_RPC_URL" \ + --payload "$PAYLOAD_DIR" +``` + +**RPC transport:** + +```bash +arc-engine-bench new-payload-fcu \ + --engine-rpc-url "$ENGINE_RPC_URL" \ + --jwt-secret "$BENCH_DATADIR/jwt.hex" \ + --target-eth-rpc-url "$TARGET_ETH_RPC_URL" \ + --payload "$PAYLOAD_DIR" +``` + +Other flags: + +- `--output ` writes artifacts to an explicit directory. By default, output goes to + `target/engine-bench/new-payload-fcu-/`. +- `--eth-rpc-timeout-ms ` sets the timeout for target Ethereum RPC requests. The + default is `10000` ms. + +## Live Metrics + +From the repo root, start the monitoring stack: + +```bash +docker compose -f deployments/monitoring.yaml up -d +``` + +The bundled Prometheus config includes an `arc_engine_bench_target` scrape job that reads the +benchmark target from `host.docker.internal:19001`. + +Open Grafana at `http://127.0.0.1:3000`, then open the provisioned `Reth` dashboard and select the +`arc_engine_bench_target` instance. + +## Output Artifacts + +Each run writes to `target/engine-bench/-/` unless you pass `--output`. + +| File | Content | +| --- | --- | +| `combined_latency.csv` | One row per replayed block with block metadata, `new_payload_ms`, `fcu_ms`, `total_ms`, per-block throughput, and cumulative throughput. | +| `summary.csv` | One-row summary with sample count, total gas and txs, wall-clock time, average throughput, and latency percentiles. | diff --git a/crates/engine-bench/src/bench/context.rs b/crates/engine-bench/src/bench/context.rs new file mode 100644 index 0000000..6e6a2bc --- /dev/null +++ b/crates/engine-bench/src/bench/context.rs @@ -0,0 +1,250 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::cli::CommonArgs; +use arc_eth_engine::{ + engine::EngineAPI, ipc::engine_ipc::EngineIPC, rpc::engine_rpc::EngineRpc, + rpc::ethereum_rpc::EthereumRPC, +}; +use chrono::Utc; +use eyre::{bail, Context}; +use reqwest::Url; +use std::{ + fmt, fs, + path::{Path, PathBuf}, + time::Duration, +}; + +const ETH_BATCH_TIMEOUT_FLOOR: Duration = Duration::from_secs(30); + +/// Which Engine API transport was selected on the CLI. +pub(crate) enum EngineTransport { + Ipc(PathBuf), + Rpc { url: String, jwt_secret: PathBuf }, +} + +impl fmt::Display for EngineTransport { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Ipc(path) => write!(f, "ipc:{}", path.display()), + Self::Rpc { url, .. } => write!(f, "rpc:{url}"), + } + } +} + +pub(crate) struct BenchContext { + transport: EngineTransport, + output_dir: PathBuf, + eth_rpc_timeout: Duration, +} + +impl fmt::Debug for BenchContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BenchContext") + .field("transport", &self.transport.to_string()) + .field("output_dir", &self.output_dir) + .field("eth_rpc_timeout", &self.eth_rpc_timeout) + .finish() + } +} + +impl BenchContext { + pub(crate) fn new(common: &CommonArgs, mode: &str) -> eyre::Result { + let transport = match (&common.engine_ipc, &common.engine_rpc_url) { + (Some(ipc), None) => EngineTransport::Ipc(ipc.clone()), + (None, Some(url)) => { + let jwt = common + .jwt_secret + .as_ref() + .ok_or_else(|| eyre::eyre!("--jwt-secret is required with --engine-rpc-url"))?; + if !jwt.exists() { + bail!("JWT secret file does not exist: {}", jwt.display()); + } + EngineTransport::Rpc { + url: url.clone(), + jwt_secret: jwt.clone(), + } + } + (None, None) => bail!( + "specify either --engine-ipc or --engine-rpc-url --jwt-secret " + ), + _ => unreachable!("clap group prevents both"), + }; + + Ok(Self { + transport, + output_dir: resolve_output_dir(common, mode)?, + eth_rpc_timeout: Duration::from_millis(common.eth_rpc_timeout_ms), + }) + } + + pub(crate) fn output_dir(&self) -> &Path { + &self.output_dir + } + + pub(crate) fn transport(&self) -> &EngineTransport { + &self.transport + } + + pub(crate) async fn engine(&self) -> eyre::Result> { + match &self.transport { + EngineTransport::Ipc(path) => { + let path_str = path.to_str().ok_or_else(|| { + eyre::eyre!( + "engine IPC socket path is not valid UTF-8: {}", + path.display() + ) + })?; + let ipc = EngineIPC::new(path_str) + .await + .wrap_err("failed to create engine IPC client")?; + Ok(Box::new(ipc) as Box) + } + EngineTransport::Rpc { url, jwt_secret } => { + let rpc = EngineRpc::new( + Url::parse(url).wrap_err("invalid engine RPC URL")?, + jwt_secret.as_path(), + ) + .wrap_err("failed to create engine RPC client")?; + Ok(Box::new(rpc) as Box) + } + } + } + + pub(crate) fn ethereum_rpc(&self, rpc_url: &str, role: &str) -> eyre::Result { + ethereum_rpc_client(rpc_url, role, self.eth_rpc_timeout) + } +} + +pub(crate) fn ethereum_rpc_client( + rpc_url: &str, + role: &str, + eth_rpc_timeout: Duration, +) -> eyre::Result { + EthereumRPC::new_with_timeouts( + Url::parse(rpc_url).wrap_err_with(|| format!("invalid {role} url"))?, + eth_rpc_timeout, + eth_rpc_timeout.max(ETH_BATCH_TIMEOUT_FLOOR), + ) + .wrap_err_with(|| format!("failed to create {role} client")) +} + +fn resolve_output_dir(common: &CommonArgs, mode: &str) -> eyre::Result { + let output = match &common.output { + Some(path) => path.clone(), + None => { + let timestamp = Utc::now().format("%Y%m%dT%H%M%SZ"); + PathBuf::from("target") + .join("engine-bench") + .join(format!("{mode}-{timestamp}")) + } + }; + fs::create_dir_all(&output) + .wrap_err_with(|| format!("failed to create benchmark output dir {}", output.display()))?; + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn common_args_ipc() -> CommonArgs { + CommonArgs { + engine_ipc: Some(PathBuf::from("/tmp/reth.ipc")), + engine_rpc_url: None, + jwt_secret: None, + eth_rpc_timeout_ms: 10_000, + output: None, + } + } + + fn common_args_rpc(jwt_path: PathBuf) -> CommonArgs { + CommonArgs { + engine_ipc: None, + engine_rpc_url: Some("http://127.0.0.1:8551".to_string()), + jwt_secret: Some(jwt_path), + eth_rpc_timeout_ms: 10_000, + output: None, + } + } + + #[test] + fn resolve_output_dir_creates_explicit_output_directory() { + let temp_dir = TempDir::new().unwrap(); + let output_dir = temp_dir.path().join("bench-output"); + let mut args = common_args_ipc(); + args.output = Some(output_dir.clone()); + + let resolved = resolve_output_dir(&args, "new-payload-fcu").unwrap(); + + assert_eq!(resolved, output_dir); + assert!(resolved.is_dir()); + } + + #[test] + fn new_context_selects_ipc_transport() { + let args = common_args_ipc(); + let ctx = BenchContext::new(&args, "test").unwrap(); + assert!(matches!(ctx.transport(), EngineTransport::Ipc(_))); + } + + #[test] + fn new_context_selects_rpc_transport() { + let temp_dir = TempDir::new().unwrap(); + let jwt_path = temp_dir.path().join("jwt.hex"); + fs::write(&jwt_path, "secret").unwrap(); + + let args = common_args_rpc(jwt_path); + let ctx = BenchContext::new(&args, "test").unwrap(); + assert!(matches!(ctx.transport(), EngineTransport::Rpc { .. })); + } + + #[test] + fn new_context_errors_when_neither_transport_specified() { + let args = CommonArgs { + engine_ipc: None, + engine_rpc_url: None, + jwt_secret: None, + eth_rpc_timeout_ms: 10_000, + output: None, + }; + + let err = BenchContext::new(&args, "test").unwrap_err(); + assert!(err.to_string().contains("--engine-ipc")); + } + + #[test] + fn new_context_errors_when_rpc_without_jwt() { + let args = CommonArgs { + engine_ipc: None, + engine_rpc_url: Some("http://127.0.0.1:8551".to_string()), + jwt_secret: None, + eth_rpc_timeout_ms: 10_000, + output: None, + }; + + let err = BenchContext::new(&args, "test").unwrap_err(); + assert!(err.to_string().contains("--jwt-secret")); + } + + #[test] + fn new_context_errors_when_jwt_file_missing() { + let args = common_args_rpc(PathBuf::from("/tmp/nonexistent-jwt.hex")); + let err = BenchContext::new(&args, "test").unwrap_err(); + assert!(err.to_string().contains("does not exist")); + } +} diff --git a/crates/engine-bench/src/bench/fixture.rs b/crates/engine-bench/src/bench/fixture.rs new file mode 100644 index 0000000..44bad6a --- /dev/null +++ b/crates/engine-bench/src/bench/fixture.rs @@ -0,0 +1,542 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::helpers::fmt_hash; +use alloy_genesis::Genesis; +use alloy_primitives::B256; +use alloy_rpc_types_engine::ExecutionPayloadV3; +use eyre::{bail, Context}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, File}, + io::{BufRead, BufReader, BufWriter, Write}, + path::{Path, PathBuf}, +}; + +pub(crate) const GENESIS_FILE_NAME: &str = "genesis.json"; +pub(crate) const METADATA_FILE_NAME: &str = "metadata.json"; +pub(crate) const PAYLOADS_FILE_NAME: &str = "payloads.jsonl"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ExpectedParentBlock { + pub block_number: u64, + pub block_hash: B256, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct PayloadFixtureMetadata { + pub from_block: u64, + pub to_block: u64, + pub payload_count: u64, + pub expected_parent: ExpectedParentBlock, +} + +impl PayloadFixtureMetadata { + pub(crate) fn validate(&self) -> eyre::Result<()> { + if self.from_block == 0 { + bail!("payload fixture from_block must be greater than 0"); + } + if self.from_block > self.to_block { + bail!("payload fixture from_block must be less than or equal to to_block"); + } + + let expected_count = self.to_block - self.from_block + 1; + if self.payload_count != expected_count { + bail!( + "payload fixture payload_count mismatch: expected {expected_count}, got {}", + self.payload_count + ); + } + + let expected_parent_block_number = self.from_block - 1; + if self.expected_parent.block_number != expected_parent_block_number { + bail!( + "payload fixture expected_parent block number mismatch: expected {expected_parent_block_number}, got {}", + self.expected_parent.block_number + ); + } + + Ok(()) + } +} + +pub(crate) struct PayloadFixtureWriter { + output_dir: PathBuf, + genesis_tmp_path: PathBuf, + metadata_tmp_path: PathBuf, + payloads_tmp_path: PathBuf, + payloads_writer: BufWriter, +} + +impl PayloadFixtureWriter { + pub(crate) fn new(output_dir: &Path) -> eyre::Result { + fs::create_dir_all(output_dir).wrap_err_with(|| { + format!( + "failed to create payload fixture directory {}", + output_dir.display() + ) + })?; + + let genesis_tmp_path = output_dir.join(format!("{GENESIS_FILE_NAME}.tmp")); + let metadata_tmp_path = output_dir.join(format!("{METADATA_FILE_NAME}.tmp")); + let payloads_tmp_path = output_dir.join(format!("{PAYLOADS_FILE_NAME}.tmp")); + remove_file_if_exists(&genesis_tmp_path)?; + remove_file_if_exists(&metadata_tmp_path)?; + remove_file_if_exists(&payloads_tmp_path)?; + + let payloads_writer = BufWriter::new( + File::create(&payloads_tmp_path) + .wrap_err_with(|| format!("failed to create {}", payloads_tmp_path.display()))?, + ); + + Ok(Self { + output_dir: output_dir.to_path_buf(), + genesis_tmp_path, + metadata_tmp_path, + payloads_tmp_path, + payloads_writer, + }) + } + + pub(crate) fn write_payload(&mut self, payload: &ExecutionPayloadV3) -> eyre::Result<()> { + serde_json::to_writer(&mut self.payloads_writer, payload) + .wrap_err("failed to serialize payload fixture entry")?; + self.payloads_writer + .write_all(b"\n") + .wrap_err("failed to write payload fixture newline")?; + Ok(()) + } + + pub(crate) fn finish( + self, + metadata: &PayloadFixtureMetadata, + genesis: &Genesis, + ) -> eyre::Result<()> { + metadata.validate()?; + + let Self { + output_dir, + genesis_tmp_path, + metadata_tmp_path, + payloads_tmp_path, + mut payloads_writer, + } = self; + + payloads_writer + .flush() + .wrap_err("failed to flush payload fixture stream")?; + drop(payloads_writer); + + write_json_file(&genesis_tmp_path, genesis, "genesis")?; + write_json_file(&metadata_tmp_path, metadata, "metadata")?; + + let genesis_path = genesis_path(&output_dir); + let metadata_path = metadata_path(&output_dir); + let payloads_path = payloads_path(&output_dir); + remove_file_if_exists(&genesis_path)?; + remove_file_if_exists(&metadata_path)?; + remove_file_if_exists(&payloads_path)?; + rename_into_place(&payloads_tmp_path, &payloads_path)?; + rename_into_place(&genesis_tmp_path, &genesis_path)?; + rename_into_place(&metadata_tmp_path, &metadata_path)?; + + Ok(()) + } +} + +pub(crate) struct PayloadFixture { + #[allow(dead_code)] + genesis: Genesis, + metadata: PayloadFixtureMetadata, + payload_reader: PayloadJsonlReader, + expected_next_block: u64, + expected_parent_hash: B256, + yielded: u64, + exhausted: bool, +} + +impl PayloadFixture { + pub(crate) fn open(payload_dir: &Path) -> eyre::Result { + let genesis = load_genesis(payload_dir)?; + let metadata = load_metadata(payload_dir)?; + let payloads_path = payloads_path(payload_dir); + if !payloads_path.is_file() { + bail!( + "payload fixture payloads file does not exist: {}", + payloads_path.display() + ); + } + + Ok(Self { + expected_next_block: metadata.from_block, + expected_parent_hash: metadata.expected_parent.block_hash, + payload_reader: PayloadJsonlReader::new(&payloads_path)?, + genesis, + metadata, + yielded: 0, + exhausted: false, + }) + } + + #[allow(dead_code)] + pub(crate) fn genesis(&self) -> &Genesis { + &self.genesis + } + + pub(crate) fn metadata(&self) -> &PayloadFixtureMetadata { + &self.metadata + } + + pub(crate) fn next_payload(&mut self) -> eyre::Result> { + if self.exhausted { + return Ok(None); + } + + if self.yielded == self.metadata.payload_count { + if let Some(extra_payload) = self.payload_reader.next_payload()? { + let extra_block = extra_payload.payload_inner.payload_inner.block_number; + bail!( + "payload fixture contains more payloads than expected: first extra block is {extra_block}" + ); + } + self.exhausted = true; + return Ok(None); + } + + let payload = self.payload_reader.next_payload()?.ok_or_else(|| { + eyre::eyre!( + "payload fixture ended early after {} payloads; expected {}", + self.yielded, + self.metadata.payload_count + ) + })?; + + let block = &payload.payload_inner.payload_inner; + if block.block_number != self.expected_next_block { + bail!( + "payload fixture block sequence mismatch: expected block {}, got {}", + self.expected_next_block, + block.block_number + ); + } + if block.parent_hash != self.expected_parent_hash { + bail!( + "payload fixture parent hash mismatch for block {}: expected parent {}, got {}", + block.block_number, + fmt_hash(self.expected_parent_hash), + fmt_hash(block.parent_hash), + ); + } + + self.yielded = self.yielded.saturating_add(1); + self.expected_next_block = self + .expected_next_block + .checked_add(1) + .ok_or_else(|| eyre::eyre!("payload fixture block number overflow"))?; + self.expected_parent_hash = block.block_hash; + + Ok(Some(payload)) + } +} + +pub(crate) fn genesis_path(payload_dir: &Path) -> PathBuf { + payload_dir.join(GENESIS_FILE_NAME) +} + +pub(crate) fn metadata_path(payload_dir: &Path) -> PathBuf { + payload_dir.join(METADATA_FILE_NAME) +} + +pub(crate) fn payloads_path(payload_dir: &Path) -> PathBuf { + payload_dir.join(PAYLOADS_FILE_NAME) +} + +fn load_genesis(payload_dir: &Path) -> eyre::Result { + let path = genesis_path(payload_dir); + load_json_file(&path) +} + +fn load_metadata(payload_dir: &Path) -> eyre::Result { + if !payload_dir.is_dir() { + bail!( + "payload fixture directory does not exist: {}", + payload_dir.display() + ); + } + let path = metadata_path(payload_dir); + let metadata: PayloadFixtureMetadata = load_json_file(&path)?; + metadata.validate()?; + Ok(metadata) +} + +fn load_json_file(path: &Path) -> eyre::Result { + if !path.is_file() { + bail!("payload fixture file does not exist: {}", path.display()); + } + let reader = BufReader::new( + File::open(path).wrap_err_with(|| format!("failed to open {}", path.display()))?, + ); + serde_json::from_reader(reader).wrap_err_with(|| format!("failed to parse {}", path.display())) +} + +fn write_json_file(path: &Path, value: &T, label: &str) -> eyre::Result<()> { + let mut writer = BufWriter::new( + File::create(path).wrap_err_with(|| format!("failed to create {}", path.display()))?, + ); + serde_json::to_writer_pretty(&mut writer, value) + .wrap_err_with(|| format!("failed to serialize payload fixture {label}"))?; + writer + .write_all(b"\n") + .wrap_err_with(|| format!("failed to write payload fixture {label} newline"))?; + writer + .flush() + .wrap_err_with(|| format!("failed to flush payload fixture {label}"))?; + Ok(()) +} + +fn rename_into_place(from: &Path, to: &Path) -> eyre::Result<()> { + fs::rename(from, to).wrap_err_with(|| { + format!( + "failed to move payload fixture file into place: {} -> {}", + from.display(), + to.display() + ) + }) +} + +fn remove_file_if_exists(path: &Path) -> eyre::Result<()> { + if path.exists() { + fs::remove_file(path).wrap_err_with(|| format!("failed to remove {}", path.display()))?; + } + Ok(()) +} + +struct PayloadJsonlReader { + path: PathBuf, + reader: BufReader, + line_number: usize, +} + +impl PayloadJsonlReader { + fn new(path: &Path) -> eyre::Result { + let file = + File::open(path).wrap_err_with(|| format!("failed to open {}", path.display()))?; + Ok(Self { + path: path.to_path_buf(), + reader: BufReader::new(file), + line_number: 0, + }) + } + + fn next_payload(&mut self) -> eyre::Result> { + let mut line = String::new(); + let bytes_read = self + .reader + .read_line(&mut line) + .wrap_err_with(|| format!("failed to read {}", self.path.display()))?; + if bytes_read == 0 { + return Ok(None); + } + + self.line_number += 1; + let line = line.trim_end_matches(&['\r', '\n'][..]); + if line.is_empty() { + bail!( + "payload fixture contains an empty line at {}:{}", + self.path.display(), + self.line_number + ); + } + + let payload = serde_json::from_str(line).wrap_err_with(|| { + format!( + "failed to parse payload fixture entry at {}:{}", + self.path.display(), + self.line_number + ) + })?; + Ok(Some(payload)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tempfile::TempDir; + + fn test_genesis() -> Genesis { + Genesis::default() + } + + fn zero_address() -> String { + format!("0x{}", "00".repeat(20)) + } + + fn zero_bloom() -> String { + format!("0x{}", "00".repeat(256)) + } + + fn hash_hex(value: u64) -> String { + format!("0x{value:064x}") + } + + fn payload(block_number: u64, parent_hash: u64, block_hash: u64) -> ExecutionPayloadV3 { + serde_json::from_value(json!({ + "parentHash": hash_hex(parent_hash), + "feeRecipient": zero_address(), + "stateRoot": hash_hex(1_000 + block_number), + "receiptsRoot": hash_hex(2_000 + block_number), + "logsBloom": zero_bloom(), + "prevRandao": hash_hex(3_000 + block_number), + "blockNumber": format!("0x{block_number:x}"), + "gasLimit": "0x1c9c380", + "gasUsed": format!("0x{:x}", block_number * 1_000), + "timestamp": format!("0x{:x}", 10_000 + block_number), + "extraData": "0x", + "baseFeePerGas": "0x1", + "blockHash": hash_hex(block_hash), + "transactions": [], + "withdrawals": [], + "blobGasUsed": "0x0", + "excessBlobGas": "0x0" + })) + .unwrap() + } + + fn metadata_for( + first_payload: &ExecutionPayloadV3, + to_block: u64, + payload_count: u64, + ) -> PayloadFixtureMetadata { + PayloadFixtureMetadata { + from_block: first_payload.payload_inner.payload_inner.block_number, + to_block, + payload_count, + expected_parent: ExpectedParentBlock { + block_number: first_payload.payload_inner.payload_inner.block_number - 1, + block_hash: first_payload.payload_inner.payload_inner.parent_hash, + }, + } + } + + #[test] + fn metadata_round_trips_through_json() { + let first_payload = payload(5, 44, 55); + let metadata = metadata_for(&first_payload, 6, 2); + + let encoded = serde_json::to_string(&metadata).unwrap(); + let decoded: PayloadFixtureMetadata = serde_json::from_str(&encoded).unwrap(); + + assert_eq!(decoded, metadata); + } + + #[test] + fn metadata_validation_rejects_count_mismatch() { + let first_payload = payload(5, 44, 55); + let mut metadata = metadata_for(&first_payload, 6, 2); + metadata.payload_count = 3; + + let err = metadata.validate().unwrap_err(); + + assert_eq!( + err.to_string(), + "payload fixture payload_count mismatch: expected 2, got 3" + ); + } + + #[test] + fn payload_fixture_writer_and_reader_round_trip() { + let temp_dir = TempDir::new().unwrap(); + let payload1 = payload(7, 66, 77); + let payload2 = payload(8, 77, 88); + let metadata = metadata_for(&payload1, 8, 2); + + let mut writer = PayloadFixtureWriter::new(temp_dir.path()).unwrap(); + writer.write_payload(&payload1).unwrap(); + writer.write_payload(&payload2).unwrap(); + writer.finish(&metadata, &test_genesis()).unwrap(); + + let mut fixture = PayloadFixture::open(temp_dir.path()).unwrap(); + + assert_eq!(fixture.metadata(), &metadata); + assert_eq!(fixture.next_payload().unwrap(), Some(payload1)); + assert_eq!(fixture.next_payload().unwrap(), Some(payload2)); + assert_eq!(fixture.next_payload().unwrap(), None); + } + + #[test] + fn payload_fixture_open_fails_when_files_are_missing() { + let temp_dir = TempDir::new().unwrap(); + + let err = match PayloadFixture::open(temp_dir.path()) { + Ok(_) => panic!("expected missing fixture files to fail"), + Err(err) => err, + }; + + assert_eq!( + err.to_string(), + format!( + "payload fixture file does not exist: {}", + temp_dir.path().join(GENESIS_FILE_NAME).display() + ) + ); + } + + #[test] + fn payload_fixture_rejects_block_gaps() { + let temp_dir = TempDir::new().unwrap(); + let payload1 = payload(10, 99, 100); + let payload3 = payload(12, 100, 101); + let metadata = metadata_for(&payload1, 11, 2); + + let mut writer = PayloadFixtureWriter::new(temp_dir.path()).unwrap(); + writer.write_payload(&payload1).unwrap(); + writer.write_payload(&payload3).unwrap(); + writer.finish(&metadata, &test_genesis()).unwrap(); + + let mut fixture = PayloadFixture::open(temp_dir.path()).unwrap(); + assert_eq!(fixture.next_payload().unwrap(), Some(payload1)); + + let err = fixture.next_payload().unwrap_err(); + assert_eq!( + err.to_string(), + "payload fixture block sequence mismatch: expected block 11, got 12" + ); + } + + #[test] + fn payload_fixture_rejects_extra_payloads() { + let temp_dir = TempDir::new().unwrap(); + let payload1 = payload(20, 199, 200); + let payload2 = payload(21, 200, 201); + let metadata = metadata_for(&payload1, 20, 1); + + let mut writer = PayloadFixtureWriter::new(temp_dir.path()).unwrap(); + writer.write_payload(&payload1).unwrap(); + writer.write_payload(&payload2).unwrap(); + writer.finish(&metadata, &test_genesis()).unwrap(); + + let mut fixture = PayloadFixture::open(temp_dir.path()).unwrap(); + assert_eq!(fixture.next_payload().unwrap(), Some(payload1)); + + let err = fixture.next_payload().unwrap_err(); + assert_eq!( + err.to_string(), + "payload fixture contains more payloads than expected: first extra block is 21" + ); + } +} diff --git a/crates/engine-bench/src/bench/helpers.rs b/crates/engine-bench/src/bench/helpers.rs new file mode 100644 index 0000000..3ab9e8b --- /dev/null +++ b/crates/engine-bench/src/bench/helpers.rs @@ -0,0 +1,27 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +pub(crate) const SUMMARY_FILE_NAME: &str = "summary.csv"; + +pub(crate) fn duration_to_ms(duration: Duration) -> f64 { + duration.as_secs_f64() * 1_000.0 +} + +pub(crate) fn fmt_hash(value: T) -> String { + format!("{value:#x}") +} diff --git a/crates/engine-bench/src/bench/mod.rs b/crates/engine-bench/src/bench/mod.rs new file mode 100644 index 0000000..545223e --- /dev/null +++ b/crates/engine-bench/src/bench/mod.rs @@ -0,0 +1,31 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::cli::Command; + +mod context; +mod fixture; +mod helpers; +pub mod new_payload_fcu; +mod output; +mod prepare_payload; + +pub async fn run(command: Command) -> eyre::Result<()> { + match command { + Command::PreparePayload(args) => prepare_payload::run(args).await, + Command::NewPayloadFcu(args) => new_payload_fcu::run(args).await, + } +} diff --git a/crates/engine-bench/src/bench/new_payload_fcu.rs b/crates/engine-bench/src/bench/new_payload_fcu.rs new file mode 100644 index 0000000..5aa284c --- /dev/null +++ b/crates/engine-bench/src/bench/new_payload_fcu.rs @@ -0,0 +1,212 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ + context::BenchContext, + fixture::{PayloadFixture, PayloadFixtureMetadata}, + helpers::{duration_to_ms, fmt_hash, SUMMARY_FILE_NAME}, + output::{ + build_summary, throughput_mgas_per_s, throughput_tx_per_s, write_csv, CombinedLatencyRow, + CsvWriter, COMBINED_LATENCY_FILE_NAME, + }, +}; +use crate::cli::NewPayloadFcuArgs; +use arc_eth_engine::{json_structures::ExecutionBlock, rpc::ethereum_rpc::EthereumRPC}; +use eyre::{bail, Context}; +use std::time::Instant; +use tracing::info; + +pub async fn run(args: NewPayloadFcuArgs) -> eyre::Result<()> { + let context = BenchContext::new(&args.common, "new-payload-fcu")?; + + info!( + payload_dir = %args.payload.display(), + target_eth_rpc_url = args.target_eth_rpc_url, + engine = %context.transport(), + output_dir = %context.output_dir().display(), + "running new-payload-fcu benchmark" + ); + + let target_eth_rpc = context.ethereum_rpc(&args.target_eth_rpc_url, "target eth rpc")?; + let engine = context.engine().await?; + let mut payload_fixture = PayloadFixture::open(args.payload.as_path())?; + let metadata = payload_fixture.metadata().clone(); + verify_target_start_state(&target_eth_rpc, &metadata).await?; + + info!( + from_block = metadata.from_block, + to_block = metadata.to_block, + payload_count = metadata.payload_count, + "starting payload replay" + ); + + let benchmark_started = Instant::now(); + let row_capacity = metadata.payload_count.min(usize::MAX as u64) as usize; + let mut rows = Vec::with_capacity(row_capacity); + let mut csv_writer = CsvWriter::new(&context.output_dir().join(COMBINED_LATENCY_FILE_NAME))?; + let mut cumulative_gas = 0_u64; + let mut cumulative_txs = 0_u64; + + while let Some(payload) = payload_fixture.next_payload()? { + let block_hash = payload.payload_inner.payload_inner.block_hash; + let parent_hash = payload.payload_inner.payload_inner.parent_hash; + let tx_count = payload.payload_inner.payload_inner.transactions.len() as u64; + let gas_used = payload.payload_inner.payload_inner.gas_used; + let block_number = payload.payload_inner.payload_inner.block_number; + + let start = Instant::now(); + let status = engine + .new_payload(&payload, Vec::new(), parent_hash) + .await + .wrap_err_with(|| format!("engine_newPayloadV4 failed for block {block_number}"))?; + let new_payload_latency = start.elapsed(); + + if !status.is_valid() { + bail!("engine_newPayloadV4 returned non-valid status for block {block_number}: {status:?}"); + } + + let fcu_result = engine + .forkchoice_updated(block_hash, None) + .await + .wrap_err_with(|| { + format!("engine_forkchoiceUpdatedV3 failed for block {block_number}") + })?; + let total_latency = start.elapsed(); + let fcu_latency = total_latency.saturating_sub(new_payload_latency); + + if !fcu_result.payload_status.is_valid() { + bail!( + "engine_forkchoiceUpdatedV3 returned non-valid status for block {block_number}: {:?}", + fcu_result.payload_status + ); + } + + cumulative_gas = cumulative_gas.saturating_add(gas_used); + cumulative_txs = cumulative_txs.saturating_add(tx_count); + let elapsed = benchmark_started.elapsed(); + + let row = CombinedLatencyRow { + block_number, + block_hash: fmt_hash(block_hash), + tx_count, + gas_used, + new_payload_ms: duration_to_ms(new_payload_latency), + fcu_ms: duration_to_ms(fcu_latency), + total_ms: duration_to_ms(total_latency), + elapsed_ms: duration_to_ms(elapsed), + mgas_per_s: throughput_mgas_per_s(gas_used, total_latency), + tx_per_s: throughput_tx_per_s(tx_count, total_latency), + cumulative_mgas_per_s: throughput_mgas_per_s(cumulative_gas, elapsed), + cumulative_tx_per_s: throughput_tx_per_s(cumulative_txs, elapsed), + }; + csv_writer.write_row(&row)?; + rows.push(row); + } + + csv_writer.finish()?; + let wall_clock = benchmark_started.elapsed(); + + let summary = build_summary("new-payload-fcu", &rows, wall_clock)?; + write_csv(&context.output_dir().join(SUMMARY_FILE_NAME), &[summary])?; + + info!( + samples = rows.len(), + wall_clock_ms = duration_to_ms(wall_clock), + output_dir = %context.output_dir().display(), + "new-payload-fcu benchmark complete" + ); + + Ok(()) +} + +async fn verify_target_start_state( + target_rpc: &EthereumRPC, + metadata: &PayloadFixtureMetadata, +) -> eyre::Result<()> { + let target_latest_block = target_rpc + .get_block_by_number("latest") + .await + .wrap_err("failed to fetch latest block from target node")? + .ok_or_else(|| eyre::eyre!("latest block not found on target node"))?; + + ensure_target_start_state(&target_latest_block, metadata)?; + + info!( + target_block_number = target_latest_block.block_number, + target_block_hash = %fmt_hash(target_latest_block.block_hash), + "verified target node replay start state" + ); + + Ok(()) +} + +fn ensure_target_start_state( + target_latest_block: &ExecutionBlock, + metadata: &PayloadFixtureMetadata, +) -> eyre::Result<()> { + if target_latest_block.block_number != metadata.expected_parent.block_number + || target_latest_block.block_hash != metadata.expected_parent.block_hash + { + bail!( + "target node is not at the expected replay start state: expected parent block {} ({}) before replaying block {}, but target latest is block {} ({})", + metadata.expected_parent.block_number, + fmt_hash(metadata.expected_parent.block_hash), + metadata.from_block, + target_latest_block.block_number, + fmt_hash(target_latest_block.block_hash), + ); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bench::fixture::ExpectedParentBlock; + + #[test] + fn ensure_target_start_state_rejects_mismatch() { + let expected_hash = format!("0x{}", "01".repeat(32)).parse().unwrap(); + let actual_hash = format!("0x{}", "02".repeat(32)).parse().unwrap(); + let target_latest_block = ExecutionBlock { + block_hash: actual_hash, + block_number: 10, + parent_hash: format!("0x{}", "00".repeat(32)).parse().unwrap(), + timestamp: 123, + }; + let metadata = PayloadFixtureMetadata { + from_block: 11, + to_block: 12, + payload_count: 2, + expected_parent: ExpectedParentBlock { + block_number: 10, + block_hash: expected_hash, + }, + }; + + let err = ensure_target_start_state(&target_latest_block, &metadata).unwrap_err(); + + assert_eq!( + err.to_string(), + format!( + "target node is not at the expected replay start state: expected parent block 10 ({}) before replaying block 11, but target latest is block 10 ({})", + fmt_hash(expected_hash), + fmt_hash(actual_hash) + ) + ); + } +} diff --git a/crates/engine-bench/src/bench/output.rs b/crates/engine-bench/src/bench/output.rs new file mode 100644 index 0000000..dca2ee9 --- /dev/null +++ b/crates/engine-bench/src/bench/output.rs @@ -0,0 +1,313 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::helpers::duration_to_ms; +use eyre::{bail, Context}; +use serde::Serialize; +use std::{ + cmp::Ordering, + path::{Path, PathBuf}, + time::Duration, +}; + +pub(crate) const COMBINED_LATENCY_FILE_NAME: &str = "combined_latency.csv"; + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct CombinedLatencyRow { + pub block_number: u64, + pub block_hash: String, + pub tx_count: u64, + pub gas_used: u64, + pub new_payload_ms: f64, + pub fcu_ms: f64, + pub total_ms: f64, + pub elapsed_ms: f64, + pub mgas_per_s: f64, + pub tx_per_s: f64, + pub cumulative_mgas_per_s: f64, + pub cumulative_tx_per_s: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct SummaryRow { + pub mode: String, + pub samples: u64, + pub total_gas: u64, + pub total_txs: u64, + pub wall_clock_ms: f64, + pub execution_ms: f64, + pub avg_total_ms: f64, + pub avg_new_payload_ms: Option, + pub avg_fcu_ms: Option, + pub avg_mgas_per_s: f64, + pub avg_tx_per_s: f64, + pub p50_new_payload_ms: Option, + pub p95_new_payload_ms: Option, + pub p99_new_payload_ms: Option, + pub p50_fcu_ms: Option, + pub p95_fcu_ms: Option, + pub p99_fcu_ms: Option, + pub p50_total_ms: f64, + pub p95_total_ms: f64, + pub p99_total_ms: f64, +} + +pub(crate) trait BenchmarkRow { + fn gas_used(&self) -> u64; + fn tx_count(&self) -> u64; + fn total_ms(&self) -> f64; + fn new_payload_ms(&self) -> Option { + None + } + fn fcu_ms(&self) -> Option { + None + } +} + +impl BenchmarkRow for CombinedLatencyRow { + fn gas_used(&self) -> u64 { + self.gas_used + } + + fn tx_count(&self) -> u64 { + self.tx_count + } + + fn total_ms(&self) -> f64 { + self.total_ms + } + + fn new_payload_ms(&self) -> Option { + Some(self.new_payload_ms) + } + + fn fcu_ms(&self) -> Option { + Some(self.fcu_ms) + } +} + +pub(crate) fn throughput_mgas_per_s(gas_used: u64, duration: Duration) -> f64 { + let seconds = duration.as_secs_f64(); + if seconds == 0.0 { + return 0.0; + } + gas_used as f64 / seconds / 1_000_000.0 +} + +pub(crate) fn throughput_tx_per_s(tx_count: u64, duration: Duration) -> f64 { + let seconds = duration.as_secs_f64(); + if seconds == 0.0 { + return 0.0; + } + tx_count as f64 / seconds +} + +pub(crate) struct CsvWriter { + writer: csv::Writer, + path: PathBuf, +} + +impl CsvWriter { + pub(crate) fn new(path: &Path) -> eyre::Result { + let writer = csv::Writer::from_path(path) + .wrap_err_with(|| format!("failed to create {}", path.display()))?; + Ok(Self { + writer, + path: path.to_path_buf(), + }) + } + + pub(crate) fn write_row(&mut self, row: &T) -> eyre::Result<()> { + self.writer + .serialize(row) + .wrap_err_with(|| format!("failed to write row to {}", self.path.display())) + } + + pub(crate) fn finish(mut self) -> eyre::Result<()> { + self.writer + .flush() + .wrap_err_with(|| format!("failed to flush {}", self.path.display())) + } +} + +pub(crate) fn write_csv(path: &Path, rows: &[T]) -> eyre::Result<()> { + let mut writer = CsvWriter::new(path)?; + for row in rows { + writer.write_row(row)?; + } + writer.finish() +} + +pub(crate) fn build_summary( + mode: &str, + rows: &[T], + wall_clock: Duration, +) -> eyre::Result { + if rows.is_empty() { + bail!("cannot build a summary for an empty benchmark result set"); + } + + let total_gas = rows.iter().map(BenchmarkRow::gas_used).sum::(); + let total_txs = rows.iter().map(BenchmarkRow::tx_count).sum::(); + let mut total_latencies = rows.iter().map(BenchmarkRow::total_ms).collect::>(); + let mut new_payload_latencies = rows + .iter() + .filter_map(BenchmarkRow::new_payload_ms) + .collect::>(); + let mut fcu_latencies = rows + .iter() + .filter_map(BenchmarkRow::fcu_ms) + .collect::>(); + let execution_ms = total_latencies.iter().sum::(); + + sort_f64(&mut total_latencies); + sort_f64(&mut new_payload_latencies); + sort_f64(&mut fcu_latencies); + + let pct_sorted_opt = |sorted: &[f64], q: f64| -> Option { + (!sorted.is_empty()).then(|| percentile_sorted(sorted, q)) + }; + + Ok(SummaryRow { + mode: mode.to_owned(), + samples: rows.len() as u64, + total_gas, + total_txs, + wall_clock_ms: duration_to_ms(wall_clock), + execution_ms, + avg_total_ms: execution_ms / rows.len() as f64, + avg_new_payload_ms: average(&new_payload_latencies), + avg_fcu_ms: average(&fcu_latencies), + avg_mgas_per_s: throughput_mgas_per_s(total_gas, wall_clock), + avg_tx_per_s: throughput_tx_per_s(total_txs, wall_clock), + p50_new_payload_ms: pct_sorted_opt(&new_payload_latencies, 0.50), + p95_new_payload_ms: pct_sorted_opt(&new_payload_latencies, 0.95), + p99_new_payload_ms: pct_sorted_opt(&new_payload_latencies, 0.99), + p50_fcu_ms: pct_sorted_opt(&fcu_latencies, 0.50), + p95_fcu_ms: pct_sorted_opt(&fcu_latencies, 0.95), + p99_fcu_ms: pct_sorted_opt(&fcu_latencies, 0.99), + p50_total_ms: percentile_sorted(&total_latencies, 0.50), + p95_total_ms: percentile_sorted(&total_latencies, 0.95), + p99_total_ms: percentile_sorted(&total_latencies, 0.99), + }) +} + +fn average(values: &[f64]) -> Option { + (!values.is_empty()).then(|| values.iter().sum::() / values.len() as f64) +} + +fn sort_f64(values: &mut [f64]) { + values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); +} + +fn percentile_sorted(sorted: &[f64], quantile: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + if sorted.len() == 1 { + return sorted[0]; + } + let rank = quantile.clamp(0.0, 1.0) * (sorted.len().saturating_sub(1)) as f64; + let lower_index = rank.floor() as usize; + let upper_index = rank.ceil() as usize; + if lower_index == upper_index { + sorted[lower_index] + } else { + sorted[lower_index] + + (sorted[upper_index] - sorted[lower_index]) * (rank - lower_index as f64) + } +} + +#[cfg(test)] +fn percentile(mut values: Vec, quantile: f64) -> f64 { + sort_f64(&mut values); + percentile_sorted(&values, quantile) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percentile_interpolates_between_samples() { + let actual = percentile(vec![10.0, 20.0, 30.0, 40.0], 0.75); + assert_eq!(actual, 32.5); + } + + #[test] + fn summary_uses_wall_clock_for_average_throughput() { + let row = CombinedLatencyRow { + block_number: 1, + block_hash: "0x01".to_string(), + tx_count: 2, + gas_used: 2_000_000, + new_payload_ms: 100.0, + fcu_ms: 50.0, + total_ms: 150.0, + elapsed_ms: 150.0, + mgas_per_s: 0.0, + tx_per_s: 0.0, + cumulative_mgas_per_s: 0.0, + cumulative_tx_per_s: 0.0, + }; + let summary = build_summary("new-payload-fcu", &[row], Duration::from_millis(200)).unwrap(); + assert_eq!(summary.execution_ms, 150.0); + assert_eq!(summary.avg_total_ms, 150.0); + assert_eq!(summary.avg_new_payload_ms, Some(100.0)); + assert_eq!(summary.avg_fcu_ms, Some(50.0)); + assert_eq!(summary.avg_mgas_per_s, 10.0); + } + + #[test] + fn summary_includes_component_latency_percentiles() { + let rows = [ + CombinedLatencyRow { + block_number: 1, + block_hash: "0x01".to_string(), + tx_count: 1, + gas_used: 1_000_000, + new_payload_ms: 10.0, + fcu_ms: 1.0, + total_ms: 11.0, + elapsed_ms: 11.0, + mgas_per_s: 0.0, + tx_per_s: 0.0, + cumulative_mgas_per_s: 0.0, + cumulative_tx_per_s: 0.0, + }, + CombinedLatencyRow { + block_number: 2, + block_hash: "0x02".to_string(), + tx_count: 1, + gas_used: 1_000_000, + new_payload_ms: 20.0, + fcu_ms: 2.0, + total_ms: 22.0, + elapsed_ms: 33.0, + mgas_per_s: 0.0, + tx_per_s: 0.0, + cumulative_mgas_per_s: 0.0, + cumulative_tx_per_s: 0.0, + }, + ]; + + let summary = build_summary("new-payload-fcu", &rows, Duration::from_millis(33)).unwrap(); + assert_eq!(summary.p50_new_payload_ms, Some(15.0)); + assert_eq!(summary.p95_new_payload_ms, Some(19.5)); + assert_eq!(summary.p50_fcu_ms, Some(1.5)); + assert_eq!(summary.p95_fcu_ms, Some(1.95)); + } +} diff --git a/crates/engine-bench/src/bench/prepare_payload.rs b/crates/engine-bench/src/bench/prepare_payload.rs new file mode 100644 index 0000000..69c5920 --- /dev/null +++ b/crates/engine-bench/src/bench/prepare_payload.rs @@ -0,0 +1,164 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ + context::ethereum_rpc_client, + fixture::{ExpectedParentBlock, PayloadFixtureMetadata, PayloadFixtureWriter}, + helpers::fmt_hash, +}; +use crate::cli::PreparePayloadArgs; +use arc_eth_engine::{engine::EthereumAPI, rpc::ethereum_rpc::EthereumRPC}; +use arc_execution_config::chainspec::ArcChainSpecParser; +use eyre::{bail, Context}; +use reth_cli::chainspec::ChainSpecParser; +use std::time::Duration; +use tracing::info; + +pub async fn run(args: PreparePayloadArgs) -> eyre::Result<()> { + if args.from > args.to { + bail!("--from must be less than or equal to --to"); + } + if args.batch_size == 0 { + bail!("--batch-size must be greater than 0"); + } + + let chain_spec = ArcChainSpecParser::parse(&args.chain) + .wrap_err_with(|| format!("failed to parse chain spec: {}", args.chain))?; + let genesis = chain_spec.inner.genesis.clone(); + + info!( + chain = args.chain, + chain_id = genesis.config.chain_id, + source_rpc_url = args.source_rpc_url, + output_dir = %args.output_dir.display(), + from_block = args.from, + to_block = args.to, + "preparing payload fixture" + ); + + let source_rpc = ethereum_rpc_client( + &args.source_rpc_url, + "source rpc", + Duration::from_millis(args.eth_rpc_timeout_ms), + )?; + let expected_parent = fetch_expected_parent(&source_rpc, args.from).await?; + let payload_count = write_payload_fixture( + &source_rpc, + &args.output_dir, + &expected_parent, + &genesis, + args.from, + args.to, + args.batch_size, + ) + .await?; + + info!( + output_dir = %args.output_dir.display(), + payload_count, + "payload fixture prepared" + ); + + Ok(()) +} + +async fn fetch_expected_parent( + source_rpc: &EthereumRPC, + from_block: u64, +) -> eyre::Result { + let expected_parent_block_number = from_block + .checked_sub(1) + .ok_or_else(|| eyre::eyre!("from_block must be greater than 0"))?; + let expected_parent_block = source_rpc + .get_block_by_number(&format!("0x{expected_parent_block_number:x}")) + .await + .wrap_err_with(|| { + format!("failed to fetch source parent block {expected_parent_block_number}") + })? + .ok_or_else(|| { + eyre::eyre!("source parent block {expected_parent_block_number} not found") + })?; + + Ok(ExpectedParentBlock { + block_number: expected_parent_block.block_number, + block_hash: expected_parent_block.block_hash, + }) +} + +async fn write_payload_fixture( + source_rpc: &EthereumRPC, + output_dir: &std::path::Path, + expected_parent: &ExpectedParentBlock, + genesis: &alloy_genesis::Genesis, + from: u64, + to: u64, + batch_size: usize, +) -> eyre::Result { + let mut writer = PayloadFixtureWriter::new(output_dir)?; + let mut payload_count = 0_u64; + let mut expected_parent_hash = expected_parent.block_hash; + + for chunk_start in (from..=to).step_by(batch_size) { + let chunk_end = chunk_start + .saturating_add(batch_size as u64) + .saturating_sub(1) + .min(to); + let block_numbers = (chunk_start..=chunk_end) + .map(|block_number| format!("0x{block_number:x}")) + .collect::>(); + let chunk = + ::get_execution_payloads(source_rpc, &block_numbers) + .await + .wrap_err_with(|| { + format!("failed to fetch source blocks {chunk_start}..={chunk_end}") + })?; + + for (idx, maybe_payload) in chunk.into_iter().enumerate() { + let expected_block = chunk_start + idx as u64; + let payload = maybe_payload + .ok_or_else(|| eyre::eyre!("source block {expected_block} not found"))?; + let block = &payload.payload_inner.payload_inner; + if block.block_number != expected_block { + bail!( + "source payload sequence mismatch: expected block {expected_block}, got {}", + block.block_number + ); + } + if block.parent_hash != expected_parent_hash { + bail!( + "source payload parent hash mismatch for block {}: expected parent {}, got {}", + block.block_number, + fmt_hash(expected_parent_hash), + fmt_hash(block.parent_hash), + ); + } + + writer.write_payload(&payload)?; + expected_parent_hash = block.block_hash; + payload_count = payload_count.saturating_add(1); + } + } + + let metadata = PayloadFixtureMetadata { + from_block: from, + to_block: to, + payload_count, + expected_parent: expected_parent.clone(), + }; + writer.finish(&metadata, genesis)?; + + Ok(payload_count) +} diff --git a/crates/engine-bench/src/cli.rs b/crates/engine-bench/src/cli.rs new file mode 100644 index 0000000..11b31be --- /dev/null +++ b/crates/engine-bench/src/cli.rs @@ -0,0 +1,94 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::{Args, Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Debug, Parser)] +#[command( + name = "arc-engine-bench", + version = arc_version::SHORT_VERSION, + long_version = arc_version::LONG_VERSION, + about = "Benchmark Arc Engine API block import via newPayload + forkchoiceUpdated" +)] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Debug, Subcommand)] +pub enum Command { + /// Prepare a local payload fixture directory from historical source blocks. + PreparePayload(PreparePayloadArgs), + /// Replay historical blocks into a target Arc node with newPayload + forkchoiceUpdated. + NewPayloadFcu(NewPayloadFcuArgs), +} + +#[derive(Debug, Args, Clone)] +pub struct CommonArgs { + /// Engine API IPC socket path. Mutually exclusive with --engine-rpc-url / --jwt-secret. + #[arg(long, value_name = "ENGINE_IPC", group = "engine_transport")] + pub engine_ipc: Option, + /// Authenticated Engine API HTTP endpoint. Requires --jwt-secret. + #[arg(long, value_name = "ENGINE_RPC_URL", group = "engine_transport")] + pub engine_rpc_url: Option, + /// JWT secret used to authenticate Engine API requests (required with --engine-rpc-url). + #[arg(long = "jwt-secret", value_name = "PATH", requires = "engine_rpc_url")] + pub jwt_secret: Option, + /// Timeout for Ethereum JSON-RPC requests used by this command, in milliseconds (must be >= 1). + #[arg(long, value_name = "MILLISECONDS", default_value_t = 10_000, value_parser = clap::value_parser!(u64).range(1..))] + pub eth_rpc_timeout_ms: u64, + /// Output directory for CSV artifacts. Defaults to target/engine-bench/-. + #[arg(long, short, value_name = "OUTPUT_DIR")] + pub output: Option, +} + +#[derive(Debug, Args, Clone)] +pub struct PreparePayloadArgs { + /// Chain name or path to a genesis JSON file. Used to store the genesis config in the fixture. + #[arg(long, default_value = "arc-localdev")] + pub chain: String, + /// Read historical payloads from this source RPC endpoint. + #[arg(long, value_name = "SOURCE_RPC_URL")] + pub source_rpc_url: String, + /// First source block number to include in the fixture (must be >= 1). + #[arg(long, value_name = "FROM_BLOCK", value_parser = clap::value_parser!(u64).range(1..))] + pub from: u64, + /// Last source block number to include in the fixture, inclusive. + #[arg(long, value_name = "TO_BLOCK")] + pub to: u64, + /// Batch size for source RPC block fetching. + #[arg(long, value_name = "BATCH_SIZE", default_value_t = 20)] + pub batch_size: usize, + /// Timeout for source Ethereum JSON-RPC requests, in milliseconds (must be >= 1). + #[arg(long, value_name = "MILLISECONDS", default_value_t = 10_000, value_parser = clap::value_parser!(u64).range(1..))] + pub eth_rpc_timeout_ms: u64, + /// Output directory for the prepared payload fixture. + #[arg(long, value_name = "OUTPUT_DIR")] + pub output_dir: PathBuf, +} + +#[derive(Debug, Args, Clone)] +pub struct NewPayloadFcuArgs { + #[command(flatten)] + pub common: CommonArgs, + /// Regular RPC endpoint of the target Arc execution node. Used to verify the target head before replay starts. + #[arg(long, value_name = "TARGET_ETH_RPC_URL")] + pub target_eth_rpc_url: String, + /// Payload fixture directory containing genesis.json, metadata.json, and payloads.jsonl. + #[arg(long, value_name = "PAYLOAD_DIR")] + pub payload: PathBuf, +} diff --git a/crates/engine-bench/src/lib.rs b/crates/engine-bench/src/lib.rs new file mode 100644 index 0000000..e4be6fe --- /dev/null +++ b/crates/engine-bench/src/lib.rs @@ -0,0 +1,27 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] + +mod bench; +pub mod cli; +pub use bench::new_payload_fcu; + +use cli::Command; + +pub async fn run(command: Command) -> eyre::Result<()> { + bench::run(command).await +} diff --git a/crates/engine-bench/src/main.rs b/crates/engine-bench/src/main.rs new file mode 100644 index 0000000..d2f4de2 --- /dev/null +++ b/crates/engine-bench/src/main.rs @@ -0,0 +1,45 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arc_engine_bench::cli::Cli; +use clap::Parser; +use color_eyre::eyre::Context; +use std::io::IsTerminal; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + color_eyre::install()?; + + let filter = EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) + .from_env() + .wrap_err("failed to initialize tracing filter")?; + let subscriber = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_ansi(std::io::stdout().is_terminal()) + .finish(); + tracing::subscriber::set_global_default(subscriber) + .wrap_err("failed to install tracing subscriber")?; + + tracing::info!( + version = arc_version::SHORT_VERSION, + commit = arc_version::GIT_COMMIT_HASH, + "arc-engine-bench starting" + ); + + arc_engine_bench::run(Cli::parse().command).await +} diff --git a/crates/eth-engine/src/constants.rs b/crates/eth-engine/src/constants.rs index 3be57b6..5adc090 100644 --- a/crates/eth-engine/src/constants.rs +++ b/crates/eth-engine/src/constants.rs @@ -74,6 +74,13 @@ pub const ENGINE_EXCHANGE_CAPABILITIES_RETRY_IPC: backon::FibonacciBuilder = .with_max_delay(Duration::from_secs(1)) .with_max_times(30); +// Retry policy for IPC Engine API calls (newPayload, forkchoiceUpdated, getPayload). +// All three are idempotent per the Engine API spec. +pub const ENGINE_API_RETRY_IPC: backon::FibonacciBuilder = backon::FibonacciBuilder::new() + .with_min_delay(Duration::from_millis(200)) + .with_max_delay(Duration::from_secs(2)) + .with_max_times(5); + // Engine API retries for RPC -- First call to `reth`, keep retrying indefinitely. pub const ENGINE_EXCHANGE_CAPABILITIES_RETRY_RPC: backon::ConstantBuilder = backon::ConstantBuilder::new() diff --git a/crates/eth-engine/src/engine.rs b/crates/eth-engine/src/engine.rs index 73510cd..c9872f3 100644 --- a/crates/eth-engine/src/engine.rs +++ b/crates/eth-engine/src/engine.rs @@ -22,6 +22,7 @@ use std::{ sync::{Arc, OnceLock}, time::{SystemTime, UNIX_EPOCH}, }; +use tokio::sync::watch; use async_trait::async_trait; use tracing::debug; @@ -113,12 +114,30 @@ pub struct Engine(Arc); impl Engine { /// Create a new engine using IPC. pub async fn new_ipc(execution_socket: &str, eth_socket: &str) -> eyre::Result { - let api = Box::new(EngineIPC::new(execution_socket).await?); - let eth = Box::new(EthereumIPC::new(eth_socket).await?); + let api = EngineIPC::new(execution_socket).await?; + let eth = EthereumIPC::new(eth_socket).await?; + + let api_disconnect = api.on_disconnect(); + let eth_disconnect = eth.on_disconnect(); + + let (disconnect_tx, disconnect_rx) = watch::channel(false); + tokio::spawn(async move { + tokio::select! { + _ = api_disconnect => {}, + _ = eth_disconnect => {}, + } + disconnect_tx.send(true).ok(); + }); + let sub_endpoint = SubscriptionEndpoint::Ipc { socket_path: eth_socket.to_owned(), }; - Ok(Self(Arc::new(Inner::new(api, eth, Some(sub_endpoint))))) + Ok(Self(Arc::new(Inner::new( + Box::new(api), + Box::new(eth), + Some(sub_endpoint), + Some(disconnect_rx), + )))) } /// Create a new engine using RPC. @@ -141,12 +160,26 @@ impl Engine { eth.check_connectivity().await?; let sub_endpoint = ws_endpoint.map(|url| SubscriptionEndpoint::Ws { url }); - Ok(Self(Arc::new(Inner::new(api, eth, sub_endpoint)))) + Ok(Self(Arc::new(Inner::new(api, eth, sub_endpoint, None)))) } /// Create a new engine with custom API implementations. pub fn new(api: Box, eth: Box) -> Self { - Self(Arc::new(Inner::new(api, eth, None))) + Self(Arc::new(Inner::new(api, eth, None, None))) + } + + /// Resolves when either IPC connection to the EL closes. + /// + /// Stays pending forever for non-IPC engines (RPC, mock), so calling code can + /// unconditionally `select!` on this without special-casing the transport. + pub async fn wait_for_disconnect(&self) { + match &self.0.disconnect_rx { + // wait_for checks the current value first, so late subscribers see a prior disconnect. + Some(rx) => { + rx.clone().wait_for(|&v| v).await.ok(); + } + None => std::future::pending().await, + } } /// Set the function that determines whether Osaka is active at a given timestamp. @@ -252,6 +285,8 @@ pub struct Inner { /// Set after construction via [`Engine::set_is_osaka_active`]. /// When `None`, defaults to `false` (use V4). is_osaka_active: OnceLock, + /// Set to `true` when either IPC connection to the EL drops. `None` for non-IPC engines. + disconnect_rx: Option>, } impl Inner { @@ -259,12 +294,14 @@ impl Inner { api: Box, eth: Box, subscription_endpoint: Option, + disconnect_rx: Option>, ) -> Self { Self { api, eth, subscription_endpoint, is_osaka_active: OnceLock::new(), + disconnect_rx, } } @@ -604,8 +641,11 @@ impl EthereumAPI for Box { #[cfg(test)] mod tests { + use std::time::Duration; + use arc_execution_config::chain_ids::*; use rstest::rstest; + use tokio::time::timeout; use super::*; @@ -616,6 +656,32 @@ mod tests { ) } + fn engine_with_watch() -> (Engine, watch::Sender) { + let (tx, rx) = watch::channel(false); + let engine = Engine(Arc::new(Inner::new( + Box::new(MockEngineAPI::new()), + Box::new(MockEthereumAPI::new()), + None, + Some(rx), + ))); + (engine, tx) + } + + /// Bind a silent IPC server at `path`. Accepts one connection and holds it open + /// until the returned sender is dropped (or sends), then drops the stream. + async fn start_silent_ipc_server(path: &str) -> tokio::sync::oneshot::Sender<()> { + use tokio::net::UnixListener; + let listener = UnixListener::bind(path).unwrap(); + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + if let Ok((stream, _)) = listener.accept().await { + let _ = rx.await; + drop(stream); + } + }); + tx + } + #[test] fn test_timestamp_now() { let now = Engine::timestamp_now(); @@ -693,4 +759,90 @@ mod tests { "use_v5(0) for chain_id {chain_id}" ); } + + #[tokio::test] + async fn wait_for_disconnect_never_resolves_for_mock_engine() { + let engine = mock_engine(); + let result = timeout(Duration::from_millis(10), engine.wait_for_disconnect()).await; + assert!(result.is_err(), "mock engine should never disconnect"); + } + + #[tokio::test] + async fn wait_for_disconnect_resolves_when_signalled() { + let (engine, tx) = engine_with_watch(); + tx.send(true).unwrap(); + timeout(Duration::from_millis(100), engine.wait_for_disconnect()) + .await + .expect("should resolve after send(true)"); + } + + #[tokio::test] + async fn wait_for_disconnect_resolves_when_sender_dropped() { + let (engine, tx) = engine_with_watch(); + drop(tx); // sender dropped without sending true — Err(RecvError) path + timeout(Duration::from_millis(10), engine.wait_for_disconnect()) + .await + .expect("should resolve when sender is dropped"); + } + + #[tokio::test] + async fn wait_for_disconnect_resolves_if_already_disconnected() { + let (engine, tx) = engine_with_watch(); + // Signal disconnect before anyone calls wait_for_disconnect. + tx.send(true).unwrap(); + drop(tx); + // Late subscriber must see the already-set state immediately. + timeout(Duration::from_millis(10), engine.wait_for_disconnect()) + .await + .expect("late subscriber should see already-disconnected state"); + } + + #[tokio::test] + async fn new_ipc_disconnect_fires_when_connection_closes() { + let tmp = tempfile::TempDir::new().unwrap(); + let engine_sock = tmp + .path() + .join("engine.sock") + .to_string_lossy() + .into_owned(); + let eth_sock = tmp.path().join("eth.sock").to_string_lossy().into_owned(); + + let close_engine = start_silent_ipc_server(&engine_sock).await; + let _close_eth = start_silent_ipc_server(ð_sock).await; + + let engine = Engine::new_ipc(&engine_sock, ð_sock) + .await + .expect("should connect to mock IPC servers"); + + // Drop closes the sender → receiver Err → stream dropped → client sees EOF + drop(close_engine); + + timeout(Duration::from_millis(500), engine.wait_for_disconnect()) + .await + .expect("disconnect should fire when engine IPC connection closes"); + } + + #[tokio::test] + async fn new_ipc_disconnect_fires_for_eth_socket() { + let tmp = tempfile::TempDir::new().unwrap(); + let engine_sock = tmp + .path() + .join("engine2.sock") + .to_string_lossy() + .into_owned(); + let eth_sock = tmp.path().join("eth2.sock").to_string_lossy().into_owned(); + + let _close_engine = start_silent_ipc_server(&engine_sock).await; + let close_eth = start_silent_ipc_server(ð_sock).await; + + let engine = Engine::new_ipc(&engine_sock, ð_sock) + .await + .expect("should connect to mock IPC servers"); + + drop(close_eth); + + timeout(Duration::from_millis(500), engine.wait_for_disconnect()) + .await + .expect("disconnect should fire when eth IPC connection closes"); + } } diff --git a/crates/eth-engine/src/ipc/engine_ipc.rs b/crates/eth-engine/src/ipc/engine_ipc.rs index 09c8872..7a65c0d 100644 --- a/crates/eth-engine/src/ipc/engine_ipc.rs +++ b/crates/eth-engine/src/ipc/engine_ipc.rs @@ -31,7 +31,6 @@ use crate::capabilities::EngineCapabilities; use crate::constants::*; use crate::engine::EngineAPI; use crate::ipc::ipc_builder::Ipc; -use crate::retry::NoRetry; /// Engine API client for connecting to Engine IPC via Unix Domain Socket. pub struct EngineIPC { @@ -56,6 +55,14 @@ impl EngineIPC { Ok(Self { ipc }) } + /// Returns a future that resolves when the IPC connection closes. + pub fn on_disconnect(&self) -> impl std::future::Future + 'static { + let client = self.ipc.client_arc(); + async move { + let _ = client.on_disconnect().await; + } + } + /// Send an RPC request to the Engine RPC endpoint via IPC. pub async fn rpc_request( &self, @@ -110,7 +117,7 @@ impl EngineAPI for EngineIPC { ENGINE_FORKCHOICE_UPDATED_V3, params, ENGINE_FORKCHOICE_UPDATED_TIMEOUT, - NoRetry, + ENGINE_API_RETRY_IPC.build(), ) .await } @@ -130,7 +137,7 @@ impl EngineAPI for EngineIPC { ENGINE_GET_PAYLOAD_V5, rpc_params![payload_id], ENGINE_GET_PAYLOAD_TIMEOUT, - NoRetry, + ENGINE_API_RETRY_IPC.build(), ) .await?; Ok(execution_payload) @@ -140,7 +147,7 @@ impl EngineAPI for EngineIPC { ENGINE_GET_PAYLOAD_V4, rpc_params![payload_id], ENGINE_GET_PAYLOAD_TIMEOUT, - NoRetry, + ENGINE_API_RETRY_IPC.build(), ) .await?; Ok(envelope_inner.execution_payload) @@ -169,7 +176,7 @@ impl EngineAPI for EngineIPC { ENGINE_NEW_PAYLOAD_V4, params, ENGINE_NEW_PAYLOAD_TIMEOUT, - NoRetry, + ENGINE_API_RETRY_IPC.build(), ) .await } diff --git a/crates/eth-engine/src/ipc/ethereum_ipc.rs b/crates/eth-engine/src/ipc/ethereum_ipc.rs index bdd409b..4376d1c 100644 --- a/crates/eth-engine/src/ipc/ethereum_ipc.rs +++ b/crates/eth-engine/src/ipc/ethereum_ipc.rs @@ -103,6 +103,14 @@ impl EthereumIPC { Ok(Self { ipc }) } + /// Returns a future that resolves when the IPC connection closes. + pub fn on_disconnect(&self) -> impl std::future::Future + 'static { + let client = self.ipc.client_arc(); + async move { + let _ = client.on_disconnect().await; + } + } + /// Send an RPC request to the Ethereum RPC endpoint via IPC. pub async fn rpc_request( &self, diff --git a/crates/eth-engine/src/ipc/ipc_builder.rs b/crates/eth-engine/src/ipc/ipc_builder.rs index 3dde18c..d52acc8 100644 --- a/crates/eth-engine/src/ipc/ipc_builder.rs +++ b/crates/eth-engine/src/ipc/ipc_builder.rs @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::time::Duration; use backon::{Backoff, Retryable}; @@ -36,7 +37,7 @@ use crate::rpc::EngineApiRpcError; /// Common IPC client functionality for connecting to IPC endpoints via Unix Domain Socket. #[derive(Debug)] pub struct Ipc { - client: Client, + client: Arc, socket_path: String, } @@ -57,7 +58,7 @@ impl Ipc { info!("🟢 Connected to IPC: {}", socket_path); Ok(Self { - client, + client: Arc::new(client), socket_path: socket_path.to_string(), }) } @@ -143,6 +144,14 @@ impl Ipc { pub fn socket_path(&self) -> &str { &self.socket_path } + + /// Returns a cloned `Arc` reference to the underlying client. + /// + /// Callers can use this to monitor connection lifetime independently of the + /// `Ipc` owner (e.g. `arc_client.on_disconnect().await`). + pub fn client_arc(&self) -> Arc { + self.client.clone() + } } use crate::retry::NoRetry; diff --git a/crates/eth-engine/src/persistence_meter.rs b/crates/eth-engine/src/persistence_meter.rs index 98b1a1d..445e10d 100644 --- a/crates/eth-engine/src/persistence_meter.rs +++ b/crates/eth-engine/src/persistence_meter.rs @@ -381,7 +381,11 @@ async fn reconnect( "Persistence meter: reconnection failed; retrying" ); tokio::time::sleep(backoff).await; - backoff = (backoff * 2).min(MAX_RECONNECT_BACKOFF); + // backoff <= MAX_RECONNECT_BACKOFF (60s); *2 fits in Duration + #[allow(clippy::arithmetic_side_effects)] + { + backoff = (backoff * 2).min(MAX_RECONNECT_BACKOFF); + } } } } @@ -414,17 +418,22 @@ async fn connect_client(endpoint: &SubscriptionEndpoint) -> eyre::Result .build(socket_path) .await .wrap_err_with(|| format!("Failed to connect to IPC socket {socket_path}")), - SubscriptionEndpoint::Ws { url } => WsClientBuilder::default() - .request_timeout(REQUEST_TIMEOUT) - .connection_timeout(CONNECT_TIMEOUT) - .enable_ws_ping( - PingConfig::new() - .ping_interval(WS_PING_INTERVAL) - .inactive_limit(WS_PING_INTERVAL * 2), - ) - .build(url) - .await - .wrap_err_with(|| format!("Failed to connect to WebSocket endpoint {url}")), + SubscriptionEndpoint::Ws { url } => { + // 30s * 2 = 60s — fits in Duration + #[allow(clippy::arithmetic_side_effects)] + let ws_inactive_limit = WS_PING_INTERVAL * 2; + WsClientBuilder::default() + .request_timeout(REQUEST_TIMEOUT) + .connection_timeout(CONNECT_TIMEOUT) + .enable_ws_ping( + PingConfig::new() + .ping_interval(WS_PING_INTERVAL) + .inactive_limit(ws_inactive_limit), + ) + .build(url) + .await + .wrap_err_with(|| format!("Failed to connect to WebSocket endpoint {url}")) + } } } diff --git a/crates/eth-engine/src/rpc/ethereum_rpc.rs b/crates/eth-engine/src/rpc/ethereum_rpc.rs index b82637e..f0a5d16 100644 --- a/crates/eth-engine/src/rpc/ethereum_rpc.rs +++ b/crates/eth-engine/src/rpc/ethereum_rpc.rs @@ -70,14 +70,27 @@ fn abi_get_consensus_params_params_rpc(block_height: u64) -> eyre::Result pub struct EthereumRPC { client: Client, url: Url, + default_timeout: Duration, + batch_request_timeout: Duration, } impl EthereumRPC { /// Create a new `EthereumRPC` struct given the URL. pub fn new(url: Url) -> eyre::Result { + Self::new_with_timeouts(url, ETH_DEFAULT_TIMEOUT, ETH_BATCH_REQUEST_TIMEOUT) + } + + /// Create a new `EthereumRPC` struct given the URL and request timeouts. + pub fn new_with_timeouts( + url: Url, + default_timeout: Duration, + batch_request_timeout: Duration, + ) -> eyre::Result { Ok(Self { client: Client::builder().build()?, url, + default_timeout, + batch_request_timeout, }) } @@ -110,7 +123,7 @@ impl EthereumRPC { /// Probe the RPC server with `net_listening` to confirm /// it is accepting requests. pub async fn check_connectivity(&self) -> eyre::Result { - self.rpc_request("net_listening", json!([]), ETH_DEFAULT_TIMEOUT) + self.rpc_request("net_listening", json!([]), self.default_timeout) .await } @@ -118,7 +131,7 @@ impl EthereumRPC { pub async fn get_chain_id(&self) -> eyre::Result { self.build_rpc_request("eth_chainId") .params(json!([])) - .timeout(ETH_DEFAULT_TIMEOUT) + .timeout(self.default_timeout) .retry(ETH_CALL_RETRY.build()) .send() .await @@ -130,7 +143,7 @@ impl EthereumRPC { .rpc_request( "eth_getBlockByNumber", json!(["0x0", false]), - ETH_DEFAULT_TIMEOUT, + self.default_timeout, ) .await?; @@ -145,7 +158,7 @@ impl EthereumRPC { let result: String = self .build_rpc_request("eth_call") .params(params) - .timeout(ETH_DEFAULT_TIMEOUT) + .timeout(self.default_timeout) .retry(ETH_CALL_RETRY.build()) .send() .await @@ -169,7 +182,7 @@ impl EthereumRPC { let result: String = self .build_rpc_request("eth_call") .params(params) - .timeout(ETH_DEFAULT_TIMEOUT) + .timeout(self.default_timeout) .retry(ETH_CALL_RETRY.build()) .send() .await @@ -191,7 +204,7 @@ impl EthereumRPC { ) -> eyre::Result> { let return_full_transaction_objects = false; let params = json!([block_number, return_full_transaction_objects]); - self.rpc_request("eth_getBlockByNumber", params, ETH_DEFAULT_TIMEOUT) + self.rpc_request("eth_getBlockByNumber", params, self.default_timeout) .await } @@ -225,7 +238,7 @@ impl EthereumRPC { .client .post(self.url.clone()) .json(&batch_requests) - .timeout(ETH_BATCH_REQUEST_TIMEOUT) + .timeout(self.batch_request_timeout) .send() .await .wrap_err("Failed to send RPC batch request")?; @@ -250,6 +263,7 @@ impl EthereumRPC { continue; }; + #[allow(clippy::cast_possible_truncation)] // bounded by block_numbers.len() below let id = id as usize; if id >= block_numbers.len() { debug!( @@ -297,13 +311,13 @@ impl EthereumRPC { /// Get the status of the transaction pool. pub async fn txpool_status(&self) -> eyre::Result { - self.rpc_request("txpool_status", json!([]), ETH_DEFAULT_TIMEOUT) + self.rpc_request("txpool_status", json!([]), self.default_timeout) .await } /// Get the contents of the transaction pool. pub async fn txpool_inspect(&self) -> eyre::Result { - self.rpc_request("txpool_inspect", json!([]), ETH_DEFAULT_TIMEOUT) + self.rpc_request("txpool_inspect", json!([]), self.default_timeout) .await } } @@ -474,10 +488,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let result = ethereum_rpc.get_genesis_block().await.unwrap(); assert_eq!(result.block_hash, genesis_hash.parse::().unwrap()); @@ -499,10 +510,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let err = ethereum_rpc.get_genesis_block().await.unwrap_err(); assert!(err.to_string().contains("Genesis block not found")); @@ -560,10 +568,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let block_numbers = vec!["0x1".to_string(), "0x2".to_string()]; let result = ethereum_rpc @@ -628,10 +633,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let block_numbers = vec!["0x1".to_string(), "0x999".to_string()]; let result = ethereum_rpc @@ -678,10 +680,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let block_numbers = vec!["0x1".to_string()]; let result = ethereum_rpc @@ -724,10 +723,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let block_numbers = vec!["0x1".to_string()]; let result = ethereum_rpc @@ -743,10 +739,7 @@ mod tests { async fn test_ethereum_rpc_batch_payloads_network_error() { // Use invalid URL to trigger network error let url = Url::parse("http://invalid-host:8545").unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let block_numbers = vec!["0x1".to_string()]; let result = ethereum_rpc.get_execution_payloads(&block_numbers).await; @@ -789,10 +782,7 @@ mod tests { .await; let url = Url::parse(&server.uri()).unwrap(); - let ethereum_rpc = EthereumRPC { - url, - client: Client::new(), - }; + let ethereum_rpc = EthereumRPC::new(url).unwrap(); let block_numbers = vec!["0x1".to_string()]; let result = ethereum_rpc diff --git a/crates/eth-engine/tests/integration.rs b/crates/eth-engine/tests/integration.rs index ad3e56b..faa908e 100644 --- a/crates/eth-engine/tests/integration.rs +++ b/crates/eth-engine/tests/integration.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] use std::path::Path; use std::time::Duration; diff --git a/crates/evm-node/Cargo.toml b/crates/evm-node/Cargo.toml index 47faca8..becb2d7 100644 --- a/crates/evm-node/Cargo.toml +++ b/crates/evm-node/Cargo.toml @@ -32,7 +32,6 @@ eyre.workspace = true jsonrpsee.workspace = true reqwest.workspace = true reth-chainspec.workspace = true -reth-engine-local.workspace = true reth-engine-primitives.workspace = true reth-ethereum = { workspace = true, features = ["node-api", "node", "provider", "network", "evm", "pool", "cli", "js-tracer"] } reth-ethereum-engine-primitives.workspace = true diff --git a/crates/evm-node/src/lib.rs b/crates/evm-node/src/lib.rs index 9c197dd..3b2db7f 100644 --- a/crates/evm-node/src/lib.rs +++ b/crates/evm-node/src/lib.rs @@ -21,6 +21,7 @@ pub mod engine; pub mod node; +pub mod payload; pub mod rpc; pub mod rpc_middleware; diff --git a/crates/evm-node/src/node.rs b/crates/evm-node/src/node.rs index b324762..cc3da1b 100644 --- a/crates/evm-node/src/node.rs +++ b/crates/evm-node/src/node.rs @@ -21,12 +21,12 @@ //! - inject our consensus ArcConsensus in ArcConsensusBuilder //! - inject ArcEngineValidatorBuilder in ArcEngineValidatorBuilder +use crate::payload::ArcLocalPayloadAttributesBuilder; use alloy_network::Ethereum; use alloy_rpc_types_engine::ExecutionData; use arc_evm::{ArcEvmConfig, ArcEvmFactory}; use arc_execution_validation::ArcConsensus; use reth_chainspec::{EthereumHardforks, Hardforks}; -use reth_engine_local::LocalPayloadAttributesBuilder; use reth_engine_primitives::EngineTypes; use reth_ethereum::{node::EthEngineTypes, node::EthEvmConfig}; use reth_ethereum_engine_primitives::{ @@ -516,7 +516,7 @@ impl> DebugNode for ArcNode { fn local_payload_attributes_builder( chain_spec: &Self::ChainSpec, ) -> impl PayloadAttributesBuilder<::PayloadAttributes> { - LocalPayloadAttributesBuilder::new(Arc::new(chain_spec.clone())) + ArcLocalPayloadAttributesBuilder::new(Arc::new(chain_spec.clone())) } } diff --git a/crates/evm-node/src/payload.rs b/crates/evm-node/src/payload.rs new file mode 100644 index 0000000..a31b66f --- /dev/null +++ b/crates/evm-node/src/payload.rs @@ -0,0 +1,138 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Arc payload attributes builder for dev-mode local mining. +//! Fork from https://github.com/paradigmxyz/reth/blob/v1.11.3/crates/engine/local/src/payload.rs +//! - Uses `max(parent.timestamp, wall_clock)` instead of `max(parent.timestamp + 1, wall_clock)` +//! to allow equal timestamps, matching Arc's relaxed validation. + +use alloy_consensus::BlockHeader; +use alloy_primitives::B256; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_ethereum_engine_primitives::EthPayloadAttributes; +use reth_payload_primitives::PayloadAttributesBuilder; +use reth_primitives_traits::SealedHeader; +use std::sync::Arc; + +/// Payload attributes builder for Arc's dev-mode miner. +/// +/// Unlike upstream's `LocalPayloadAttributesBuilder` which enforces strictly +/// increasing timestamps (`parent + 1`), this uses `max(wall_clock, parent.timestamp)` +/// to allow equal timestamps — matching Arc's sub-second block production. +#[derive(Debug)] +pub struct ArcLocalPayloadAttributesBuilder { + chain_spec: Arc, +} + +impl ArcLocalPayloadAttributesBuilder { + /// Creates a new instance of the builder. + pub const fn new(chain_spec: Arc) -> Self { + Self { chain_spec } + } +} + +impl PayloadAttributesBuilder + for ArcLocalPayloadAttributesBuilder +where + ChainSpec: EthChainSpec + EthereumHardforks + 'static, +{ + fn build(&self, parent: &SealedHeader) -> EthPayloadAttributes { + let timestamp = std::cmp::max( + parent.timestamp(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("Clock is before UNIX epoch") + .as_secs(), + ); + + EthPayloadAttributes { + timestamp, + prev_randao: B256::random(), + // Mock CL uses genesis coinbase as suggested fee recipient + suggested_fee_recipient: self.chain_spec.genesis_header().beneficiary(), + withdrawals: self + .chain_spec + .is_shanghai_active_at_timestamp(timestamp) + .then(Default::default), + parent_beacon_block_root: self + .chain_spec + .is_cancun_active_at_timestamp(timestamp) + .then(B256::random), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_chainspec::ChainSpecBuilder; + use reth_ethereum::primitives::Header; + use reth_primitives_traits::SealedHeader; + + fn builder() -> ArcLocalPayloadAttributesBuilder { + ArcLocalPayloadAttributesBuilder::new(Arc::new(ChainSpecBuilder::mainnet().build())) + } + + #[test] + fn timestamp_uses_parent_when_ahead_of_clock() { + let b = builder(); + let parent = SealedHeader::seal_slow(Header { + timestamp: u64::MAX / 2, + ..Default::default() + }); + let attrs = b.build(&parent); + assert_eq!(attrs.timestamp, u64::MAX / 2); + } + + #[test] + fn timestamp_equal_to_parent_not_incremented() { + let b = builder(); + let future = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + + 10; + let parent = SealedHeader::seal_slow(Header { + timestamp: future, + ..Default::default() + }); + let attrs = b.build(&parent); + assert_eq!(attrs.timestamp, future); + } + + #[test] + fn timestamp_uses_wall_clock_when_parent_is_old() { + let b = builder(); + let parent = SealedHeader::seal_slow(Header { + timestamp: 0, + ..Default::default() + }); + let attrs = b.build(&parent); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + assert!(attrs.timestamp >= now.saturating_sub(1)); + } + + #[test] + fn withdrawals_populated_for_shanghai() { + let b = builder(); + let parent = SealedHeader::seal_slow(Header::default()); + let attrs = b.build(&parent); + assert_eq!(attrs.withdrawals, Some(vec![])); // empty array + } +} diff --git a/crates/evm-specs-tests/Cargo.toml b/crates/evm-specs-tests/Cargo.toml new file mode 100644 index 0000000..17bdedc --- /dev/null +++ b/crates/evm-specs-tests/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "arc-evm-specs-tests" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish.workspace = true + +[[bin]] +name = "arc-evm-specs-tests" +path = "src/main.rs" + +[dependencies] +alloy-primitives = { workspace = true, features = ["std", "rlp"] } +alloy-rlp = { workspace = true } + +alloy-trie = { workspace = true, features = ["ethereum"] } + +arc-evm = { workspace = true } +arc-execution-config = { workspace = true, features = ["test-utils"] } + +clap = { workspace = true, features = ["derive"] } + +hash-db = { workspace = true } +plain_hasher = { workspace = true } +reth-evm = { workspace = true } + +revm = { workspace = true, features = ["std", "serde"] } +revm-context-interface = { workspace = true } +revm-database = { workspace = true, features = ["std"] } +revm-inspector = { workspace = true, features = ["tracer"] } +revm-primitives = { workspace = true } +revm-statetest-types = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +thiserror = { workspace = true } +triehash = { workspace = true } + +[lints] +workspace = true diff --git a/crates/evm-specs-tests/README.md b/crates/evm-specs-tests/README.md new file mode 100644 index 0000000..503f211 --- /dev/null +++ b/crates/evm-specs-tests/README.md @@ -0,0 +1,179 @@ +# arc-evm-specs-tests + +`arc-evm-specs-tests` is a fixture-consumer binary for `arc-execution-specs`. + +Its purpose is: +- execute Ethereum state-test fixtures through the ARC EVM execution path +- expose that execution via a CLI binary that `arc-execution-specs` can call through `consume direct` + +## How It Simulates ARC EVM Execution + +`arc-evm-specs-tests` does not implement a second EVM. It is a harness that drives the existing ARC execution layer from `arc-evm` (`crates/evm`). + +The runner implementation and design are sourced from upstream `revm`: + +- source: +- statetest runner shape: [`revme/src/cmd/statetest/runner.rs`](https://github.com/bluealloy/revm/blob/main/bins/revme/src/cmd/statetest/runner.rs) +- root helper shape: [`revme/src/cmd/statetest/merkle_trie.rs`](https://github.com/bluealloy/revm/blob/main/bins/revme/src/cmd/statetest/merkle_trie.rs) + +Execution flow: +1. `arc-evm-specs-tests` command entrypoint loads fixtures (`statetest` command). +2. It builds an `ArcEvmFactory` from `arc-evm`. +3. For each fixture test case, it builds env/tx input and calls `factory.create_evm(...)`. +4. It executes with `evm.transact_commit(tx)`. +5. It validates logs hash and state root against fixture expectations and returns normalized error classes/kinds. + +That means the transaction execution engine is ARC EVM code (from `crates/evm/src/evm.rs`), while `arc-evm-specs-tests` is the harness around it. + +## Execution Mode + +Current adapter setup uses `LOCAL_DEV`, which means ARC localdev executes with the ARC hardforks active in that chain spec. + +The fixture fork name still selects the Ethereum `cfg.spec`, but the underlying executor is ARC localdev. + +This is an ARC-mode harness, not a pure Ethereum fork-isolated runner: +- the fixture fork still chooses the REVM `cfg.spec` +- the chain-level execution context is still ARC `LOCAL_DEV` +- ARC localdev behavior and ARC hardfork activation can therefore influence results even when the fixture fork is pre-ARC or pre-feature + +Interpretation rule: +- use this runner to measure how ARC localdev behaves when driven by Ethereum statetest fixtures +- do not interpret the results as “stock Ethereum execution for that fixture fork in isolation” + +If you need pure Ethereum fork-isolated validation, that should be a separate runner mode with a different chain-spec contract. + + +## Temporary Upstream Workarounds + +This crate currently uses a two-pass fixture parse because `revm-statetest-types` does not yet expose all fields needed by ARC's harness flow. + +- Verified against: `revm-statetest-types = 14.x` +- Workaround details: + - extract `config.chainid` from raw JSON before typed deserialization + - sanitize unsupported fields (`receipt`, `state` -> `postState`) before deserializing into `TestSuite` +- Removal condition: delete this workaround once upstream typed deserialization includes `config.chainid` and fixture fields needed for direct `TestSuite` parsing. + +## How `arc-execution-specs` Consumes It + +`arc-execution-specs` runs this binary through: + +```bash +uv --project packages/testing run --python 3.12 consume direct \ + --bin $HOME/crcl/arc-node-project-name/target/release/arc-evm-specs-tests \ + --input $HOME/crcl/arc-execution-specs/fixtures/state_tests/for_prague \ + -m state_test -q +``` + +Integration contract: +- producer side (arc-node repo): build `arc-evm-specs-tests` binary and keep Rust health (`cargo test`, `cargo build`) +- consumer side (`arc-execution-specs`): drive fixtures, collect pass/fail output, and report compatibility metrics + +## Build + +```bash +cd $HOME/crcl/arc-node-project-name +cargo build --release -p arc-evm-specs-tests +``` + +Binary: + +```text +$HOME/crcl/arc-node-project-name/target/release/arc-evm-specs-tests +``` + +## Consume Prague State Tests + +```bash +cd $HOME/crcl/arc-execution-specs +uv --project packages/testing run --python 3.12 consume direct \ + --bin $HOME/crcl/arc-node-project-name/target/release/arc-evm-specs-tests \ + --input $HOME/crcl/arc-execution-specs/fixtures/state_tests/for_prague \ + -m state_test -q +``` + +## Focused Retest Examples + +`extcodehash_via_call`: + +```bash +cd $HOME/crcl/arc-execution-specs +uv --project packages/testing run --python 3.12 consume direct \ + --bin $HOME/crcl/arc-node-project-name/target/release/arc-evm-specs-tests \ + --input $HOME/crcl/arc-execution-specs/fixtures/state_tests/for_prague \ + -m state_test -k "test_extcodehash_via_call and fork_Prague" --maxfail=1 -ra +``` + +`precompile_absence`: + +```bash +cd $HOME/crcl/arc-execution-specs +uv --project packages/testing run --python 3.12 consume direct \ + --bin $HOME/crcl/arc-node-project-name/target/release/arc-evm-specs-tests \ + --input $HOME/crcl/arc-execution-specs/fixtures/state_tests/for_prague \ + -m state_test -k "test_precompile_absence and fork_Prague" --maxfail=1 -ra +``` + +## Error Classes in Adaptor Output + +`arc-evm-specs-tests` emits normalized tags in failure details: + +- `error_class=HARNESS_PRECONDITION` +- `error_class=EXECUTION_MISMATCH` +- `error_kind=TX_ENV_BUILD_FAILED` +- `error_kind=UNEXPECTED_EXCEPTION` +- `error_kind=UNEXPECTED_SUCCESS` +- `error_kind=WRONG_EXCEPTION` +- `error_kind=LOGS_HASH_MISMATCH` +- `error_kind=STATE_ROOT_MISMATCH` + +This is intended to be stable and reusable across suites, rather than requiring per-test string mapping. + +## Output Metadata + +Consume-direct JSON output includes structured variant metadata: + +- `data_index`: index into fixture `transaction.data` +- `gas_index`: index into fixture `transaction.gasLimit` +- `value_index`: index into fixture `transaction.value` + +The `variantId` string is still emitted for compatibility with existing +consume-direct tooling. Its suffix remains in the historical `d..._g..._v...` +format even though the structured JSON fields use the clearer names above. + +## Reporting Artifacts + +`consume direct` exposes multiple reporting layers, and they operate at +different granularities: + +- `report_consume.html` is the pytest HTML report. Its primary unit is the + collected pytest item, which is suite-level from the fixture consumer's + point of view. +- the aggregate JSON written to stdout by `arc-evm-specs-tests` is the + fixture-consumer contract consumed by `arc-execution-specs`; it summarizes + each executed result as `name` / `variantId` / `pass` / `error`, with the + optional JSON outcome attached alongside it. +- `per_test_outcomes.jsonl` is the ARC runner's variant-level artifact. Each + line is one concrete fixture variant emitted through `--json-outcome`. + +That means the HTML report is useful for high-level triage, but it does not +model fixture-internal transaction variants as first-class rows. The variant +detail lives in the runner output artifacts. + +Practical interpretation: + +- use `report_consume.html` to answer "which pytest cases failed?" +- use the aggregate JSON output to answer "what did the fixture consumer + return for this run as a whole?" +- use `per_test_outcomes.jsonl` to answer "which exact fixture variant failed, + and what were its state root, logs root, gas used, and normalized error?" + +The stable link between those two views is the runner's formatted test ID: + +- pytest failure details surface the full variant identifier +- per-test JSON lines emit the same identifier in the `test` field +- `variantId` carries the same concrete variant identity in aggregate + consume-direct JSON output + +This is why `format_test_id(...)` intentionally remains stable even though the +structured metadata fields were renamed to `data_index`, `gas_index`, and +`value_index`: the reporting contract depends on a stable concrete variant key. diff --git a/crates/evm-specs-tests/src/adapter.rs b/crates/evm-specs-tests/src/adapter.rs new file mode 100644 index 0000000..e9b4159 --- /dev/null +++ b/crates/evm-specs-tests/src/adapter.rs @@ -0,0 +1,305 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use arc_evm::ArcEvmFactory; +use arc_execution_config::chainspec::{ArcChainSpec, LOCAL_DEV}; +use reth_evm::EvmEnv; +use revm::context::CfgEnv; +use revm_statetest_types::{SpecName, TestUnit}; + +use crate::error::EvmSpecsTestError; + +/// Extract a test_name → chain_id map from raw JSON before revm-statetest-types +/// deserialization. +/// +/// EEST StateFixture JSON has `"config": {"chainid": "0x01"}` at the test-unit +/// level. `revm-statetest-types` v14 does NOT parse this field (it silently +/// drops unknown fields). We parse the raw JSON as a generic map to extract +/// `config.chainid` per test name, then let revm-statetest-types handle the +/// execution-relevant fields. +/// +/// TODO(arc-evm-specs-tests): remove this raw JSON extraction once `revm-statetest-types` +/// natively deserializes `config.chainid` for EEST fixtures. +/// Verified workaround requirement against `revm-statetest-types = 14.x`. +pub fn extract_chain_ids( + raw_json: &serde_json::Value, +) -> Result, EvmSpecsTestError> { + let mut map = HashMap::new(); + if let Some(obj) = raw_json.as_object() { + for (test_name, test_value) in obj { + if let Some(raw_chain_id) = test_value + .get("config") + .and_then(|c| c.get("chainid")) + .and_then(|v| v.as_str()) + { + let chain_id = parse_chain_id(raw_chain_id).ok_or_else(|| { + EvmSpecsTestError::MalformedChainId { + test_name: test_name.clone(), + raw_value: raw_chain_id.to_string(), + } + })?; + map.insert(test_name.clone(), chain_id); + } + } + } + Ok(map) +} + +fn parse_chain_id(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + if let Some(hex) = trimmed + .strip_prefix("0x") + .or_else(|| trimmed.strip_prefix("0X")) + { + return u64::from_str_radix(hex, 16).ok(); + } + + trimmed.parse::().ok() +} + +/// Resolve chain ID for a test with precedence: +/// 1. config.chainid (from raw JSON map — EEST source of truth) +/// 2. env.current_chain_id (revm-statetest-types, older reference format) +/// 3. Error (do NOT silently default) +pub fn resolve_chain_id( + test_name: &str, + chain_id_map: &HashMap, + unit: &TestUnit, +) -> Result { + if let Some(&id) = chain_id_map.get(test_name) { + return Ok(id); + } + if let Some(id) = unit.env.current_chain_id { + return id + .try_into() + .map_err(|_| EvmSpecsTestError::MissingChainId { + test_name: test_name.to_string(), + }); + } + Err(EvmSpecsTestError::MissingChainId { + test_name: test_name.to_string(), + }) +} + +/// Build the default ARC chain spec used for statetest execution. +/// +/// This runner always executes against ARC localdev with all ARC hardforks +/// active. Ethereum fixture fork names still choose the `cfg.spec`, but the +/// underlying ARC executor is the full local ARC chain configuration. +/// +/// This is intentional ARC-mode execution, not pure Ethereum fork isolation. +/// The fixture chooses the EVM spec id; the chain-level execution context +/// remains ARC `LOCAL_DEV`. +pub fn build_default_arc_chain_spec() -> Arc { + LOCAL_DEV.clone() +} + +/// Build the ArcEvmFactory from a chain spec. +/// +/// Note: ArcEvmFactory::new takes a single arg (chain_spec). +/// The struct is #[non_exhaustive] so the API may expand in the future. +pub fn build_evm_factory(chain_spec: Arc) -> ArcEvmFactory { + ArcEvmFactory::new(chain_spec) +} + +/// Build CfgEnv + BlockEnv from a TestUnit for a given SpecName + chain_id. +/// +/// Chain ID is resolved externally via `resolve_chain_id()` before calling +/// this function, so it takes chain_id as an explicit parameter. +pub fn build_evm_env( + unit: &TestUnit, + spec_name: &SpecName, + chain_id: u64, +) -> Result { + let spec_id = spec_name.to_spec_id(); + let mut cfg = CfgEnv::default(); + + cfg.chain_id = chain_id; + cfg.spec = spec_id; + + let block = unit.block_env(&mut cfg); + + Ok(EvmEnv { + cfg_env: cfg, + block_env: block, + }) +} + +/// Phase 1: Only standard Ethereum forks are supported. +/// `SpecName::Unknown` (custom ARC names) is unsupported. +pub fn is_supported_spec(spec_name: &SpecName) -> bool { + !matches!(spec_name, SpecName::Unknown) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{Address, U256}; + use revm_primitives::hardfork::SpecId; + use revm_statetest_types::{Env, TestUnit, TransactionParts}; + use std::collections::BTreeMap; + + fn unit_with_chain_id(chain_id: Option) -> TestUnit { + TestUnit { + info: None, + env: Env { + current_chain_id: chain_id, + current_coinbase: Address::ZERO, + current_difficulty: U256::ZERO, + current_gas_limit: U256::from(30_000_000), + current_number: U256::from(1), + current_timestamp: U256::from(1), + current_base_fee: Some(U256::ZERO), + previous_hash: None, + current_random: None, + current_beacon_root: None, + current_withdrawals_root: None, + current_excess_blob_gas: None, + }, + pre: alloy_primitives::map::HashMap::default(), + post: BTreeMap::default(), + transaction: TransactionParts { + tx_type: None, + data: vec![], + gas_limit: vec![], + gas_price: None, + nonce: U256::ZERO, + secret_key: Default::default(), + sender: None, + to: None, + value: vec![], + max_fee_per_gas: None, + max_priority_fee_per_gas: None, + initcodes: None, + access_lists: vec![], + authorization_list: None, + blob_versioned_hashes: vec![], + max_fee_per_blob_gas: None, + }, + out: None, + } + } + + #[test] + fn extracts_hex_and_decimal_chain_ids() { + let json = serde_json::json!({ + "hex_case": { "config": { "chainid": "0x10" } }, + "dec_case": { "config": { "chainid": "10" } } + }); + let ids = extract_chain_ids(&json).expect("chain ids should parse"); + assert_eq!(ids.get("hex_case"), Some(&16)); + assert_eq!(ids.get("dec_case"), Some(&10)); + } + + #[test] + fn rejects_malformed_chain_id_with_context() { + let json = serde_json::json!({ + "bad_case": { "config": { "chainid": "xyz" } } + }); + let err = extract_chain_ids(&json).expect_err("invalid chain id should fail"); + assert!(matches!( + err, + EvmSpecsTestError::MalformedChainId { test_name, raw_value } + if test_name == "bad_case" && raw_value == "xyz" + )); + } + + #[test] + fn extract_chain_ids_ignores_missing_or_non_string_chain_ids() { + let json = serde_json::json!({ + "missing": { "config": {} }, + "non_string": { "config": { "chainid": 1 } }, + "good": { "config": { "chainid": "0X2a" } } + }); + + let ids = extract_chain_ids(&json).expect("chain ids should parse"); + + assert_eq!(ids.len(), 1); + assert_eq!(ids.get("good"), Some(&42)); + } + + #[test] + fn supported_spec_rejects_unknown_only() { + assert!(is_supported_spec(&SpecName::Prague)); + assert!(!is_supported_spec(&SpecName::Unknown)); + } + + #[test] + fn resolve_chain_id_prefers_explicit_map_over_env() { + let unit = unit_with_chain_id(Some(U256::from(7))); + let ids = HashMap::from([(String::from("fixture"), 42_u64)]); + + let resolved = resolve_chain_id("fixture", &ids, &unit).expect("chain id should resolve"); + + assert_eq!(resolved, 42); + } + + #[test] + fn resolve_chain_id_falls_back_to_env_chain_id() { + let unit = unit_with_chain_id(Some(U256::from(7))); + + let resolved = resolve_chain_id("fixture", &HashMap::default(), &unit) + .expect("chain id should resolve"); + + assert_eq!(resolved, 7); + } + + #[test] + fn resolve_chain_id_errors_when_no_sources_exist() { + let unit = unit_with_chain_id(None); + + let err = resolve_chain_id("fixture", &HashMap::default(), &unit) + .expect_err("missing chain id should fail"); + + assert!(matches!( + err, + EvmSpecsTestError::MissingChainId { test_name } if test_name == "fixture" + )); + } + + #[test] + fn build_default_arc_chain_spec_returns_local_dev() { + let chain_spec = build_default_arc_chain_spec(); + + assert_eq!(chain_spec.inner.chain.id(), LOCAL_DEV.inner.chain.id()); + } + + #[test] + fn build_evm_env_sets_requested_chain_id_and_spec() { + let unit = unit_with_chain_id(Some(U256::from(1))); + + let env = build_evm_env(&unit, &SpecName::Prague, 5042).expect("env should build"); + + assert_eq!(env.cfg_env.chain_id, 5042); + assert_eq!(env.cfg_env.spec, SpecId::PRAGUE); + assert_eq!(env.block_env.number.to::(), 1); + } + + #[test] + fn build_evm_factory_uses_supplied_chain_spec() { + let chain_spec = build_default_arc_chain_spec(); + let _factory = build_evm_factory(chain_spec); + + // Construction succeeds with the supplied chain spec. + } +} diff --git a/crates/evm-specs-tests/src/cmd/mod.rs b/crates/evm-specs-tests/src/cmd/mod.rs new file mode 100644 index 0000000..9d7d1d9 --- /dev/null +++ b/crates/evm-specs-tests/src/cmd/mod.rs @@ -0,0 +1,17 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod statetest; diff --git a/crates/evm-specs-tests/src/cmd/statetest.rs b/crates/evm-specs-tests/src/cmd/statetest.rs new file mode 100644 index 0000000..1b0d723 --- /dev/null +++ b/crates/evm-specs-tests/src/cmd/statetest.rs @@ -0,0 +1,58 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::PathBuf; + +use crate::error::EvmSpecsTestError; +use crate::result::RunStatus; + +/// Thin CLI wrapper over the ARC-backed statetest runner harness. +pub fn run( + path: PathBuf, + filter_name: Option, + strict_exit: bool, + trace: bool, + json_outcome: bool, +) -> Result { + crate::runner::run(path, filter_name, strict_exit, trace, json_outcome) +} + +#[cfg(test)] +mod tests { + use super::run; + use crate::error::EvmSpecsTestError; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn run_forwards_runner_no_json_files_error() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!("arc_evm_specs_cmd_empty_{nonce}")); + std::fs::create_dir_all(&root).expect("temp dir should be created"); + + let err = run(root.clone(), None, false, false, false) + .expect_err("empty directory should fail through wrapper"); + + assert!(matches!( + err, + EvmSpecsTestError::NoJsonFiles { path } if path == root.display().to_string() + )); + + std::fs::remove_dir_all(root).expect("temp dir should be removed"); + } +} diff --git a/crates/evm-specs-tests/src/error.rs b/crates/evm-specs-tests/src/error.rs new file mode 100644 index 0000000..32ad0a3 --- /dev/null +++ b/crates/evm-specs-tests/src/error.rs @@ -0,0 +1,147 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use thiserror::Error; + +/// Errors that can occur during arc-evm-specs-tests execution. +#[derive(Debug, Error)] +pub enum EvmSpecsTestError { + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + #[error("JSON parse error: {path}: {source}")] + JsonParse { + path: String, + source: serde_json::Error, + }, + + #[error("Test error in '{name}': {kind}")] + TestFailure { name: String, kind: TestErrorKind }, + #[error("No JSON fixture files found at: {path}")] + NoJsonFiles { path: String }, + #[error("Missing chain_id for test '{test_name}': neither config.chainid nor env.current_chain_id is available")] + MissingChainId { test_name: String }, + #[error("Malformed config.chainid for test '{test_name}': '{raw_value}' (expected decimal like '1' or hex like '0x1')")] + MalformedChainId { + test_name: String, + raw_value: String, + }, + #[error("Runner queue mutex poisoned")] + RunnerQueuePoisoned, + #[error("Failed to spawn runner worker thread: {0}")] + WorkerSpawn(std::io::Error), + #[error("Runner worker thread panicked")] + WorkerPanic, +} + +/// Specific kinds of test failures. +#[derive(Debug, Error)] +pub enum TestErrorKind { + #[error("EVM execution error: error_class={error_class}; error_kind={error_kind}; {detail}")] + EvmError { + error_class: &'static str, + error_kind: &'static str, + detail: String, + }, +} + +impl TestErrorKind { + pub fn evm( + error_class: &'static str, + error_kind: &'static str, + detail: impl Into, + ) -> Self { + Self::EvmError { + error_class, + error_kind, + detail: detail.into(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn evm_error_kind_formats_with_class_kind_and_detail() { + let err = TestErrorKind::evm("EXECUTION_MISMATCH", "WRONG_EXCEPTION", "details"); + + assert_eq!( + err.to_string(), + "EVM execution error: error_class=EXECUTION_MISMATCH; error_kind=WRONG_EXCEPTION; details" + ); + } + + #[test] + fn top_level_error_wraps_test_failure_context() { + let err = EvmSpecsTestError::TestFailure { + name: "fixture/Prague/d0_g0_v0".to_string(), + kind: TestErrorKind::evm("EXECUTION_MISMATCH", "STATE_ROOT_MISMATCH", "nope"), + }; + + let rendered = err.to_string(); + assert!(rendered.contains("Test error in 'fixture/Prague/d0_g0_v0'")); + assert!(rendered.contains("error_class=EXECUTION_MISMATCH")); + assert!(rendered.contains("error_kind=STATE_ROOT_MISMATCH")); + } + + #[test] + fn missing_chain_id_error_mentions_expected_sources() { + let err = EvmSpecsTestError::MissingChainId { + test_name: "fixture".to_string(), + }; + + assert_eq!( + err.to_string(), + "Missing chain_id for test 'fixture': neither config.chainid nor env.current_chain_id is available" + ); + } + + #[test] + fn error_variants_render_expected_messages() { + let io_err = EvmSpecsTestError::Io(std::io::Error::other("disk")); + assert!(io_err.to_string().contains("IO error: disk")); + + let json_err = EvmSpecsTestError::JsonParse { + path: "fixture.json".to_string(), + source: serde_json::from_str::("{").expect_err("invalid json"), + }; + assert!(json_err + .to_string() + .contains("JSON parse error: fixture.json")); + + let malformed = EvmSpecsTestError::MalformedChainId { + test_name: "fixture".to_string(), + raw_value: "xyz".to_string(), + }; + assert!(malformed.to_string().contains("Malformed config.chainid")); + + assert_eq!( + EvmSpecsTestError::RunnerQueuePoisoned.to_string(), + "Runner queue mutex poisoned" + ); + assert_eq!( + EvmSpecsTestError::WorkerPanic.to_string(), + "Runner worker thread panicked" + ); + + let spawn = EvmSpecsTestError::WorkerSpawn(std::io::Error::other("thread")); + assert!(spawn + .to_string() + .contains("Failed to spawn runner worker thread: thread")); + } +} diff --git a/crates/evm-specs-tests/src/exception_match.rs b/crates/evm-specs-tests/src/exception_match.rs new file mode 100644 index 0000000..61dfc9e --- /dev/null +++ b/crates/evm-specs-tests/src/exception_match.rs @@ -0,0 +1,457 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Temporary exception-bucket matching helpers. +//! +//! This module is intentionally stricter than upstream `revme` today. +//! `revme` currently treats any execution error as satisfying +//! `expectException`, and during tx env construction it also accepts: +//! +//! ```ignore +//! let tx = match test.tx_env(&unit) { +//! Ok(tx) => tx, +//! Err(_) if test.expect_exception.is_some() => continue, +//! Err(e) => ... +//! }; +//! ``` +//! +//! See `bins/revme/src/cmd/statetest/runner.rs` in the `bluealloy/revm` +//! repository. +//! +//! We are not following that behavior yet because this stage of the work is +//! focused on debugging expectation mismatches across different error buckets. +//! For that reason, this file normalizes and classifies error strings so the +//! runner can distinguish "wrong exception bucket" from "some exception +//! happened". +//! +//! Once that debugging stage is complete, we should remove this stricter +//! bucket-matching behavior and follow `revme`'s exception handling semantics. + +use std::borrow::Cow; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExceptionCategory { + IntrinsicGasTooLow, + IntrinsicGasBelowFloorGasCost, + InsufficientAccountFunds, + GasLimitPriceProductOverflow, + SenderNotEoa, + Type3TxBlobCountExceeded, + Type1TxPreFork, + Type2TxPreFork, + Type3TxInvalidBlobVersionedHash, + Type3TxZeroBlobs, + Type4EmptyAuthorizationList, + Type4TxPreFork, + InsufficientMaxFeePerGas, + InsufficientMaxFeePerBlobGas, + InitcodeSizeExceeded, + GasLimitExceedsMaximum, + PriorityGreaterThanMaxFeePerGas, + GasAllowanceExceeded, +} + +pub(crate) fn exception_matches(expected: &str, actual: &str) -> bool { + let actual_norm = normalize_exception_phrase(actual); + if actual_norm.is_empty() { + return false; + } + + let actual_category = actual_exception_category(&actual_norm); + expected + .split('|') + .map(str::trim) + .filter(|candidate| !candidate.is_empty()) + .any(|candidate| expected_alternative_matches(candidate, &actual_norm, actual_category)) +} + +pub(crate) fn normalize_exception_phrase(value: &str) -> String { + let mut normalized = String::with_capacity(value.len()); + let mut prev_was_alnum = false; + let mut prev_was_lower = false; + let mut prev_was_digit = false; + + for ch in value.chars() { + if !ch.is_ascii_alphanumeric() { + normalized.push(' '); + prev_was_alnum = false; + prev_was_lower = false; + prev_was_digit = false; + continue; + } + + let is_upper = ch.is_ascii_uppercase(); + let is_lower = ch.is_ascii_lowercase(); + let is_digit = ch.is_ascii_digit(); + + if prev_was_alnum + && ((prev_was_lower && is_upper) + || (prev_was_digit && !is_digit) + || (!prev_was_digit && is_digit)) + { + normalized.push(' '); + } + + normalized.push(ch.to_ascii_lowercase()); + prev_was_alnum = true; + prev_was_lower = is_lower; + prev_was_digit = is_digit; + } + + normalized.split_whitespace().collect::>().join(" ") +} + +pub(crate) fn tx_env_actual_exception(error: &str) -> Option<&str> { + let got_marker = "got Some(\""; + let start = error.find(got_marker)?.checked_add(got_marker.len())?; + let rest = &error[start..]; + let end = rest.find("\")")?; + Some(&rest[..end]) +} + +pub(crate) fn tx_env_exception_matches(expected: &str, actual: &str) -> bool { + if exception_matches(expected, actual) { + return true; + } + + let expected_norm = normalize_exception_phrase(expected); + let actual_norm = normalize_exception_phrase(actual); + + let expected_key = normalize_expected_key(&expected_norm); + matches!( + (expected_key.as_ref(), actual_norm.as_str()), + ("type 3 tx contract creation", "invalid transaction type") + | ("type 4 tx contract creation", "invalid transaction type") + ) +} + +fn normalize_expected_key(expected_norm: &str) -> Cow<'_, str> { + let trimmed = expected_norm + .trim_start_matches("transaction exception ") + .trim_start_matches("exception ") + .trim(); + Cow::Borrowed(trimmed) +} + +fn expected_alternative_matches( + expected: &str, + actual_norm: &str, + actual_category: Option, +) -> bool { + let expected_norm = normalize_exception_phrase(expected); + if expected_norm.is_empty() { + return false; + } + + if let Some(expected_category) = expected_exception_category(&expected_norm) { + return actual_category.is_some_and(|actual| actual == expected_category); + } + + if expected_norm == actual_norm { + return true; + } + if expected_norm.split_whitespace().count() < 2 { + return false; + } + + let bounded_actual = format!(" {actual_norm} "); + let bounded_expected = format!(" {expected_norm} "); + bounded_actual.contains(&bounded_expected) +} + +fn expected_exception_category(expected_norm: &str) -> Option { + match normalize_expected_key(expected_norm).as_ref() { + "intrinsic gas too low" => Some(ExceptionCategory::IntrinsicGasTooLow), + "intrinsic gas below floor gas cost" => { + Some(ExceptionCategory::IntrinsicGasBelowFloorGasCost) + } + "insufficient account funds" => Some(ExceptionCategory::InsufficientAccountFunds), + "gaslimit price product overflow" => Some(ExceptionCategory::GasLimitPriceProductOverflow), + "sender not eoa" => Some(ExceptionCategory::SenderNotEoa), + "type 3 tx max blob gas allowance exceeded" | "type 3 tx blob count exceeded" => { + Some(ExceptionCategory::Type3TxBlobCountExceeded) + } + "type 1 tx pre fork" => Some(ExceptionCategory::Type1TxPreFork), + "type 2 tx pre fork" => Some(ExceptionCategory::Type2TxPreFork), + "type 3 tx invalid blob versioned hash" => { + Some(ExceptionCategory::Type3TxInvalidBlobVersionedHash) + } + "type 3 tx zero blobs" => Some(ExceptionCategory::Type3TxZeroBlobs), + "type 4 empty authorization list" => Some(ExceptionCategory::Type4EmptyAuthorizationList), + "type 4 tx pre fork" => Some(ExceptionCategory::Type4TxPreFork), + "insufficient max fee per gas" => Some(ExceptionCategory::InsufficientMaxFeePerGas), + "insufficient max fee per blob gas" => { + Some(ExceptionCategory::InsufficientMaxFeePerBlobGas) + } + "initcode size exceeded" => Some(ExceptionCategory::InitcodeSizeExceeded), + "gas limit exceeds maximum" => Some(ExceptionCategory::GasLimitExceedsMaximum), + "priority greater than max fee per gas" => { + Some(ExceptionCategory::PriorityGreaterThanMaxFeePerGas) + } + "gas allowance exceeded" => Some(ExceptionCategory::GasAllowanceExceeded), + _ => None, + } +} + +fn actual_exception_category(actual_norm: &str) -> Option { + actual_exception_rules() + .iter() + .find(|rule| rule.matches(actual_norm)) + .map(|rule| rule.category) +} + +struct ActualExceptionRule { + category: ExceptionCategory, + required_substrings: &'static [&'static str], +} + +impl ActualExceptionRule { + fn matches(&self, actual_norm: &str) -> bool { + self.required_substrings + .iter() + .all(|needle| actual_norm.contains(needle)) + } +} + +fn actual_exception_rules() -> &'static [ActualExceptionRule] { + &[ + ActualExceptionRule { + category: ExceptionCategory::IntrinsicGasTooLow, + required_substrings: &["call gas cost", "exceeds the gas limit"], + }, + ActualExceptionRule { + category: ExceptionCategory::IntrinsicGasBelowFloorGasCost, + required_substrings: &["gas floor", "exceeds the gas limit"], + }, + ActualExceptionRule { + category: ExceptionCategory::InsufficientAccountFunds, + required_substrings: &["lack of funds", "for max fee"], + }, + ActualExceptionRule { + category: ExceptionCategory::GasLimitPriceProductOverflow, + required_substrings: &["overflow payment in transaction"], + }, + ActualExceptionRule { + category: ExceptionCategory::SenderNotEoa, + required_substrings: &["senders with deployed code"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type3TxBlobCountExceeded, + required_substrings: &["too many blobs"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type1TxPreFork, + required_substrings: &["eip 2930", "not supported"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type2TxPreFork, + required_substrings: &["eip 1559", "not supported"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type3TxInvalidBlobVersionedHash, + required_substrings: &["blob version not supported"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type3TxZeroBlobs, + required_substrings: &["empty blobs"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type4EmptyAuthorizationList, + required_substrings: &["empty authorization list"], + }, + ActualExceptionRule { + category: ExceptionCategory::Type4TxPreFork, + required_substrings: &["eip 7702", "not supported"], + }, + ActualExceptionRule { + category: ExceptionCategory::InsufficientMaxFeePerGas, + required_substrings: &["gas price is less than basefee"], + }, + ActualExceptionRule { + category: ExceptionCategory::InsufficientMaxFeePerBlobGas, + required_substrings: &["blob gas price", "greater than max fee per blob gas"], + }, + ActualExceptionRule { + category: ExceptionCategory::InitcodeSizeExceeded, + required_substrings: &["create initcode size limit"], + }, + ActualExceptionRule { + category: ExceptionCategory::GasLimitExceedsMaximum, + required_substrings: &["transaction gas limit", "greater than the cap"], + }, + ActualExceptionRule { + category: ExceptionCategory::PriorityGreaterThanMaxFeePerGas, + required_substrings: &["priority fee is greater than max fee"], + }, + ActualExceptionRule { + category: ExceptionCategory::GasAllowanceExceeded, + required_substrings: &["caller gas limit exceeds the block gas limit"], + }, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exception_matching_requires_the_expected_error() { + assert!(exception_matches( + "TR_TypeNotSupported", + "transaction validation error: tr type not supported" + )); + assert!(!exception_matches( + "TR_TypeNotSupported", + "transaction validation error: nonce too low" + )); + } + + #[test] + fn exception_matching_rejects_substring_word_matches() { + assert!(!exception_matches("gas", "IntrinsicGasTooLow")); + } + + #[test] + fn exception_matching_accepts_expected_alternatives() { + assert!(exception_matches( + "TransactionException.INSUFFICIENT_ACCOUNT_FUNDS|TransactionException.INTRINSIC_GAS_TOO_LOW", + "transaction validation error: call gas cost (53000) exceeds the gas limit (21000)" + )); + } + + #[test] + fn exception_matching_keeps_unrecognized_exact_alternative() { + assert!(exception_matches( + "TransactionException.INTRINSIC_GAS_TOO_LOW|custom exact mismatch text", + "custom exact mismatch text" + )); + } + + #[test] + fn exception_matching_maps_high_volume_transaction_aliases() { + let cases = [ + ( + "TransactionException.INTRINSIC_GAS_TOO_LOW", + "transaction validation error: call gas cost (53000) exceeds the gas limit (21000)", + ), + ( + "TransactionException.INSUFFICIENT_ACCOUNT_FUNDS", + "transaction validation error: lack of funds (100) for max fee (101)", + ), + ( + "TransactionException.INSUFFICIENT_ACCOUNT_FUNDS|TransactionException.GASLIMIT_PRICE_PRODUCT_OVERFLOW", + "transaction validation error: overflow payment in transaction", + ), + ( + "TransactionException.INTRINSIC_GAS_BELOW_FLOOR_GAS_COST", + "transaction validation error: gas floor (53000) exceeds the gas limit (21000)", + ), + ( + "TransactionException.SENDER_NOT_EOA", + "transaction validation error: reject transactions from senders with deployed code", + ), + ( + "TransactionException.TYPE_2_TX_PRE_FORK", + "transaction validation error: Eip1559 is not supported", + ), + ( + "TransactionException.TYPE_1_TX_PRE_FORK", + "transaction validation error: Eip2930 is not supported", + ), + ( + "TransactionException.TYPE_4_TX_PRE_FORK", + "transaction validation error: Eip7702 is not supported", + ), + ( + "TransactionException.TYPE_3_TX_INVALID_BLOB_VERSIONED_HASH", + "transaction validation error: blob version not supported", + ), + ( + "TransactionException.TYPE_3_TX_MAX_BLOB_GAS_ALLOWANCE_EXCEEDED|TransactionException.TYPE_3_TX_BLOB_COUNT_EXCEEDED", + "transaction validation error: too many blobs, have 7, max 6", + ), + ( + "TransactionException.INSUFFICIENT_MAX_FEE_PER_GAS", + "transaction validation error: gas price is less than basefee", + ), + ( + "TransactionException.INITCODE_SIZE_EXCEEDED", + "transaction validation error: create initcode size limit exceeded", + ), + ( + "TransactionException.GAS_LIMIT_EXCEEDS_MAXIMUM", + "transaction validation error: transaction gas limit 30000001 greater than the cap", + ), + ( + "TransactionException.PRIORITY_GREATER_THAN_MAX_FEE_PER_GAS", + "transaction validation error: priority fee is greater than max fee", + ), + ( + "TransactionException.TYPE_3_TX_ZERO_BLOBS", + "transaction validation error: empty blobs are not allowed", + ), + ( + "TransactionException.TYPE_4_EMPTY_AUTHORIZATION_LIST", + "transaction validation error: empty authorization list", + ), + ( + "TransactionException.INSUFFICIENT_MAX_FEE_PER_BLOB_GAS", + "transaction validation error: blob gas price 5 greater than max fee per blob gas 4", + ), + ( + "TransactionException.GAS_ALLOWANCE_EXCEEDED", + "transaction validation error: caller gas limit exceeds the block gas limit", + ), + ]; + + for (expected, actual) in cases { + assert!( + exception_matches(expected, actual), + "expected {expected} to match {actual}" + ); + } + } + + #[test] + fn normalize_exception_phrase_splits_camel_case_and_digits() { + assert_eq!( + normalize_exception_phrase("IntrinsicGasTooLow123"), + "intrinsic gas too low 123" + ); + } + + #[test] + fn tx_env_actual_exception_extracts_inner_message() { + assert_eq!( + tx_env_actual_exception( + "unexpected exception: got Some(\"Invalid transaction type\"), expected Some(\"TransactionException.TYPE_3_TX_CONTRACT_CREATION\")" + ), + Some("Invalid transaction type") + ); + } + + #[test] + fn tx_env_exception_matches_contract_creation_aliases() { + assert!(tx_env_exception_matches( + "TransactionException.TYPE_3_TX_CONTRACT_CREATION", + "Invalid transaction type" + )); + assert!(tx_env_exception_matches( + "TransactionException.TYPE_4_TX_CONTRACT_CREATION", + "Invalid transaction type" + )); + } +} diff --git a/crates/evm-specs-tests/src/fixture_sanitizer.rs b/crates/evm-specs-tests/src/fixture_sanitizer.rs new file mode 100644 index 0000000..3cc04ba --- /dev/null +++ b/crates/evm-specs-tests/src/fixture_sanitizer.rs @@ -0,0 +1,154 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Narrow compatibility shim for fixture fields that current typed upstream +/// parsing does not accept yet. +/// +/// This intentionally rewrites only the supported top-level `state` field and +/// direct `post.[]` entries. Nested objects are left untouched so schema +/// drift surfaces instead of being recursively mutated into shape. +pub(crate) fn strip_unsupported_fields(value: &mut serde_json::Value) { + let Some(root) = value.as_object_mut() else { + return; + }; + + for test_unit in root.values_mut() { + let Some(test_map) = test_unit.as_object_mut() else { + continue; + }; + + if !test_map.contains_key("postState") + && let Some(state) = test_map.remove("state") + { + test_map.insert("postState".to_string(), state); + } + + if let Some(post) = test_map.get_mut("post") { + strip_unsupported_fields_in_post(post); + } + } +} + +fn strip_unsupported_fields_in_post(post: &mut serde_json::Value) { + let Some(forks) = post.as_object_mut() else { + return; + }; + + for entries in forks.values_mut() { + let Some(items) = entries.as_array_mut() else { + continue; + }; + + for item in items { + let Some(map) = item.as_object_mut() else { + continue; + }; + + if !map.contains_key("postState") + && let Some(state) = map.remove("state") + { + map.insert("postState".to_string(), state); + } + map.remove("receipt"); + } + } +} + +#[cfg(test)] +mod tests { + use super::strip_unsupported_fields; + + #[test] + fn strip_unsupported_fields_removes_receipt_and_sets_post_state() { + let mut value = serde_json::json!({ + "suite": { + "receipt": { "status": "0x1" }, + "state": { "0x1": { "balance": "0x01" } }, + "post": { + "Cancun": [ + { + "receipt": { "status": "0x1" } + } + ] + } + } + }); + strip_unsupported_fields(&mut value); + let suite = value + .get("suite") + .and_then(serde_json::Value::as_object) + .unwrap(); + assert!(suite.contains_key("receipt")); + assert!(!suite.contains_key("state")); + assert!(suite.contains_key("postState")); + let post_entry = suite + .get("post") + .and_then(|v| v.get("Cancun")) + .and_then(serde_json::Value::as_array) + .and_then(|arr| arr.first()) + .and_then(serde_json::Value::as_object) + .unwrap(); + assert!(!post_entry.contains_key("receipt")); + } + + #[test] + fn strip_unsupported_fields_preserves_existing_post_state() { + let mut value = serde_json::json!({ + "suite": { + "state": { "old": true }, + "postState": { "kept": true }, + "post": { + "Prague": [ + { + "receipt": { "status": "0x1" }, + "nested": { + "receipt": { "status": "0x1" } + } + } + ] + } + } + }); + + strip_unsupported_fields(&mut value); + let suite = value.get("suite").unwrap(); + + assert_eq!( + suite.get("postState").unwrap(), + &serde_json::json!({ "kept": true }) + ); + assert_eq!( + suite.get("state").unwrap(), + &serde_json::json!({ "old": true }) + ); + let post_entry = suite + .get("post") + .and_then(|v| v.get("Prague")) + .and_then(serde_json::Value::as_array) + .and_then(|arr| arr.first()) + .and_then(serde_json::Value::as_object) + .unwrap(); + assert!(!post_entry.contains_key("receipt")); + assert!(suite + .get("post") + .and_then(|v| v.get("Prague")) + .and_then(serde_json::Value::as_array) + .and_then(|arr| arr.first()) + .and_then(|entry| entry.get("nested")) + .and_then(serde_json::Value::as_object) + .is_some_and(|nested| nested.contains_key("receipt"))); + } +} diff --git a/crates/evm-specs-tests/src/lib.rs b/crates/evm-specs-tests/src/lib.rs new file mode 100644 index 0000000..a4d93dd --- /dev/null +++ b/crates/evm-specs-tests/src/lib.rs @@ -0,0 +1,24 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod adapter; +pub mod cmd; +pub mod error; +pub mod exception_match; +pub mod fixture_sanitizer; +pub mod result; +pub mod roots; +pub mod runner; diff --git a/crates/evm-specs-tests/src/main.rs b/crates/evm-specs-tests/src/main.rs new file mode 100644 index 0000000..0a4ffdf --- /dev/null +++ b/crates/evm-specs-tests/src/main.rs @@ -0,0 +1,137 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arc_evm_specs_tests::result::RunStatus; +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "arc-evm-specs-tests", + version, + about = "ARC EVM specs state-test runner" +)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Run state tests from EEST fixtures + Statetest { + /// Path to a JSON fixture file or directory + path: std::path::PathBuf, + + /// Run only the test with this name + #[arg(long)] + run: Option, + + /// Rerun failing tests with EIP-3155 trace output on stderr + #[arg(long)] + trace: bool, + + /// Emit upstream-style per-test JSON outcome fields alongside name/pass/error + #[arg(long)] + json_outcome: bool, + + /// Exit with non-zero code if any test fails + #[arg(long)] + strict_exit: bool, + }, +} + +fn main() { + if emit_compat_version_if_requested() { + return; + } + + let cli = Cli::parse(); + + match cli.command { + Commands::Statetest { + path, + run: filter_name, + trace, + json_outcome, + strict_exit, + } => match arc_evm_specs_tests::cmd::statetest::run( + path, + filter_name, + strict_exit, + trace, + json_outcome, + ) { + Ok(RunStatus::Success) => {} + Ok(status) => std::process::exit(status as i32), + Err(e) => { + eprintln!("Fatal error: {e}"); + std::process::exit(2); + } + }, + } +} + +fn emit_compat_version_if_requested() -> bool { + emit_compat_version_if_requested_from_args(std::env::args().skip(1)) +} + +fn emit_compat_version_if_requested_from_args(mut args: impl Iterator) -> bool { + let Some(first) = args.next() else { + return false; + }; + + if args.next().is_some() { + return false; + } + + if matches!(first.as_str(), "-v" | "--version" | "version") { + println!("evm version {}", env!("CARGO_PKG_VERSION")); + return true; + } + + false +} + +#[cfg(test)] +mod tests { + use super::emit_compat_version_if_requested_from_args; + + #[test] + fn compat_version_accepts_single_version_flags() { + assert!(emit_compat_version_if_requested_from_args( + ["-v".to_string()].into_iter() + )); + assert!(emit_compat_version_if_requested_from_args( + ["--version".to_string()].into_iter() + )); + assert!(emit_compat_version_if_requested_from_args( + ["version".to_string()].into_iter() + )); + } + + #[test] + fn compat_version_rejects_other_shapes() { + assert!(!emit_compat_version_if_requested_from_args( + std::iter::empty() + )); + assert!(!emit_compat_version_if_requested_from_args( + ["statetest".to_string()].into_iter() + )); + assert!(!emit_compat_version_if_requested_from_args( + ["--version".to_string(), "extra".to_string()].into_iter() + )); + } +} diff --git a/crates/evm-specs-tests/src/result.rs b/crates/evm-specs-tests/src/result.rs new file mode 100644 index 0000000..e899207 --- /dev/null +++ b/crates/evm-specs-tests/src/result.rs @@ -0,0 +1,328 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//! Result-model types for the ARC state-test runner. +//! +//! This module owns the JSON-facing data structures emitted by the runner: +//! `TestResult` for aggregate stdout output, `JsonOutcome` for per-variant +//! execution metadata, and `TestSummary` / `RunStatus` for run-level reporting. +//! +//! Artifact mapping: +//! - `TestResult`: aggregate JSON written to stdout and consumed by +//! `arc-execution-specs` +//! - `JsonOutcome`: embedded into `TestResult` and also emitted as per-variant +//! lines in `per_test_outcomes.jsonl` +//! - `TestSummary`: CLI stderr summary text for the overall run +//! - `RunStatus`: process/report status used to derive command exit behavior +//! +//! The pytest HTML report is generated on the Python side and does not +//! serialize these Rust types directly. + +use alloy_primitives::{Bytes, B256}; +use serde::Serialize; +/// A single test result, serialized to JSON stdout. +#[derive(Debug, Clone, Serialize)] +pub struct TestResult { + /// Consumer-facing fixture identity. This may be fixture-level for + /// compatibility with existing consume-direct integrations. + pub name: String, + /// Stable unique identifier for a concrete test variant when available. + /// + /// The `variantId` string currently keeps the historical `d{}_g{}_v{}` + /// suffix inherited from the upstream `revm` statetest runner shape + /// (`bins/revme/src/cmd/statetest/runner.rs`). Structured JSON fields use + /// the clearer `data_index` / `gas_index` / `value_index` names, while the + /// per-test stderr artifact still carries legacy `d` / `g` / `v` aliases + /// for compatibility with existing revm-style parsers. + #[serde(rename = "variantId", skip_serializing_if = "Option::is_none")] + pub variant_id: Option, + pub pass: bool, + #[serde(skip_serializing_if = "String::is_empty")] + pub error: String, + #[serde(rename = "stateRoot", skip_serializing_if = "Option::is_none")] + pub state_root: Option, + #[serde(rename = "logsRoot", skip_serializing_if = "Option::is_none")] + pub logs_root: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + #[serde(rename = "gasUsed", skip_serializing_if = "Option::is_none")] + pub gas_used: Option, + #[serde(rename = "errorMsg", skip_serializing_if = "Option::is_none")] + pub error_msg: Option, + #[serde(rename = "evmResult", skip_serializing_if = "Option::is_none")] + pub evm_result: Option, + #[serde(rename = "postLogsHash", skip_serializing_if = "Option::is_none")] + pub post_logs_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub fork: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub test: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Index into fixture `transaction.data`. + pub data_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Index into fixture `transaction.gasLimit`. + pub gas_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Index into fixture `transaction.value`. + pub value_index: Option, +} + +impl TestResult { + pub fn passed(name: String) -> Self { + Self { + name, + variant_id: None, + pass: true, + error: String::new(), + state_root: None, + logs_root: None, + output: None, + gas_used: None, + error_msg: None, + evm_result: None, + post_logs_hash: None, + fork: None, + test: None, + data_index: None, + gas_index: None, + value_index: None, + } + } + + pub fn failed(name: String, error: String) -> Self { + Self { + name, + variant_id: None, + pass: false, + error, + state_root: None, + logs_root: None, + output: None, + gas_used: None, + error_msg: None, + evm_result: None, + post_logs_hash: None, + fork: None, + test: None, + data_index: None, + gas_index: None, + value_index: None, + } + } + + pub fn with_json_outcome(mut self, outcome: JsonOutcome) -> Self { + self.state_root = Some(outcome.state_root); + self.logs_root = Some(outcome.logs_root); + self.output = Some(outcome.output); + self.gas_used = Some(outcome.gas_used); + self.error_msg = Some(outcome.error_msg); + self.evm_result = Some(outcome.evm_result); + self.post_logs_hash = Some(outcome.post_logs_hash); + self.fork = Some(outcome.fork); + self.test = Some(outcome.test); + self.data_index = Some(outcome.data_index); + self.gas_index = Some(outcome.gas_index); + self.value_index = Some(outcome.value_index); + self + } + + pub fn with_variant_id(mut self, variant_id: impl Into) -> Self { + self.variant_id = Some(variant_id.into()); + self + } +} + +#[derive(Debug, Clone)] +pub struct JsonOutcome { + pub state_root: B256, + pub logs_root: B256, + pub output: Bytes, + pub gas_used: u64, + pub error_msg: String, + pub evm_result: String, + pub post_logs_hash: B256, + pub fork: String, + pub test: String, + /// Index into fixture `transaction.data`. + pub data_index: usize, + /// Index into fixture `transaction.gasLimit`. + pub gas_index: usize, + /// Index into fixture `transaction.value`. + pub value_index: usize, +} + +/// Summary printed to stderr after all tests. +#[derive(Debug, Default)] +pub struct TestSummary { + pub files_processed: usize, + pub tests_total: usize, + pub tests_passed: usize, + pub tests_failed: usize, + pub tests_skipped_by_spec: usize, +} + +impl std::fmt::Display for TestSummary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "--- arc-evm-specs-tests summary ---")?; + writeln!(f, "files processed: {}", self.files_processed)?; + writeln!(f, "tests total: {}", self.tests_total)?; + writeln!(f, "tests passed: {}", self.tests_passed)?; + writeln!(f, "tests failed: {}", self.tests_failed)?; + writeln!(f, "tests skipped (spec): {}", self.tests_skipped_by_spec) + } +} + +impl TestSummary { + pub fn add_files_processed(&mut self, delta: usize) { + self.files_processed = self + .files_processed + .checked_add(delta) + .expect("files_processed overflow"); + } + + pub fn add_tests_total(&mut self, delta: usize) { + self.tests_total = self + .tests_total + .checked_add(delta) + .expect("tests_total overflow"); + } + + pub fn add_tests_passed(&mut self, delta: usize) { + self.tests_passed = self + .tests_passed + .checked_add(delta) + .expect("tests_passed overflow"); + } + + pub fn add_tests_failed(&mut self, delta: usize) { + self.tests_failed = self + .tests_failed + .checked_add(delta) + .expect("tests_failed overflow"); + } + + pub fn add_tests_skipped_by_spec(&mut self, delta: usize) { + self.tests_skipped_by_spec = self + .tests_skipped_by_spec + .checked_add(delta) + .expect("tests_skipped_by_spec overflow"); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunStatus { + Success = 0, + TestsFailed = 1, + FatalFileErrors = 2, +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{bytes, B256}; + + #[test] + fn passed_result_has_empty_error() { + let result = TestResult::passed("fixture/Prague/d0_g0_v0".to_string()); + + assert!(result.pass); + assert_eq!(result.name, "fixture/Prague/d0_g0_v0"); + assert!(result.error.is_empty()); + } + + #[test] + fn failed_result_preserves_error() { + let result = TestResult::failed("fixture".to_string(), "boom".to_string()); + + assert!(!result.pass); + assert_eq!(result.name, "fixture"); + assert_eq!(result.error, "boom"); + } + + #[test] + fn json_outcome_fields_serialize_only_when_present() { + let result = TestResult::passed("fixture".to_string()) + .with_variant_id("fixture/Berlin/d0_g0_v0") + .with_json_outcome(JsonOutcome { + state_root: B256::ZERO, + logs_root: B256::ZERO, + output: bytes!("01"), + gas_used: 21000, + error_msg: String::new(), + evm_result: "Success: Stop".to_string(), + post_logs_hash: B256::ZERO, + fork: "BERLIN".to_string(), + test: "fixture/Berlin/d0_g0_v0".to_string(), + data_index: 0, + gas_index: 0, + value_index: 0, + }); + + let json = serde_json::to_value(result).unwrap(); + assert_eq!(json.get("name").unwrap(), "fixture"); + assert_eq!(json.get("variantId").unwrap(), "fixture/Berlin/d0_g0_v0"); + assert_eq!(json.get("gasUsed").unwrap(), 21000); + assert_eq!(json.get("fork").unwrap(), "BERLIN"); + assert!(json.get("stateRoot").is_some()); + assert_eq!(json.get("data_index").unwrap(), 0); + assert_eq!(json.get("gas_index").unwrap(), 0); + assert_eq!(json.get("value_index").unwrap(), 0); + } + + #[test] + fn summary_display_includes_all_counts() { + let summary = TestSummary { + files_processed: 2, + tests_total: 7, + tests_passed: 5, + tests_failed: 1, + tests_skipped_by_spec: 1, + }; + + let rendered = summary.to_string(); + + assert!(rendered.contains("--- arc-evm-specs-tests summary ---")); + assert!(rendered.contains("files processed: 2")); + assert!(rendered.contains("tests total: 7")); + assert!(rendered.contains("tests passed: 5")); + assert!(rendered.contains("tests failed: 1")); + assert!(rendered.contains("tests skipped (spec): 1")); + } + + #[test] + fn summary_adders_accumulate_counts() { + let mut summary = TestSummary::default(); + + summary.add_files_processed(2); + summary.add_tests_total(7); + summary.add_tests_passed(5); + summary.add_tests_failed(1); + summary.add_tests_skipped_by_spec(1); + + assert_eq!(summary.files_processed, 2); + assert_eq!(summary.tests_total, 7); + assert_eq!(summary.tests_passed, 5); + assert_eq!(summary.tests_failed, 1); + assert_eq!(summary.tests_skipped_by_spec, 1); + } + + #[test] + fn run_status_exit_codes_are_stable() { + assert_eq!(RunStatus::Success as i32, 0); + assert_eq!(RunStatus::TestsFailed as i32, 1); + assert_eq!(RunStatus::FatalFileErrors as i32, 2); + } +} diff --git a/crates/evm-specs-tests/src/roots.rs b/crates/evm-specs-tests/src/roots.rs new file mode 100644 index 0000000..1541c12 --- /dev/null +++ b/crates/evm-specs-tests/src/roots.rs @@ -0,0 +1,295 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::convert::Infallible; + +use alloy_primitives::{address, keccak256, Address, B256, U256}; +use alloy_rlp::{RlpEncodable, RlpMaxEncodedLen}; +use alloy_trie::{ + root::{state_root_unhashed, storage_root_unhashed}, + TrieAccount as FixtureTrieAccount, EMPTY_ROOT_HASH, +}; +use hash_db::Hasher; +use plain_hasher::PlainHasher; +use revm::{ + context::result::{EVMError, ExecutionResult, HaltReason, InvalidTransaction}, + database::{bal::EvmDatabaseError, EmptyDB, PlainAccount, State}, +}; +use revm_primitives::Log; +use revm_statetest_types::AccountInfo as FixtureAccountInfo; +use triehash::sec_trie_root; + +pub struct TestValidationResult { + pub logs_root: B256, + pub state_root: B256, +} + +const ARC_NATIVE_COIN_AUTHORITY: Address = address!("1800000000000000000000000000000000000000"); + +pub fn compute_test_roots( + exec_result: &Result< + ExecutionResult, + EVMError, InvalidTransaction>, + >, + db: &State, +) -> TestValidationResult { + TestValidationResult { + logs_root: compute_logs_hash( + filter_validation_logs( + exec_result + .as_ref() + .map(|result| result.logs()) + .unwrap_or_default(), + ) + .as_slice(), + ), + state_root: state_merkle_trie_root(db.cache.trie_account()), + } +} + +pub fn compute_logs_hash(logs: &[Log]) -> B256 { + let mut encoded = Vec::new(); + alloy_rlp::encode_list(logs, &mut encoded); + keccak256(&encoded) +} + +pub fn filter_validation_logs(logs: &[Log]) -> Vec { + logs.iter() + .filter(|log| log.address != ARC_NATIVE_COIN_AUTHORITY) + .cloned() + .collect() +} + +pub fn state_merkle_trie_root<'a>( + accounts: impl IntoIterator, +) -> B256 { + trie_root( + accounts + .into_iter() + .filter(|(_, account)| !is_empty_plain_account(account)) + .map(|(address, account)| { + ( + address, + alloy_rlp::encode_fixed_size(&TrieAccount::new(account)), + ) + }), + ) +} + +#[derive(RlpEncodable, RlpMaxEncodedLen)] +struct TrieAccount { + nonce: u64, + balance: U256, + root_hash: B256, + code_hash: B256, +} + +impl TrieAccount { + fn new(account: &PlainAccount) -> Self { + Self { + nonce: account.info.nonce, + balance: account.info.balance, + root_hash: sec_trie_root::( + account + .storage + .iter() + .filter(|(_slot, value)| !value.is_zero()) + .map(|(slot, value)| { + ( + slot.to_be_bytes::<32>(), + alloy_rlp::encode_fixed_size(value), + ) + }), + ), + code_hash: account.info.code_hash, + } + } +} + +fn is_empty_plain_account(account: &PlainAccount) -> bool { + account.info.is_empty() && account.storage.values().all(|value| value.is_zero()) +} + +#[inline] +fn trie_root(input: I) -> B256 +where + I: IntoIterator, + A: AsRef<[u8]>, + B: AsRef<[u8]>, +{ + sec_trie_root::(input) +} + +#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] +struct KeccakHasher; + +impl Hasher for KeccakHasher { + type Out = B256; + type StdHasher = PlainHasher; + const LENGTH: usize = 32; + + fn hash(x: &[u8]) -> Self::Out { + keccak256(x) + } +} + +pub fn compute_state_root_from_fixture_accounts( + accounts: &alloy_primitives::map::HashMap, +) -> B256 { + state_root_unhashed(accounts.iter().map(|(address, account)| { + let storage_root = if account.storage.is_empty() { + EMPTY_ROOT_HASH + } else { + storage_root_unhashed( + account + .storage + .iter() + .map(|(slot, value)| (B256::from(slot.to_be_bytes::<32>()), *value)), + ) + }; + ( + *address, + FixtureTrieAccount { + nonce: account.nonce, + balance: account.balance, + storage_root, + code_hash: keccak256(&account.code), + }, + ) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{address, Bytes}; + use revm::state::AccountInfo; + use revm_database::states::CacheState; + + #[test] + fn logs_hash_changes_when_log_payload_changes() { + let baseline_log = Log { + address: address!("1000000000000000000000000000000000000001"), + data: revm_primitives::LogData::new_unchecked( + vec![B256::ZERO], + Bytes::from(vec![1, 2]), + ), + }; + let changed_log = Log { + address: baseline_log.address, + data: revm_primitives::LogData::new_unchecked( + vec![B256::repeat_byte(0x11)], + Bytes::from(vec![1, 2, 3]), + ), + }; + + let baseline_hash = compute_logs_hash(std::slice::from_ref(&baseline_log)); + assert_eq!( + baseline_hash, + compute_logs_hash(std::slice::from_ref(&baseline_log)) + ); + assert_ne!( + baseline_hash, + compute_logs_hash(std::slice::from_ref(&changed_log)) + ); + } + + #[test] + fn filter_validation_logs_excludes_arc_authority_log() { + let authority_log = Log { + address: ARC_NATIVE_COIN_AUTHORITY, + data: revm_primitives::LogData::new_unchecked(vec![B256::ZERO], Bytes::from(vec![1])), + }; + let user_log = Log { + address: address!("1000000000000000000000000000000000000001"), + data: revm_primitives::LogData::new_unchecked( + vec![B256::repeat_byte(0x11)], + Bytes::from(vec![2, 3]), + ), + }; + + assert_eq!( + filter_validation_logs(&[authority_log, user_log.clone()]), + vec![user_log] + ); + } + + #[test] + fn revm_state_root_ignores_zero_storage_slots() { + let address = address!("2000000000000000000000000000000000000002"); + let mut cache = CacheState::new(true); + cache.insert_account_with_storage( + address, + AccountInfo::default(), + std::collections::HashMap::from_iter([ + (U256::from(1), U256::ZERO), + (U256::from(2), U256::from(22)), + ]), + ); + + let root = state_merkle_trie_root(cache.trie_account()); + + let expected = + compute_state_root_from_fixture_accounts(&alloy_primitives::map::HashMap::from_iter([ + ( + address, + FixtureAccountInfo { + balance: U256::ZERO, + code: Bytes::default(), + nonce: 0, + storage: alloy_primitives::map::HashMap::from_iter([( + U256::from(2), + U256::from(22), + )]), + }, + ), + ])); + + assert_eq!(root, expected); + } + + #[test] + fn state_root_ignores_empty_accounts() { + let address = address!("3000000000000000000000000000000000000003"); + let mut cache = CacheState::new(true); + cache.insert_account(address, AccountInfo::default()); + + assert_eq!( + state_merkle_trie_root(cache.trie_account()), + EMPTY_ROOT_HASH + ); + } + + #[test] + fn fixture_state_root_matches_expected_fixture_accounts() { + let mut accounts = alloy_primitives::map::HashMap::default(); + accounts.insert( + address!("1000000000000000000000000000000000000001"), + FixtureAccountInfo { + balance: U256::from(7), + code: Bytes::default(), + nonce: 2, + storage: alloy_primitives::map::HashMap::from_iter([ + (U256::from(1), U256::from(11)), + (U256::from(2), U256::from(22)), + ]), + }, + ); + + let root = compute_state_root_from_fixture_accounts(&accounts); + assert_ne!(root, B256::ZERO); + } +} diff --git a/crates/evm-specs-tests/src/runner.rs b/crates/evm-specs-tests/src/runner.rs new file mode 100644 index 0000000..c145f3f --- /dev/null +++ b/crates/evm-specs-tests/src/runner.rs @@ -0,0 +1,1854 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ARC-backed statetest runner and harness. +//! +//! This module owns both per-suite execution and file-level orchestration, +//! closer to upstream `revme`, while preserving ARC execution semantics and +//! the structured consume-direct output contract. +//! +//! Important execution-mode note: +//! - fixture fork names still choose the REVM `cfg.spec` +//! - the executor chain spec is always ARC `LOCAL_DEV` +//! - this runner therefore measures ARC localdev behavior under Ethereum +//! fixture inputs, not pure Ethereum fork-isolated execution + +use std::{ + collections::{BTreeSet, HashMap}, + io::stderr, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, +}; + +use alloy_primitives::{address, Address, Bytes, B256}; +use arc_evm::ArcEvmFactory; +use reth_evm::{Evm, EvmEnv, EvmFactory}; +use revm::{context::CfgEnv, database::State}; +use revm_inspector::{inspectors::TracerEip3155, InspectCommitEvm}; +use revm_primitives::hardfork::SpecId; +use revm_statetest_types::{SpecName, Test, TestSuite, TestUnit}; + +use crate::adapter::{ + build_default_arc_chain_spec, build_evm_env, build_evm_factory, extract_chain_ids, + is_supported_spec, resolve_chain_id, +}; +use crate::error::{EvmSpecsTestError, TestErrorKind}; +use crate::exception_match::{ + exception_matches, tx_env_actual_exception, tx_env_exception_matches, +}; +use crate::fixture_sanitizer::strip_unsupported_fields; +use crate::result::{JsonOutcome, TestResult, TestSummary}; +use crate::roots::{ + compute_state_root_from_fixture_accounts, compute_test_roots, state_merkle_trie_root, + TestValidationResult, +}; + +const ARC_NATIVE_COIN_AUTHORITY: Address = address!("1800000000000000000000000000000000000000"); +const ARC_NATIVE_COIN_CONTROL: Address = address!("1800000000000000000000000000000000000001"); + +const SIGNAL_ARC_NATIVE_COIN_AUTHORITY_LOG_PRESENT: &str = "arc_native_coin_authority_log_present"; +const SIGNAL_ARC_NATIVE_COIN_CONTROL_STATE_TOUCHED: &str = "arc_native_coin_control_state_touched"; +const SIGNAL_ARC_SYSTEM_ACCOUNT_TOUCHED: &str = "arc_system_account_touched"; +const SIGNAL_PRECOMPILE_ADDRESS_TOUCHED: &str = "precompile_address_touched"; +const SIGNAL_COINBASE_TOUCHED: &str = "coinbase_touched"; +const SIGNAL_FIXTURE_ORACLE_ROOT_MATCHES_FIXTURE_HASH: &str = + "fixture_oracle_root_matches_fixture_hash"; + +#[derive(Default)] +struct FileRunOutput { + results: Vec, + summary: TestSummary, + fatal_file_errors: usize, +} + +struct RunnerConfig { + filter_name: Option, + trace: bool, + json_outcome: bool, +} + +#[derive(Clone)] +struct RunnerState { + completed_files: Arc, + queue: Arc)>>, + total_files: usize, +} + +impl RunnerState { + fn new(files: Vec) -> Self { + let total_files = files.len(); + Self { + completed_files: Arc::new(AtomicUsize::new(0)), + queue: Arc::new(Mutex::new((0, files))), + total_files, + } + } + + fn next_file(&self) -> Result, EvmSpecsTestError> { + let (next_idx, files) = &mut *self + .queue + .lock() + .map_err(|_| EvmSpecsTestError::RunnerQueuePoisoned)?; + let index = *next_idx; + let Some(path) = files.get(index).cloned() else { + return Ok(None); + }; + *next_idx = index.checked_add(1).expect("runner queue index overflow"); + Ok(Some((index, path))) + } +} + +struct TestExecutionContext<'a> { + factory: &'a ArcEvmFactory, + cache_state: &'a revm_database::states::CacheState, + evm_env: &'a EvmEnv, + unit: &'a TestUnit, + test: &'a Test, + test_id: &'a str, +} + +struct DebugContext<'a> { + factory: &'a ArcEvmFactory, + cache_state: &'a revm_database::states::CacheState, + evm_env: &'a EvmEnv, + unit: &'a TestUnit, + test: &'a Test, + test_id: &'a str, + error: &'a EvmSpecsTestError, +} + +struct TestExecutionReport { + error: Option, + json_outcome: Option, +} + +struct LoadedFixtureFile { + suite: TestSuite, + chain_id_map: HashMap, +} + +struct TestUnitExecution<'a, 'b> { + name: &'a str, + unit: &'a TestUnit, + factory: &'a ArcEvmFactory, + chain_id_map: &'a HashMap, + filter_name: Option<&'a str>, + trace: bool, + json_outcome: bool, + summary: &'b mut TestSummary, + results: &'b mut Vec, +} + +pub fn run( + path: PathBuf, + filter_name: Option, + strict_exit: bool, + trace: bool, + json_outcome: bool, +) -> Result { + use crate::result::RunStatus; + + let json_files = find_json_files(&path)?; + if json_files.is_empty() { + return Err(EvmSpecsTestError::NoJsonFiles { + path: path.display().to_string(), + }); + } + + let state = RunnerState::new(json_files); + let config = Arc::new(RunnerConfig { + filter_name, + trace, + json_outcome, + }); + let num_threads = determine_thread_count(state.total_files, trace); + + let mut handles = Vec::with_capacity(num_threads); + for worker_id in 0..num_threads { + let state = state.clone(); + let config = Arc::clone(&config); + let thread = std::thread::Builder::new() + .name(format!("arc-evm-specs-tests-runner-{worker_id}")) + .spawn(move || run_file_worker(state, config)) + .map_err(EvmSpecsTestError::WorkerSpawn)?; + handles.push(thread); + } + + let mut indexed_outputs = Vec::new(); + for handle in handles { + let output = handle + .join() + .map_err(|_| EvmSpecsTestError::WorkerPanic)??; + indexed_outputs.extend(output); + } + indexed_outputs.sort_by_key(|(index, _)| *index); + + let mut all_results = Vec::new(); + let mut total_summary = TestSummary::default(); + let mut fatal_file_errors = 0usize; + + for (_, output) in indexed_outputs { + all_results.extend(output.results); + fatal_file_errors = fatal_file_errors + .checked_add(output.fatal_file_errors) + .expect("fatal_file_errors overflow"); + total_summary.add_files_processed(output.summary.files_processed); + total_summary.add_tests_total(output.summary.tests_total); + total_summary.add_tests_passed(output.summary.tests_passed); + total_summary.add_tests_failed(output.summary.tests_failed); + total_summary.add_tests_skipped_by_spec(output.summary.tests_skipped_by_spec); + } + + let json_output = + serde_json::to_string_pretty(&all_results).expect("Failed to serialize results"); + println!("{json_output}"); + eprintln!("{total_summary}"); + + if fatal_file_errors > 0 { + return Ok(RunStatus::FatalFileErrors); + } + if strict_exit && total_summary.tests_failed > 0 { + return Ok(RunStatus::TestsFailed); + } + + Ok(RunStatus::Success) +} + +/// Execute all tests in a deserialized TestSuite. +/// +/// `chain_id_map` is built from raw JSON via `extract_chain_ids()` before +/// calling this function (two-pass parse pattern). +/// +/// Returns a vec of TestResult (one per test variant) and updates the summary. +pub fn execute_test_suite( + suite: &TestSuite, + factory: &ArcEvmFactory, + chain_id_map: &HashMap, + filter_name: Option<&str>, + trace: bool, + json_outcome: bool, +) -> (Vec, TestSummary) { + let mut summary = TestSummary::default(); + let mut results = Vec::new(); + + for (name, unit) in &suite.0 { + execute_test_unit(TestUnitExecution { + name, + unit, + factory, + chain_id_map, + filter_name, + trace, + json_outcome, + summary: &mut summary, + results: &mut results, + }); + } + + (results, summary) +} + +fn run_file_worker( + state: RunnerState, + config: Arc, +) -> Result, EvmSpecsTestError> { + let mut outputs = Vec::new(); + + while let Some((index, file)) = state.next_file()? { + let output = process_fixture_file( + &file, + config.filter_name.as_deref(), + config.trace, + config.json_outcome, + &state.completed_files, + state.total_files, + ); + outputs.push((index, output)); + } + + Ok(outputs) +} + +fn process_fixture_file( + file: &Path, + filter_name: Option<&str>, + trace: bool, + json_outcome: bool, + completed_files: &AtomicUsize, + total_files: usize, +) -> FileRunOutput { + let loaded_fixture = match load_fixture_file(file) { + Ok(loaded_fixture) => loaded_fixture, + Err(error) => { + report_progress(completed_files, total_files); + return file_error_result(file, error); + } + }; + + let chain_spec = build_default_arc_chain_spec(); + let factory = build_evm_factory(chain_spec); + let (results, mut summary) = execute_test_suite( + &loaded_fixture.suite, + &factory, + &loaded_fixture.chain_id_map, + filter_name, + trace, + json_outcome, + ); + summary.files_processed = 1; + + report_progress(completed_files, total_files); + + FileRunOutput { + results, + summary, + fatal_file_errors: 0, + } +} + +fn load_fixture_file(file: &Path) -> Result { + let bytes = std::fs::read(file).map_err(|e| format!("read error: {e}"))?; + let raw_json: serde_json::Value = serde_json::from_slice(&bytes).map_err(|source| { + EvmSpecsTestError::JsonParse { + path: file.display().to_string(), + source, + } + .to_string() + })?; + + if !looks_like_statetest_fixture(&raw_json) { + return Err(format!( + "unsupported or malformed statetest fixture shape in {}", + file.display() + )); + } + + let chain_id_map = extract_chain_ids(&raw_json).map_err(|e| e.to_string())?; + let mut sanitized_json = raw_json; + strip_unsupported_fields(&mut sanitized_json); + let suite = serde_json::from_value(sanitized_json).map_err(|source| { + EvmSpecsTestError::JsonParse { + path: file.display().to_string(), + source, + } + .to_string() + })?; + + Ok(LoadedFixtureFile { + suite, + chain_id_map, + }) +} + +fn execute_test_unit(exec: TestUnitExecution<'_, '_>) { + let TestUnitExecution { + name, + unit, + factory, + chain_id_map, + filter_name, + trace, + json_outcome, + summary, + results, + } = exec; + let cache_state = unit.state(); + + let chain_id = + match resolve_test_unit_chain_id(name, unit, chain_id_map, filter_name, summary, results) { + Some(chain_id) => chain_id, + None => return, + }; + + for (spec_name, tests) in &unit.post { + execute_spec_tests( + name, + unit, + factory, + &cache_state, + chain_id, + spec_name, + tests, + filter_name, + trace, + json_outcome, + summary, + results, + ); + } +} + +fn resolve_test_unit_chain_id( + name: &str, + unit: &TestUnit, + chain_id_map: &HashMap, + filter_name: Option<&str>, + summary: &mut TestSummary, + results: &mut Vec, +) -> Option { + match resolve_chain_id(name, chain_id_map, unit) { + Ok(chain_id) => Some(chain_id), + Err(error) => { + let rendered_error = error.to_string(); + for (spec_name, tests) in &unit.post { + push_failures_for_spec_variants( + name, + spec_name, + tests, + filter_name, + &rendered_error, + summary, + results, + ); + } + None + } + } +} + +#[allow(clippy::too_many_arguments)] +fn execute_spec_tests( + name: &str, + unit: &TestUnit, + factory: &ArcEvmFactory, + cache_state: &revm_database::states::CacheState, + chain_id: u64, + spec_name: &SpecName, + tests: &[Test], + filter_name: Option<&str>, + trace: bool, + json_outcome: bool, + summary: &mut TestSummary, + results: &mut Vec, +) { + if !is_supported_spec(spec_name) { + record_skipped_spec_tests(name, spec_name, tests, filter_name, summary); + return; + } + + let evm_env = match build_spec_evm_env(unit, spec_name, chain_id) { + Ok(evm_env) => evm_env, + Err(error) => { + push_failures_for_spec_variants( + name, + spec_name, + tests, + filter_name, + &error, + summary, + results, + ); + return; + } + }; + + for test in tests { + execute_spec_test(TestCaseExecution { + name, + unit, + factory, + cache_state, + evm_env: &evm_env, + spec_name, + test, + filter_name, + trace, + json_outcome, + summary, + results, + }); + } +} + +fn build_spec_evm_env( + unit: &TestUnit, + spec_name: &SpecName, + chain_id: u64, +) -> Result { + let mut evm_env = build_evm_env(unit, spec_name, chain_id).map_err(|e| e.to_string())?; + configure_blob_limits(&mut evm_env.cfg_env); + Ok(evm_env) +} + +struct TestCaseExecution<'a, 'b> { + name: &'a str, + unit: &'a TestUnit, + factory: &'a ArcEvmFactory, + cache_state: &'a revm_database::states::CacheState, + evm_env: &'a EvmEnv, + spec_name: &'a SpecName, + test: &'a Test, + filter_name: Option<&'a str>, + trace: bool, + json_outcome: bool, + summary: &'b mut TestSummary, + results: &'b mut Vec, +} + +fn execute_spec_test(exec: TestCaseExecution<'_, '_>) { + let test_id = format_test_id(exec.name, exec.spec_name, &exec.test.indexes); + if let Some(filter) = exec.filter_name + && !matches_filter(filter, &test_id, exec.name, exec.spec_name) + { + return; + } + + let result_name = output_name_for_filter(exec.filter_name, &test_id, exec.name, exec.spec_name); + exec.summary.add_tests_total(1); + + let ctx = TestExecutionContext { + factory: exec.factory, + cache_state: exec.cache_state, + evm_env: exec.evm_env, + unit: exec.unit, + test: exec.test, + test_id: &test_id, + }; + + let report = execute_single_test(ctx, exec.json_outcome); + push_test_execution_result(exec, test_id, result_name, report); +} + +fn push_test_execution_result( + exec: TestCaseExecution<'_, '_>, + test_id: String, + result_name: String, + report: TestExecutionReport, +) { + match report { + TestExecutionReport { + error: None, + json_outcome, + } => { + exec.summary.add_tests_passed(1); + if let Some(outcome) = json_outcome.as_ref() { + emit_json_outcome_line(outcome, true); + } + let result = TestResult::passed(result_name).with_variant_id(test_id); + exec.results.push(match json_outcome { + Some(outcome) => result.with_json_outcome(outcome), + None => result, + }); + } + TestExecutionReport { + error: Some(error), + json_outcome, + } => { + if exec.trace { + debug_failed_test(DebugContext { + factory: exec.factory, + cache_state: exec.cache_state, + evm_env: exec.evm_env, + unit: exec.unit, + test: exec.test, + test_id: &test_id, + error: &error, + }); + } + exec.summary.add_tests_failed(1); + if let Some(outcome) = json_outcome.as_ref() { + emit_json_outcome_line(outcome, false); + } + let result = + TestResult::failed(result_name, error.to_string()).with_variant_id(test_id); + exec.results.push(match json_outcome { + Some(outcome) => result.with_json_outcome(outcome), + None => result, + }); + } + } +} + +fn emit_json_outcome_line(outcome: &JsonOutcome, pass: bool) { + eprintln!("{}", build_json_outcome_report(outcome, pass)); +} + +fn build_json_outcome_report(outcome: &JsonOutcome, pass: bool) -> serde_json::Value { + serde_json::json!({ + "stateRoot": outcome.state_root, + "logsRoot": outcome.logs_root, + "output": outcome.output, + "gasUsed": outcome.gas_used, + "pass": pass, + "errorMsg": outcome.error_msg, + "evmResult": outcome.evm_result, + "postLogsHash": outcome.post_logs_hash, + "fork": outcome.fork, + "test": outcome.test, + // Keep the legacy aliases alongside the readable names so existing + // revm-style parsers do not silently break when consume-direct starts + // persisting `per_test_outcomes.jsonl`. + "d": outcome.data_index, + "g": outcome.gas_index, + "v": outcome.value_index, + "data_index": outcome.data_index, + "gas_index": outcome.gas_index, + "value_index": outcome.value_index, + }) +} + +fn record_skipped_spec_tests( + name: &str, + spec_name: &SpecName, + tests: &[Test], + filter_name: Option<&str>, + summary: &mut TestSummary, +) { + if let Some(filter) = filter_name { + for test in tests { + let test_id = format_test_id(name, spec_name, &test.indexes); + if matches_filter(filter, &test_id, name, spec_name) { + summary.add_tests_skipped_by_spec(1); + } + } + return; + } + summary.add_tests_skipped_by_spec(tests.len()); +} + +fn push_failures_for_spec_variants( + name: &str, + spec_name: &SpecName, + tests: &[Test], + filter_name: Option<&str>, + error: &str, + summary: &mut TestSummary, + results: &mut Vec, +) { + for test in tests { + let test_id = format_test_id(name, spec_name, &test.indexes); + if let Some(filter) = filter_name + && !matches_filter(filter, &test_id, name, spec_name) + { + continue; + } + let result_name = output_name_for_filter(filter_name, &test_id, name, spec_name); + summary.add_tests_total(1); + summary.add_tests_failed(1); + results.push(TestResult::failed(result_name, error.to_string()).with_variant_id(test_id)); + } +} + +fn configure_blob_limits(cfg: &mut CfgEnv) { + if cfg.spec.is_enabled_in(SpecId::OSAKA) { + cfg.set_max_blobs_per_tx(6); + } else if cfg.spec.is_enabled_in(SpecId::PRAGUE) { + cfg.set_max_blobs_per_tx(9); + } else { + cfg.set_max_blobs_per_tx(6); + } +} + +/// Execute a single test variant and validate results. +fn execute_single_test(ctx: TestExecutionContext<'_>, json_outcome: bool) -> TestExecutionReport { + let tx = match ctx.test.tx_env(ctx.unit) { + Ok(tx) => tx, + Err(err) => match handle_tx_env_error(&ctx, &err.to_string()) { + Ok(()) => { + return TestExecutionReport { + error: None, + json_outcome: None, + }; + } + Err(error) => { + return TestExecutionReport { + error: Some(error), + json_outcome: None, + }; + } + }, + }; + + let state = State::builder() + .with_cached_prestate(ctx.cache_state.clone()) + .with_bundle_update() + .build(); + + let mut evm = ctx.factory.create_evm(state, ctx.evm_env.clone()); + let exec_result = evm.transact_commit(tx); + let db = &*evm.db_mut(); + + let validation = compute_test_roots(&exec_result, db); + let error = evaluate_evm_execution(&ctx, ctx.unit.out.as_ref(), &exec_result, db, &validation); + let json_outcome = json_outcome.then(|| { + build_json_outcome( + &ctx, + &exec_result, + &validation, + error.as_ref().map(std::string::ToString::to_string), + ) + }); + + TestExecutionReport { + error, + json_outcome, + } +} + +fn handle_tx_env_error(ctx: &TestExecutionContext<'_>, err: &str) -> Result<(), EvmSpecsTestError> { + let actual_exception = tx_env_actual_exception(err).unwrap_or(err); + + if let Some(expected) = &ctx.test.expect_exception { + if tx_env_exception_matches(expected, actual_exception) { + return Ok(()); + } + + return Err(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "WRONG_EXCEPTION", + format!("expected_exception={expected}, got_exception={actual_exception}"), + ), + }); + } + + Err(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "HARNESS_PRECONDITION", + "TX_ENV_BUILD_FAILED", + err.to_string(), + ), + }) +} + +fn validate_expected_exception( + ctx: &TestExecutionContext<'_>, + exec_result: &Result< + revm::context::result::ExecutionResult, + revm::context::result::EVMError< + revm::database::bal::EvmDatabaseError, + revm::context::result::InvalidTransaction, + >, + >, +) -> Result { + match (&ctx.test.expect_exception, exec_result) { + (None, Err(e)) => Err(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "UNEXPECTED_EXCEPTION", + format!("expected_exception=None, got_exception={}", e), + ), + }), + (Some(expected), Ok(_)) => Err(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "UNEXPECTED_SUCCESS", + format!("expected_exception={expected}, got_exception=None"), + ), + }), + (Some(expected), Err(actual)) => { + let actual = actual.to_string(); + if exception_matches(expected, &actual) { + Ok(true) + } else { + Err(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "WRONG_EXCEPTION", + format!("expected_exception={expected}, got_exception={actual}"), + ), + }) + } + } + (None, Ok(_)) => Ok(false), + } +} + +fn validate_output( + ctx: &TestExecutionContext<'_>, + expected_output: Option<&Bytes>, + actual_result: &revm::context::result::ExecutionResult, +) -> Result<(), EvmSpecsTestError> { + if let Some((expected, actual)) = expected_output.zip(actual_result.output()) + && expected != actual + { + return Err(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "UNEXPECTED_OUTPUT", + format!("expected_output={expected:?}, got_output={actual:?}"), + ), + }); + } + + Ok(()) +} + +fn evaluate_evm_execution( + ctx: &TestExecutionContext<'_>, + expected_output: Option<&Bytes>, + exec_result: &Result< + revm::context::result::ExecutionResult, + revm::context::result::EVMError< + revm::database::bal::EvmDatabaseError, + revm::context::result::InvalidTransaction, + >, + >, + db: &State, + validation: &TestValidationResult, +) -> Option { + let logs = exec_result + .as_ref() + .map(|result| result.logs()) + .unwrap_or_default(); + + match validate_expected_exception(ctx, exec_result) { + Ok(true) => return None, + Ok(false) => {} + Err(error) => return Some(error), + } + + if let Ok(result) = exec_result + && let Err(error) = validate_output(ctx, expected_output, result) + { + return Some(error); + } + + if validation.logs_root != ctx.test.logs { + let logs_preview = summarize_logs(logs); + let signals = summarize_arc_signals(logs, db, None, ctx.unit.env.current_coinbase); + return Some(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "LOGS_HASH_MISMATCH", + format!( + "expected={}, got={}; logs_count={}, {}; signals={signals}", + ctx.test.logs, + validation.logs_root, + logs.len(), + logs_preview + ), + ), + }); + } + + if validation.state_root != ctx.test.hash { + let diagnostic = build_state_root_diagnostic(ctx, db, logs); + return Some(EvmSpecsTestError::TestFailure { + name: ctx.test_id.to_string(), + kind: TestErrorKind::evm( + "EXECUTION_MISMATCH", + "STATE_ROOT_MISMATCH", + format!( + "expected={}, got={}; diagnostic: {diagnostic}", + ctx.test.hash, validation.state_root + ), + ), + }); + } + + None +} + +fn build_state_root_diagnostic( + ctx: &TestExecutionContext<'_>, + db: &State, + logs: &[revm_primitives::Log], +) -> String { + let coinbase = ctx.unit.env.current_coinbase; + let coinbase_delta = db + .cache + .accounts + .get(&coinbase) + .and_then(|account| account.account.as_ref()) + .map(|account| { + format!( + "nonce={},balance={}", + account.info.nonce, account.info.balance + ) + }) + .unwrap_or_else(|| "missing".to_string()); + let touched_accounts = summarize_touched_accounts(db); + let actual_trie_accounts: BTreeSet<_> = db + .cache + .trie_account() + .into_iter() + .map(|(address, _)| address) + .collect(); + let filtered_arc_system_root = state_merkle_trie_root( + db.cache + .trie_account() + .into_iter() + .filter(|(address, _)| *address != ARC_NATIVE_COIN_CONTROL), + ); + let filtered_arc_system_and_coinbase_root = state_merkle_trie_root( + db.cache + .trie_account() + .into_iter() + .filter(|(address, _)| *address != ARC_NATIVE_COIN_CONTROL && *address != coinbase), + ); + + if ctx.test.post_state.is_empty() { + let signals = summarize_arc_signals(logs, db, None, coinbase); + return format!( + "fixture postState unavailable; actual_trie_accounts={}, filtered_arc_system_root={filtered_arc_system_root}, filtered_arc_system_and_coinbase_root={filtered_arc_system_and_coinbase_root}, fixture_hash={}, coinbase={}, coinbase_delta={coinbase_delta}, touched_accounts={touched_accounts}, signals={signals}", + actual_trie_accounts.len(), + ctx.test.hash, + coinbase + ); + } + + let oracle_root = compute_state_root_from_fixture_accounts(&ctx.test.post_state); + let expected_trie_accounts: BTreeSet<_> = ctx.test.post_state.keys().copied().collect(); + let extra_actual_accounts = actual_trie_accounts + .difference(&expected_trie_accounts) + .take(6) + .map(|address| address.to_string()) + .collect::>() + .join("|"); + let missing_expected_accounts = expected_trie_accounts + .difference(&actual_trie_accounts) + .take(6) + .map(|address| address.to_string()) + .collect::>() + .join("|"); + let signals = summarize_arc_signals(logs, db, Some((oracle_root, ctx.test.hash)), coinbase); + + format!( + "fixture_postState_root={oracle_root}, actual_trie_accounts={}, expected_trie_accounts={}, extra_actual_accounts={}, missing_expected_accounts={}, filtered_arc_system_root={filtered_arc_system_root}, filtered_arc_system_and_coinbase_root={filtered_arc_system_and_coinbase_root}, fixture_hash={}, coinbase={}, coinbase_delta={coinbase_delta}, touched_accounts={touched_accounts}, signals={signals}", + actual_trie_accounts.len(), + expected_trie_accounts.len(), + if extra_actual_accounts.is_empty() { "none" } else { &extra_actual_accounts }, + if missing_expected_accounts.is_empty() { "none" } else { &missing_expected_accounts }, + ctx.test.hash, + coinbase + ) +} + +fn build_json_outcome( + ctx: &TestExecutionContext<'_>, + exec_result: &Result< + revm::context::result::ExecutionResult, + revm::context::result::EVMError< + revm::database::bal::EvmDatabaseError, + revm::context::result::InvalidTransaction, + >, + >, + validation: &TestValidationResult, + error: Option, +) -> JsonOutcome { + JsonOutcome { + state_root: validation.state_root, + logs_root: validation.logs_root, + output: exec_result + .as_ref() + .ok() + .and_then(|result| result.output().cloned()) + .unwrap_or_default(), + gas_used: exec_result + .as_ref() + .ok() + .map(|result| result.gas_used()) + .unwrap_or_default(), + error_msg: error.unwrap_or_default(), + evm_result: format_evm_result(exec_result), + post_logs_hash: validation.logs_root, + fork: format!("{:?}", ctx.evm_env.cfg_env.spec), + test: ctx.test_id.to_string(), + data_index: ctx.test.indexes.data, + gas_index: ctx.test.indexes.gas, + value_index: ctx.test.indexes.value, + } +} + +fn format_evm_result( + exec_result: &Result< + revm::context::result::ExecutionResult, + revm::context::result::EVMError< + revm::database::bal::EvmDatabaseError, + revm::context::result::InvalidTransaction, + >, + >, +) -> String { + match exec_result { + Ok(result) => match result { + revm::context::result::ExecutionResult::Success { reason, .. } => { + format!("Success: {reason:?}") + } + revm::context::result::ExecutionResult::Revert { .. } => "Revert".to_string(), + revm::context::result::ExecutionResult::Halt { reason, .. } => { + format!("Halt: {reason:?}") + } + }, + Err(error) => error.to_string(), + } +} + +/// Format the stable identifier for a concrete fixture variant. +/// +/// This ID ties together the suite-level pytest HTML report and the +/// variant-level `per_test_outcomes.jsonl` artifact emitted during +/// `consume direct`, so keep it stable unless the downstream reporting +/// contract is updated in lockstep. +fn format_test_id( + name: &str, + spec: &SpecName, + indexes: &revm_statetest_types::TxPartIndices, +) -> String { + // Keep the historical `d{}_g{}_v{}` suffix inherited from the upstream + // `revm` statetest runner shape (`bins/revme/src/cmd/statetest/runner.rs`). + format!( + "{name}/{spec:?}/d{}_g{}_v{}", + indexes.data, indexes.gas, indexes.value + ) +} + +fn summarize_logs(logs: &[revm_primitives::Log]) -> String { + if logs.is_empty() { + return "first_log=none".to_string(); + } + + let first = &logs[0]; + let topics_len = first.data.topics().len(); + let data_len = first.data.data.len(); + format!( + "first_log=address={},topics={},data_len={}", + first.address, topics_len, data_len + ) +} + +fn summarize_arc_signals( + logs: &[revm_primitives::Log], + db: &State, + fixture_roots: Option<(B256, B256)>, + coinbase: Address, +) -> String { + let mut signals = Vec::new(); + + if logs + .iter() + .any(|log| log.address == ARC_NATIVE_COIN_AUTHORITY) + { + signals.push(SIGNAL_ARC_NATIVE_COIN_AUTHORITY_LOG_PRESENT.to_string()); + } + + if db.cache.accounts.contains_key(&ARC_NATIVE_COIN_CONTROL) { + signals.push(format!( + "{SIGNAL_ARC_NATIVE_COIN_CONTROL_STATE_TOUCHED}={ARC_NATIVE_COIN_CONTROL}" + )); + signals.push(format!( + "{SIGNAL_ARC_SYSTEM_ACCOUNT_TOUCHED}={ARC_NATIVE_COIN_CONTROL}" + )); + } + + if db.cache.accounts.contains_key(&coinbase) { + signals.push(format!("{SIGNAL_COINBASE_TOUCHED}={coinbase}")); + } + + if let Some(precompile_address) = first_touched_precompile_address(db) { + signals.push(format!( + "{SIGNAL_PRECOMPILE_ADDRESS_TOUCHED}={precompile_address}" + )); + } + + if let Some((oracle_root, fixture_hash)) = fixture_roots + && oracle_root == fixture_hash + { + signals.push(SIGNAL_FIXTURE_ORACLE_ROOT_MATCHES_FIXTURE_HASH.to_string()); + } + + if signals.is_empty() { + "none".to_string() + } else { + signals.join("|") + } +} + +fn summarize_touched_accounts(db: &State) -> String { + let mut entries: Vec<_> = db.cache.accounts.iter().collect(); + entries.sort_by_key(|(address, _)| *address); + + let preview = entries + .into_iter() + .take(4) + .filter_map(|(address, account)| account.account.as_ref().map(|plain| (address, plain))) + .map(|(address, account)| { + format!( + "{address}:nonce={},balance={},code_hash={},storage_slots={},selfdestructed={}", + account.info.nonce, + account.info.balance, + account.info.code_hash, + account.storage.len(), + false + ) + }) + .collect::>() + .join("|"); + + if preview.is_empty() { + "none".to_string() + } else { + preview + } +} + +fn first_touched_precompile_address(db: &State) -> Option
{ + let mut addresses: Vec<_> = db.cache.accounts.keys().copied().collect(); + addresses.sort(); + addresses + .into_iter() + .find(|address| is_precompile_address(*address)) +} + +fn is_precompile_address(address: Address) -> bool { + let bytes = address.as_slice(); + if !bytes[..18].iter().all(|byte| *byte == 0) { + return false; + } + + let index = u16::from_be_bytes([bytes[18], bytes[19]]); + matches!(index, 0x0001..=0x0011 | 0x0100) +} + +fn matches_filter(filter: &str, test_id: &str, fixture_name: &str, _spec: &SpecName) -> bool { + let filter_norm = normalize_fixture_name(filter); + let fixture_norm = normalize_fixture_name(fixture_name); + filter == test_id + || filter == fixture_name + || filter_norm == fixture_norm + || is_consume_state_filter_for_fixture(filter, fixture_name) +} + +fn output_name_for_filter( + filter_name: Option<&str>, + test_id: &str, + fixture_name: &str, + _spec: &SpecName, +) -> String { + match filter_name { + Some(filter) if filter == fixture_name => test_id.to_string(), + Some(filter) if normalize_fixture_name(filter) == normalize_fixture_name(fixture_name) => { + test_id.to_string() + } + Some(filter) if is_consume_state_filter_for_fixture(filter, fixture_name) => { + filter.to_string() + } + _ => fixture_name.to_string(), + } +} + +fn is_consume_state_filter_for_fixture(filter: &str, fixture_name: &str) -> bool { + let Some(suffix) = filter.strip_prefix(fixture_name) else { + return false; + }; + suffix.starts_with("[fork_") && suffix.ends_with("-state_test]") +} + +fn normalize_fixture_name(name: &str) -> &str { + name.strip_prefix("./").unwrap_or(name) +} + +fn debug_failed_test(ctx: DebugContext<'_>) { + eprintln!("\nTraces:"); + + let tx = match ctx.test.tx_env(ctx.unit) { + Ok(tx) => tx, + Err(err) => { + eprintln!("Unable to rebuild tx for trace rerun: {err}"); + eprintln!( + "\nTest name: {:?} failed before trace rerun:\n{}", + ctx.test_id, ctx.error + ); + return; + } + }; + + let state = State::builder() + .with_cached_prestate(ctx.cache_state.clone()) + .with_bundle_update() + .build(); + + let tracer = TracerEip3155::buffered(stderr()).without_summary(); + let mut evm = ctx + .factory + .create_evm_with_inspector(state, ctx.evm_env.clone(), tracer); + let exec_result = evm.inner.inspect_tx_commit(tx.clone()); + + eprintln!("\nExecution result: {exec_result:#?}"); + eprintln!("\nExpected exception: {:?}", ctx.test.expect_exception); + eprintln!("\nState before:\n{}", ctx.cache_state.pretty_print()); + eprintln!("\nState after:\n{}", evm.db_mut().cache.pretty_print()); + eprintln!("\nSpecification: {:?}", ctx.evm_env.cfg_env.spec); + eprintln!("\nTx: {tx:#?}"); + eprintln!("Block: {:#?}", ctx.evm_env.block_env); + eprintln!("Cfg: {:#?}", ctx.evm_env.cfg_env); + eprintln!("\nTest name: {:?} failed:\n{}", ctx.test_id, ctx.error); +} + +fn determine_thread_count(total_files: usize, trace: bool) -> usize { + if trace { + return 1; + } + std::thread::available_parallelism() + .map(|count| count.get().min(total_files)) + .unwrap_or(1) + .max(1) +} + +fn find_json_files(path: &Path) -> Result, EvmSpecsTestError> { + let mut files = Vec::new(); + if path.is_file() { + if path.extension().is_some_and(|ext| ext == "json") { + files.push(path.to_path_buf()); + } + } else if path.is_dir() { + collect_json_files_recursive(path, &mut files)?; + } + Ok(files) +} + +fn collect_json_files_recursive( + dir: &Path, + files: &mut Vec, +) -> Result<(), EvmSpecsTestError> { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_json_files_recursive(&path, files)?; + } else if path.extension().is_some_and(|ext| ext == "json") { + files.push(path); + } + } + Ok(()) +} + +fn file_error_result(file: &Path, error: String) -> FileRunOutput { + let summary = TestSummary { + files_processed: 1, + ..TestSummary::default() + }; + + FileRunOutput { + results: vec![TestResult::failed( + format!("__file_error__/{}", file.display()), + error, + )], + summary, + fatal_file_errors: 1, + } +} + +fn looks_like_statetest_fixture(value: &serde_json::Value) -> bool { + let Some(obj) = value.as_object() else { + return false; + }; + obj.values().any(|entry| { + entry.is_object() + && entry.get("env").is_some() + && entry.get("post").is_some() + && entry.get("pre").is_some() + }) +} + +fn report_progress(completed_files: &AtomicUsize, total_files: usize) { + let done = completed_files + .fetch_add(1, Ordering::Relaxed) + .checked_add(1) + .expect("completed_files overflow"); + if done == 1 || done == total_files || done.is_multiple_of(100) { + eprintln!("[arc-evm-specs-tests] processed {done}/{total_files} fixture files"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{bytes, LogData}; + use reth_evm::EvmEnv; + use revm::context::result::{ + EVMError, ExecutionResult, HaltReason, InvalidTransaction, OutOfGasError, Output, + SuccessReason, + }; + use revm::database::{EmptyDB, State}; + use revm::state::AccountInfo; + use revm_database::states::CacheState; + use revm_statetest_types::{Env, Test, TestSuite, TransactionParts}; + use std::{ + collections::BTreeMap, + time::{SystemTime, UNIX_EPOCH}, + }; + + fn touched_state(addresses: &[Address]) -> State { + let mut cache = CacheState::new(true); + for address in addresses { + cache.insert_account(*address, AccountInfo::default()); + } + State::builder() + .with_cached_prestate(cache) + .with_bundle_update() + .build() + } + + fn log_at(address: Address) -> revm_primitives::Log { + revm_primitives::Log { + address, + data: LogData::new_unchecked(vec![B256::ZERO], bytes!("01")), + } + } + + fn fixture_test(expect_exception: Option<&str>) -> Test { + serde_json::from_value(serde_json::json!({ + "expectException": expect_exception, + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "hash": format!("{:#066x}", B256::ZERO), + "postState": {}, + "logs": format!("{:#066x}", B256::ZERO) + })) + .expect("test fixture should deserialize") + } + + fn fixture_unit(spec_name: SpecName) -> TestUnit { + let mut post = BTreeMap::new(); + post.insert(spec_name, vec![fixture_test(None)]); + + TestUnit { + info: None, + env: Env { + current_chain_id: Some(alloy_primitives::U256::from(1)), + current_coinbase: Address::ZERO, + current_difficulty: alloy_primitives::U256::ZERO, + current_gas_limit: alloy_primitives::U256::from(30_000_000), + current_number: alloy_primitives::U256::from(1), + current_timestamp: alloy_primitives::U256::from(1), + current_base_fee: Some(alloy_primitives::U256::ZERO), + previous_hash: None, + current_random: None, + current_beacon_root: None, + current_withdrawals_root: None, + current_excess_blob_gas: None, + }, + pre: alloy_primitives::map::HashMap::default(), + post, + transaction: TransactionParts { + tx_type: None, + data: vec![bytes!("")], + gas_limit: vec![alloy_primitives::U256::from(21_000)], + gas_price: Some(alloy_primitives::U256::from(1)), + nonce: alloy_primitives::U256::ZERO, + secret_key: B256::ZERO, + sender: Some(Address::ZERO), + to: Some(Address::ZERO), + value: vec![alloy_primitives::U256::ZERO], + max_fee_per_gas: None, + max_priority_fee_per_gas: None, + initcodes: None, + access_lists: vec![], + authorization_list: None, + blob_versioned_hashes: vec![], + max_fee_per_blob_gas: None, + }, + out: None, + } + } + + fn fixture_context<'a>( + unit: &'a TestUnit, + test: &'a Test, + evm_env: &'a EvmEnv, + test_id: &'a str, + ) -> TestExecutionContext<'a> { + let factory = + crate::adapter::build_evm_factory(crate::adapter::build_default_arc_chain_spec()); + let cache_state = CacheState::new(true); + + TestExecutionContext { + factory: Box::leak(Box::new(factory)), + cache_state: Box::leak(Box::new(cache_state)), + evm_env, + unit, + test, + test_id, + } + } + + fn call_result(output: Bytes) -> ExecutionResult { + ExecutionResult::Success { + reason: SuccessReason::Return, + gas_used: 21_000, + gas_refunded: 0, + logs: Vec::new(), + output: Output::Call(output), + } + } + + fn revert_result(output: Bytes) -> ExecutionResult { + ExecutionResult::Revert { + gas_used: 21_000, + output, + } + } + + #[test] + fn detects_arc_authority_log_signals() { + let signals = summarize_arc_signals( + &[log_at(ARC_NATIVE_COIN_AUTHORITY)], + &touched_state(&[]), + None, + Address::ZERO, + ); + + assert!(signals.contains(SIGNAL_ARC_NATIVE_COIN_AUTHORITY_LOG_PRESENT)); + } + + #[test] + fn detects_native_coin_control_touched_signals() { + let state = touched_state(&[ARC_NATIVE_COIN_CONTROL]); + let signals = summarize_arc_signals(&[], &state, None, Address::ZERO); + + assert!(signals.contains(SIGNAL_ARC_NATIVE_COIN_CONTROL_STATE_TOUCHED)); + assert!(signals.contains(SIGNAL_ARC_SYSTEM_ACCOUNT_TOUCHED)); + } + + #[test] + fn detects_touched_precompile_signal() { + let state = touched_state(&[address!("000000000000000000000000000000000000000a")]); + let signals = summarize_arc_signals(&[], &state, None, Address::ZERO); + + assert!(signals + .contains("precompile_address_touched=0x000000000000000000000000000000000000000A")); + } + + #[test] + fn ignores_non_precompile_low_address_signal() { + let state = touched_state(&[address!("000000000000000000000000000000000000006a")]); + let signals = summarize_arc_signals(&[], &state, None, Address::ZERO); + + assert!(!signals.contains(SIGNAL_PRECOMPILE_ADDRESS_TOUCHED)); + } + + #[test] + fn detects_p256_precompile_signal() { + let state = touched_state(&[address!("0000000000000000000000000000000000000100")]); + let signals = summarize_arc_signals(&[], &state, None, Address::ZERO); + + assert!(signals + .contains("precompile_address_touched=0x0000000000000000000000000000000000000100")); + } + + #[test] + fn detects_fixture_oracle_match_signal() { + let root = B256::repeat_byte(0x11); + let signals = + summarize_arc_signals(&[], &touched_state(&[]), Some((root, root)), Address::ZERO); + + assert!(signals.contains(SIGNAL_FIXTURE_ORACLE_ROOT_MATCHES_FIXTURE_HASH)); + } + + #[test] + fn filter_matching_does_not_use_substrings() { + let fixture_name = "foo_bar_1"; + let test_id = "foo_bar_1/Prague/d0_g0_v0"; + + assert!(matches_filter( + fixture_name, + test_id, + fixture_name, + &SpecName::Prague + )); + assert!(!matches_filter( + "foo_bar_10", + test_id, + fixture_name, + &SpecName::Prague + )); + } + + #[test] + fn output_name_prefers_fixture_filter_variants() { + let test_id = "fixture/Prague/d0_g0_v0"; + let fixture_name = "./fixture"; + + assert_eq!( + output_name_for_filter(Some("fixture"), test_id, fixture_name, &SpecName::Prague), + test_id + ); + assert_eq!( + output_name_for_filter( + Some("fixture[fork_Prague-state_test]"), + test_id, + "fixture", + &SpecName::Prague + ), + "fixture[fork_Prague-state_test]" + ); + assert_eq!( + output_name_for_filter(None, test_id, fixture_name, &SpecName::Prague), + fixture_name + ); + } + + #[test] + fn summarize_logs_and_touched_accounts_produce_previews() { + let log_summary = summarize_logs(&[log_at(ARC_NATIVE_COIN_AUTHORITY)]); + assert!(log_summary.contains("first_log=address=")); + assert!(log_summary.contains("topics=1")); + + let state = touched_state(&[ARC_NATIVE_COIN_CONTROL]); + let touched_summary = summarize_touched_accounts(&state); + assert!(touched_summary.contains("0x1800000000000000000000000000000000000001")); + assert!(touched_summary.contains("storage_slots=0")); + } + + #[test] + fn execute_test_suite_records_missing_chain_id_as_failures() { + let mut unit = fixture_unit(SpecName::Prague); + unit.env.current_chain_id = None; + let suite = TestSuite(BTreeMap::from([(String::from("fixture"), unit)])); + let factory = + crate::adapter::build_evm_factory(crate::adapter::build_default_arc_chain_spec()); + + let (results, summary) = + execute_test_suite(&suite, &factory, &HashMap::new(), None, false, false); + + assert_eq!(summary.tests_total, 1); + assert_eq!(summary.tests_failed, 1); + assert_eq!(results.len(), 1); + assert!(results[0].error.contains("Missing chain_id")); + } + + #[test] + fn execute_test_suite_skips_unknown_specs() { + let suite = TestSuite(BTreeMap::from([( + String::from("fixture"), + fixture_unit(SpecName::Unknown), + )])); + let factory = + crate::adapter::build_evm_factory(crate::adapter::build_default_arc_chain_spec()); + + let (results, summary) = + execute_test_suite(&suite, &factory, &HashMap::new(), None, false, false); + + assert!(results.is_empty()); + assert_eq!(summary.tests_total, 0); + assert_eq!(summary.tests_failed, 0); + assert_eq!(summary.tests_skipped_by_spec, 1); + } + + #[test] + fn execute_test_suite_reports_tx_env_build_failures() { + let mut unit = fixture_unit(SpecName::Prague); + unit.transaction.to = None; + unit.transaction.max_fee_per_blob_gas = Some(alloy_primitives::U256::from(1)); + let suite = TestSuite(BTreeMap::from([(String::from("fixture"), unit)])); + let factory = + crate::adapter::build_evm_factory(crate::adapter::build_default_arc_chain_spec()); + + let (results, summary) = + execute_test_suite(&suite, &factory, &HashMap::new(), None, false, false); + + assert_eq!(summary.tests_total, 1); + assert_eq!(summary.tests_failed, 1); + assert_eq!(results.len(), 1); + assert!(results[0].error.contains("HARNESS_PRECONDITION")); + assert!(results[0].error.contains("TX_ENV_BUILD_FAILED")); + } + + #[test] + fn handle_tx_env_error_accepts_fixture_declared_exception() { + let unit = fixture_unit(SpecName::Prague); + let test = fixture_test(Some("PriorityGreaterThanMaxFeePerGas")); + let evm_env = crate::adapter::build_evm_env(&unit, &SpecName::Prague, 1) + .expect("test env should build"); + let ctx = fixture_context(&unit, &test, &evm_env, "fixture/Prague/d0_g0_v0"); + + let result = handle_tx_env_error( + &ctx, + "tx env build failed: got Some(\"priority fee is greater than max fee\")", + ); + + assert!(result.is_ok()); + } + + #[test] + fn handle_tx_env_error_reports_harness_precondition_without_expected_exception() { + let unit = fixture_unit(SpecName::Prague); + let test = fixture_test(None); + let evm_env = crate::adapter::build_evm_env(&unit, &SpecName::Prague, 1) + .expect("test env should build"); + let ctx = fixture_context(&unit, &test, &evm_env, "fixture/Prague/d0_g0_v0"); + + let error = handle_tx_env_error(&ctx, "tx env build failed: missing to").unwrap_err(); + let rendered = error.to_string(); + + assert!(rendered.contains("HARNESS_PRECONDITION")); + assert!(rendered.contains("TX_ENV_BUILD_FAILED")); + assert!(rendered.contains("missing to")); + } + + #[test] + fn validate_expected_exception_distinguishes_expected_and_unexpected_outcomes() { + let unit = fixture_unit(SpecName::Prague); + let evm_env = crate::adapter::build_evm_env(&unit, &SpecName::Prague, 1) + .expect("test env should build"); + + let matching_test = fixture_test(Some("PriorityGreaterThanMaxFeePerGas")); + let matching_ctx = + fixture_context(&unit, &matching_test, &evm_env, "fixture/Prague/d0_g0_v0"); + let matching_error = Err(EVMError::Transaction( + InvalidTransaction::PriorityFeeGreaterThanMaxFee, + )); + assert!(validate_expected_exception(&matching_ctx, &matching_error) + .expect("expected exception should match")); + + let unexpected_success_test = fixture_test(Some("PriorityGreaterThanMaxFeePerGas")); + let unexpected_success_ctx = fixture_context( + &unit, + &unexpected_success_test, + &evm_env, + "fixture/Prague/d0_g0_v0", + ); + let unexpected_success = + validate_expected_exception(&unexpected_success_ctx, &Ok(call_result(bytes!("01")))) + .unwrap_err() + .to_string(); + assert!(unexpected_success.contains("UNEXPECTED_SUCCESS")); + + let unexpected_exception_test = fixture_test(None); + let unexpected_exception_ctx = fixture_context( + &unit, + &unexpected_exception_test, + &evm_env, + "fixture/Prague/d0_g0_v0", + ); + let unexpected_exception = validate_expected_exception( + &unexpected_exception_ctx, + &Err(EVMError::Transaction( + InvalidTransaction::PriorityFeeGreaterThanMaxFee, + )), + ) + .unwrap_err() + .to_string(); + assert!(unexpected_exception.contains("UNEXPECTED_EXCEPTION")); + } + + #[test] + fn validate_output_flags_only_mismatched_call_data() { + let mut unit = fixture_unit(SpecName::Prague); + unit.out = Some(bytes!("0102")); + let test = fixture_test(None); + let evm_env = crate::adapter::build_evm_env(&unit, &SpecName::Prague, 1) + .expect("test env should build"); + let ctx = fixture_context(&unit, &test, &evm_env, "fixture/Prague/d0_g0_v0"); + + validate_output(&ctx, unit.out.as_ref(), &call_result(bytes!("0102"))) + .expect("matching output should pass"); + + let error = validate_output(&ctx, unit.out.as_ref(), &call_result(bytes!("03"))) + .unwrap_err() + .to_string(); + assert!(error.contains("UNEXPECTED_OUTPUT")); + + validate_output( + &ctx, + unit.out.as_ref(), + &ExecutionResult::Halt { + reason: HaltReason::OutOfGas(OutOfGasError::Basic), + gas_used: 21_000, + }, + ) + .expect("halted executions do not have output to compare"); + } + + #[test] + fn configure_blob_limits_tracks_fork_specific_limits() { + let mut pre_prague = CfgEnv::default(); + pre_prague.spec = SpecId::CANCUN; + configure_blob_limits(&mut pre_prague); + assert_eq!(pre_prague.max_blobs_per_tx, Some(6)); + + let mut prague = CfgEnv::default(); + prague.spec = SpecId::PRAGUE; + configure_blob_limits(&mut prague); + assert_eq!(prague.max_blobs_per_tx, Some(9)); + + let mut osaka = CfgEnv::default(); + osaka.spec = SpecId::OSAKA; + configure_blob_limits(&mut osaka); + assert_eq!(osaka.max_blobs_per_tx, Some(6)); + } + + #[test] + fn format_evm_result_preserves_result_kind_for_json_consumers() { + assert_eq!( + format_evm_result(&Ok(call_result(bytes!("010203")))), + "Success: Return" + ); + assert_eq!( + format_evm_result(&Ok(revert_result(bytes!("deadbeef")))), + "Revert" + ); + assert_eq!( + format_evm_result(&Ok(ExecutionResult::Halt { + reason: HaltReason::OutOfGas(OutOfGasError::Basic), + gas_used: 21_000, + })), + "Halt: OutOfGas(Basic)" + ); + assert_eq!( + format_evm_result(&Err(EVMError::Transaction( + InvalidTransaction::PriorityFeeGreaterThanMaxFee, + ))), + "transaction validation error: priority fee is greater than max fee" + ); + } + + #[test] + fn json_outcome_report_preserves_legacy_aliases_and_readable_index_names() { + let outcome = JsonOutcome { + state_root: B256::repeat_byte(0x11), + logs_root: B256::repeat_byte(0x22), + output: bytes!("0102"), + gas_used: 21_000, + error_msg: "boom".to_string(), + evm_result: "Success: Return".to_string(), + post_logs_hash: B256::repeat_byte(0x33), + fork: "BERLIN".to_string(), + test: "fixture/Berlin/d0_g0_v0".to_string(), + data_index: 0, + gas_index: 1, + value_index: 2, + }; + + let report = build_json_outcome_report(&outcome, false); + + assert_eq!(report.get("pass").unwrap(), false); + assert_eq!(report.get("gasUsed").unwrap(), 21_000); + assert_eq!(report.get("fork").unwrap(), "BERLIN"); + assert_eq!(report.get("errorMsg").unwrap(), "boom"); + assert_eq!(report.get("d").unwrap(), 0); + assert_eq!(report.get("g").unwrap(), 1); + assert_eq!(report.get("v").unwrap(), 2); + assert_eq!(report.get("data_index").unwrap(), 0); + assert_eq!(report.get("gas_index").unwrap(), 1); + assert_eq!(report.get("value_index").unwrap(), 2); + } + + #[test] + fn file_errors_are_counted_in_summary() { + let output = file_error_result(Path::new("/tmp/bad.json"), "boom".to_string()); + + assert_eq!(output.summary.tests_total, 0); + assert_eq!(output.summary.tests_failed, 0); + assert_eq!(output.fatal_file_errors, 1); + } + + #[test] + fn malformed_fixture_shape_becomes_file_error() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let file = std::env::temp_dir().join(format!("arc_evm_specs_tests_bad_shape_{nonce}.json")); + std::fs::write( + &file, + serde_json::json!({ + "my_test": { + "env": {}, + "post": {} + } + }) + .to_string(), + ) + .unwrap(); + + let completed = AtomicUsize::new(0); + let output = process_fixture_file(&file, None, false, false, &completed, 1); + + assert_eq!(output.fatal_file_errors, 1); + assert_eq!(output.summary.files_processed, 1); + assert_eq!(output.summary.tests_total, 0); + assert_eq!(output.results.len(), 1); + assert!(output.results[0] + .error + .contains("unsupported or malformed statetest fixture shape")); + + std::fs::remove_file(file).unwrap(); + } + + #[test] + fn looks_like_statetest_fixture_checks_required_keys() { + let good = serde_json::json!({ + "my_test": { + "env": {}, + "post": {}, + "pre": {} + } + }); + let bad = serde_json::json!({ + "config": { "name": "not-a-fixture" } + }); + assert!(looks_like_statetest_fixture(&good)); + assert!(!looks_like_statetest_fixture(&bad)); + } + + #[test] + fn find_json_files_recurses_and_filters_extensions() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("arc_evm_specs_tests_find_json_{nonce}")); + let nested = root.join("nested"); + std::fs::create_dir_all(&nested).unwrap(); + + let a = root.join("a.json"); + let b = nested.join("b.json"); + let c = nested.join("c.txt"); + std::fs::write(&a, "{}").unwrap(); + std::fs::write(&b, "{}").unwrap(); + std::fs::write(&c, "ignore").unwrap(); + + let mut files = find_json_files(&root).unwrap(); + files.sort(); + + assert_eq!(files, vec![a, b]); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn run_returns_no_json_files_for_empty_directory() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("arc_evm_specs_tests_empty_{nonce}")); + std::fs::create_dir_all(&root).unwrap(); + + let err = + run(root.clone(), None, false, false, false).expect_err("empty directory should fail"); + assert!(matches!( + err, + EvmSpecsTestError::NoJsonFiles { path } if path == root.display().to_string() + )); + + std::fs::remove_dir_all(root).unwrap(); + } +} diff --git a/crates/evm/src/evm.rs b/crates/evm/src/evm.rs index cdc4a4b..4cd4ca2 100644 --- a/crates/evm/src/evm.rs +++ b/crates/evm/src/evm.rs @@ -18,10 +18,7 @@ extern crate alloc; use crate::assembler::ArcBlockAssembler; use crate::executor::ArcBlockExecutor; -use crate::frame_result::{ - create_blocklisted_frame_result, create_frame_result, create_oog_frame_result, - BeforeFrameInitResult, BLOCKLISTED_GAS_PENALTY, -}; +use crate::frame_result::{create_frame_result, create_oog_frame_result, BeforeFrameInitResult}; use crate::log::{create_eip7708_transfer_log, create_native_transfer_log}; use alloc::sync::Arc; use alloy_evm::eth::EthEvmContext; @@ -37,7 +34,8 @@ use arc_execution_config::native_coin_control::{ compute_is_blocklisted_storage_slot, is_blocklisted_status, }; use arc_precompiles::helpers::{ - ERR_SELFDESTRUCTED_BALANCE_INCREASED, ERR_ZERO_ADDRESS, PRECOMPILE_SLOAD_GAS_COST, + ERR_BLOCKED_ADDRESS, ERR_SELFDESTRUCTED_BALANCE_INCREASED, ERR_ZERO_ADDRESS, + PRECOMPILE_SLOAD_GAS_COST, }; use arc_precompiles::NATIVE_COIN_CONTROL_ADDRESS; use core::fmt::Debug; @@ -93,7 +91,7 @@ use crate::opcode::{arc_network_selfdestruct, arc_network_selfdestruct_zero4}; use crate::subcall::{SubcallContinuation, SubcallRegistry}; use arc_execution_config::chainspec::{ArcChainSpec, BlockGasLimitProvider}; use arc_execution_config::protocol_config::{ - expected_gas_limit, retrieve_fee_params, retrieve_reward_beneficiary, + expected_gas_limit, retrieve_fee_params, retrieve_reward_beneficiary, ProtocolConfigError, }; use arc_precompiles::call_from::{CallFromPrecompile, CALL_FROM_ADDRESS}; use arc_precompiles::precompile_provider::ArcPrecompileProvider; @@ -101,8 +99,8 @@ use arc_precompiles::subcall::SubcallPrecompile; use revm::interpreter::interpreter_action::CallInputs; /// Flat gas cost charged for rejected subcall dispatches (unauthorized caller, wrong scheme, -/// static context, value attached, init_subcall errors). Charged by `init_subcall_revert` calls. -/// Prevents zero-cost probing of subcall precompile addresses. +/// static context, value attached, sender spoofing, init_subcall errors). Charged by +/// `init_subcall_revert` calls. Prevents zero-cost probing of subcall precompile addresses. const SUBCALL_DISPATCH_COST: u64 = 100; /// Construct a revert `FrameResult` for a subcall precompile rejection. @@ -184,6 +182,7 @@ fn extract_call_transfer_params( /// Deducts a gas cost from a frame's gas limit. Returns `Some(oog_result)` if the /// frame has insufficient gas, `None` on success. +#[allow(clippy::arithmetic_side_effects)] // Subtractions are guarded by the `< cost` checks. fn deduct_gas_from_frame(frame_input: &mut FrameInit, cost: u64) -> Option { match &mut frame_input.frame_input { FrameInput::Call(inputs) => { @@ -203,6 +202,30 @@ fn deduct_gas_from_frame(frame_input: &mut FrameInit, cost: u64) -> Option u64 { + match &frame_input.frame_input { + FrameInput::Call(inputs) => inputs.gas_limit, + FrameInput::Create(inputs) => inputs.gas_limit(), + FrameInput::Empty => 0, + } +} + +/// Creates a revert that charges the actual SLOAD gas cost when metered, or OOGs if the +/// frame's gas budget is insufficient. +fn metered_revert( + frame_input: &FrameInit, + meter_sloads: bool, + gas_cost: u64, + reason: &str, +) -> BeforeFrameInitResult { + let gas_spent = if meter_sloads { gas_cost } else { 0 }; + if gas_spent > 0 && frame_gas_limit(frame_input) < gas_spent { + BeforeFrameInitResult::Reverted(create_oog_frame_result(frame_input)) + } else { + BeforeFrameInitResult::Reverted(create_frame_result(frame_input, reason, gas_spent)) + } +} + /// Defensive revert for when a subcall interception fires on a non-Call frame. /// This should never happen (the caller in `frame_init` checks for Call), but /// avoids panicking in production. @@ -344,6 +367,18 @@ impl ArcEvm { Ok((is_blocklisted_status(state_load.data), state_load.is_cold)) } + fn sload_cost(&self, is_cold: bool) -> u64 { + if self.hardfork_flags.is_active(ArcHardfork::Zero6) { + if is_cold { + revm_interpreter::gas::COLD_SLOAD_COST + } else { + revm_interpreter::gas::WARM_STORAGE_READ_COST + } + } else { + PRECOMPILE_SLOAD_GAS_COST + } + } + /// Extracts transfer parameters (from, to, amount) from a Create frame input. /// Returns None if value is zero or scheme is Custom. fn extract_create_transfer_params( @@ -399,6 +434,11 @@ impl ArcEvm { amount: U256, frame_input: &FrameInit, ) -> Result> { + // Meter SLOAD gas on revert for nested frames (depth > 0) with Zero6 active. + // Depth 0 is covered by `validate_initial_tx_gas` which charges fixed cold SLOAD costs. + let meter_sloads = + frame_input.depth > 0 && self.hardfork_flags.is_active(ArcHardfork::Zero6); + // Zero5: reject CALL/CREATE value transfers involving the zero address. // This prevents accidental burn/mint semantics at the EVM execution layer. // @@ -412,48 +452,35 @@ impl ArcEvm { return Ok(BeforeFrameInitResult::Reverted(create_frame_result( frame_input, ERR_ZERO_ADDRESS, - BLOCKLISTED_GAS_PENALTY, + 0, ))); } let (from_blocklisted, from_is_cold) = self.is_address_blocklisted(from)?; - - // Compute gas cost for the `from` sload - let from_sload_cost = if self.hardfork_flags.is_active(ArcHardfork::Zero6) { - if from_is_cold { - revm_interpreter::gas::COLD_SLOAD_COST - } else { - revm_interpreter::gas::WARM_STORAGE_READ_COST - } - } else { - PRECOMPILE_SLOAD_GAS_COST - }; + let from_sload_cost = self.sload_cost(from_is_cold); if from_blocklisted { - // Short-circuit: only the `from` sload was performed - return Ok(BeforeFrameInitResult::Reverted( - create_blocklisted_frame_result(frame_input), + return Ok(metered_revert( + frame_input, + meter_sloads, + from_sload_cost, + ERR_BLOCKED_ADDRESS, )); } let (to_blocklisted, to_is_cold) = self.is_address_blocklisted(to)?; + let to_sload_cost = self.sload_cost(to_is_cold); - // Compute gas cost for the `to` sload - let to_sload_cost = if self.hardfork_flags.is_active(ArcHardfork::Zero6) { - if to_is_cold { - revm_interpreter::gas::COLD_SLOAD_COST - } else { - revm_interpreter::gas::WARM_STORAGE_READ_COST - } - } else { - PRECOMPILE_SLOAD_GAS_COST - }; - + // Both are PRECOMPILE_SLOAD_GAS_COST (2,100); sum fits in u64 + #[allow(clippy::arithmetic_side_effects)] let total_sload_cost = from_sload_cost + to_sload_cost; if to_blocklisted { - return Ok(BeforeFrameInitResult::Reverted( - create_blocklisted_frame_result(frame_input), + return Ok(metered_revert( + frame_input, + meter_sloads, + total_sload_cost, + ERR_BLOCKED_ADDRESS, )); } if self.hardfork_flags.is_active(ArcHardfork::Zero5) { @@ -462,11 +489,12 @@ impl ArcEvm { // execution, so this usually has no practical gas impact even if the first load is cold. let target_account = self.inner.journal_mut().load_account(to)?; if target_account.is_selfdestructed() { - return Ok(BeforeFrameInitResult::Reverted(create_frame_result( + return Ok(metered_revert( frame_input, + meter_sloads, + total_sload_cost, ERR_SELFDESTRUCTED_BALANCE_INCREASED, - BLOCKLISTED_GAS_PENALTY, - ))); + )); } } @@ -851,6 +879,17 @@ where } }; + // Prevent sender spoofing by contracts: if the precompile changes the caller + // (e.g. callFrom), the new caller must be tx.origin (the signing EOA). + if init_result.child_inputs.caller != call_inputs.caller + && init_result.child_inputs.caller != self.inner.ctx.tx().caller() + { + return Ok(ItemOrResult::Result(init_subcall_revert( + "sender spoofing requires tx.origin as sender", + call_inputs, + ))); + } + let return_memory_offset = call_inputs.return_memory_offset.clone(); // Store continuation keyed by the precompile call's depth. @@ -876,9 +915,6 @@ where }, ); - // Construct child frame input - let child_inputs = &init_result.child_inputs; - // Pre-load the child's caller and target accounts into the journal. The normal EVM // execution path has these already loaded (caller is the executing frame, target was // loaded by the CALL opcode handler), but we're constructing a synthetic child frame @@ -892,23 +928,68 @@ where // Side effect: these `load_account` calls create `AccountWarmed` journal entries // outside the child frame's checkpoint scope. If the child reverts, the child's // journal entries are rolled back but these pre-loads persist — the addresses stay - // warm for the rest of the transaction. For CallFrom this is bounded by the - // allowlist and the addresses are typically already warm. Future subcall precompiles - // with `AllowedCallers::Unrestricted` targeting user-supplied addresses should - // account for this. + // warm for the rest of the transaction. + // + // The target's cold/warm status is captured to charge the EIP-2929 account access + // cost, mirroring the normal CALL opcode's gas metering. Note: when caller==target, + // `load_account(caller)` warms the address first, so `target_load.is_cold` is false + // and only the warm cost (100) is charged — matching normal EVM CALL behavior. + let mut child_inputs = init_result.child_inputs; self.inner .ctx .journal_mut() .load_account(child_inputs.caller)?; - self.inner + let target_load = self + .inner .ctx .journal_mut() .load_account(child_inputs.target_address)?; + // EIP-2929 account access cost for the child target. Our `load_account` call + // above pre-warms the target, so revm's internal CALL handler won't charge cold + // access. We charge it explicitly to match the normal CALL opcode's gas metering. + let account_access_cost = if target_load.is_cold { + revm_interpreter::gas::COLD_ACCOUNT_ACCESS_COST + } else { + revm_interpreter::gas::WARM_STORAGE_READ_COST + }; + + let total_overhead = init_result.gas_overhead.saturating_add(account_access_cost); + + let Some(available) = call_inputs.gas_limit.checked_sub(total_overhead) else { + // OOG: total overhead exceeds the caller's gas budget. Consume all gas. + // Remove the continuation inserted above — no child frame will run. + self.subcall_continuations.remove(&depth); + let mut gas = Gas::new(call_inputs.gas_limit); + gas.spend_all(); + return Ok(ItemOrResult::Result(FrameResult::Call(CallOutcome { + result: InterpreterResult::new(InstructionResult::OutOfGas, Bytes::new(), gas), + memory_offset: call_inputs.return_memory_offset.clone(), + was_precompile_called: true, + precompile_call_logs: Default::default(), + }))); + }; + + // Recalculate child gas with EIP-150 (63/64ths) applied to the gas remaining + // after total overhead. This overwrites the gas_limit set by the trait's + // `init_subcall`, which only accounted for the ABI decode overhead. + // available / 64 <= available, so the subtraction cannot underflow. + #[allow(clippy::arithmetic_side_effects)] + let child_gas_limit = available - (available / 64); + child_inputs.gas_limit = child_gas_limit; + + // Update the continuation with the total overhead (ABI decode + account access). + self.subcall_continuations + .get_mut(&depth) + .expect("continuation was inserted above") + .init_subcall_gas_overhead = total_overhead; + let child_frame_input = FrameInit { + // Call depth is bounded by the EVM stack limit (1024) + #[allow(clippy::arithmetic_side_effects)] depth: depth + 1, memory: frame_input.memory, - frame_input: FrameInput::Call(init_result.child_inputs), + frame_input: FrameInput::Call(child_inputs), }; // Initialize the child frame through checked_frame_init so that @@ -1585,6 +1666,23 @@ impl BlockExecutorFactory for ArcEvmConfig { } } +/// Pick the fee recipient for the next block, given the ProtocolConfig lookup result and the +/// CL-provided recipient. The contract value wins when explicitly set; zero or lookup failure +/// defers to the CL. +fn select_fee_recipient( + protocol_config_beneficiary: Result>, + cl_suggested_recipient: Address, +) -> Address { + match protocol_config_beneficiary { + Ok(addr) if !addr.is_zero() => addr, + Ok(_) => cl_suggested_recipient, + Err(err) => { + tracing::warn!(error = %err, "ProtocolConfig rewardBeneficiary() failed; using CL-provided fee recipient"); + cl_suggested_recipient + } + } +} + impl ConfigureEvm for ArcEvmConfig { type Primitives = ::Primitives; type Error = ::Error; @@ -1629,13 +1727,10 @@ impl ConfigureEvm for ArcEvmConfig { })?, ); - // Override the suggested fee recipient with the reward beneficiary from the ProtocolConfig contract. - attributes.suggested_fee_recipient = retrieve_reward_beneficiary(&mut system_evm) - .unwrap_or_else(|err| { - tracing::warn!(error = ?err, "Failed to get reward beneficiary from ProtocolConfig"); - // fallback - the coinbase address from the parent block - parent.beneficiary - }); + attributes.suggested_fee_recipient = select_fee_recipient( + retrieve_reward_beneficiary(&mut system_evm), + attributes.suggested_fee_recipient, + ); // Override the gas limit with the gas limit from the ProtocolConfig contract. // ADR-0003: use chainspec bounds; fall back to chainspec default when ProtocolConfig @@ -1644,7 +1739,7 @@ impl ConfigureEvm for ArcEvmConfig { let fee_params = retrieve_fee_params(&mut system_evm).inspect_err(|err| { tracing::warn!(error = ?err, "Failed to get fee params from ProtocolConfig, using default gas limit"); }).ok(); - let next_block_height = parent.number + 1; + let next_block_height = parent.number.checked_add(1).expect("block number overflow"); let gas_limit_config = chain_spec.block_gas_limit_config(next_block_height); attributes.gas_limit = expected_gas_limit(fee_params.as_ref(), &gas_limit_config); @@ -1707,7 +1802,7 @@ impl ConfigureEngineEvm for ArcEvmConfig { mod tests { use super::*; use crate::frame_result::BeforeFrameInitResult; - use crate::frame_result::BLOCKLISTED_GAS_PENALTY; + use crate::log::NativeCoinTransferred; use alloy_consensus::Block; use alloy_primitives::{address, Bytes, B256, U256}; @@ -2000,40 +2095,151 @@ mod tests { let evm_config = ArcEvmConfig::new(inner_config); let mut db = State::builder().build(); - // Create a minimal sealed parent header with a known beneficiary - let fallback_beneficiary = Address::repeat_byte(0x99); + // ProtocolConfig contract is absent from `db`, so `retrieve_reward_beneficiary` errors + // and `select_fee_recipient` falls back to `attributes.suggested_fee_recipient`. let parent_header = Header { number: 1, - gas_limit: 30_000_000, // Standard gas limit + gas_limit: 30_000_000, gas_used: 21_000, base_fee_per_gas: Some(1_000_000_000), // 1 gwei timestamp: 1000, - beneficiary: fallback_beneficiary, ..Default::default() }; let sealed_parent = SealedHeader::new(parent_header, B256::ZERO); - // Create next block attributes let attributes = NextBlockEnvAttributes { timestamp: 1001, prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, // Different from parent beneficiary + suggested_fee_recipient: Address::repeat_byte(0x42), gas_limit: 30_000_000, parent_beacon_block_root: None, withdrawals: None, extra_data: Default::default(), }; - // Test builder_for_next_block - should succeed and use fallback beneficiary let result = evm_config.builder_for_next_block(&mut db, &sealed_parent, attributes); assert!( result.is_ok(), - "builder_for_next_block should succeed with fallback" + "builder_for_next_block should succeed when ProtocolConfig is absent" + ); + } + + #[test] + fn test_retrieve_reward_beneficiary_nonzero_address() { + use arc_execution_config::protocol_config::{ + retrieve_reward_beneficiary, PROTOCOL_CONFIG_ADDRESS, + }; + use revm::bytecode::opcode; + use revm::state::AccountInfo; + + let expected = Address::repeat_byte(0xAB); + + // Mock contract at PROTOCOL_CONFIG_ADDRESS that returns a non-zero address, + // exercising the `Ok(addr) if !addr.is_zero() => addr` branch. + // rewardBeneficiary() returns (address) ABI-encodes as [12 zero bytes | 20 addr bytes]; + // MSTORE right-aligns the stack value to produce exactly that. + // + // Equivalent assembly: + // ```text + // PUSH20 ; push address as uint256 (zero-fills upper 12 bytes on stack) + // PUSH1 0 ; memory offset for MSTORE + // MSTORE ; memory[0..32] = [12 zero bytes][20 addr bytes] + // PUSH1 0x20 ; return size = 32 + // PUSH1 0 ; return offset + // RETURN + // ``` + let mut code = vec![opcode::PUSH20]; + code.extend_from_slice(expected.as_slice()); + code.extend_from_slice(&[opcode::PUSH1, 0x00, opcode::MSTORE]); + code.extend_from_slice(&[opcode::PUSH1, 0x20, opcode::PUSH1, 0x00, opcode::RETURN]); + let raw = Bytes::from(code); + + let mut db = InMemoryDB::default(); + db.insert_account_info( + PROTOCOL_CONFIG_ADDRESS, + AccountInfo { + balance: U256::ZERO, + nonce: 1, + code_hash: keccak256(&raw), + code: Some(Bytecode::new_raw(raw)), + account_id: None, + }, + ); + + let mut evm = create_test_evm(db, ArcHardforkFlags::default()); + assert_eq!(retrieve_reward_beneficiary(&mut evm).unwrap(), expected); + } + + #[test] + fn test_retrieve_reward_beneficiary_zero_address() { + use arc_execution_config::protocol_config::{ + retrieve_reward_beneficiary, PROTOCOL_CONFIG_ADDRESS, + }; + use revm::bytecode::opcode; + use revm::state::AccountInfo; + + // Mock contract returning address(0): rewardBeneficiary() ABI-encodes as 32 zero + // bytes, which EVM memory provides by default — no MSTORE needed. + // + // Equivalent assembly: + // ```text + // PUSH1 0x20 ; return size = 32 + // PUSH1 0 ; return offset (memory[0..32] is all zeros by default) + // RETURN ; returns 32 zero bytes = abi.encode(address(0)) + // ``` + let raw = Bytes::from(vec![ + opcode::PUSH1, + 0x20, + opcode::PUSH1, + 0x00, + opcode::RETURN, + ]); + + let mut db = InMemoryDB::default(); + db.insert_account_info( + PROTOCOL_CONFIG_ADDRESS, + AccountInfo { + balance: U256::ZERO, + nonce: 1, + code_hash: keccak256(&raw), + code: Some(Bytecode::new_raw(raw)), + account_id: None, + }, + ); + + let mut evm = create_test_evm(db, ArcHardforkFlags::default()); + assert_eq!( + retrieve_reward_beneficiary(&mut evm).unwrap(), + Address::ZERO + ); + } + + #[test] + fn select_fee_recipient_uses_protocol_config_when_nonzero() { + let pc_beneficiary = Address::repeat_byte(0x11); + let cl_suggested = Address::repeat_byte(0x22); + assert_eq!( + select_fee_recipient(Ok(pc_beneficiary), cl_suggested), + pc_beneficiary, + ); + } + + #[test] + fn select_fee_recipient_falls_back_to_cl_suggested_when_protocol_config_returns_zero() { + let cl_suggested = Address::repeat_byte(0x22); + assert_eq!( + select_fee_recipient(Ok(Address::ZERO), cl_suggested), + cl_suggested, ); + } - // The function should have set suggested_fee_recipient to parent.beneficiary - // We can't directly inspect the builder's internal state, but we know it succeeded - // and the fallback logic was triggered since ProtocolConfig contract doesn't exist + #[test] + fn select_fee_recipient_falls_back_to_cl_suggested_on_error() { + let cl_suggested = Address::repeat_byte(0x22); + assert_eq!( + select_fee_recipient(Err(ProtocolConfigError::EmptyOutput), cl_suggested), + cl_suggested, + ); } /// Under Zero5, Arc self-emits EIP-7708 Transfer logs for CALL/CREATE value transfers. @@ -2434,8 +2640,8 @@ mod tests { if let BeforeFrameInitResult::Reverted(reverted) = transfer_result { assert_eq!( reverted.gas().spent(), - BLOCKLISTED_GAS_PENALTY, - "{} (hardfork: {:?}): expect reverted gas spent BLOCKLISTED_GAS_PENALTY", + 0, + "{} (hardfork: {:?}): depth-0 reverts should have zero gas spent", test_case.name, hardfork ); @@ -3524,6 +3730,7 @@ mod tests { const REVERT_CONTRACT: Address = address!("c000000000000000000000000000000000000003"); const CALLER_CONTRACT: Address = address!("c000000000000000000000000000000000000004"); const WRAPPER_INNER: Address = address!("c000000000000000000000000000000000000005"); + const SPOOFED_SENDER: Address = address!("a000000000000000000000000000000000000001"); // ----- Integration tests ----- @@ -5475,6 +5682,51 @@ mod tests { ); } + // ================================================================ + // tx.origin sender validation tests + // ================================================================ + + /// EOA → WRAPPER → callFrom(sender=SPOOFED_SENDER, target=ECHO, data) + /// SPOOFED_SENDER is neither tx.origin (EOA) nor the actual caller (WRAPPER), + /// so the sender validation rejects it. + #[test] + fn test_call_from_contract_sender_spoofing_rejected() { + let contract_a_code = wrapper_call_bytecode(CALL_FROM_ADDRESS); + let contract_b_code = echo_double_bytecode(); + const CONTRACT_A: Address = WRAPPER; + const CONTRACT_B: Address = ECHO_CONTRACT; + + let mut evm = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[(CONTRACT_A, contract_a_code), (CONTRACT_B, contract_b_code)], + &[CONTRACT_A], + ); + + let inner_calldata = U256::from(42).to_be_bytes::<32>().to_vec(); + let call_from_input = + encode_call_from_input(SPOOFED_SENDER, CONTRACT_B, &inner_calldata); + + let tx = TxEnv { + caller: EOA, + kind: TxKind::Call(CONTRACT_A), + value: U256::ZERO, + gas_limit: 1_000_000, + gas_price: 0, + chain_id: Some(LOCAL_DEV.chain_id()), + data: call_from_input, + ..Default::default() + }; + + let result = evm.transact_one(tx).expect("transact_one should succeed"); + match &result { + ExecutionResult::Success { output, .. } => { + let reason = decode_revert_reason(output.data()); + assert_eq!(reason, "sender spoofing requires tx.origin as sender"); + } + other => panic!("expected Success (wrapper catches revert), got {other:?}"), + } + } + /// complete_subcall error should consume all gas allocated to the subcall. #[test] fn test_complete_subcall_error_consumes_all_gas() { @@ -5602,173 +5854,1121 @@ mod tests { normal path ({gas_used_normal}) because all subcall gas is consumed" ); } - } - fn create_test_evm_with_spec( - db: InMemoryDB, - hardfork_flags: ArcHardforkFlags, - spec: SpecId, - ) -> ArcEvm< - EthEvmContext, - NoOpInspector, - EthInstructions>, - PrecompilesMap, - > { - let ctx = Context::new(db, spec); - let instruction = EthInstructions::default(); - let precompiles = ArcPrecompileProvider::create_precompiles_map(spec, hardfork_flags); - ArcEvm::new( - ctx, - NoOpInspector {}, - precompiles, - instruction, - false, - hardfork_flags, - Arc::new(SubcallRegistry::default()), - ) - } - #[test] - fn test_zero5_emits_eip7708_transfer_log() { - use revm::handler::SYSTEM_ADDRESS; + // These tests verify that `init_subcall` correctly charges EIP-2929 account + // access costs for the child target address. - let db = CacheDB::new(EmptyDB::default()); - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); - let mut evm = create_test_evm(db, flags); + /// Integration test: CallFrom targeting a cold account should cost more gas + /// than targeting a warm one (pre-warmed via access_list). + /// + /// Uses two separate EVM instances so each transaction starts with a fresh + /// journal — reusing one EVM would leave addresses warm from the first tx. + #[test] + fn test_call_from_cold_target_costs_more_gas_than_warm() { + let contract_a_code = wrapper_call_bytecode(CALL_FROM_ADDRESS); + let contract_b_code = echo_double_bytecode(); + const CONTRACT_A: Address = WRAPPER; + const CONTRACT_B: Address = ECHO_CONTRACT; - evm.ctx_mut() - .journal_mut() - .load_account(NATIVE_COIN_CONTROL_ADDRESS) - .unwrap(); + let inner_calldata = U256::from(42).to_be_bytes::<32>().to_vec(); + let call_from_input = encode_call_from_input(EOA, CONTRACT_B, &inner_calldata); - let frame = FrameInit { - frame_input: FrameInput::Call(call_input( - CallScheme::Call, - U256::from(100), - ADDRESS_A, - ADDRESS_B, - )), - memory: SharedMemory::default(), - depth: 1, - }; + // Cold target: fresh EVM, no access_list + let mut evm_cold = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[ + (CONTRACT_A, contract_a_code.clone()), + (CONTRACT_B, contract_b_code.clone()), + ], + &[CONTRACT_A], + ); + let tx_cold = TxEnv { + tx_type: 1, // EIP-2930 + caller: EOA, + kind: TxKind::Call(CONTRACT_A), + value: U256::ZERO, + gas_limit: 200_000, + gas_price: 0, + chain_id: Some(LOCAL_DEV.chain_id()), + data: call_from_input.clone(), + access_list: Default::default(), + ..Default::default() + }; + let result_cold = evm_cold + .transact_one(tx_cold) + .expect("cold tx should succeed"); + let gas_cold = match &result_cold { + ExecutionResult::Success { gas_used, .. } => *gas_used, + other => panic!("expected Success, got {other:?}"), + }; - let result = evm.before_frame_init(&frame).unwrap(); - match result { - BeforeFrameInitResult::Log(log, gas) => { - assert!(gas > 0, "Should have SLOAD gas cost"); - assert_eq!( - log.address, SYSTEM_ADDRESS, - "Zero5 should emit EIP-7708 Transfer log from system address" - ); - } - other => panic!( - "Expected Log result with EIP-7708 Transfer under Zero5, got {:?}", - other - ), + // Warm target: fresh EVM, pre-warm CONTRACT_B via access_list. + // tx_type must be EIP-2930 (1) so revm processes the access_list warmup; + // the default Legacy type skips access list handling entirely. + let mut evm_warm = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[(CONTRACT_A, contract_a_code), (CONTRACT_B, contract_b_code)], + &[CONTRACT_A], + ); + let tx_warm = TxEnv { + tx_type: 1, // EIP-2930 + caller: EOA, + kind: TxKind::Call(CONTRACT_A), + value: U256::ZERO, + gas_limit: 200_000, + gas_price: 0, + chain_id: Some(LOCAL_DEV.chain_id()), + data: call_from_input, + access_list: vec![alloy_eips::eip2930::AccessListItem { + address: CONTRACT_B, + storage_keys: vec![], + }] + .into(), + ..Default::default() + }; + let result_warm = evm_warm + .transact_one(tx_warm) + .expect("warm tx should succeed"); + let gas_warm = match &result_warm { + ExecutionResult::Success { gas_used, .. } => *gas_used, + other => panic!("expected Success, got {other:?}"), + }; + + // Both txs are EIP-2930; the only difference is the access_list entry. + // Delta = COLD_ACCOUNT_ACCESS_COST (2600) − ACCESS_LIST_ADDRESS_COST (2400) + // - WARM_STORAGE_READ_COST (100) = 100. + assert_eq!( + gas_cold - gas_warm, + 100, + "cold/warm gas delta should be exactly 100 (COLD_ACCOUNT_ACCESS_COST 2600 \ + - ACCESS_LIST_ADDRESS_COST 2400 - WARM_STORAGE_READ_COST 100)" + ); } - } - #[test] - fn test_zero5_self_transfer_no_log() { + /// Unit test: init_subcall with a cold target should set + /// init_subcall_gas_overhead = abi_decode_gas + COLD_ACCOUNT_ACCESS_COST. + #[test] + fn test_call_from_cold_target_recalculates_child_gas() { + use alloy_sol_types::SolCall; + use arc_precompiles::call_from::{abi_decode_gas, ICallFrom}; + use revm_interpreter::gas::COLD_ACCOUNT_ACCESS_COST; + + let echo_code = echo_double_bytecode(); + let mut evm = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[(ECHO_CONTRACT, echo_code)], + &[WRAPPER], + ); + + // Clear journal state and load required accounts + evm.ctx_mut().journal_mut().clear(); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + // tx.origin must match the spoofed sender for the origin check. + evm.inner.ctx.set_tx(TxEnv { + caller: EOA, + ..Default::default() + }); + + let child_data: Vec = vec![0x42]; + let calldata = ICallFrom::callFromCall { + sender: EOA, + target: ECHO_CONTRACT, + data: child_data.clone().into(), + } + .abi_encode(); + + let gas_limit: u64 = 100_000; + let call_inputs = CallInputs { + scheme: CallScheme::Call, + target_address: CALL_FROM_ADDRESS, + bytecode_address: CALL_FROM_ADDRESS, + known_bytecode: None, + value: CallValue::Transfer(U256::ZERO), + input: CallInput::Bytes(Bytes::from(calldata)), + gas_limit, + is_static: false, + caller: WRAPPER, + return_memory_offset: 0..0, + }; + + let frame_input = FrameInit { + frame_input: FrameInput::Call(Box::new(call_inputs)), + memory: SharedMemory::default(), + depth: 1, + }; + + let precompile: Arc = + Arc::new(CallFromPrecompile); + let result = evm + .init_subcall(frame_input, precompile) + .expect("init_subcall should succeed"); + assert!( + matches!(result, ItemOrResult::Item(_)), + "expected child frame, got immediate result" + ); + + let continuation = evm + .subcall_continuations + .get(&1) + .expect("continuation should be stored at depth 1"); + + let expected_overhead = abi_decode_gas(child_data.len()) + COLD_ACCOUNT_ACCESS_COST; + assert_eq!( + continuation.init_subcall_gas_overhead, + expected_overhead, + "overhead should be abi_decode ({}) + cold access ({COLD_ACCOUNT_ACCESS_COST}), \ + got {}", + abi_decode_gas(child_data.len()), + continuation.init_subcall_gas_overhead + ); + } + + /// Unit test: when the target is already warm, init_subcall should use + /// WARM_STORAGE_READ_COST (100) instead of COLD_ACCOUNT_ACCESS_COST (2600). + #[test] + fn test_call_from_warm_target_uses_warm_cost() { + use alloy_sol_types::SolCall; + use arc_precompiles::call_from::{abi_decode_gas, ICallFrom}; + use revm_interpreter::gas::WARM_STORAGE_READ_COST; + + let echo_code = echo_double_bytecode(); + let mut evm = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[(ECHO_CONTRACT, echo_code)], + &[WRAPPER], + ); + + // Clear journal state and load required accounts + evm.ctx_mut().journal_mut().clear(); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + // Pre-warm the target account + evm.ctx_mut() + .journal_mut() + .load_account(ECHO_CONTRACT) + .unwrap(); + + evm.inner.ctx.set_tx(TxEnv { + caller: EOA, + ..Default::default() + }); + + let child_data: Vec = vec![0x42]; + let calldata = ICallFrom::callFromCall { + sender: EOA, + target: ECHO_CONTRACT, + data: child_data.clone().into(), + } + .abi_encode(); + + let gas_limit: u64 = 100_000; + let call_inputs = CallInputs { + scheme: CallScheme::Call, + target_address: CALL_FROM_ADDRESS, + bytecode_address: CALL_FROM_ADDRESS, + known_bytecode: None, + value: CallValue::Transfer(U256::ZERO), + input: CallInput::Bytes(Bytes::from(calldata)), + gas_limit, + is_static: false, + caller: WRAPPER, + return_memory_offset: 0..0, + }; + + let frame_input = FrameInit { + frame_input: FrameInput::Call(Box::new(call_inputs)), + memory: SharedMemory::default(), + depth: 1, + }; + + let precompile: Arc = + Arc::new(CallFromPrecompile); + let result = evm + .init_subcall(frame_input, precompile) + .expect("init_subcall should succeed"); + assert!( + matches!(result, ItemOrResult::Item(_)), + "expected child frame, got immediate result" + ); + + let continuation = evm + .subcall_continuations + .get(&1) + .expect("continuation should be stored at depth 1"); + + let expected_overhead = abi_decode_gas(child_data.len()) + WARM_STORAGE_READ_COST; + assert_eq!( + continuation.init_subcall_gas_overhead, + expected_overhead, + "overhead should be abi_decode ({}) + warm read ({WARM_STORAGE_READ_COST}), \ + got {}", + abi_decode_gas(child_data.len()), + continuation.init_subcall_gas_overhead + ); + } + + /// When caller == target, `load_account(caller)` warms the address before + /// `load_account(target)`, so only the warm access cost (100) is charged. + #[test] + fn test_call_from_caller_equals_target_uses_warm_cost() { + use alloy_sol_types::SolCall; + use arc_precompiles::call_from::{abi_decode_gas, ICallFrom}; + use revm_interpreter::gas::WARM_STORAGE_READ_COST; + + let echo_code = echo_double_bytecode(); + // ECHO_CONTRACT is both the sender and target — give it balance and code. + let mut evm = setup_test_evm( + &[(ECHO_CONTRACT, U256::from(10_000_000))], + &[(ECHO_CONTRACT, echo_code)], + &[WRAPPER], + ); + + evm.ctx_mut().journal_mut().clear(); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + evm.inner.ctx.set_tx(TxEnv { + caller: ECHO_CONTRACT, + ..Default::default() + }); + + let child_data: Vec = vec![0x42]; + let calldata = ICallFrom::callFromCall { + sender: ECHO_CONTRACT, + target: ECHO_CONTRACT, + data: child_data.clone().into(), + } + .abi_encode(); + + let gas_limit: u64 = 100_000; + let call_inputs = CallInputs { + scheme: CallScheme::Call, + target_address: CALL_FROM_ADDRESS, + bytecode_address: CALL_FROM_ADDRESS, + known_bytecode: None, + value: CallValue::Transfer(U256::ZERO), + input: CallInput::Bytes(Bytes::from(calldata)), + gas_limit, + is_static: false, + caller: WRAPPER, + return_memory_offset: 0..0, + }; + + let frame_input = FrameInit { + frame_input: FrameInput::Call(Box::new(call_inputs)), + memory: SharedMemory::default(), + depth: 1, + }; + + let precompile: Arc = + Arc::new(CallFromPrecompile); + let result = evm + .init_subcall(frame_input, precompile) + .expect("init_subcall should succeed"); + assert!( + matches!(result, ItemOrResult::Item(_)), + "expected child frame, got immediate result" + ); + + let continuation = evm + .subcall_continuations + .get(&1) + .expect("continuation should be stored at depth 1"); + + let expected_overhead = abi_decode_gas(child_data.len()) + WARM_STORAGE_READ_COST; + assert_eq!( + continuation.init_subcall_gas_overhead, + expected_overhead, + "caller==target: overhead should be abi_decode ({}) + warm read \ + ({WARM_STORAGE_READ_COST}), got {}", + abi_decode_gas(child_data.len()), + continuation.init_subcall_gas_overhead + ); + } + + /// Unit test: when gas_limit is just below abi_decode + COLD_ACCOUNT_ACCESS_COST, + /// init_subcall should OOG. + #[test] + fn test_call_from_oog_with_cold_account_access() { + use alloy_sol_types::SolCall; + use arc_precompiles::call_from::{abi_decode_gas, ICallFrom}; + use revm_interpreter::gas::COLD_ACCOUNT_ACCESS_COST; + + let echo_code = echo_double_bytecode(); + let mut evm = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[(ECHO_CONTRACT, echo_code)], + &[WRAPPER], + ); + + evm.ctx_mut().journal_mut().clear(); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + evm.inner.ctx.set_tx(TxEnv { + caller: EOA, + ..Default::default() + }); + + let child_data: Vec = vec![0x42]; + let calldata = ICallFrom::callFromCall { + sender: EOA, + target: ECHO_CONTRACT, + data: child_data.clone().into(), + } + .abi_encode(); + + // Gas is enough for ABI decode but NOT enough for ABI decode + cold access + let insufficient_gas = abi_decode_gas(child_data.len()) + COLD_ACCOUNT_ACCESS_COST - 1; + let call_inputs = CallInputs { + scheme: CallScheme::Call, + target_address: CALL_FROM_ADDRESS, + bytecode_address: CALL_FROM_ADDRESS, + known_bytecode: None, + value: CallValue::Transfer(U256::ZERO), + input: CallInput::Bytes(Bytes::from(calldata)), + gas_limit: insufficient_gas, + is_static: false, + caller: WRAPPER, + return_memory_offset: 0..0, + }; + + let frame_input = FrameInit { + frame_input: FrameInput::Call(Box::new(call_inputs)), + memory: SharedMemory::default(), + depth: 1, + }; + + let precompile: Arc = + Arc::new(CallFromPrecompile); + let result = evm + .init_subcall(frame_input, precompile) + .expect("init_subcall should not return db error"); + + match result { + ItemOrResult::Result(FrameResult::Call(outcome)) => { + assert_eq!( + outcome.result.result, + InstructionResult::OutOfGas, + "should OOG when gas is insufficient for account access cost" + ); + assert_eq!( + outcome.result.gas.spent(), + insufficient_gas, + "OOG should consume all allocated gas" + ); + assert!( + !evm.subcall_continuations.contains_key(&1), + "continuation should be removed after OOG" + ); + } + ItemOrResult::Result(other) => { + panic!("expected Call result, got {other:?}"); + } + ItemOrResult::Item(_) => { + panic!( + "expected OutOfGas when gas_limit ({insufficient_gas}) < \ + abi_decode ({}) + cold access ({COLD_ACCOUNT_ACCESS_COST})", + abi_decode_gas(child_data.len()) + ); + } + } + } + + /// Boundary: gas_limit == abi_decode + COLD_ACCOUNT_ACCESS_COST should succeed + /// with child_gas_limit = 0 (child will OOG when it runs, but init_subcall itself + /// should not reject it). + #[test] + fn test_call_from_exact_overhead_gas_succeeds_with_zero_child_gas() { + use alloy_sol_types::SolCall; + use arc_precompiles::call_from::{abi_decode_gas, ICallFrom}; + use revm_interpreter::gas::COLD_ACCOUNT_ACCESS_COST; + + let echo_code = echo_double_bytecode(); + let mut evm = setup_test_evm( + &[(EOA, U256::from(1_000_000))], + &[(ECHO_CONTRACT, echo_code)], + &[WRAPPER], + ); + + evm.ctx_mut().journal_mut().clear(); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + evm.inner.ctx.set_tx(TxEnv { + caller: EOA, + ..Default::default() + }); + + let child_data: Vec = vec![0x42]; + let calldata = ICallFrom::callFromCall { + sender: EOA, + target: ECHO_CONTRACT, + data: child_data.clone().into(), + } + .abi_encode(); + + let exact_gas = abi_decode_gas(child_data.len()) + COLD_ACCOUNT_ACCESS_COST; + let call_inputs = CallInputs { + scheme: CallScheme::Call, + target_address: CALL_FROM_ADDRESS, + bytecode_address: CALL_FROM_ADDRESS, + known_bytecode: None, + value: CallValue::Transfer(U256::ZERO), + input: CallInput::Bytes(Bytes::from(calldata)), + gas_limit: exact_gas, + is_static: false, + caller: WRAPPER, + return_memory_offset: 0..0, + }; + + let frame_input = FrameInit { + frame_input: FrameInput::Call(Box::new(call_inputs)), + memory: SharedMemory::default(), + depth: 1, + }; + + let precompile: Arc = + Arc::new(CallFromPrecompile); + let result = evm + .init_subcall(frame_input, precompile) + .expect("init_subcall should not return db error"); + + assert!( + matches!(result, ItemOrResult::Item(_)), + "exact overhead gas should push a child frame, not OOG" + ); + + let continuation = evm + .subcall_continuations + .get(&1) + .expect("continuation should exist at depth 1"); + assert_eq!( + continuation.init_subcall_gas_overhead, exact_gas, + "overhead should equal the full gas budget" + ); + } + } + fn create_test_evm_with_spec( + db: InMemoryDB, + hardfork_flags: ArcHardforkFlags, + spec: SpecId, + ) -> ArcEvm< + EthEvmContext, + NoOpInspector, + EthInstructions>, + PrecompilesMap, + > { + let ctx = Context::new(db, spec); + let instruction = EthInstructions::default(); + let precompiles = ArcPrecompileProvider::create_precompiles_map(spec, hardfork_flags); + ArcEvm::new( + ctx, + NoOpInspector {}, + precompiles, + instruction, + false, + hardfork_flags, + Arc::new(SubcallRegistry::default()), + ) + } + + #[test] + fn test_zero5_emits_eip7708_transfer_log() { + use revm::handler::SYSTEM_ADDRESS; + + let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let mut evm = create_test_evm(db, flags); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + ADDRESS_A, + ADDRESS_B, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + match result { + BeforeFrameInitResult::Log(log, gas) => { + assert!(gas > 0, "Should have SLOAD gas cost"); + assert_eq!( + log.address, SYSTEM_ADDRESS, + "Zero5 should emit EIP-7708 Transfer log from system address" + ); + } + other => panic!( + "Expected Log result with EIP-7708 Transfer under Zero5, got {:?}", + other + ), + } + } + + #[test] + fn test_zero5_self_transfer_no_log() { + let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let mut evm = create_test_evm(db, flags); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + // Self-transfer: from == to + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + ADDRESS_A, + ADDRESS_A, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + match result { + BeforeFrameInitResult::Checked(gas) => { + assert!(gas > 0, "Should have SLOAD gas cost"); + } + other => panic!( + "Expected Checked result for self-transfer under Zero5, got {:?}", + other + ), + } + } + + #[test] + fn test_pre_zero5_still_emits_custom_log() { + let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero4]); + let mut evm = create_test_evm(db, flags); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + ADDRESS_A, + ADDRESS_B, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + match result { + BeforeFrameInitResult::Log(log, _gas) => { + assert_eq!(log.address, NATIVE_COIN_AUTHORITY_ADDRESS); + } + other => panic!("Expected Log result pre-Zero5, got {:?}", other), + } + } + + /// Verifies that AMSTERDAM SpecId enables EIP-7708 (is_enabled_in returns true). + /// Once REVM is upgraded to a version with EIP-7708 journal support, the journal's + /// `transfer` method will emit Transfer logs when SpecId >= AMSTERDAM. + #[test] + fn test_amsterdam_spec_enables_eip7708() { + // AMSTERDAM is after PRAGUE in the SpecId ordering + assert!( + SpecId::AMSTERDAM.is_enabled_in(SpecId::AMSTERDAM), + "AMSTERDAM should be enabled in AMSTERDAM" + ); + assert!( + !SpecId::PRAGUE.is_enabled_in(SpecId::AMSTERDAM), + "PRAGUE should NOT be enabled in AMSTERDAM (AMSTERDAM comes after PRAGUE)" + ); + + // Verify that an EVM can be created with AMSTERDAM spec (for future use) + let db = create_db(&[(ADDRESS_A, 1000)]); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let evm = create_test_evm_with_spec(db, flags, SpecId::AMSTERDAM); + assert_eq!( + evm.inner.ctx.cfg.spec, + SpecId::AMSTERDAM, + "EVM should be configured with AMSTERDAM spec" + ); + } + + /// Verifies that Zero5 emits EIP-7708 Transfer logs regardless of SpecId. + /// Arc self-implements EIP-7708 log emission, so PRAGUE vs AMSTERDAM doesn't matter. + #[test] + fn test_zero5_emits_eip7708_regardless_of_spec() { + use revm::handler::SYSTEM_ADDRESS; + + // Zero5 + PRAGUE: Arc emits EIP-7708 Transfer logs itself + let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let mut evm = create_test_evm_with_spec(db, flags, SpecId::PRAGUE); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + ADDRESS_A, + ADDRESS_B, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + match result { + BeforeFrameInitResult::Log(log, _gas) => { + assert_eq!( + log.address, SYSTEM_ADDRESS, + "Zero5 + PRAGUE: should emit EIP-7708 Transfer log" + ); + } + other => panic!( + "Expected Log result with EIP-7708 Transfer, got {:?}", + other + ), + } + } + + /// Zero5: CALL with value to Address::ZERO should revert. + #[test] + fn test_zero5_call_to_zero_address_reverts() { + let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let mut evm = create_test_evm(db, flags); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + ADDRESS_A, + Address::ZERO, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + assert!( + matches!(result, BeforeFrameInitResult::Reverted(_)), + "Zero5 should revert CALL with value to zero address, got {:?}", + result, + ); + } + + /// Zero5: CALL from Address::ZERO with value should revert. + #[test] + fn test_zero5_call_from_zero_address_reverts() { + let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let mut evm = create_test_evm(db, flags); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + Address::ZERO, + ADDRESS_B, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + assert!( + matches!(result, BeforeFrameInitResult::Reverted(_)), + "Zero5 should revert CALL with value from zero address, got {:?}", + result, + ); + } + + /// Pre-Zero5: CALL to Address::ZERO is NOT blocked (backwards compatible). + #[test] + fn test_pre_zero5_call_to_zero_address_allowed() { let db = CacheDB::new(EmptyDB::default()); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero4]); + let mut evm = create_test_evm(db, flags); + + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + ADDRESS_A, + Address::ZERO, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + assert!( + matches!(result, BeforeFrameInitResult::Log(_, _)), + "Pre-Zero5 should allow CALL to zero address, got {:?}", + result, + ); + } + + /// Regression test for phantom EIP-7708 logs: + /// an inner value-transferring CALL that reverts must not leave a Transfer log behind. + #[test] + fn test_zero5_reverted_call_with_value_emits_no_eip7708_log() { + use arc_execution_config::{chainspec::localdev_with_hardforks, hardforks::ArcHardfork}; + use revm_primitives::TxKind; + + let chain_spec = localdev_with_hardforks(&[ + (ArcHardfork::Zero3, 0), + (ArcHardfork::Zero4, 0), + (ArcHardfork::Zero5, 0), + ]); + let sender = Address::repeat_byte(0x11); + let caller_contract = Address::repeat_byte(0x22); + let reverting_contract = Address::repeat_byte(0x33); + let amount = U256::from(100); + + // Runtime bytecode: PUSH1 0x00 PUSH1 0x00 REVERT + let revert_runtime: Bytes = vec![0x60, 0x00, 0x60, 0x00, 0xfd].into(); + let caller_runtime = call_with_value_bytecode(reverting_contract, amount); + + let mut db = create_db(&[(sender, 1000)]); + db.insert_account_info( + caller_contract, + revm::state::AccountInfo { + balance: U256::from(1000), + nonce: 1, + code_hash: keccak256(caller_runtime.bytecode()), + code: Some(caller_runtime), + account_id: None, + }, + ); + db.insert_account_info( + reverting_contract, + revm::state::AccountInfo { + balance: U256::ZERO, + nonce: 1, + code_hash: keccak256(&revert_runtime), + code: Some(Bytecode::new_raw(revert_runtime)), + account_id: None, + }, + ); + + let mut evm = create_arc_evm(chain_spec.clone(), db); + let tx = TxEnv { + caller: sender, + kind: TxKind::Call(caller_contract), + value: U256::ZERO, + gas_limit: 100_000, + gas_price: 0, + chain_id: Some(chain_spec.chain_id()), + ..Default::default() + }; + + let result = evm.transact_one(tx).expect("nested CALL should execute"); + assert!( + result.is_success(), + "Outer transaction should succeed; only the inner CALL should revert, got {:?}", + result + ); + assert_eq!( + result.logs().len(), + 0, + "Reverted inner CALL with value must not leave an EIP-7708 Transfer log behind" + ); + } + + /// Regression test for phantom EIP-7708 logs: + /// an inner CREATE with endowment whose initcode reverts must not leave a Transfer log behind. + #[test] + fn test_zero5_reverted_create_with_value_emits_no_eip7708_log() { + use arc_execution_config::{chainspec::localdev_with_hardforks, hardforks::ArcHardfork}; + use revm_primitives::TxKind; + + let chain_spec = localdev_with_hardforks(&[ + (ArcHardfork::Zero3, 0), + (ArcHardfork::Zero4, 0), + (ArcHardfork::Zero5, 0), + ]); + let sender = Address::repeat_byte(0x33); + let factory_contract = Address::repeat_byte(0x44); + let amount = U256::from(100); + + // Initcode: PUSH1 0x00 PUSH1 0x00 REVERT + let revert_initcode = vec![0x60, 0x00, 0x60, 0x00, 0xfd]; + let factory_runtime = create_with_value_bytecode(&revert_initcode, amount); + + let mut db = create_db(&[(sender, 1000)]); + db.insert_account_info( + factory_contract, + revm::state::AccountInfo { + balance: U256::from(1000), + nonce: 1, + code_hash: keccak256(factory_runtime.bytecode()), + code: Some(factory_runtime), + account_id: None, + }, + ); + let mut evm = create_arc_evm(chain_spec.clone(), db); + let tx = TxEnv { + caller: sender, + kind: TxKind::Call(factory_contract), + value: U256::ZERO, + gas_limit: 120_000, + gas_price: 0, + chain_id: Some(chain_spec.chain_id()), + ..Default::default() + }; + + let result = evm.transact_one(tx).expect("nested CREATE should execute"); + assert!( + result.is_success(), + "Outer transaction should succeed; only the inner CREATE should revert, got {:?}", + result + ); + assert_eq!( + result.logs().len(), + 0, + "Reverted inner CREATE with value must not leave an EIP-7708 Transfer log behind" + ); + } + + /// Zero5: EIP-7708 Transfer log must precede precompile logs in the journal. + /// + /// When a CALL with value targets a precompile that emits its own logs, the + /// EIP-7708 Transfer log from the value transfer must appear before any logs + /// produced by the precompile execution. + #[test] + fn test_zero5_transfer_log_precedes_precompile_log() { + use alloy_primitives::{Log as PrimLog, LogData}; + use reth_evm::precompiles::DynPrecompile; + use revm::handler::SYSTEM_ADDRESS; + use revm::precompile::{PrecompileId, PrecompileOutput}; + + // A distinctive address for the mock precompile, not overlapping with real ones. + const MOCK_PRECOMPILE: Address = address!("ff00000000000000000000000000000000000099"); + + // A distinctive log address so we can tell precompile logs from transfer logs. + const MOCK_LOG_ADDRESS: Address = address!("aa00000000000000000000000000000000000001"); + + // Build a PrecompilesMap that includes the mock precompile. + let spec = SpecId::PRAGUE; let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); - let mut evm = create_test_evm(db, flags); + let mut precompile_map = ArcPrecompileProvider::create_precompiles_map(spec, flags); + precompile_map.set_precompile_lookup(move |address: &Address| { + if *address == MOCK_PRECOMPILE { + Some(DynPrecompile::new_stateful( + PrecompileId::Custom("MOCK_LOG_EMITTER".into()), + move |mut input| { + // Emit a log via the journal so it appears in the journal log list. + input.internals.log(PrimLog { + address: MOCK_LOG_ADDRESS, + data: LogData::new_unchecked(vec![], Bytes::new()), + }); + Ok(PrecompileOutput::new(0, Bytes::new())) + }, + )) + } else { + None + } + }); + + // Build the ArcEvm with our custom precompile map. + let db = create_db(&[(ADDRESS_A, 10_000)]); + let mut evm = create_test_evm_with_precompiles(db, flags, precompile_map); + // Warm-load accounts so journal state is populated for transfer_loaded. evm.ctx_mut() .journal_mut() .load_account(NATIVE_COIN_CONTROL_ADDRESS) .unwrap(); + evm.ctx_mut().journal_mut().load_account(ADDRESS_A).unwrap(); + evm.ctx_mut() + .journal_mut() + .load_account(MOCK_PRECOMPILE) + .unwrap(); - // Self-transfer: from == to + // CALL with value from ADDRESS_A to MOCK_PRECOMPILE. + // bytecode_address must equal the precompile address so PrecompilesMap::run finds it. let frame = FrameInit { - frame_input: FrameInput::Call(call_input( - CallScheme::Call, - U256::from(100), - ADDRESS_A, - ADDRESS_A, - )), + frame_input: FrameInput::Call(Box::new(CallInputs { + scheme: CallScheme::Call, + target_address: MOCK_PRECOMPILE, + bytecode_address: MOCK_PRECOMPILE, + known_bytecode: None, + value: CallValue::Transfer(U256::from(100)), + input: CallInput::Bytes(Bytes::new()), + gas_limit: 500_000, + is_static: false, + caller: ADDRESS_A, + return_memory_offset: 0..0, + })), memory: SharedMemory::default(), depth: 1, }; - let result = evm.before_frame_init(&frame).unwrap(); - match result { - BeforeFrameInitResult::Checked(gas) => { - assert!(gas > 0, "Should have SLOAD gas cost"); - } - other => panic!( - "Expected Checked result for self-transfer under Zero5, got {:?}", - other - ), - } + // Record the log count before frame_init. + let logs_before = evm.ctx().journal().logs().len(); + + let result = evm.frame_init(frame); + assert!(result.is_ok(), "frame_init should succeed"); + + let ctx = evm.ctx(); + let logs = ctx.journal().logs(); + let new_logs = &logs[logs_before..]; + + assert!( + new_logs.len() >= 2, + "Expected at least 2 logs (transfer + precompile), got {}", + new_logs.len() + ); + + // First log must be the EIP-7708 Transfer log from the value transfer. + assert_eq!( + new_logs[0].address, SYSTEM_ADDRESS, + "First log should be the EIP-7708 Transfer log from system address, got {:?}", + new_logs[0].address + ); + + // Second log must be the mock precompile's log. + assert_eq!( + new_logs[1].address, MOCK_LOG_ADDRESS, + "Second log should be the mock precompile log, got {:?}", + new_logs[1].address + ); } + /// Regression test: a CALL with value to a precompile that reverts must roll back + /// the EIP-7708 Transfer log via Arc's checkpoint_revert (the precompile path at + /// line 507, distinct from the non-precompile path tested by + /// `test_zero5_reverted_call_with_value_emits_no_eip7708_log`). #[test] - fn test_pre_zero5_still_emits_custom_log() { - let db = CacheDB::new(EmptyDB::default()); - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero4]); - let mut evm = create_test_evm(db, flags); + fn test_zero5_reverted_precompile_call_with_value_emits_no_eip7708_log() { + use reth_evm::precompiles::DynPrecompile; + use revm::precompile::{PrecompileError, PrecompileId}; + + const MOCK_PRECOMPILE: Address = address!("ff00000000000000000000000000000000000099"); + + let spec = SpecId::PRAGUE; + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + let mut precompile_map = ArcPrecompileProvider::create_precompiles_map(spec, flags); + precompile_map.set_precompile_lookup(move |address: &Address| { + if *address == MOCK_PRECOMPILE { + Some(DynPrecompile::new_stateful( + PrecompileId::Custom("MOCK_REVERTER".into()), + move |_input| Err(PrecompileError::other("authorization failed")), + )) + } else { + None + } + }); + + let db = create_db(&[(ADDRESS_A, 10_000)]); + let mut evm = create_test_evm_with_precompiles(db, flags, precompile_map); evm.ctx_mut() .journal_mut() .load_account(NATIVE_COIN_CONTROL_ADDRESS) .unwrap(); + evm.ctx_mut().journal_mut().load_account(ADDRESS_A).unwrap(); + evm.ctx_mut() + .journal_mut() + .load_account(MOCK_PRECOMPILE) + .unwrap(); + + let logs_before = evm.ctx().journal().logs().len(); let frame = FrameInit { - frame_input: FrameInput::Call(call_input( - CallScheme::Call, - U256::from(100), - ADDRESS_A, - ADDRESS_B, - )), + frame_input: FrameInput::Call(Box::new(CallInputs { + scheme: CallScheme::Call, + target_address: MOCK_PRECOMPILE, + bytecode_address: MOCK_PRECOMPILE, + known_bytecode: None, + value: CallValue::Transfer(U256::from(100)), + input: CallInput::Bytes(Bytes::new()), + gas_limit: 500_000, + is_static: false, + caller: ADDRESS_A, + return_memory_offset: 0..0, + })), memory: SharedMemory::default(), depth: 1, }; - let result = evm.before_frame_init(&frame).unwrap(); - match result { - BeforeFrameInitResult::Log(log, _gas) => { - assert_eq!(log.address, NATIVE_COIN_AUTHORITY_ADDRESS); - } - other => panic!("Expected Log result pre-Zero5, got {:?}", other), - } - } - - /// Verifies that AMSTERDAM SpecId enables EIP-7708 (is_enabled_in returns true). - /// Once REVM is upgraded to a version with EIP-7708 journal support, the journal's - /// `transfer` method will emit Transfer logs when SpecId >= AMSTERDAM. - #[test] - fn test_amsterdam_spec_enables_eip7708() { - // AMSTERDAM is after PRAGUE in the SpecId ordering - assert!( - SpecId::AMSTERDAM.is_enabled_in(SpecId::AMSTERDAM), - "AMSTERDAM should be enabled in AMSTERDAM" - ); + let result = evm.frame_init(frame); assert!( - !SpecId::PRAGUE.is_enabled_in(SpecId::AMSTERDAM), - "PRAGUE should NOT be enabled in AMSTERDAM (AMSTERDAM comes after PRAGUE)" + result.is_ok(), + "frame_init should succeed (precompile failure is a Result, not an Err)" ); - // Verify that an EVM can be created with AMSTERDAM spec (for future use) - let db = create_db(&[(ADDRESS_A, 1000)]); - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); - let evm = create_test_evm_with_spec(db, flags, SpecId::AMSTERDAM); + let ctx = evm.ctx(); + let new_logs = &ctx.journal().logs()[logs_before..]; assert_eq!( - evm.inner.ctx.cfg.spec, - SpecId::AMSTERDAM, - "EVM should be configured with AMSTERDAM spec" + new_logs.len(), + 0, + "Reverting precompile CALL with value must not leave an EIP-7708 Transfer log behind, got {} logs", + new_logs.len() ); } - /// Verifies that Zero5 emits EIP-7708 Transfer logs regardless of SpecId. - /// Arc self-implements EIP-7708 log emission, so PRAGUE vs AMSTERDAM doesn't matter. #[test] - fn test_zero5_emits_eip7708_regardless_of_spec() { - use revm::handler::SYSTEM_ADDRESS; + fn test_zero6_nested_from_blocklisted_charges_sload_gas() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); - // Zero5 + PRAGUE: Arc emits EIP-7708 Transfer logs itself - let db = CacheDB::new(EmptyDB::default()); - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); - let mut evm = create_test_evm_with_spec(db, flags, SpecId::PRAGUE); + let mut db = CacheDB::new(EmptyDB::default()); + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(sender); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); + + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); + let mut evm = create_test_evm(db, flags); evm.ctx_mut() .journal_mut() @@ -5779,33 +6979,47 @@ mod tests { frame_input: FrameInput::Call(call_input( CallScheme::Call, U256::from(100), - ADDRESS_A, - ADDRESS_B, + sender, + recipient, )), memory: SharedMemory::default(), depth: 1, }; let result = evm.before_frame_init(&frame).unwrap(); - match result { - BeforeFrameInitResult::Log(log, _gas) => { - assert_eq!( - log.address, SYSTEM_ADDRESS, - "Zero5 + PRAGUE: should emit EIP-7708 Transfer log" - ); - } - other => panic!( - "Expected Log result with EIP-7708 Transfer, got {:?}", - other - ), + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.gas().spent(), + revm_interpreter::gas::COLD_SLOAD_COST, + "Zero6 nested from-blocklisted revert should charge one cold SLOAD" + ); + let expected_revert = + arc_precompiles::helpers::revert_message_to_bytes(ERR_BLOCKED_ADDRESS); + assert_eq!( + reverted.interpreter_result().output, + expected_revert, + "revert reason should be ERR_BLOCKED_ADDRESS" + ); + } else { + panic!("Expected Reverted result for blocklisted sender"); } } - /// Zero5: CALL with value to Address::ZERO should revert. #[test] - fn test_zero5_call_to_zero_address_reverts() { - let db = CacheDB::new(EmptyDB::default()); - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); + fn test_zero6_nested_to_blocklisted_charges_sload_gas() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); + + let mut db = CacheDB::new(EmptyDB::default()); + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(recipient); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); + + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); let mut evm = create_test_evm(db, flags); evm.ctx_mut() @@ -5817,25 +7031,39 @@ mod tests { frame_input: FrameInput::Call(call_input( CallScheme::Call, U256::from(100), - ADDRESS_A, - Address::ZERO, + sender, + recipient, )), memory: SharedMemory::default(), depth: 1, }; let result = evm.before_frame_init(&frame).unwrap(); - assert!( - matches!(result, BeforeFrameInitResult::Reverted(_)), - "Zero5 should revert CALL with value to zero address, got {:?}", - result, - ); + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.gas().spent(), + 2 * revm_interpreter::gas::COLD_SLOAD_COST, + "Zero6 nested to-blocklisted revert should charge two cold SLOADs" + ); + } else { + panic!("Expected Reverted result for blocklisted recipient"); + } } - /// Zero5: CALL from Address::ZERO with value should revert. #[test] - fn test_zero5_call_from_zero_address_reverts() { - let db = CacheDB::new(EmptyDB::default()); + fn test_pre_zero6_nested_blocklisted_charges_zero_gas() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); + + let mut db = CacheDB::new(EmptyDB::default()); + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(sender); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); let mut evm = create_test_evm(db, flags); @@ -5848,26 +7076,40 @@ mod tests { frame_input: FrameInput::Call(call_input( CallScheme::Call, U256::from(100), - Address::ZERO, - ADDRESS_B, + sender, + recipient, )), memory: SharedMemory::default(), depth: 1, }; let result = evm.before_frame_init(&frame).unwrap(); - assert!( - matches!(result, BeforeFrameInitResult::Reverted(_)), - "Zero5 should revert CALL with value from zero address, got {:?}", - result, - ); + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.gas().spent(), + 0, + "Pre-Zero6 nested revert should charge zero gas (unchanged behavior)" + ); + } else { + panic!("Expected Reverted result for blocklisted sender"); + } } - /// Pre-Zero5: CALL to Address::ZERO is NOT blocked (backwards compatible). #[test] - fn test_pre_zero5_call_to_zero_address_allowed() { - let db = CacheDB::new(EmptyDB::default()); - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero4]); + fn test_zero6_depth0_blocklisted_charges_zero_gas() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); + + let mut db = CacheDB::new(EmptyDB::default()); + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(sender); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); + + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); let mut evm = create_test_evm(db, flags); evm.ctx_mut() @@ -5879,323 +7121,320 @@ mod tests { frame_input: FrameInput::Call(call_input( CallScheme::Call, U256::from(100), - ADDRESS_A, - Address::ZERO, + sender, + recipient, )), memory: SharedMemory::default(), - depth: 1, + depth: 0, }; let result = evm.before_frame_init(&frame).unwrap(); - assert!( - matches!(result, BeforeFrameInitResult::Log(_, _)), - "Pre-Zero5 should allow CALL to zero address, got {:?}", - result, - ); + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.gas().spent(), + 0, + "Depth-0 Zero6 revert should charge zero gas (covered by validate_initial_tx_gas)" + ); + } else { + panic!("Expected Reverted result for blocklisted sender"); + } } - /// Regression test for phantom EIP-7708 logs: - /// an inner value-transferring CALL that reverts must not leave a Transfer log behind. #[test] - fn test_zero5_reverted_call_with_value_emits_no_eip7708_log() { - use arc_execution_config::{chainspec::localdev_with_hardforks, hardforks::ArcHardfork}; - use revm_primitives::TxKind; - - let chain_spec = localdev_with_hardforks(&[ - (ArcHardfork::Zero3, 0), - (ArcHardfork::Zero4, 0), - (ArcHardfork::Zero5, 0), - ]); - let sender = Address::repeat_byte(0x11); - let caller_contract = Address::repeat_byte(0x22); - let reverting_contract = Address::repeat_byte(0x33); - let amount = U256::from(100); - - // Runtime bytecode: PUSH1 0x00 PUSH1 0x00 REVERT - let revert_runtime: Bytes = vec![0x60, 0x00, 0x60, 0x00, 0xfd].into(); - let caller_runtime = call_with_value_bytecode(reverting_contract, amount); - - let mut db = create_db(&[(sender, 1000)]); - db.insert_account_info( - caller_contract, - revm::state::AccountInfo { - balance: U256::from(1000), - nonce: 1, - code_hash: keccak256(caller_runtime.bytecode()), - code: Some(caller_runtime), - account_id: None, - }, - ); - db.insert_account_info( - reverting_contract, - revm::state::AccountInfo { - balance: U256::ZERO, - nonce: 1, - code_hash: keccak256(&revert_runtime), - code: Some(Bytecode::new_raw(revert_runtime)), - account_id: None, - }, - ); - - let mut evm = create_arc_evm(chain_spec.clone(), db); - let tx = TxEnv { - caller: sender, - kind: TxKind::Call(caller_contract), - value: U256::ZERO, - gas_limit: 100_000, - gas_price: 0, - chain_id: Some(chain_spec.chain_id()), - ..Default::default() - }; + fn test_zero6_nested_from_blocklisted_warm_sload_gas() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); - let result = evm.transact_one(tx).expect("nested CALL should execute"); - assert!( - result.is_success(), - "Outer transaction should succeed; only the inner CALL should revert, got {:?}", - result - ); - assert_eq!( - result.logs().len(), - 0, - "Reverted inner CALL with value must not leave an EIP-7708 Transfer log behind" - ); - } + let mut db = CacheDB::new(EmptyDB::default()); + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(sender); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); - /// Regression test for phantom EIP-7708 logs: - /// an inner CREATE with endowment whose initcode reverts must not leave a Transfer log behind. - #[test] - fn test_zero5_reverted_create_with_value_emits_no_eip7708_log() { - use arc_execution_config::{chainspec::localdev_with_hardforks, hardforks::ArcHardfork}; - use revm_primitives::TxKind; + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); + let mut evm = create_test_evm(db, flags); - let chain_spec = localdev_with_hardforks(&[ - (ArcHardfork::Zero3, 0), - (ArcHardfork::Zero4, 0), - (ArcHardfork::Zero5, 0), - ]); - let sender = Address::repeat_byte(0x33); - let factory_contract = Address::repeat_byte(0x44); - let amount = U256::from(100); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); - // Initcode: PUSH1 0x00 PUSH1 0x00 REVERT - let revert_initcode = vec![0x60, 0x00, 0x60, 0x00, 0xfd]; - let factory_runtime = create_with_value_bytecode(&revert_initcode, amount); + // Warm the slot by reading it first + evm.inner + .ctx + .journal_mut() + .sload(NATIVE_COIN_CONTROL_ADDRESS, storage_slot.into()) + .unwrap(); - let mut db = create_db(&[(sender, 1000)]); - db.insert_account_info( - factory_contract, - revm::state::AccountInfo { - balance: U256::from(1000), - nonce: 1, - code_hash: keccak256(factory_runtime.bytecode()), - code: Some(factory_runtime), - account_id: None, - }, - ); - let mut evm = create_arc_evm(chain_spec.clone(), db); - let tx = TxEnv { - caller: sender, - kind: TxKind::Call(factory_contract), - value: U256::ZERO, - gas_limit: 120_000, - gas_price: 0, - chain_id: Some(chain_spec.chain_id()), - ..Default::default() + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + sender, + recipient, + )), + memory: SharedMemory::default(), + depth: 1, }; - let result = evm.transact_one(tx).expect("nested CREATE should execute"); - assert!( - result.is_success(), - "Outer transaction should succeed; only the inner CREATE should revert, got {:?}", - result - ); - assert_eq!( - result.logs().len(), - 0, - "Reverted inner CREATE with value must not leave an EIP-7708 Transfer log behind" - ); + let result = evm.before_frame_init(&frame).unwrap(); + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.gas().spent(), + revm_interpreter::gas::WARM_STORAGE_READ_COST, + "Zero6 nested from-blocklisted warm revert should charge one warm SLOAD" + ); + } else { + panic!("Expected Reverted result for blocklisted sender"); + } } - /// Zero5: EIP-7708 Transfer log must precede precompile logs in the journal. - /// - /// When a CALL with value targets a precompile that emits its own logs, the - /// EIP-7708 Transfer log from the value transfer must appear before any logs - /// produced by the precompile execution. #[test] - fn test_zero5_transfer_log_precedes_precompile_log() { - use alloy_primitives::{Log as PrimLog, LogData}; - use reth_evm::precompiles::DynPrecompile; - use revm::handler::SYSTEM_ADDRESS; - use revm::precompile::{PrecompileId, PrecompileOutput}; - - // A distinctive address for the mock precompile, not overlapping with real ones. - const MOCK_PRECOMPILE: Address = address!("ff00000000000000000000000000000000000099"); + fn test_zero6_nested_to_blocklisted_mixed_warm_cold_sload_gas() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); - // A distinctive log address so we can tell precompile logs from transfer logs. - const MOCK_LOG_ADDRESS: Address = address!("aa00000000000000000000000000000000000001"); + let mut db = CacheDB::new(EmptyDB::default()); + let recipient_slot = native_coin_control::compute_is_blocklisted_storage_slot(recipient); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + recipient_slot.into(), + U256::from(1), + ) + .unwrap(); - // Build a PrecompilesMap that includes the mock precompile. - let spec = SpecId::PRAGUE; - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); - let mut precompile_map = ArcPrecompileProvider::create_precompiles_map(spec, flags); - precompile_map.set_precompile_lookup(move |address: &Address| { - if *address == MOCK_PRECOMPILE { - Some(DynPrecompile::new_stateful( - PrecompileId::Custom("MOCK_LOG_EMITTER".into()), - move |mut input| { - // Emit a log via the journal so it appears in the journal log list. - input.internals.log(PrimLog { - address: MOCK_LOG_ADDRESS, - data: LogData::new_unchecked(vec![], Bytes::new()), - }); - Ok(PrecompileOutput::new(0, Bytes::new())) - }, - )) - } else { - None - } - }); + let sender_slot = native_coin_control::compute_is_blocklisted_storage_slot(sender); + db.insert_account_storage(NATIVE_COIN_CONTROL_ADDRESS, sender_slot.into(), U256::ZERO) + .unwrap(); - // Build the ArcEvm with our custom precompile map. - let db = create_db(&[(ADDRESS_A, 10_000)]); - let mut evm = create_test_evm_with_precompiles(db, flags, precompile_map); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); + let mut evm = create_test_evm(db, flags); - // Warm-load accounts so journal state is populated for transfer_loaded. evm.ctx_mut() .journal_mut() .load_account(NATIVE_COIN_CONTROL_ADDRESS) .unwrap(); - evm.ctx_mut().journal_mut().load_account(ADDRESS_A).unwrap(); - evm.ctx_mut() + + // Warm the sender's slot by reading it + evm.inner + .ctx .journal_mut() - .load_account(MOCK_PRECOMPILE) + .sload(NATIVE_COIN_CONTROL_ADDRESS, sender_slot.into()) .unwrap(); - // CALL with value from ADDRESS_A to MOCK_PRECOMPILE. - // bytecode_address must equal the precompile address so PrecompilesMap::run finds it. + // Recipient's slot stays cold + + let frame = FrameInit { + frame_input: FrameInput::Call(call_input( + CallScheme::Call, + U256::from(100), + sender, + recipient, + )), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + if let BeforeFrameInitResult::Reverted(reverted) = result { + // Warm from-SLOAD (100) + cold to-SLOAD (2100) = 2200 + assert_eq!( + reverted.gas().spent(), + revm_interpreter::gas::WARM_STORAGE_READ_COST + + revm_interpreter::gas::COLD_SLOAD_COST, + "Mixed warm/cold: warm from-SLOAD + cold to-SLOAD should be 2200" + ); + } else { + panic!("Expected Reverted result for blocklisted recipient"); + } + } + + #[test] + fn test_zero6_nested_selfdestructed_target_charges_sload_gas() { + let db = CacheDB::new(EmptyDB::default()); + let flags = + ArcHardforkFlags::with(&[ArcHardfork::Zero4, ArcHardfork::Zero5, ArcHardfork::Zero6]); + let mut evm = create_test_evm(db, flags); + + let spec_id = evm.ctx().cfg.spec; + let journal = evm.ctx_mut().journal_mut(); + + journal.load_account(NATIVE_COIN_CONTROL_ADDRESS).unwrap(); + + journal + .load_account_mut_optional_code(ADDRESS_A, false) + .expect("load ADDRESS_A") + .set_balance(U256::from(100)); + + journal.load_account(ADDRESS_B).expect("load ADDRESS_B"); + journal + .create_account_checkpoint(ADDRESS_A, ADDRESS_B, U256::from(100), spec_id) + .unwrap(); + journal + .selfdestruct(ADDRESS_B, ADDRESS_A, false) + .expect("selfdestruct"); + let frame = FrameInit { frame_input: FrameInput::Call(Box::new(CallInputs { scheme: CallScheme::Call, - target_address: MOCK_PRECOMPILE, - bytecode_address: MOCK_PRECOMPILE, + target_address: ADDRESS_B, + bytecode_address: ADDRESS_A, known_bytecode: None, value: CallValue::Transfer(U256::from(100)), input: CallInput::Bytes(Bytes::new()), - gas_limit: 500_000, - is_static: false, + gas_limit: 100_000, caller: ADDRESS_A, + is_static: false, return_memory_offset: 0..0, })), memory: SharedMemory::default(), depth: 1, }; - // Record the log count before frame_init. - let logs_before = evm.ctx().journal().logs().len(); + let result = evm.before_frame_init(&frame); + match result { + Ok(BeforeFrameInitResult::Reverted(reverted)) => { + assert_eq!( + reverted.gas().spent(), + 2 * revm_interpreter::gas::COLD_SLOAD_COST, + "Zero6 nested selfdestructed-target revert should charge two cold SLOADs" + ); + let expected_revert = arc_precompiles::helpers::revert_message_to_bytes( + ERR_SELFDESTRUCTED_BALANCE_INCREASED, + ); + assert_eq!( + reverted.interpreter_result().output, + expected_revert, + "revert reason should be ERR_SELFDESTRUCTED_BALANCE_INCREASED" + ); + } + other => panic!( + "Expected Reverted for selfdestructed target, got {:?}", + other + ), + } + } - let result = evm.frame_init(frame); - assert!(result.is_ok(), "frame_init should succeed"); + #[test] + fn test_zero6_nested_blocklisted_oog_when_gas_below_sload_cost() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); - let ctx = evm.ctx(); - let logs = ctx.journal().logs(); - let new_logs = &logs[logs_before..]; + let mut db = CacheDB::new(EmptyDB::default()); + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(sender); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); - assert!( - new_logs.len() >= 2, - "Expected at least 2 logs (transfer + precompile), got {}", - new_logs.len() - ); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); + let mut evm = create_test_evm(db, flags); - // First log must be the EIP-7708 Transfer log from the value transfer. - assert_eq!( - new_logs[0].address, SYSTEM_ADDRESS, - "First log should be the EIP-7708 Transfer log from system address, got {:?}", - new_logs[0].address - ); + evm.ctx_mut() + .journal_mut() + .load_account(NATIVE_COIN_CONTROL_ADDRESS) + .unwrap(); - // Second log must be the mock precompile's log. - assert_eq!( - new_logs[1].address, MOCK_LOG_ADDRESS, - "Second log should be the mock precompile log, got {:?}", - new_logs[1].address - ); + // gas_limit just below COLD_SLOAD_COST — frame can't afford the SLOAD + let frame = FrameInit { + frame_input: FrameInput::Call(Box::new(CallInputs { + scheme: CallScheme::Call, + target_address: recipient, + bytecode_address: recipient, + known_bytecode: None, + value: CallValue::Transfer(U256::from(100)), + input: CallInput::Bytes(Bytes::new()), + gas_limit: revm_interpreter::gas::COLD_SLOAD_COST - 1, + is_static: false, + caller: sender, + return_memory_offset: 0..0, + })), + memory: SharedMemory::default(), + depth: 1, + }; + + let result = evm.before_frame_init(&frame).unwrap(); + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.instruction_result(), + InstructionResult::OutOfGas, + "Should OOG when gas_limit < SLOAD cost" + ); + assert_eq!( + reverted.gas().spent(), + revm_interpreter::gas::COLD_SLOAD_COST - 1, + "OOG should consume all available gas" + ); + } else { + panic!("Expected Reverted result for blocklisted sender with insufficient gas"); + } } - /// Regression test: a CALL with value to a precompile that reverts must roll back - /// the EIP-7708 Transfer log via Arc's checkpoint_revert (the precompile path at - /// line 507, distinct from the non-precompile path tested by - /// `test_zero5_reverted_call_with_value_emits_no_eip7708_log`). #[test] - fn test_zero5_reverted_precompile_call_with_value_emits_no_eip7708_log() { - use reth_evm::precompiles::DynPrecompile; - use revm::precompile::{PrecompileError, PrecompileId}; - - const MOCK_PRECOMPILE: Address = address!("ff00000000000000000000000000000000000099"); + fn test_zero6_nested_to_blocklisted_oog_when_gas_between_one_and_two_sloads() { + let sender = address!("A000000000000000000000000000000000000001"); + let recipient = address!("B000000000000000000000000000000000000002"); - let spec = SpecId::PRAGUE; - let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5]); - let mut precompile_map = ArcPrecompileProvider::create_precompiles_map(spec, flags); - precompile_map.set_precompile_lookup(move |address: &Address| { - if *address == MOCK_PRECOMPILE { - Some(DynPrecompile::new_stateful( - PrecompileId::Custom("MOCK_REVERTER".into()), - move |_input| Err(PrecompileError::other("authorization failed")), - )) - } else { - None - } - }); + let mut db = CacheDB::new(EmptyDB::default()); + // Only recipient is blocklisted — requires 2 SLOADs (from check + to check) + let storage_slot = native_coin_control::compute_is_blocklisted_storage_slot(recipient); + db.insert_account_storage( + NATIVE_COIN_CONTROL_ADDRESS, + storage_slot.into(), + U256::from(1), + ) + .unwrap(); - let db = create_db(&[(ADDRESS_A, 10_000)]); - let mut evm = create_test_evm_with_precompiles(db, flags, precompile_map); + let flags = ArcHardforkFlags::with(&[ArcHardfork::Zero5, ArcHardfork::Zero6]); + let mut evm = create_test_evm(db, flags); evm.ctx_mut() .journal_mut() .load_account(NATIVE_COIN_CONTROL_ADDRESS) .unwrap(); - evm.ctx_mut().journal_mut().load_account(ADDRESS_A).unwrap(); - evm.ctx_mut() - .journal_mut() - .load_account(MOCK_PRECOMPILE) - .unwrap(); - - let logs_before = evm.ctx().journal().logs().len(); + // gas_limit between COLD_SLOAD_COST and 2*COLD_SLOAD_COST — enough for 1 SLOAD but not 2 + let gas_limit = revm_interpreter::gas::COLD_SLOAD_COST + 100; let frame = FrameInit { frame_input: FrameInput::Call(Box::new(CallInputs { scheme: CallScheme::Call, - target_address: MOCK_PRECOMPILE, - bytecode_address: MOCK_PRECOMPILE, + target_address: recipient, + bytecode_address: recipient, known_bytecode: None, value: CallValue::Transfer(U256::from(100)), input: CallInput::Bytes(Bytes::new()), - gas_limit: 500_000, + gas_limit, is_static: false, - caller: ADDRESS_A, + caller: sender, return_memory_offset: 0..0, })), memory: SharedMemory::default(), depth: 1, }; - let result = evm.frame_init(frame); - assert!( - result.is_ok(), - "frame_init should succeed (precompile failure is a Result, not an Err)" - ); - - let ctx = evm.ctx(); - let new_logs = &ctx.journal().logs()[logs_before..]; - assert_eq!( - new_logs.len(), - 0, - "Reverting precompile CALL with value must not leave an EIP-7708 Transfer log behind, got {} logs", - new_logs.len() - ); + let result = evm.before_frame_init(&frame).unwrap(); + if let BeforeFrameInitResult::Reverted(reverted) = result { + assert_eq!( + reverted.instruction_result(), + InstructionResult::OutOfGas, + "Should OOG when gas_limit < 2 * SLOAD cost for to-blocklisted" + ); + assert_eq!( + reverted.gas().spent(), + gas_limit, + "OOG should consume all available gas" + ); + } else { + panic!("Expected Reverted result for blocklisted recipient with insufficient gas"); + } } - // ----- Revm upgrade checklist ----- - + /// ----- Revm upgrade checklist ----- /// Guard against silent drift when upgrading revm. If this test fails, the revm /// dependency version has changed and the forked/mirrored functions below must be /// reviewed for upstream behavioral changes: diff --git a/crates/evm/src/executor.rs b/crates/evm/src/executor.rs index ba2f42d..d33186c 100644 --- a/crates/evm/src/executor.rs +++ b/crates/evm/src/executor.rs @@ -292,7 +292,9 @@ where ); } - let base_fee_config = self.chain_spec.base_fee_config(block_number + 1); + let base_fee_config = self + .chain_spec + .base_fee_config(block_number.checked_add(1).expect("block number overflow")); let calc = base_fee_config.resolve_calc_params(fee_params.as_ref()); let parent_block_number = block_number.saturating_sub(1); @@ -429,7 +431,12 @@ where // The sum of the transaction's gas limit, Tg, and the gas utilized in this block prior, // must be no greater than the block's gasLimit. - let block_available_gas = self.evm.block().gas_limit() - self.gas_used; + let block_available_gas = self + .evm + .block() + .gas_limit() + .checked_sub(self.gas_used) + .expect("gas_used must not exceed block gas_limit"); if tx.tx().gas_limit() > block_available_gas { return Err( @@ -467,7 +474,10 @@ where let gas_used = result.gas_used(); // append gas used - self.gas_used += gas_used; + self.gas_used = self + .gas_used + .checked_add(gas_used) + .expect("cumulative gas overflow"); // Cancun is always active for arc self.blob_gas_used = self.blob_gas_used.saturating_add(blob_gas_used); @@ -681,8 +691,8 @@ mod tests { .expect("Should be able to query reward beneficiary") } - /// Patch ProtocolConfig storage so rewardBeneficiary() returns 0x00. - fn set_beneficiary_to_zero_address(db: &mut InMemoryDB) { + /// Patch ProtocolConfig storage so rewardBeneficiary() returns the given address. + fn set_reward_beneficiary(db: &mut InMemoryDB, beneficiary: Address) { // ProtocolConfig ERC-7201 base slot from the contract (see ProtocolConfig.sol). const PROTOCOL_CONFIG_STORAGE_LOCATION_HEX: &str = "668f09ce856848ead6cb1ddee963f15ef833cea8958030868f867aec84385200"; @@ -693,11 +703,16 @@ mod tests { db.insert_account_storage( protocol_config::PROTOCOL_CONFIG_ADDRESS, reward_beneficiary_slot, - StorageValue::from(0u64), + StorageValue::from(U256::from_be_slice(beneficiary.as_slice())), ) .expect("Insert storage"); } + /// Patch ProtocolConfig storage so rewardBeneficiary() returns 0x00. + fn set_beneficiary_to_zero_address(db: &mut InMemoryDB) { + set_reward_beneficiary(db, Address::ZERO); + } + fn mark_address_as_blocklisted(db: &mut InMemoryDB, beneficiary: Address) { let storage_slot = compute_is_blocklisted_storage_slot(beneficiary).into(); db.insert_account_storage( @@ -1116,9 +1131,13 @@ mod tests { let mut db = InMemoryDB::default(); insert_alloc_into_db(&mut db, chain_spec.genesis()); + // Set a non-zero expected beneficiary so the mismatch check is active. + let expected_beneficiary = address!("0000000000000000000000000000000000001234"); + set_reward_beneficiary(&mut db, expected_beneficiary); + let evm_config = create_evm_config(chain_spec.clone()); - // Use a wrong beneficiary address + // Use a wrong beneficiary address (different from expected_beneficiary above) let wrong_beneficiary = address!("0000000000000000000000000000000000000bad"); let mut block_env = get_mock_block_env(); @@ -1444,13 +1463,11 @@ mod tests { let mut db = InMemoryDB::default(); insert_alloc_into_db(&mut db, chain_spec.genesis()); + // Set a non-zero expected beneficiary so the mismatch check is active, then blocklist it. + let expected_beneficiary = address!("0000000000000000000000000000000000001234"); + set_reward_beneficiary(&mut db, expected_beneficiary); + let block_number = 10; - let expected_beneficiary = - query_expected_beneficiary(chain_spec.clone(), &mut db, block_number); - assert!( - !expected_beneficiary.is_zero(), - "LOCAL_DEV rewardBeneficiary should be non-zero for this test" - ); mark_address_as_blocklisted(&mut db, expected_beneficiary); diff --git a/crates/evm/src/frame_result.rs b/crates/evm/src/frame_result.rs index a3e4e27..58565c0 100644 --- a/crates/evm/src/frame_result.rs +++ b/crates/evm/src/frame_result.rs @@ -16,7 +16,7 @@ //! Frame result utilities for Arc EVM handler -use arc_precompiles::helpers::{revert_message_to_bytes, ERR_BLOCKED_ADDRESS}; +use arc_precompiles::helpers::revert_message_to_bytes; use reth_ethereum::primitives::Log; use revm::{ handler::FrameResult, @@ -40,8 +40,6 @@ pub enum BeforeFrameInitResult { None, } -pub(crate) const BLOCKLISTED_GAS_PENALTY: u64 = 0; - /// Creates a new FrameResult for out-of-gas during blocklist checks. /// This is used when a nested frame doesn't have enough gas to cover the SLOAD costs /// for blocklist verification. @@ -120,15 +118,11 @@ pub fn create_frame_result( } } -/// Creates a new FrameResult for blocklist violations from a FrameInit. -pub fn create_blocklisted_frame_result(frame_init: &FrameInit) -> FrameResult { - create_frame_result(frame_init, ERR_BLOCKED_ADDRESS, BLOCKLISTED_GAS_PENALTY) -} - #[cfg(test)] mod tests { use super::*; use alloy_primitives::{Address, Bytes, U256}; + use arc_precompiles::helpers::ERR_BLOCKED_ADDRESS; use revm::interpreter::{ interpreter_action::{CreateInputs, FrameInit, FrameInput}, SharedMemory, @@ -136,7 +130,7 @@ mod tests { use revm_interpreter::CreateScheme; #[test] - fn test_create_blocklisted_frame_result_returns_none_address_for_create() { + fn test_create_frame_result_returns_none_address_for_create() { let create_inputs = CreateInputs::new( Address::repeat_byte(0x42), CreateScheme::Create, @@ -151,7 +145,7 @@ mod tests { frame_input: FrameInput::Create(Box::new(create_inputs)), }; - let result = create_blocklisted_frame_result(&frame_init); + let result = create_frame_result(&frame_init, ERR_BLOCKED_ADDRESS, 0); match result { FrameResult::Create(outcome) => { diff --git a/crates/evm/src/handler.rs b/crates/evm/src/handler.rs index a667d2a..cb506a9 100644 --- a/crates/evm/src/handler.rs +++ b/crates/evm/src/handler.rs @@ -91,6 +91,8 @@ where let effective_gas_price = ctx.tx().effective_gas_price(basefee); let gas_used = exec_result.gas().used(); + // u128 * u64 fits in U256 (max 192 bits). + #[allow(clippy::arithmetic_side_effects)] let total_fee_amount = U256::from(effective_gas_price) * U256::from(gas_used); // Transfer the total fee to the beneficiary (both base fee and priority fee) @@ -126,7 +128,11 @@ where // Charge for recipient blocklist check if value > 0 // This applies to both Call and Create transactions with value if !tx.value().is_zero() { - extra_gas += PRECOMPILE_SLOAD_GAS_COST; + // 2,100 + 2,100 = 4,200; fits in u64 + #[allow(clippy::arithmetic_side_effects)] + { + extra_gas += PRECOMPILE_SLOAD_GAS_COST; + } } // Add extra gas to initial gas diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index fdb771d..e924fb9 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -14,6 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr( + test, + allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation) +)] + //! Arc EVM configuration, executor, and handler. //! //! This crate contains the EVM customization for Arc Network: block assembler, diff --git a/crates/evm/src/subcall.rs b/crates/evm/src/subcall.rs index 1c25519..321046f 100644 --- a/crates/evm/src/subcall.rs +++ b/crates/evm/src/subcall.rs @@ -34,7 +34,7 @@ pub struct SubcallContinuation { pub(crate) precompile: Arc, /// The original call's gas limit (budget allocated by the parent frame). pub(crate) gas_limit: u64, - /// Gas consumed by `init_subcall` (ABI decoding, validation). + /// Gas consumed by `init_subcall` (ABI decoding, validation, and EIP-2929 account access). pub(crate) init_subcall_gas_overhead: u64, /// The original call's return memory offset. pub(crate) return_memory_offset: std::ops::Range, diff --git a/crates/execution-config/src/follow.rs b/crates/execution-config/src/follow.rs index 861f4fe..30acc85 100644 --- a/crates/execution-config/src/follow.rs +++ b/crates/execution-config/src/follow.rs @@ -21,10 +21,10 @@ use reth_network_peers::TrustedPeer; use arc_shared::chain_ids::{LOCALDEV_CHAIN_ID, TESTNET_CHAIN_ID}; -/// Returns the RPC URL for the given chain ID. -pub fn url_for_chain_id(chain_id: u64) -> Result { +/// Returns the WebSocket URL for the given chain ID. +pub fn ws_url_for_chain_id(chain_id: u64) -> Result { let url = match chain_id { - TESTNET_CHAIN_ID => "https://rpc.quicknode.testnet.arc.network/", + TESTNET_CHAIN_ID => "wss://rpc.quicknode.testnet.arc.network", LOCALDEV_CHAIN_ID => "ws://localhost:8546", _ => return Err(eyre!("Unsupported chain for follow mode: {}", chain_id)), }; @@ -50,26 +50,26 @@ mod tests { use arc_shared::chain_ids::DEVNET_CHAIN_ID; #[test] - fn test_url_for_chain_id_localdev() { - let url = url_for_chain_id(LOCALDEV_CHAIN_ID).unwrap(); + fn test_ws_url_for_chain_id_localdev() { + let url = ws_url_for_chain_id(LOCALDEV_CHAIN_ID).unwrap(); assert_eq!(url, "ws://localhost:8546"); } #[test] - fn test_url_for_chain_id_devnet() { - let result = url_for_chain_id(DEVNET_CHAIN_ID); + fn test_ws_url_for_chain_id_devnet() { + let result = ws_url_for_chain_id(DEVNET_CHAIN_ID); assert!(result.is_err()); } #[test] - fn test_url_for_chain_id_testnet() { - let url = url_for_chain_id(TESTNET_CHAIN_ID).unwrap(); - assert_eq!(url, "https://rpc.quicknode.testnet.arc.network/"); + fn test_ws_url_for_chain_id_testnet() { + let url = ws_url_for_chain_id(TESTNET_CHAIN_ID).unwrap(); + assert_eq!(url, "wss://rpc.quicknode.testnet.arc.network"); } #[test] - fn test_url_for_chain_id_unsupported() { - let result = url_for_chain_id(999); + fn test_ws_url_for_chain_id_unsupported() { + let result = ws_url_for_chain_id(999); assert!(result.is_err()); assert_eq!( result.unwrap_err().to_string(), diff --git a/crates/execution-config/src/gas_fee.rs b/crates/execution-config/src/gas_fee.rs index 1ad8340..37b944c 100644 --- a/crates/execution-config/src/gas_fee.rs +++ b/crates/execution-config/src/gas_fee.rs @@ -36,7 +36,10 @@ pub fn determine_ema_parent_gas_used( // (1-α) * G[t-1] + α * G[t] // α is expressed as an integer value [0, 100] - let left = (ALPHA_MAX - a).checked_mul(smoothed)?; + // a <= ALPHA_MAX is guaranteed by the guard above. + #[allow(clippy::arithmetic_side_effects)] + let complement = ALPHA_MAX - a; + let left = complement.checked_mul(smoothed)?; let right = a.checked_mul(raw)?; let together = left.checked_add(right)?; @@ -70,14 +73,24 @@ pub fn arc_calc_next_block_base_fee( k_rate: u64, // 2500 => 25% inverse_elasticity_multiplier: u64, // 7500 => 75% ) -> u64 { - // Calculate the target gas by dividing the gas limit by the elasticity multiplier. - let gas_target = (gas_limit as u128 * inverse_elasticity_multiplier as u128 - / ARC_BASE_FEE_FIXED_POINT_SCALE) as u64; + // All u64×u64 products fit in u128 without overflow. + #[allow(clippy::arithmetic_side_effects)] + let gas_target_u128 = + gas_limit as u128 * inverse_elasticity_multiplier as u128 / ARC_BASE_FEE_FIXED_POINT_SCALE; + let gas_target = u64::try_from(gas_target_u128).unwrap_or(u64::MAX); if gas_target == 0 || k_rate == 0 { return base_fee; } + // k_rate != 0 checked above; gas_target (u64) × 10_000 fits in u128 + #[allow(clippy::arithmetic_side_effects)] + let denominator = gas_target as u128 * ARC_BASE_FEE_FIXED_POINT_SCALE / k_rate as u128; + + if denominator == 0 { + return base_fee; + } + match gas_used.cmp(&gas_target) { // If the gas used in the current block is equal to the gas target, the base fee remains the // same (no increase). @@ -86,22 +99,22 @@ pub fn arc_calc_next_block_base_fee( // increased base fee. core::cmp::Ordering::Greater => { // Calculate the increase in base fee based on the formula defined by EIP-1559. - base_fee.saturating_add(core::cmp::max( - // Ensure a minimum increase of 1. - 1, - base_fee as u128 * (gas_used - gas_target) as u128 - / (gas_target as u128 * ARC_BASE_FEE_FIXED_POINT_SCALE / k_rate as u128), - ) as u64) + // gas_used > gas_target in this arm, so subtraction is safe + #[allow(clippy::arithmetic_side_effects)] + let increase = base_fee as u128 * (gas_used - gas_target) as u128 / denominator; + let increase = u64::try_from(increase).unwrap_or(u64::MAX); + // Ensure a minimum increase of 1. + base_fee.saturating_add(core::cmp::max(1, increase)) } // If the gas used in the current block is less than the gas target, calculate a new // decreased base fee. core::cmp::Ordering::Less => { // Calculate the decrease in base fee based on the formula defined by EIP-1559. - base_fee.saturating_sub( - (base_fee as u128 * (gas_target - gas_used) as u128 - / (gas_target as u128 * ARC_BASE_FEE_FIXED_POINT_SCALE / k_rate as u128)) - as u64, - ) + // gas_target > gas_used in this arm, so subtraction is safe + #[allow(clippy::arithmetic_side_effects)] + let decrease = base_fee as u128 * (gas_target - gas_used) as u128 / denominator; + let decrease = u64::try_from(decrease).unwrap_or(u64::MAX); + base_fee.saturating_sub(decrease) } } } @@ -300,6 +313,7 @@ mod tests { .next_block_base_fee(gas_used, gas_limit, base_fee); // use kRate 12.5% and inverse elasticity multiplier 50% to compute the same value as eip1559 + #[allow(clippy::cast_possible_truncation)] // 10_000u128 fits in u64 let next_base_fee = arc_calc_next_block_base_fee( gas_used, gas_limit, diff --git a/crates/execution-e2e/src/lib.rs b/crates/execution-e2e/src/lib.rs index 16ba2b7..ae1c735 100644 --- a/crates/execution-e2e/src/lib.rs +++ b/crates/execution-e2e/src/lib.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] //! Arc E2E Test Framework //! diff --git a/crates/execution-e2e/tests/base_fee.rs b/crates/execution-e2e/tests/base_fee.rs index 102e37d..0f37241 100644 --- a/crates/execution-e2e/tests/base_fee.rs +++ b/crates/execution-e2e/tests/base_fee.rs @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] + use alloy_primitives::{Bytes, U256}; use alloy_rpc_types_engine::PayloadStatusEnum; use arc_execution_config::{gas_fee::decode_base_fee_from_bytes, hardforks::ArcHardfork}; diff --git a/crates/execution-e2e/tests/beneficiary_mismatch.rs b/crates/execution-e2e/tests/beneficiary_mismatch.rs index 5d67952..a9e0ef9 100644 --- a/crates/execution-e2e/tests/beneficiary_mismatch.rs +++ b/crates/execution-e2e/tests/beneficiary_mismatch.rs @@ -54,13 +54,16 @@ async fn submit_with_overridden_beneficiary( /// Ensure `apply_pre_execution_changes` rejects payloads whose beneficiary does /// not match `ProtocolConfig.rewardBeneficiary()`. -/// The LOCAL_DEV genesis has rewardBeneficiary set to 0xa0Ee7A142d267C1f36714E4a8F75612F20a79720. #[tokio::test] async fn test_beneficiary_mismatch_rejected_with_error() -> Result<()> { reth_tracing::init_test_tracing(); + // Set a non-zero expected beneficiary so the mismatch check is active. + let expected_beneficiary = address!("0x65E0a200006D4FF91bD59F9694220dafc49dbBC1"); + let chain_spec = localdev_with_storage_override(expected_beneficiary, None); + let status = submit_with_overridden_beneficiary( - ArcSetup::new(), + ArcSetup::new().with_chain_spec(chain_spec), address!("0xbad0000000000000000000000000000000000000"), "submit_payload should return Ok for beneficiary mismatch", ) diff --git a/crates/execution-e2e/tests/denylist.rs b/crates/execution-e2e/tests/denylist.rs index 3bf296e..56b8dc0 100644 --- a/crates/execution-e2e/tests/denylist.rs +++ b/crates/execution-e2e/tests/denylist.rs @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] + //! E2E tests for the addresses denylist. //! //! Covers: diff --git a/crates/execution-e2e/tests/native_transfer_balance.rs b/crates/execution-e2e/tests/native_transfer_balance.rs index 83fccf3..1f656ee 100644 --- a/crates/execution-e2e/tests/native_transfer_balance.rs +++ b/crates/execution-e2e/tests/native_transfer_balance.rs @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] + //! E2E tests verifying that native value transfers produce correct balance changes. use alloy_primitives::{address, Address, U256}; diff --git a/crates/execution-payload/src/payload.rs b/crates/execution-payload/src/payload.rs index 4975b53..b5b8775 100644 --- a/crates/execution-payload/src/payload.rs +++ b/crates/execution-payload/src/payload.rs @@ -363,11 +363,11 @@ fn dump_tx_data(bytes: &[u8]) -> String { let mut offset = 0usize; let mut lines = 0usize; while offset < bytes.len() && lines < MAX_LINES { - let end = (offset + BYTES_PER_LINE).min(bytes.len()); + let end = (offset.saturating_add(BYTES_PER_LINE)).min(bytes.len()); let slice = &bytes[offset..end]; out.push_str(&format!("{:04x}: {}\n", offset, hex::encode(slice))); offset = end; - lines += 1; + lines = lines.saturating_add(1); } if offset < bytes.len() { out.push_str(&format!( @@ -539,7 +539,7 @@ where let chain_spec = client.chain_spec(); info!(target: "payload_builder", id=%attributes.id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "(arc) building new payload"); - let mut cumulative_gas_used = 0; + let mut cumulative_gas_used = 0u64; let block_gas_limit: u64 = builder.evm_mut().block().gas_limit(); let base_fee = builder.evm_mut().block().basefee(); @@ -556,7 +556,7 @@ where })?; PayloadBuildMetrics::record_stage_pre_execution(stage_start.elapsed()); - let mut block_transactions_rlp_length = 0; + let mut block_transactions_rlp_length = 0usize; let is_osaka = chain_spec.is_osaka_active_at_timestamp(attributes.timestamp); let withdrawals_rlp_length = attributes.withdrawals().length(); @@ -567,16 +567,15 @@ where // Break early if loop time budget exhausted if let Some(limit) = loop_time_limit { if loop_started.elapsed() >= limit { - warn!( - elapsed_ms = loop_started.elapsed().as_millis() as u64, - "(arc) loop time budget reached; sealing early" - ); + #[allow(clippy::cast_possible_truncation)] + let elapsed_ms = loop_started.elapsed().as_millis() as u64; + warn!(elapsed_ms, "(arc) loop time budget reached; sealing early"); break; } } // ensure we still have capacity for this transaction - if cumulative_gas_used + pool_tx.gas_limit() > block_gas_limit { + if cumulative_gas_used.saturating_add(pool_tx.gas_limit()) > block_gas_limit { // we can't fit this transaction into the block, so we need to mark it as invalid // which also removes all dependent transaction from the iterator before we can // continue @@ -600,8 +599,10 @@ where let tx_rlp_len = tx.inner().length(); - let estimated_block_size_with_tx = - block_transactions_rlp_length + tx_rlp_len + withdrawals_rlp_length + 1024; // 1Kb of overhead for the block header + let estimated_block_size_with_tx = block_transactions_rlp_length + .saturating_add(tx_rlp_len) + .saturating_add(withdrawals_rlp_length) + .saturating_add(1024); // 1Kb of overhead for the block header if is_osaka && estimated_block_size_with_tx > MAX_RLP_BLOCK_SIZE { best_txs.mark_invalid( @@ -650,14 +651,18 @@ where } }; - block_transactions_rlp_length += tx_rlp_len; + block_transactions_rlp_length = block_transactions_rlp_length.saturating_add(tx_rlp_len); // update and add to total fees let miner_fee = tx .effective_tip_per_gas(base_fee) .expect("fee is always valid; execution succeeded"); - total_fees += U256::from(miner_fee) * U256::from(gas_used); - cumulative_gas_used += gas_used; + // u128 * u64 fits in U256 (max 192 bits); total_fees is bounded by block gas limit * max fee. + #[allow(clippy::arithmetic_side_effects)] + { + total_fees += U256::from(miner_fee) * U256::from(gas_used); + } + cumulative_gas_used = cumulative_gas_used.saturating_add(gas_used); } PayloadBuildMetrics::record_stage_tx_execution(loop_started.elapsed()); diff --git a/crates/execution-txpool/src/error.rs b/crates/execution-txpool/src/error.rs index 83e9978..0158117 100644 --- a/crates/execution-txpool/src/error.rs +++ b/crates/execution-txpool/src/error.rs @@ -34,7 +34,8 @@ impl PoolTransactionError for ArcTransactionValidatorError { match self { Self::BlocklistedError => true, Self::InvalidTxError => true, - Self::DenylistedAddressError(_) => true, + // Node-local policy — peers can't know our denylist config + Self::DenylistedAddressError(_) => false, } } @@ -66,11 +67,11 @@ mod tests { } #[test] - fn denylisted_address_variant_is_bad_transaction() { + fn denylisted_address_variant_is_not_bad_transaction() { let err = ArcTransactionValidatorError::DenylistedAddressError(Address::ZERO); assert!( - err.is_bad_transaction(), - "DenylistedAddressError must be classified as bad transaction" + !err.is_bad_transaction(), + "DenylistedAddressError must not be classified as bad transaction" ); } } diff --git a/crates/execution-txpool/src/validator.rs b/crates/execution-txpool/src/validator.rs index 13c40e2..80fd70a 100644 --- a/crates/execution-txpool/src/validator.rs +++ b/crates/execution-txpool/src/validator.rs @@ -134,7 +134,11 @@ where if !map.insert(hash, ()) { success = false; } - hashes_count += 1; + // Bounded by the iterator length + #[allow(clippy::arithmetic_side_effects)] + { + hashes_count += 1; + } } map.len() }; diff --git a/crates/execution-validation/src/consensus.rs b/crates/execution-validation/src/consensus.rs index d470e31..e6d890b 100644 --- a/crates/execution-validation/src/consensus.rs +++ b/crates/execution-validation/src/consensus.rs @@ -280,7 +280,7 @@ fn arc_validate_header_timestamp_with_time( local_time: u64, ) -> Result<(), ConsensusError> { // Validate that the header's timestamp is not too far in the future - if header.timestamp() > local_time + ARC_PROPOSER_CLOCK_SKEW_THRESHOLD { + if header.timestamp() > local_time.saturating_add(ARC_PROPOSER_CLOCK_SKEW_THRESHOLD) { return Err(ConsensusError::TimestampIsInFuture { timestamp: header.timestamp(), present_timestamp: local_time, diff --git a/crates/malachite-app/README.md b/crates/malachite-app/README.md index 24bff73..691f2db 100644 --- a/crates/malachite-app/README.md +++ b/crates/malachite-app/README.md @@ -60,6 +60,7 @@ A validator is a node that participates in consensus, proposes and votes on bloc arc-node-consensus start \ --home=~/.arc/consensus \ --moniker=validator-1 \ + --validator \ --eth-socket=/tmp/reth.ipc \ --execution-socket=/tmp/auth.ipc \ --minimal @@ -71,6 +72,7 @@ arc-node-consensus start \ arc-node-consensus start \ --home=~/.arc/consensus \ --moniker=validator-1 \ + --validator \ --p2p.addr=/ip4/172.19.0.5/tcp/27000 \ --p2p.persistent-peers=/ip4/172.19.0.6/tcp/27000,/ip4/172.19.0.7/tcp/27000 \ --metrics=172.19.0.5:29000 \ @@ -86,6 +88,7 @@ arc-node-consensus start \ arc-node-consensus start \ --home=~/.arc/consensus \ --moniker=validator-1 \ + --validator \ --p2p.addr=/ip4/172.19.0.5/tcp/27000 \ --p2p.persistent-peers=/ip4/172.19.0.6/tcp/27000,/ip4/172.19.0.7/tcp/27000 \ --metrics=0.0.0.0:29000 \ @@ -112,10 +115,11 @@ arc-node-consensus start \ --moniker=full-1 \ --eth-socket=/tmp/reth.ipc \ --execution-socket=/tmp/auth.ipc \ - --no-consensus \ --full ``` +To run as a sync-only node that does not subscribe to consensus gossip topics, pass `--no-consensus`. + #### c) Follow mode (RPC sync) Follow mode syncs blocks from trusted RPC endpoints instead of participating in P2P consensus. The node fetches blocks via HTTP, verifies commit certificates, and applies them locally. This is useful for read-only nodes that sync from validators without joining the P2P network. @@ -148,7 +152,8 @@ https://example.com,wss=ws.example.com:1212 - `--p2p.persistent-peers` - Comma-separated list of persistent peer multiaddrs - `--p2p.persistent-peers-only` - Only allow connections to/from persistent peers (default: false). Useful for sentry node setups where a validator should only communicate with known trusted peers. -- `--no-consensus` - Run as a full node that syncs with the network but does not participate in consensus as a validator (no block proposing or voting). +- `--validator` - Run as a validator: load the consensus signing key, sign the validator proof (ADR-006), and advertise a validator identity. Without this flag the node runs as a full node (no signing, ephemeral consensus key). Mutually exclusive with `--no-consensus` and `--follow`. +- `--no-consensus` - Run as a sync-only node that does not subscribe to consensus gossip topics. Mutually exclusive with `--validator`. - `--discovery` - Enable peer discovery (default: false) - `--discovery.num-outbound-peers` - Number of outbound peers (default: 20) - `--discovery.num-inbound-peers` - Number of inbound peers (default: 20) @@ -169,7 +174,7 @@ https://example.com,wss=ws.example.com:1212 - `--runtime.worker-threads ` - Number of worker threads for the multi-threaded runtime (default: number of CPU cores; ignored with single-threaded) - `--private-key ` - Path to private validator key file. Used for P2P identity and (when not using `--signing.remote`) consensus signing. Default: `{home}/config/priv_validator_key.json` - `--db.skip-upgrade` - Skip database schema upgrade on startup -- `--signing.remote` - Use remote signing with specified endpoint URL (if not provided, uses local signing) +- `--signing.remote` - Use remote signing with specified endpoint URL (if not provided, uses local signing). Requires `--validator`. - `--signing.tls-cert-path` - Path to TLS certificate file for remote signing; auto-enables TLS (requires `--signing.remote`) #### Remote Signing @@ -180,6 +185,7 @@ For validator nodes that use a remote signing service instead of local private k arc-node-consensus start \ --home=~/.arc/consensus \ --moniker=validator-1 \ + --validator \ --eth-socket=/tmp/reth.ipc \ --execution-socket=/tmp/auth.ipc \ --minimal \ diff --git a/crates/malachite-app/src/app.rs b/crates/malachite-app/src/app.rs index ce0c200..58b4bb2 100644 --- a/crates/malachite-app/src/app.rs +++ b/crates/malachite-app/src/app.rs @@ -254,7 +254,7 @@ async fn handle_consensus( AppMsg::GetHistoryMinHeight { reply } => { let _guard = state.metrics.start_msg_process_timer("GetHistoryMinHeight"); - get_history_min_height::handle(state, reply).await?; + get_history_min_height::handle(state, engine, reply).await?; } // Request to re-stream a proposal that was previously seen at valid_round or round (if valid_round is Nil). @@ -306,7 +306,7 @@ async fn handle_app_request(req: AppRequest, state: &State, engine: &Engine) -> })?; let info = match result { - Some(certificate) => get_certificate_info(state, engine, certificate).await, + Some(certificate) => get_certificate_info(&state.ctx, engine, certificate).await, None => None, }; @@ -369,31 +369,38 @@ async fn handle_app_request(req: AppRequest, state: &State, engine: &Engine) -> error!("GetHealth: Failed to reply: {e:?}"); } } + + AppRequest::GetSyncState(reply) => { + if let Err(e) = reply.send(state.sync_state) { + error!("GetSyncState: Failed to reply: {e:?}"); + } + } } Ok(()) } async fn get_certificate_info( - state: &State, + ctx: &ArcContext, engine: &Engine, stored: StoredCommitCertificate, ) -> Option { + // The validator set that signed the certificate is the one *before* executing that block, + // since the block itself could contain validator set changes. + let prev_height = stored.certificate.height.as_u64().saturating_sub(1); let validator_set = engine .eth - .get_active_validator_set(stored.certificate.height.as_u64()) + .get_active_validator_set(prev_height) .await .ok()?; let proposer = stored.proposer.unwrap_or_else(|| { - state - .ctx - .select_proposer( - &validator_set, - stored.certificate.height, - stored.certificate.round, - ) - .address + ctx.select_proposer( + &validator_set, + stored.certificate.height, + stored.certificate.round, + ) + .address }); Some(CommitCertificateInfo { @@ -402,3 +409,73 @@ async fn get_certificate_info( proposer, }) } + +#[cfg(test)] +mod tests { + use super::*; + + use arc_consensus_types::{ + Address, BlockHash, CommitCertificate, CommitCertificateType, Height, Round, ValueId, + }; + use arc_eth_engine::engine::{MockEngineAPI, MockEthereumAPI}; + use mockall::predicate::eq; + + fn stored_cert(height: u64) -> StoredCommitCertificate { + StoredCommitCertificate { + certificate: CommitCertificate::new( + Height::new(height), + Round::new(0), + ValueId::new(BlockHash::new([0xAA; 32])), + vec![], + ), + certificate_type: CommitCertificateType::Minimal, + // Set so get_certificate_info skips select_proposer (keeps the test focused + // on the validator set lookup). + proposer: Some(Address::new([0x42; 20])), + } + } + + /// get_certificate_info must fetch the validator set at `certificate.height - 1` — the + /// set that signed the certificate, i.e. the state *before* executing the certified block. + #[tokio::test] + async fn get_certificate_info_queries_validator_set_at_prev_height() { + let cert_height = 42u64; + + let mut mock_eth = MockEthereumAPI::new(); + mock_eth + .expect_get_active_validator_set() + .with(eq(cert_height - 1)) + .once() + .returning(|_| Ok(Default::default())); + + let engine = Engine::new(Box::new(MockEngineAPI::new()), Box::new(mock_eth)); + let ctx = ArcContext::default(); + + let info = get_certificate_info(&ctx, &engine, stored_cert(cert_height)) + .await + .expect("should return Some"); + + assert_eq!(info.certificate.height, Height::new(cert_height)); + } + + /// At genesis (height 0), the saturating subtraction must keep the query at 0 rather + /// than underflowing. + #[tokio::test] + async fn get_certificate_info_handles_genesis_height() { + let mut mock_eth = MockEthereumAPI::new(); + mock_eth + .expect_get_active_validator_set() + .with(eq(0u64)) + .once() + .returning(|_| Ok(Default::default())); + + let engine = Engine::new(Box::new(MockEngineAPI::new()), Box::new(mock_eth)); + let ctx = ArcContext::default(); + + let info = get_certificate_info(&ctx, &engine, stored_cert(0)) + .await + .expect("should return Some"); + + assert_eq!(info.certificate.height, Height::new(0)); + } +} diff --git a/crates/malachite-app/src/config.rs b/crates/malachite-app/src/config.rs index cecd53f..ada5eef 100644 --- a/crates/malachite-app/src/config.rs +++ b/crates/malachite-app/src/config.rs @@ -21,13 +21,15 @@ use std::net::SocketAddr; use arc_consensus_types::rpc_sync::SyncEndpointUrl; use backon::{BackoffBuilder, Retryable}; -use tracing::warn; +use eyre::eyre; +use tracing::{info, warn}; use url::Url; use malachitebft_app_channel::app::consensus::Multiaddr; use arc_consensus_db::DbUpgrade; use arc_consensus_types::Address; +use arc_shared::chain_ids::{LOCALDEV_CHAIN_ID, TESTNET_CHAIN_ID}; use crate::hardcoded_config::GossipSubOverrides; use arc_eth_engine::{engine::Engine, INITIAL_RETRY_DELAY}; @@ -125,6 +127,9 @@ pub struct StartConfig { /// Skip database schema upgrade on startup pub skip_db_upgrade: bool, + /// Run as a validator (load consensus key, sign validator proof) + pub validator: bool, + /// Enable RPC sync mode, a.k.a. follow (fetch blocks via HTTP RPC instead of P2P) pub rpc_sync_enabled: bool, /// RPC endpoints to fetch blocks from (only used in RPC sync mode) @@ -134,11 +139,19 @@ pub struct StartConfig { impl StartConfig { /// Check if RPC sync mode is enabled pub fn is_rpc_sync_mode(&self) -> bool { - if self.rpc_sync_enabled && self.rpc_sync_endpoints.is_empty() { - warn!("RPC sync mode is enabled but no RPC sync endpoints are configured. Falling back to P2P sync mode."); + self.rpc_sync_enabled + } + + /// Populate `rpc_sync_endpoints` with chain-specific defaults when the user + /// enabled `--follow` without explicit `--follow.endpoint` arguments. + pub fn resolve_default_rpc_sync_endpoints(&mut self, chain_id: u64) -> eyre::Result<()> { + if !self.rpc_sync_enabled || !self.rpc_sync_endpoints.is_empty() { + return Ok(()); } - self.rpc_sync_enabled && !self.rpc_sync_endpoints.is_empty() + let url = default_rpc_sync_endpoint(chain_id)?; + self.rpc_sync_endpoints.push(url); + Ok(()) } pub fn engine_config(&'_ self) -> Option> { @@ -179,6 +192,24 @@ impl StartConfig { } } +/// Returns the default RPC sync endpoint for the given chain ID. +fn default_rpc_sync_endpoint(chain_id: u64) -> eyre::Result { + let url = match chain_id { + TESTNET_CHAIN_ID => "https://rpc.quicknode.testnet.arc.network/", + LOCALDEV_CHAIN_ID => "http://localhost:8545", + _ => { + return Err(eyre!( + "No default follow endpoint for chain ID {chain_id}. \ + Use --follow.endpoint to specify one explicitly." + )) + } + }; + + info!("Using default follow endpoint for chain {chain_id}: {url}"); + url.parse() + .map_err(|e| eyre!("Failed to parse default follow endpoint: {e}")) +} + /// Derive a WebSocket URL from an HTTP RPC URL using the reth convention: /// `http(s)://host:port` → `ws(s)://host:(port+1)`. /// @@ -231,4 +262,141 @@ mod tests { let ftp = Url::parse("ftp://localhost:8545").unwrap(); assert!(derive_ws_url(&ftp).is_none()); } + + #[test] + fn default_rpc_sync_endpoint_testnet() { + let endpoint = default_rpc_sync_endpoint(TESTNET_CHAIN_ID).unwrap(); + assert_eq!( + endpoint.http().as_str(), + "https://rpc.quicknode.testnet.arc.network/" + ); + } + + #[test] + fn default_rpc_sync_endpoint_localdev() { + let endpoint = default_rpc_sync_endpoint(LOCALDEV_CHAIN_ID).unwrap(); + assert_eq!(endpoint.http().as_str(), "http://localhost:8545/"); + } + + #[test] + fn default_rpc_sync_endpoint_unsupported_chain() { + let result = default_rpc_sync_endpoint(999); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("No default follow endpoint")); + } + + #[test] + fn resolve_defaults_populates_empty_endpoints() { + let mut config = StartConfig { + persistent_peers: Vec::new(), + persistent_peers_only: false, + gossipsub_overrides: Default::default(), + eth_socket: None, + execution_socket: None, + eth_rpc_endpoint: None, + execution_endpoint: None, + execution_ws_endpoint: None, + execution_jwt: None, + pprof_bind_address: None, + suggested_fee_recipient: None, + skip_db_upgrade: false, + validator: false, + rpc_sync_enabled: true, + rpc_sync_endpoints: Vec::new(), + }; + + config + .resolve_default_rpc_sync_endpoints(TESTNET_CHAIN_ID) + .unwrap(); + assert_eq!(config.rpc_sync_endpoints.len(), 1); + assert_eq!( + config.rpc_sync_endpoints[0].http().as_str(), + "https://rpc.quicknode.testnet.arc.network/" + ); + } + + #[test] + fn resolve_defaults_preserves_explicit_endpoints() { + let explicit: SyncEndpointUrl = "http://my-validator:8545".parse().unwrap(); + let mut config = StartConfig { + persistent_peers: Vec::new(), + persistent_peers_only: false, + gossipsub_overrides: Default::default(), + eth_socket: None, + execution_socket: None, + eth_rpc_endpoint: None, + execution_endpoint: None, + execution_ws_endpoint: None, + execution_jwt: None, + pprof_bind_address: None, + suggested_fee_recipient: None, + skip_db_upgrade: false, + validator: false, + rpc_sync_enabled: true, + rpc_sync_endpoints: vec![explicit.clone()], + }; + + config + .resolve_default_rpc_sync_endpoints(TESTNET_CHAIN_ID) + .unwrap(); + assert_eq!(config.rpc_sync_endpoints.len(), 1); + assert_eq!(config.rpc_sync_endpoints[0], explicit); + } + + #[test] + fn resolve_defaults_noop_when_disabled() { + let mut config = StartConfig { + persistent_peers: Vec::new(), + persistent_peers_only: false, + gossipsub_overrides: Default::default(), + eth_socket: None, + execution_socket: None, + eth_rpc_endpoint: None, + execution_endpoint: None, + execution_ws_endpoint: None, + execution_jwt: None, + pprof_bind_address: None, + suggested_fee_recipient: None, + skip_db_upgrade: false, + validator: false, + rpc_sync_enabled: false, + rpc_sync_endpoints: Vec::new(), + }; + + config + .resolve_default_rpc_sync_endpoints(TESTNET_CHAIN_ID) + .unwrap(); + assert!(config.rpc_sync_endpoints.is_empty()); + } + + #[test] + fn resolve_defaults_errors_on_unsupported_chain() { + let mut config = StartConfig { + persistent_peers: Vec::new(), + persistent_peers_only: false, + gossipsub_overrides: Default::default(), + eth_socket: None, + execution_socket: None, + eth_rpc_endpoint: None, + execution_endpoint: None, + execution_ws_endpoint: None, + execution_jwt: None, + pprof_bind_address: None, + suggested_fee_recipient: None, + skip_db_upgrade: false, + validator: false, + rpc_sync_enabled: true, + rpc_sync_endpoints: Vec::new(), + }; + + let result = config.resolve_default_rpc_sync_endpoints(999); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("No default follow endpoint")); + } } diff --git a/crates/malachite-app/src/handlers/consensus_ready.rs b/crates/malachite-app/src/handlers/consensus_ready.rs index 240a282..5c3655b 100644 --- a/crates/malachite-app/src/handlers/consensus_ready.rs +++ b/crates/malachite-app/src/handlers/consensus_ready.rs @@ -367,10 +367,18 @@ async fn handshake_and_replay( .unwrap_or_default(); if latest_height_el > latest_height_cons { + if latest_height_cons == Height::default() { + return Err(eyre!( + "Handshake: EL has blocks (height {latest_height_el}) but CL has no committed \ + state (height 0). The CL snapshot is missing. \ + Download one with: `arc-node-consensus download`" + )); + } return Err(eyre!( "Handshake: inconsistent state: EL latest height ({latest_height_el}) \ is greater than CL latest committed height ({latest_height_cons}). \ - Something is wrong with CL DB", + This may indicate CL database corruption or a partial snapshot restore. \ + Try re-downloading both snapshots with: `arc-snapshots download`" )); } @@ -463,7 +471,10 @@ async fn handshake_and_replay( /// Defined to be equal to the size of the consensus input buffer, /// which is itself sized to handle all in-flight sync responses. fn max_pending_proposals(config: &ValueSyncConfig) -> usize { - let limit = config.parallel_requests * config.batch_size; + let limit = config + .parallel_requests + .checked_mul(config.batch_size) + .expect("max_pending_proposals overflow"); assert!(limit > 0, "max_pending_proposals must be greater than 0"); limit } @@ -1048,7 +1059,7 @@ mod tests { assert_eq!(next_height, Height::new(1)); } - // Test 5: EL ahead of CL + // Test 5: EL ahead of CL (generic case — both have data but EL is further) #[tokio::test] async fn test_el_ahead_of_cl_error() { let el_height = 10u64; @@ -1082,8 +1093,54 @@ mod tests { let err_msg = result.unwrap_err().to_string(); assert!(err_msg.contains("inconsistent state")); - assert!(err_msg.contains("EL latest height")); - assert!(err_msg.contains("CL latest committed height")); + assert!( + err_msg.contains("arc-snapshots download"), + "should suggest re-downloading snapshots" + ); + } + + // Test 5b: EL has data but CL is at height 0 (missing CL snapshot) + #[tokio::test] + async fn test_el_ahead_of_cl_missing_snapshot() { + let el_height = 10u64; + let latest_block_el = test_execution_block(el_height, 1000); + + let payload_validator = MockPayloadValidator::new(); + let block_finalizer = MockBlockFinalizer::new(); + let engine_api = setup_mock_engine_api_success(); + let ethereum_api = + setup_mock_ethereum_api_with_block(latest_block_el, Height::default().as_u64()); + + // CL has no data — max_height returns None → defaults to Height(0) + let mut certificates_repo = MockCertificatesRepository::new(); + certificates_repo + .expect_max_height() + .return_once(move || Ok(None)); + + let payloads_repo = MockPayloadsRepository::new(); + let metrics = test_metrics(); + + let result = handshake_and_replay( + &payload_validator, + &block_finalizer, + &engine_api, + ðereum_api, + &certificates_repo, + &payloads_repo, + NoopPersistenceMeter, + &metrics, + ) + .await; + + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("CL snapshot is missing"), + "should identify missing CL snapshot, got: {err_msg}" + ); + assert!( + err_msg.contains("arc-node-consensus download"), + "should suggest downloading CL snapshot, got: {err_msg}" + ); } // Test 6: Missing latest block from EL diff --git a/crates/malachite-app/src/handlers/get_decided_values.rs b/crates/malachite-app/src/handlers/get_decided_values.rs index ccbf0fc..8ae4a94 100644 --- a/crates/malachite-app/src/handlers/get_decided_values.rs +++ b/crates/malachite-app/src/handlers/get_decided_values.rs @@ -156,7 +156,13 @@ async fn get_decided_values( // // Moreover, Malachite will perform a very similar over-approximation when checking // the response to GetDecidedValues, so this keeps our behavior consistent. - if total_bytes + raw_bytes_len > max_response_size { + #[allow(clippy::arithmetic_side_effects)] + // Equivalent to `total_bytes + raw_bytes_len > max_response_size`, + // but rearranged so the subtraction cannot overflow (raw_bytes_len <= max_response_size + // is checked first, and max_response_size.0 - raw_bytes_len.0 is then non-negative). + if raw_bytes_len > max_response_size + || total_bytes.as_u64() > max_response_size.as_u64() - raw_bytes_len.as_u64() + { warn!( %height, %max_response_size, %raw_bytes_len, "GetDecidedValues: Reached max total bytes limit for response, stopping here", @@ -165,7 +171,10 @@ async fn get_decided_values( break; } - total_bytes += raw_bytes_len; + #[allow(clippy::arithmetic_side_effects)] // Guarded by the comparison above + { + total_bytes += raw_bytes_len; + } values.push(raw_value); } @@ -205,6 +214,8 @@ async fn get_raw_decided_value( )); }; + // encoded_len returns usize; on 64-bit targets this fits in u64 + #[allow(clippy::cast_possible_truncation)] Ok((raw_value, ByteSize::b(raw_bytes_len as u64))) } @@ -264,9 +275,17 @@ fn get_clamped_request_range( return None; } - let requested_count = end.as_u64() - start.as_u64() + 1; + // start <= end guaranteed by clamp logic above; +1 cannot overflow after clamping + // to real block heights, but we handle it gracefully regardless. + #[allow(clippy::arithmetic_side_effects)] + let Some(requested_count) = (end.as_u64() - start.as_u64()).checked_add(1) else { + warn!("GetDecidedValues: height range count overflow"); + return None; + }; + // batch_size > 0 checked above; fits in u64 on 64-bit targets + #[allow(clippy::cast_possible_truncation)] if requested_count > batch_size as u64 { - end = start.increment_by((batch_size - 1) as u64); + end = start.increment_by(batch_size.saturating_sub(1) as u64); warn!( requested = %requested_count, max = %batch_size, diff --git a/crates/malachite-app/src/handlers/get_history_min_height.rs b/crates/malachite-app/src/handlers/get_history_min_height.rs index 029a3ab..28e6e28 100644 --- a/crates/malachite-app/src/handlers/get_history_min_height.rs +++ b/crates/malachite-app/src/handlers/get_history_min_height.rs @@ -15,9 +15,10 @@ // limitations under the License. use eyre::Context; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; use arc_consensus_types::Height; +use arc_eth_engine::engine::Engine; use malachitebft_app_channel::Reply; use crate::state::State; @@ -25,44 +26,78 @@ use crate::state::State; /// Handles the `GetHistoryMinHeight` message from the consensus engine. /// /// This is called when the consensus engine requests the minimum height of the application's -/// history. The application retrieves the earliest height from its state, and if a target halt -/// height is configured and is less than the latest height, it caps the minimum height to the -/// target halt height. This ensures that the consensus engine does not request history below the -/// configured halt height which typically corresponds to hard fork at the consensus level. -pub async fn handle(state: &State, reply: Reply) -> eyre::Result<()> { - let earliest_height = state +/// history. The application retrieves the earliest height from the consensus store and the +/// earliest available block from the execution layer, taking the maximum of the two as the +/// floor. If a target halt height is configured and is less than the latest height, it further +/// caps the minimum height to the target halt height. This ensures that the consensus engine +/// does not request history at heights where either the CL store or EL no longer has data, +/// and does not go below the configured halt height which typically corresponds to a hard fork +/// at the consensus level. +pub async fn handle(state: &State, engine: &Engine, reply: Reply) -> eyre::Result<()> { + let cl_earliest_height = state .store() .min_height() .await .wrap_err("Failed to get earliest height from the store")? .unwrap_or_default(); - let latest_height = state + let cl_latest_height = state .store() .max_height() .await .wrap_err("Failed to get latest height from the store")? .unwrap_or_default(); + let el_earliest_height = match engine.eth.get_block_by_number("earliest").await { + Ok(Some(block)) => Height::new(block.block_number), + Ok(None) => { + warn!("EL returned no block for 'earliest', falling back to CL-only"); + cl_earliest_height + } + Err(e) => { + warn!("Failed to get EL earliest block, falling back to CL-only: {e:#}"); + cl_earliest_height + } + }; + let halt_height = state.env_config().halt_height; - debug!(min_height = ?earliest_height, max_height = ?latest_height, halt_height = ?halt_height, "GetHistoryMinHeight: min/max heights"); - let min_height = get_history_min_height(earliest_height, latest_height, halt_height); + + debug!( + %cl_earliest_height, + %el_earliest_height, + %cl_latest_height, + halt_height = halt_height.map(|h| h.to_string()).unwrap_or_else(|| "None".to_string()), + "History bounds" + ); + + let min_height = get_history_min_height( + cl_earliest_height, + el_earliest_height, + cl_latest_height, + halt_height, + ); if let Err(e) = reply.send(min_height) { - error!("🔴 GetHistoryMinHeight: Failed to send reply: {e:?}"); + error!("🔴 Failed to send reply: {e:?}"); } Ok(()) } pub fn get_history_min_height( - earliest_height: Height, - latest_height: Height, + cl_earliest_height: Height, + el_earliest_height: Height, + cl_latest_height: Height, target_halt_height: Option, ) -> Height { - match target_halt_height { - Some(halt_height) if halt_height < latest_height => earliest_height.max(halt_height), - _ => earliest_height, + let floor = cl_earliest_height.max(el_earliest_height); + + if let Some(halt_height) = target_halt_height + && halt_height < cl_latest_height + { + floor.max(halt_height) + } else { + floor } } @@ -76,16 +111,63 @@ mod tests { #[test] fn test_get_history_min_height() { - // No halt height configured - assert_eq!(get_history_min_height(h(10), h(100), None), h(10)); + // No halt height configured, CL and EL agree + assert_eq!(get_history_min_height(h(10), h(10), h(100), None), h(10)); // Halt height greater than max decided block height - assert_eq!(get_history_min_height(h(10), h(100), Some(h(150))), h(10)); + assert_eq!( + get_history_min_height(h(10), h(10), h(100), Some(h(150))), + h(10) + ); // Halt height less than max decided block height, but greater than earliest height - assert_eq!(get_history_min_height(h(10), h(100), Some(h(50))), h(50)); + assert_eq!( + get_history_min_height(h(10), h(10), h(100), Some(h(50))), + h(50) + ); // Halt height less than both max decided block height and earliest height - assert_eq!(get_history_min_height(h(60), h(100), Some(h(50))), h(60)); + assert_eq!( + get_history_min_height(h(60), h(60), h(100), Some(h(50))), + h(60) + ); + } + + #[test] + fn test_el_earliest_higher_than_cl() { + // EL has pruned blocks, so its earliest is higher than CL's + assert_eq!(get_history_min_height(h(10), h(50), h(100), None), h(50)); + } + + #[test] + fn test_cl_earliest_higher_than_el() { + // CL earliest is higher than EL earliest + assert_eq!(get_history_min_height(h(50), h(10), h(100), None), h(50)); + } + + #[test] + fn test_el_earliest_with_halt_height() { + // EL earliest (50) > halt height (30), halt height < latest (100) + // floor = max(10, 50) = 50, then max(50, 30) = 50 + assert_eq!( + get_history_min_height(h(10), h(50), h(100), Some(h(30))), + h(50) + ); + + // EL earliest (20) < halt height (50), halt height < latest (100) + // floor = max(10, 20) = 20, then max(20, 50) = 50 + assert_eq!( + get_history_min_height(h(10), h(20), h(100), Some(h(50))), + h(50) + ); + } + + #[test] + fn test_halt_height_equals_latest_height() { + // halt_height == cl_latest_height: halt has not been passed yet, so it is ignored + assert_eq!( + get_history_min_height(h(10), h(10), h(100), Some(h(100))), + h(10) + ); } } diff --git a/crates/malachite-app/src/handlers/received_proposal_part.rs b/crates/malachite-app/src/handlers/received_proposal_part.rs index 2a86399..c7481d6 100644 --- a/crates/malachite-app/src/handlers/received_proposal_part.rs +++ b/crates/malachite-app/src/handlers/received_proposal_part.rs @@ -31,7 +31,9 @@ use arc_signer::ArcSigningProvider; use crate::block::ConsensusBlock; use crate::metrics::AppMetrics; use crate::payload::{validate_consensus_block, EnginePayloadValidator}; -use crate::proposal_parts::{assemble_block_from_parts, validate_proposal_parts}; +use crate::proposal_parts::{ + assemble_block_from_parts, resolve_expected_proposer, validate_proposal_parts, +}; use crate::state::State; use crate::store::Store; use crate::streaming::{InsertResult, PartStreamsMap}; @@ -315,10 +317,9 @@ async fn process_proposal_parts( debug_assert_eq!(parts_height, ctx.current_height); - // Proposal is for the current height, validate its proposer and signature + // Proposal is for the current height, validate its proposer and signature. let expected_proposer = - ctx.proposer_selector - .select_proposer(ctx.current_validator_set, parts_height, parts_round); + resolve_expected_proposer(ctx.proposer_selector, ctx.current_validator_set, &parts); if !validate_proposal_parts(&parts, expected_proposer, ctx.signing_provider).await { return Ok(None); @@ -354,6 +355,8 @@ async fn maybe_store_pending_proposal( max_pending_proposals: usize, parts: ProposalParts, ) -> eyre::Result<()> { + // max_pending_proposals > 0 (asserted at construction); fits in u64 on 64-bit targets + #[allow(clippy::cast_possible_truncation, clippy::arithmetic_side_effects)] let max_future_height = current_height.increment_by(max_pending_proposals as u64 - 1); // Check that proposal is not for a height too far in the future diff --git a/crates/malachite-app/src/handlers/started_round.rs b/crates/malachite-app/src/handlers/started_round.rs index 5eec1ee..ec9ae37 100644 --- a/crates/malachite-app/src/handlers/started_round.rs +++ b/crates/malachite-app/src/handlers/started_round.rs @@ -14,21 +14,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -use eyre::{eyre, Context}; +use eyre::Context; use tracing::{error, info, warn}; use malachitebft_app_channel::app::consensus::Role; use malachitebft_app_channel::app::types::ProposedValue; use malachitebft_app_channel::Reply; -use arc_consensus_types::{Address, ArcContext, Height, ProposalParts, Round, Validator}; +use arc_consensus_types::proposer::ProposerSelector; +use arc_consensus_types::{Address, ArcContext, Height, ProposalParts, Round, ValidatorSet}; use arc_eth_engine::engine::Engine; use arc_signer::ArcSigningProvider; use crate::block::ConsensusBlock; use crate::metrics::AppMetrics; use crate::payload::{validate_consensus_block, EnginePayloadValidator}; -use crate::proposal_parts::{assemble_block_from_parts, validate_proposal_parts}; +use crate::proposal_parts::{ + assemble_block_from_parts, resolve_expected_proposer, validate_proposal_parts, +}; use crate::state::State; use crate::store::Store; use arc_consensus_db::invalid_payloads::InvalidPayload; @@ -95,20 +98,11 @@ async fn on_started_round( state.current_round = round; state.current_proposer = Some(proposer); - let proposer = state - .validator_set() - .get_by_address(&proposer) - .ok_or_else(|| { - eyre!( - "Current proposer not found in validator set for height {height}, \ - cannot process pending proposal parts" - ) - })?; - fetch_and_process_pending_proposals( height, round, - proposer, + state.validator_set(), + &state.ctx.proposer_selector, state.store(), engine, state.signing_provider(), @@ -121,7 +115,8 @@ async fn on_started_round( async fn fetch_and_process_pending_proposals( height: Height, round: Round, - proposer: &Validator, + validator_set: &ValidatorSet, + proposer_selector: &dyn ProposerSelector, store: &Store, engine: &Engine, signing_provider: &ArcSigningProvider, @@ -141,7 +136,8 @@ async fn fetch_and_process_pending_proposals( pending_parts, height, round, - proposer, + validator_set, + proposer_selector, signing_provider, ) .await @@ -159,12 +155,14 @@ async fn fetch_and_process_pending_proposals( /// /// ## Important /// This function assumes that the pending parts are for the current height and round. +#[allow(clippy::too_many_arguments)] async fn process_pending_proposal_parts( store: &Store, pending_parts: Vec, current_height: Height, current_round: Round, - current_proposer: &Validator, + validator_set: &ValidatorSet, + proposer_selector: &dyn ProposerSelector, signing_provider: &ArcSigningProvider, ) -> eyre::Result<()> { for parts in pending_parts { @@ -173,9 +171,9 @@ async fn process_pending_proposal_parts( debug_assert_eq!(height, current_height, "Pending parts height mismatch"); debug_assert_eq!(round, current_round, "Pending parts round mismatch"); - // Validate the proposal parts, ensuring they are from the expected proposer - // for their height/round, and have a valid signature. - if !validate_proposal_parts(&parts, current_proposer, signing_provider).await { + let expected_proposer = resolve_expected_proposer(proposer_selector, validator_set, &parts); + + if !validate_proposal_parts(&parts, expected_proposer, signing_provider).await { continue; } diff --git a/crates/malachite-app/src/lib.rs b/crates/malachite-app/src/lib.rs index 3397747..e4a3068 100644 --- a/crates/malachite-app/src/lib.rs +++ b/crates/malachite-app/src/lib.rs @@ -14,6 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr( + test, + allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation) +)] + mod app; mod block; mod config; @@ -26,7 +31,7 @@ mod proposal_parts; mod state; mod stats; mod streaming; -mod utils; +pub mod utils; mod validator_proof; pub mod hardcoded_config; diff --git a/crates/malachite-app/src/main.rs b/crates/malachite-app/src/main.rs index 8c5d691..ae37336 100644 --- a/crates/malachite-app/src/main.rs +++ b/crates/malachite-app/src/main.rs @@ -64,14 +64,10 @@ fn main() -> Result<()> { // Load command-line arguments and possible configuration file. let args = Args::new(); - // Override logging configuration (if exists) with optional command-line parameters. - let mut logging = config::LoggingConfig::default(); - if let Some(log_level) = args.log_level { - logging.log_level = log_level; - } - if let Some(log_format) = args.log_format { - logging.log_format = log_format; - } + let logging = config::LoggingConfig { + log_level: args.log_level, + log_format: args.log_format, + }; // This is a drop guard responsible for flushing any remaining logs when the program terminates. // It must be assigned to a binding that is not _, as _ will result in the guard being dropped immediately. @@ -234,6 +230,7 @@ fn start(args: &Args, cmd: &StartCmd, logging: config::LoggingConfig) -> Result< pprof_bind_address: Some(cmd.pprof_addr.parse()?), suggested_fee_recipient: cmd.suggested_fee_recipient, skip_db_upgrade: cmd.skip_db_upgrade, + validator: cmd.validator, rpc_sync_enabled: cmd.follow, rpc_sync_endpoints: cmd.follow_endpoints.clone(), }; @@ -286,7 +283,18 @@ fn db_migrate(args: &Args, cmd: &MigrateCmd) -> Result<()> { } if cmd.dry_run { - info!("Dry-run mode: would perform migration but not committing"); + let stats = coordinator + .preview_migrate() + .map_err(|e| eyre!("Dry-run migration scan failed: {e}"))?; + + info!( + tables = stats.tables_migrated, + scanned = stats.records_scanned, + would_upgrade = stats.records_upgraded, + skipped = stats.records_skipped, + duration = ?stats.duration, + "Dry-run mode: migration scan complete (no changes committed)" + ); return Ok(()); } diff --git a/crates/malachite-app/src/metrics/app.rs b/crates/malachite-app/src/metrics/app.rs index 9801079..3ce932d 100644 --- a/crates/malachite-app/src/metrics/app.rs +++ b/crates/malachite-app/src/metrics/app.rs @@ -447,7 +447,11 @@ struct AsLabelValue(T); impl EncodeLabelValue for AsLabelValue
{ fn encode(&self, encoder: &mut LabelValueEncoder) -> Result<(), std::fmt::Error> { - encoder.write_fmt(format_args!("{}", self.0)) + // Preserve legacy uppercase-no-prefix format for Prometheus label continuity + for byte in self.0.into_inner() { + encoder.write_fmt(format_args!("{byte:02X}"))?; + } + Ok(()) } } @@ -528,3 +532,39 @@ impl Drop for MetricsGuard { (self.callback)(&self.inner, self.label, elapsed); } } + +#[cfg(test)] +mod tests { + use super::*; + use prometheus_client::encoding::text::encode; + use prometheus_client::metrics::gauge::Gauge; + use prometheus_client::registry::Registry; + + #[test] + fn test_address_label_preserves_legacy_format() { + let address = Address::new([ + 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, + 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, + ]); + + let mut registry = Registry::default(); + let family = prometheus_client::metrics::family::Family::::default(); + registry.register("test_metric", "test", family.clone()); + + family.get_or_create(&AddressLabel::new(address)).set(1); + + let mut buf = String::new(); + encode(&mut buf, ®istry).unwrap(); + + // Prometheus labels must use legacy uppercase-no-prefix format, quoted + assert!( + buf.contains("\"123456789ABCDEF0112233445566778899AABBCC\""), + "Expected uppercase-no-prefix address in metrics, got: {buf}" + ); + // No 0x prefix anywhere in the output + assert!( + !buf.contains("0x"), + "Metrics should not contain 0x prefix: {buf}" + ); + } +} diff --git a/crates/malachite-app/src/metrics/process.rs b/crates/malachite-app/src/metrics/process.rs index 152b03c..112b553 100644 --- a/crates/malachite-app/src/metrics/process.rs +++ b/crates/malachite-app/src/metrics/process.rs @@ -281,8 +281,12 @@ impl IoMetricsInner { // Update stats metrics if let Ok(stat) = process.stat() { - // CPU time in seconds (user + system) + // CPU time in seconds (user + system). + // Kernel tick counters fit in u64; sum of two won't overflow for any real process. + // Seconds as i64 covers ~292 billion years of CPU time. + #[allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] let cpu_time = (stat.utime + stat.stime) as f64 / procfs::ticks_per_second() as f64; + #[allow(clippy::cast_possible_truncation)] self.process_cpu_seconds_total.set(cpu_time as i64); // Number of threads diff --git a/crates/malachite-app/src/node.rs b/crates/malachite-app/src/node.rs index 08e82f7..5b8bf9f 100644 --- a/crates/malachite-app/src/node.rs +++ b/crates/malachite-app/src/node.rs @@ -37,7 +37,7 @@ use bytesize::ByteSize; use eyre::Context; use rand::rngs::OsRng; use tokio::signal::unix::SignalKind; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; @@ -101,6 +101,8 @@ pub struct Handle { pub store_monitor: JoinHandle<()>, pub tx_event: TxEvent, pub cancel_token: CancellationToken, + /// Fires when the EL IPC watchdog triggered shutdown (as opposed to SIGTERM or normal halt). + el_watchdog_triggered: oneshot::Receiver<()>, /// Kept alive to prevent the app request channel from closing when RPC is disabled. _tx_app_req: mpsc::Sender, } @@ -256,7 +258,12 @@ impl App { } let p2p_identity = self.p2p_identity()?; - let consensus_identity = self.consensus_identity().await?; + + let consensus_identity = if self.start_config.validator { + self.consensus_identity().await? + } else { + self.ephemeral_consensus_identity() + }; Ok(NodeIdentity::new( self.config.moniker.clone(), @@ -303,6 +310,28 @@ impl App { ) } + /// Generate an ephemeral consensus identity for full nodes. + /// + /// Full nodes participate in gossip but do not sign votes or proposals, + /// so they do not need a persistent consensus key. + fn ephemeral_consensus_identity(&self) -> ConsensusIdentity { + let consensus_key = PrivateKey::generate(OsRng); + let local_provider = LocalSigningProvider::new(consensus_key); + let public_key = local_provider.public_key(); + let address = Address::from_public_key(&public_key); + + info!( + %address, + "Using ephemeral consensus identity for full node (no signing will occur)" + ); + + ConsensusIdentity::new( + address, + public_key, + ArcSigningProvider::Local(local_provider), + ) + } + fn p2p_identity(&self) -> eyre::Result { let private_key = self.load_private_key()?; Ok(P2pIdentity::new(&private_key)) @@ -429,18 +458,14 @@ impl App { // Streaming will start when Sync actor receives first StartedHeight from Consensus self.start_rpc_sync_engine(ctx, identity, wal_path).await } else { - // Create validator proof (ADR-006) binding public key to peer ID - // - // TODO: Currently all non-RPC-sync nodes create a validator proof, but this should - // only apply to actual validators. Non-validator full nodes should use - // `NetworkIdentity::new(moniker, keypair, None)` instead of `new_validator`. - // This requires a way to detect if the node is configured as a validator - let network_identity = self - .create_network_identity_with_proof(&identity) - .await - .wrap_err("Failed to create network identity with validator proof")?; + let network_identity = if self.start_config.validator { + self.create_network_identity_with_proof(&identity) + .await + .wrap_err("Failed to create network identity with validator proof")? + } else { + NetworkIdentity::new(identity.moniker.clone(), identity.p2p.keypair.clone(), None) + }; - // Use default engine for consensus mode let (channels, engine_handle) = malachitebft_app_channel::start_engine( ctx, self.config.clone(), @@ -621,6 +646,11 @@ impl App { let consensus_spec = ConsensusSpec::from(chain_id); + // Resolve default follow endpoints when --follow is used without explicit endpoints + self.start_config + .resolve_default_rpc_sync_endpoints(chain_id.as_u64()) + .wrap_err("Failed to resolve default follow endpoints")?; + // Configure Engine API version selection (V4 vs V5) based on the chainspec. // When ARC_GENESIS_FILE_PATH is set, parse the same genesis.json the EL uses // so that patched hardfork timestamps (e.g. nightly-upgrade) are picked up @@ -666,6 +696,27 @@ impl App { let tx_event = channels.events.clone(); let cancel_token = CancellationToken::new(); + // Watchdog: cancel the app task if the EL IPC connection closes unexpectedly. + // run() will detect the signal and return an error, letting the tokio runtime + // unwind naturally (running all Drop implementations) instead of process::exit. + let engine_for_watchdog = engine.clone(); + let (el_watchdog_tx, el_watchdog_rx) = oneshot::channel::<()>(); + tokio::spawn({ + let cancel_token = cancel_token.clone(); + async move { + tokio::select! { + _ = engine_for_watchdog.wait_for_disconnect() => { + tracing::error!("EL IPC connection closed; shutting down"); + // Send before cancel so the oneshot is filled before the app task + // can observe cancellation and exit, eliminating a try_recv race. + el_watchdog_tx.send(()).ok(); + cancel_token.cancel(); + } + _ = cancel_token.cancelled() => {} + } + } + }); + // Start the application task let app_handle = tokio::spawn({ let cancel_token = cancel_token.clone(); @@ -685,6 +736,7 @@ impl App { tx_event, store, cancel_token, + el_watchdog_triggered: el_watchdog_rx, _tx_app_req: tx_app_req, }) } @@ -695,7 +747,7 @@ impl App { } // Start the application - let handles = match self.start().await { + let mut handles = match self.start().await { Ok(handles) => handles, Err(e) => { let startup_error = e.wrap_err("Node failed to start"); @@ -715,6 +767,13 @@ impl App { // Wait for the application to finish let result = handles.app.await?; + // If the EL IPC watchdog triggered the shutdown, propagate an error so the + // caller (main) exits with a non-zero code. The tokio runtime unwinds naturally + // after run() returns, running all Drop implementations — no process::exit needed. + if handles.el_watchdog_triggered.try_recv().is_ok() { + return Err(eyre::eyre!("EL IPC connection closed unexpectedly")); + } + if let Err(e) = &result { // If the application halted due to reaching a configured height, // we stop the consensus engine and wait indefinitely for a termination signal. @@ -820,3 +879,83 @@ fn spawn_pprof_server(bind_address: std::net::SocketAddr) { #[cfg(not(feature = "pprof"))] fn spawn_pprof_server(_bind_address: std::net::SocketAddr) {} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn write_key_file(dir: &std::path::Path) -> (PathBuf, PrivateKey) { + let key = PrivateKey::generate(OsRng); + let path = dir.join("priv_validator_key.json"); + let json = serde_json::to_string(&key).expect("serialize key"); + std::fs::write(&path, json).expect("write key file"); + (path, key) + } + + fn test_app(private_key_file: PathBuf, validator: bool) -> App { + let config = Config { + moniker: "test-node".to_string(), + ..Default::default() + }; + let start_config = StartConfig { + validator, + ..Default::default() + }; + App::new( + config, + PathBuf::from("/tmp"), + private_key_file, + start_config, + ) + } + + #[tokio::test] + async fn full_node_uses_ephemeral_consensus_identity() { + let dir = tempdir().unwrap(); + let (key_path, original_key) = write_key_file(dir.path()); + + let app = test_app(key_path, false); + let identity = app.setup_node_identity().await.unwrap(); + + // P2P identity uses the key from file + let expected_keypair = + Keypair::ed25519_from_bytes(original_key.inner().to_bytes()).unwrap(); + assert_eq!( + identity.p2p.keypair.public().to_peer_id(), + expected_keypair.public().to_peer_id(), + ); + + // Consensus identity is ephemeral (address differs from the file key) + let file_provider = LocalSigningProvider::new(original_key); + let file_address = Address::from_public_key(&file_provider.public_key()); + assert_ne!(identity.consensus.address(), file_address); + } + + #[tokio::test] + async fn validator_loads_consensus_identity_from_key_file() { + let dir = tempdir().unwrap(); + let (key_path, original_key) = write_key_file(dir.path()); + + let app = test_app(key_path, true); + let identity = app.setup_node_identity().await.unwrap(); + + // Both P2P and consensus derive from the same key file + let file_provider = LocalSigningProvider::new(original_key); + let file_address = Address::from_public_key(&file_provider.public_key()); + assert_eq!(identity.consensus.address(), file_address); + } + + #[test] + fn ephemeral_consensus_identity_generates_valid_identity() { + let dir = tempdir().unwrap(); + let (key_path, _) = write_key_file(dir.path()); + + let app = test_app(key_path, false); + let id1 = app.ephemeral_consensus_identity(); + let id2 = app.ephemeral_consensus_identity(); + + // Each call produces a different address + assert_ne!(id1.address(), id2.address()); + } +} diff --git a/crates/malachite-app/src/payload.rs b/crates/malachite-app/src/payload.rs index 65a1052..efd254b 100644 --- a/crates/malachite-app/src/payload.rs +++ b/crates/malachite-app/src/payload.rs @@ -53,9 +53,11 @@ pub async fn generate_payload_with_retry( let timestamp = std::cmp::max(previous_block.timestamp, now); if previous_block.timestamp > now { + // timestamp >= now (since max chose previous_block.timestamp > now) + let skew = timestamp.saturating_sub(now); warn!( timestamp = timestamp, - skew = timestamp - now, + skew = skew, "Clock skew detected: using parent timestamp", ); } @@ -67,13 +69,17 @@ pub async fn generate_payload_with_retry( .await }; - let mut attempt_num = 0; + let mut attempt_num = 0usize; call_once .retry(RETRY_POLICY.build()) .sleep(tokio::time::sleep) // give reth time to breathe .notify(|_e, dur| { - attempt_num += 1; + // Bounded by MAX_RETRIES (5) + #[allow(clippy::arithmetic_side_effects)] + { + attempt_num += 1; + } let attempts_left = MAX_RETRIES.saturating_sub(attempt_num); error!( attempt = attempt_num, diff --git a/crates/malachite-app/src/proposal_parts.rs b/crates/malachite-app/src/proposal_parts.rs index 50420ca..78c921c 100644 --- a/crates/malachite-app/src/proposal_parts.rs +++ b/crates/malachite-app/src/proposal_parts.rs @@ -28,10 +28,11 @@ use malachitebft_app_channel::app::types::core::{Round, Validity}; use malachitebft_app_channel::NetworkMsg; use alloy_rpc_types_engine::ExecutionPayloadV3; +use arc_consensus_types::proposer::ProposerSelector; use arc_consensus_types::signing::{Signature, SigningError, SigningProvider, VerificationResult}; use arc_consensus_types::{ ArcContext, Height, ProposalData, ProposalFin, ProposalInit, ProposalPart, ProposalParts, - Validator, + Validator, ValidatorSet, }; use crate::block::ConsensusBlock; @@ -104,12 +105,18 @@ pub async fn prepare_stream( .await .wrap_err("Failed to construct proposal parts")?; + // +1 for the Fin message; Vec length <= isize::MAX, so +1 cannot overflow usize + #[allow(clippy::arithmetic_side_effects)] let mut msgs = Vec::with_capacity(parts.len() + 1); - let mut sequence = 0; + let mut sequence = 0u64; for part in parts { let msg = StreamMessage::new(stream_id.clone(), sequence, StreamContent::Data(part)); - sequence += 1; + // Bounded by parts.len() which is bounded by MAX_MESSAGES_PER_STREAM + #[allow(clippy::arithmetic_side_effects)] + { + sequence += 1; + } msgs.push(msg); } @@ -238,6 +245,25 @@ pub async fn validate_proposal_parts( } } +/// Resolves the expected proposer for a set of proposal parts. +/// +/// When `pol_round` (proof-of-lock round) is set, the proposal is a re-stream +/// of a locked block. The proposer embedded in the parts is the original proposer +/// from `pol_round`, not the proposer for `parts.round()` (the restream round). +pub fn resolve_expected_proposer<'a>( + proposer_selector: &dyn ProposerSelector, + validator_set: &'a ValidatorSet, + parts: &ProposalParts, +) -> &'a Validator { + let pol_round = parts.init().pol_round; + let proposer_round = if pol_round != Round::Nil { + pol_round + } else { + parts.round() + }; + proposer_selector.select_proposer(validator_set, parts.height(), proposer_round) +} + /// Re-assemble a [`ConsensusBlock`] from its [`ProposalParts`]. pub fn assemble_block_from_parts(parts: &ProposalParts) -> eyre::Result { // Calculate total size and allocate buffer @@ -256,7 +282,7 @@ pub fn assemble_block_from_parts(parts: &ProposalParts) -> eyre::Result eyre::Result (Vec, ValidatorSet) { + let mut rng = rand::thread_rng(); + let keys: Vec = (0..n).map(|_| PrivateKey::generate(&mut rng)).collect(); + let validators: Vec = keys + .iter() + .map(|k| Validator::new(k.public_key(), 1)) + .collect(); + (keys, ValidatorSet::new(validators)) + } + + /// Build minimal ProposalParts with the given init fields and sign with the given key. + async fn make_signed_parts( + height: Height, + round: Round, + pol_round: Round, + proposer_pub: PublicKey, + signing_key: &PrivateKey, + ) -> ProposalParts { + use sha3::Digest; + + let proposer = Address::from_public_key(&proposer_pub); + let init = ProposalInit::new(height, round, pol_round, proposer); + + let mut hasher = sha3::Keccak256::new(); + hasher.update(height.as_u64().to_be_bytes()); + hasher.update(round.as_i64().to_be_bytes()); + let hash = hasher.finalize().to_vec(); + + let provider = LocalSigningProvider::new(signing_key.clone()); + let signature = provider.sign_bytes(&hash).await.unwrap(); + + ProposalParts::new(vec![ + ProposalPart::Init(init), + ProposalPart::Fin(ProposalFin::new(signature)), + ]) + .unwrap() + } + + #[test] + fn resolve_proposer_without_pol_round_uses_parts_round() { + let selector = RoundRobin; + let (_keys, validator_set) = make_validator_set(3); + + let height = Height::new(1); + let round = Round::new(2); + + // Build minimal parts with pol_round = Nil + let init = ProposalInit::new( + height, + round, + Round::Nil, + validator_set.get_by_index(0).unwrap().address, + ); + let fin = ProposalFin::new(arc_consensus_types::signing::Signature::test()); + let parts = + ProposalParts::new(vec![ProposalPart::Init(init), ProposalPart::Fin(fin)]).unwrap(); + + let expected = resolve_expected_proposer(&selector, &validator_set, &parts); + let round_proposer = selector.select_proposer(&validator_set, height, round); + + assert_eq!(expected.address, round_proposer.address); + } + + #[test] + fn resolve_proposer_with_pol_round_uses_original_round() { + let selector = RoundRobin; + let (_keys, validator_set) = make_validator_set(3); + + let height = Height::new(1); + let restream_round = Round::new(2); + let pol_round = Round::new(0); + + let original_proposer = selector.select_proposer(&validator_set, height, pol_round); + let restream_proposer = selector.select_proposer(&validator_set, height, restream_round); + + // Ensure they differ so the test is meaningful + assert_ne!( + original_proposer.address, restream_proposer.address, + "Test requires different proposers for pol_round and restream_round" + ); + + // Build parts as if restreamed: round=2, pol_round=0, proposer=original + let init = ProposalInit::new(height, restream_round, pol_round, original_proposer.address); + let fin = ProposalFin::new(arc_consensus_types::signing::Signature::test()); + let parts = + ProposalParts::new(vec![ProposalPart::Init(init), ProposalPart::Fin(fin)]).unwrap(); + + let expected = resolve_expected_proposer(&selector, &validator_set, &parts); + + // Should resolve to the pol_round proposer, not the restream round proposer + assert_eq!(expected.address, original_proposer.address); + assert_ne!(expected.address, restream_proposer.address); + } + + /// End-to-end: restreamed proposal parts signed by the original proposer + /// pass validation when expected_proposer is resolved via pol_round. + #[tokio::test] + async fn restreamed_parts_pass_validation_with_pol_round_proposer() { + let selector = RoundRobin; + let (keys, validator_set) = make_validator_set(3); + + let height = Height::new(1); + let pol_round = Round::new(0); + let restream_round = Round::new(2); + + let original_proposer = selector.select_proposer(&validator_set, height, pol_round); + + // Find the signing key for the original proposer + let signing_key = keys + .iter() + .find(|k| Address::from_public_key(&k.public_key()) == original_proposer.address) + .unwrap(); + + let parts = make_signed_parts( + height, + restream_round, + pol_round, + signing_key.public_key(), + signing_key, + ) + .await; + + // Resolve via pol_round (the fix) — should match and verify + let expected = resolve_expected_proposer(&selector, &validator_set, &parts); + let provider = LocalSigningProvider::new(signing_key.clone()); + assert!(validate_proposal_parts(&parts, expected, &provider).await); + } + + /// Restreamed parts would fail validation if we used parts_round instead + /// of pol_round to resolve the expected proposer (the old buggy behavior). + #[tokio::test] + async fn restreamed_parts_fail_validation_with_wrong_round_proposer() { + let selector = RoundRobin; + let (keys, validator_set) = make_validator_set(3); + + let height = Height::new(1); + let pol_round = Round::new(0); + let restream_round = Round::new(2); + + let original_proposer = selector.select_proposer(&validator_set, height, pol_round); + let wrong_proposer = selector.select_proposer(&validator_set, height, restream_round); + + assert_ne!(original_proposer.address, wrong_proposer.address); + + let signing_key = keys + .iter() + .find(|k| Address::from_public_key(&k.public_key()) == original_proposer.address) + .unwrap(); + + let parts = make_signed_parts( + height, + restream_round, + pol_round, + signing_key.public_key(), + signing_key, + ) + .await; + + // Using parts_round (the old bug) resolves to the wrong proposer → validation fails + let provider = LocalSigningProvider::new(signing_key.clone()); + assert!(!validate_proposal_parts(&parts, wrong_proposer, &provider).await); + } + + /// assemble_block_from_parts must preserve pol_round as valid_round. + #[tokio::test] + async fn assemble_block_preserves_valid_round_from_pol_round() { + use alloy_rpc_types_engine::ExecutionPayloadV3; + use arbitrary::{Arbitrary, Unstructured}; + + let mut u = Unstructured::new(&[0u8; 512]); + let payload = ExecutionPayloadV3::arbitrary(&mut u).unwrap(); + + let (keys, _) = make_validator_set(1); + let signing_key = &keys[0]; + let proposer = Address::from_public_key(&signing_key.public_key()); + + let pol_round = Round::new(1); + + let block = ConsensusBlock { + height: Height::new(10), + round: Round::new(3), + valid_round: pol_round, + proposer, + validity: Validity::Valid, + execution_payload: payload, + signature: None, + }; + + let provider = LocalSigningProvider::new(signing_key.clone()); + let (raw_parts, _sig) = make_proposal_parts(&provider, &block).await.unwrap(); + let parts = ProposalParts::new(raw_parts).unwrap(); + + // Sanity: Init carries the pol_round we set + assert_eq!(parts.init().pol_round, pol_round); + + let assembled = assemble_block_from_parts(&parts).unwrap(); + assert_eq!( + assembled.valid_round, pol_round, + "assemble_block_from_parts must propagate pol_round as valid_round" + ); + } + + #[tokio::test] + async fn assemble_block_preserves_nil_valid_round() { + use alloy_rpc_types_engine::ExecutionPayloadV3; + use arbitrary::{Arbitrary, Unstructured}; + + let mut u = Unstructured::new(&[0u8; 512]); + let payload = ExecutionPayloadV3::arbitrary(&mut u).unwrap(); + + let (keys, _) = make_validator_set(1); + let signing_key = &keys[0]; + let proposer = Address::from_public_key(&signing_key.public_key()); + + let block = ConsensusBlock { + height: Height::new(5), + round: Round::new(0), + valid_round: Round::Nil, + proposer, + validity: Validity::Valid, + execution_payload: payload, + signature: None, + }; + + let provider = LocalSigningProvider::new(signing_key.clone()); + let (raw_parts, _sig) = make_proposal_parts(&provider, &block).await.unwrap(); + let parts = ProposalParts::new(raw_parts).unwrap(); + + assert_eq!(parts.init().pol_round, Round::Nil); + + let assembled = assemble_block_from_parts(&parts).unwrap(); + assert_eq!(assembled.valid_round, Round::Nil); + } +} diff --git a/crates/malachite-app/src/request.rs b/crates/malachite-app/src/request.rs index 316265d..304eab8 100644 --- a/crates/malachite-app/src/request.rs +++ b/crates/malachite-app/src/request.rs @@ -22,12 +22,15 @@ use tracing::error; use arc_consensus_db::invalid_payloads::StoredInvalidPayloads; use arc_consensus_types::evidence::StoredMisbehaviorEvidence; use arc_consensus_types::{ - Address, ArcContext, BlockHash, CommitCertificateType, Height, Round, ValidatorSet, + signing::PublicKey, Address, ArcContext, BlockHash, CommitCertificateType, Height, Round, + ValidatorSet, }; use malachitebft_core_types::CommitCertificate; use arc_consensus_types::proposal_monitor::ProposalMonitor; +use crate::utils::sync_state::SyncState; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum AppRequestError { Closed, @@ -55,6 +58,7 @@ pub struct Status { pub height: Height, pub round: Round, pub address: Address, + pub public_key: PublicKey, pub proposer: Option
, pub height_start_time: SystemTime, pub prev_payload_hash: Option, @@ -63,6 +67,7 @@ pub struct Status { pub undecided_blocks_count: usize, pub pending_proposal_parts: Vec<(Height, usize)>, pub validator_set: ValidatorSet, + pub sync_state: SyncState, } #[derive(Debug)] @@ -106,6 +111,8 @@ pub enum AppRequest { GetStatus(oneshot::Sender), /// Check if the application is healthy GetHealth(oneshot::Sender<()>), + /// Get the current sync state (lightweight, no DB queries) + GetSyncState(oneshot::Sender), } pub type TxAppReq = mpsc::Sender; @@ -216,6 +223,23 @@ impl AppRequest { Ok(status) } + /// Get the current sync state. Lightweight — reads an in-memory field, no DB queries. + pub async fn get_sync_state( + tx_app_req: &mpsc::Sender, + ) -> Result { + let (tx, rx) = oneshot::channel(); + + tx_app_req + .try_send(Self::GetSyncState(tx)) + .inspect_err(|e| error!("Failed to send GetSyncState request to consensus: {e}"))?; + + let sync_state = rx.await.inspect_err(|e| { + error!("Failed to receive GetSyncState response from consensus: {e}") + })?; + + Ok(sync_state) + } + /// Get node's health. Returns unit type. Used to check whether the node is responsive. /// /// If the request fails, return an error. diff --git a/crates/malachite-app/src/rpc/handlers.rs b/crates/malachite-app/src/rpc/handlers.rs index 7ddd6c1..81b45a5 100644 --- a/crates/malachite-app/src/rpc/handlers.rs +++ b/crates/malachite-app/src/rpc/handlers.rs @@ -31,6 +31,7 @@ use crate::rpc::types::persistent_peer_error_to_response; use crate::rpc::types::request_error_to_response; use crate::rpc::types::RpcVersion; use crate::rpc::version::ApiVersion; +use crate::utils::sync_state::SyncState; use super::types::{ AddOrRemovePersistentPeerBody, GetCertificateParams, RpcAppStatus, RpcCommitCertificate, @@ -171,6 +172,24 @@ pub(crate) async fn get_health( .map_err(request_error_to_response) } +pub(crate) async fn get_ready( + tx_app_req: State, + Extension(version): Extension, +) -> impl IntoResponse { + tracing::debug!(?version, "get_ready called"); + + match AppRequest::get_sync_state(&tx_app_req).await { + Ok(sync_state) => { + let status_code = match sync_state { + SyncState::InSync => StatusCode::OK, + SyncState::CatchingUp => StatusCode::SERVICE_UNAVAILABLE, + }; + (status_code, Json(json!({ "sync_state": sync_state }))).into_response() + } + Err(e) => request_error_to_response(e).into_response(), + } +} + pub(crate) async fn get_version(Extension(version): Extension) -> impl IntoResponse { tracing::debug!(?version, "get_version called"); diff --git a/crates/malachite-app/src/rpc/routes.rs b/crates/malachite-app/src/rpc/routes.rs index b9526f7..a51797f 100644 --- a/crates/malachite-app/src/rpc/routes.rs +++ b/crates/malachite-app/src/rpc/routes.rs @@ -86,6 +86,12 @@ routes![ crate::rpc::handlers::get_health, "Returns empty value. Used to check the node's health" ), + route!( + get, + "/ready", + crate::rpc::handlers::get_ready, + "Readiness probe. Returns 200 when in sync, 503 when catching up" + ), route!( get, "/version", @@ -233,6 +239,7 @@ mod tests { use crate::rpc::types::{ RpcAppStatus, RpcCommitCertificate, RpcConsensusStateDump, RpcNetworkStateDump, }; + use crate::utils::sync_state::SyncState; enum MockValue { Present, @@ -242,6 +249,7 @@ mod tests { enum MockConfig { AppGetHealth, AppGetStatus, + AppGetSyncState(SyncState), AppGetCertificate(MockValue), ConsensusDumpState(MockValue), NetworkDumpState(MockValue), @@ -332,6 +340,12 @@ mod tests { }; let _ = reply_port.send(Self::a_status()); } + MockConfig::AppGetSyncState(state) => { + let Some(AppRequest::GetSyncState(reply)) = msg else { + panic!("Unexpected msg"); + }; + let _ = reply.send(state); + } MockConfig::AppGetCertificate(ret) => { let Some(AppRequest::GetCertificate(None, reply_port)) = msg else { panic!("Unexpected msg"); @@ -456,13 +470,15 @@ mod tests { let pending_proposal_parts = vec![]; let sk = PrivateKey::from([0x33; 32]); - let v = Validator::new(sk.public_key(), 1234); + let public_key = sk.public_key(); + let v = Validator::new(public_key, 1234); let validator_set = ValidatorSet::new(vec![v]); Status { height, round, address, + public_key, proposer, height_start_time, prev_payload_hash, @@ -471,6 +487,7 @@ mod tests { undecided_blocks_count, pending_proposal_parts, validator_set, + sync_state: SyncState::InSync, } } @@ -560,6 +577,7 @@ mod tests { "/persistent-peers", "/persistent-peers", "/proposal-monitor", + "/ready", "/status", "/version", ] @@ -682,6 +700,26 @@ mod tests { assert_eq!(val, json!({"status": "ok"})); } + #[tokio::test] + async fn test_ready_in_sync() { + let (tx_cons_req, tx_app_req, tx_nw_req) = + MockBackend::spawn_new(MockConfig::AppGetSyncState(SyncState::InSync)); + let (status, val) = + build_router_and_request(tx_cons_req, tx_app_req, tx_nw_req, "/ready").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(val, json!({"sync_state": "InSync"})); + } + + #[tokio::test] + async fn test_ready_catching_up() { + let (tx_cons_req, tx_app_req, tx_nw_req) = + MockBackend::spawn_new(MockConfig::AppGetSyncState(SyncState::CatchingUp)); + let (status, val) = + build_router_and_request(tx_cons_req, tx_app_req, tx_nw_req, "/ready").await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(val, json!({"sync_state": "CatchingUp"})); + } + #[tokio::test] async fn test_status_success() { let (tx_cons_req, tx_app_req, tx_nw_req) = MockBackend::spawn_new(MockConfig::AppGetStatus); diff --git a/crates/malachite-app/src/rpc/types.rs b/crates/malachite-app/src/rpc/types.rs index cce4e29..4f5217e 100644 --- a/crates/malachite-app/src/rpc/types.rs +++ b/crates/malachite-app/src/rpc/types.rs @@ -45,6 +45,7 @@ use malachitebft_core_types::{CommitSignature, NilOrVal, VoteType}; use malachitebft_network::PersistentPeerError; use crate::request::{AppRequestError, CommitCertificateInfo, Status, TxAppReq}; +use crate::utils::sync_state::SyncState; use alloy_rpc_types_engine::ExecutionPayloadV3; use arc_consensus_db::invalid_payloads::{InvalidPayload, StoredInvalidPayloads}; @@ -224,6 +225,8 @@ pub(crate) struct RpcNwValidatorInfo { } pub(crate) fn build_network_validator_set(vs: &[(String, u64)]) -> RpcNwValidatorSet { + // Voting powers are small protocol values; sum fits in u64 + #[allow(clippy::arithmetic_side_effects)] let total_voting_power: u64 = vs.iter().map(|(_, vp)| *vp).sum(); let validators = vs .iter() @@ -273,6 +276,7 @@ impl From for RpcProposalMonitorData { data.proposal_receive_time.map(Into::into); // It can be negative + #[allow(clippy::cast_possible_truncation, clippy::arithmetic_side_effects)] let proposal_delay_ms = data.proposal_receive_time.map(|receive| { let start_ms = data .start_time @@ -304,6 +308,7 @@ pub(crate) struct RpcAppStatus { height: u64, round: i64, address: Address, + public_key: String, proposer: Address, height_start_time: DateTime, prev_payload_hash: Option, @@ -312,6 +317,7 @@ pub(crate) struct RpcAppStatus { undecided_blocks_count: usize, pending_proposal_parts: Vec, validator_set: RpcValidatorSet, + sync_state: SyncState, } impl From for RpcAppStatus { @@ -320,6 +326,7 @@ impl From for RpcAppStatus { height: status.height.as_u64(), round: status.round.as_i64(), address: status.address, + public_key: format!("0x{}", hex::encode(status.public_key.as_bytes())), proposer: status.proposer.unwrap_or_else(|| Address::repeat_byte(0)), height_start_time: status.height_start_time.into(), prev_payload_hash: status.prev_payload_hash, @@ -335,6 +342,7 @@ impl From for RpcAppStatus { count, }) .collect(), + sync_state: status.sync_state, } } } @@ -411,6 +419,7 @@ struct RpcValidator { address: Address, voting_power: u64, public_key: PublicKey, + public_key_hex: String, } #[derive(Serialize)] @@ -484,6 +493,7 @@ struct VotesDetails { } impl VotesDetails { + #[allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] fn new(val_set: &ValidatorSet, voted: &[RpcVote]) -> Self { const X_GROUP: usize = 5; @@ -583,6 +593,7 @@ impl From<&Validator> for RpcValidator { RpcValidator { address: v.address, voting_power: v.voting_power, + public_key_hex: format!("0x{}", hex::encode(v.public_key.as_bytes())), public_key: v.public_key, } } diff --git a/crates/malachite-app/src/rpc_sync/network.rs b/crates/malachite-app/src/rpc_sync/network.rs index 29c103c..598650d 100644 --- a/crates/malachite-app/src/rpc_sync/network.rs +++ b/crates/malachite-app/src/rpc_sync/network.rs @@ -337,7 +337,11 @@ fn on_outgoing_request( ); let request_id = OutboundRequestId::new(state.next_request_id); - state.next_request_id += 1; + // Request IDs are sequential and won't reach u64::MAX in practice + #[allow(clippy::arithmetic_side_effects)] + { + state.next_request_id += 1; + } // Send request ID back immediately if let Err(e) = reply.send(request_id.clone()) { @@ -395,7 +399,7 @@ fn on_outgoing_request( output_port.send(event); } Err(e) => { - warn!(%request_id, error = %e, "Fetch failed"); + warn!(%request_id, error = format!("{e:#}"), "Fetch failed"); let event = NetworkEvent::SyncResponse(request_id, peer_id, None); output_port.send(event); } diff --git a/crates/malachite-app/src/rpc_sync/ws_subscription.rs b/crates/malachite-app/src/rpc_sync/ws_subscription.rs index 8917bd0..9cb2085 100644 --- a/crates/malachite-app/src/rpc_sync/ws_subscription.rs +++ b/crates/malachite-app/src/rpc_sync/ws_subscription.rs @@ -109,10 +109,14 @@ async fn run_ws_subscription( update_tx, }) .notify(|error, delay| { - retry_attempts += 1; + // Unbounded retries, but at ~60s backoff, overflow takes ~10^10 years + #[allow(clippy::arithmetic_side_effects)] + { + retry_attempts += 1; + } warn!( - %ws_url, %error, attempt = retry_attempts, + %ws_url, error = format!("{error:#}"), attempt = retry_attempts, "WebSocket connection failed, retrying in {delay:?}" ); }); @@ -124,7 +128,7 @@ async fn run_ws_subscription( } Some((_, Err(error))) => { // The task failed with an error that was not retried (should not happen since we have no max attempts) - error!(%ws_url, %error, "WebSocket subscription failed with unrecoverable error"); + error!(%ws_url, error = format!("{error:#}"), "WebSocket subscription failed with unrecoverable error"); } None => { // The WebSocket subscription task was cancelled diff --git a/crates/malachite-app/src/spec.rs b/crates/malachite-app/src/spec.rs index cf75e24..aca1041 100644 --- a/crates/malachite-app/src/spec.rs +++ b/crates/malachite-app/src/spec.rs @@ -209,6 +209,8 @@ impl ConsensusSpec { pub fn fork_version_at(&self, height: Height, timestamp: BlockTimestamp) -> ForkVersion { match &self.next_fork_condition { None => self.current_fork_version, + // Fork versions increment once per hardfork; u32::MAX is unreachable + #[allow(clippy::arithmetic_side_effects)] Some(cond) if cond.active_at(height, timestamp) => self.current_fork_version + 1, Some(_) => self.current_fork_version, } @@ -264,7 +266,7 @@ pub const LOCALDEV: ConsensusSpec = ConsensusSpec { /// Error returned when the chain ID is not recognized. #[derive(Debug, Error)] -#[error("Unknown chain ID {chain_id}; expected one of MAINNET (5042000), TESTNET (5042002), DEVNET (5042001), LOCALDEV (1337)")] +#[error("Unknown chain ID {chain_id}; expected one of MAINNET (5042), TESTNET (5042002), DEVNET (5042001), LOCALDEV (1337)")] pub struct UnknownChainId { pub chain_id: String, } diff --git a/crates/malachite-app/src/state.rs b/crates/malachite-app/src/state.rs index ad32dcc..7f14a06 100644 --- a/crates/malachite-app/src/state.rs +++ b/crates/malachite-app/src/state.rs @@ -190,7 +190,7 @@ impl State { validator_set: ValidatorSet::default(), // initially empty, will be updated from reth store, stream_nonce: 0, - streams_map: PartStreamsMap::new(initial_height), + streams_map: PartStreamsMap::new(initial_height, 0), config, env_config, stats: Stats::default(), @@ -269,6 +269,7 @@ impl State { /// Sets the current validator set and updates metrics pub fn set_validator_set(&mut self, val_set: ValidatorSet) { self.metrics.update_validator_set(&val_set); + self.streams_map.set_num_validators(val_set.len()); self.validator_set = val_set; } @@ -378,7 +379,12 @@ impl State { /// Defined to be equal to the size of the consensus input buffer, /// which is itself sized to handle all in-flight sync responses. pub fn max_pending_proposals(&self) -> usize { - let limit = self.config.value_sync.parallel_requests * self.config.value_sync.batch_size; + let limit = self + .config + .value_sync + .parallel_requests + .checked_mul(self.config.value_sync.batch_size) + .expect("max_pending_proposals overflow"); assert!(limit > 0, "max_pending_proposals must be greater than 0"); limit } @@ -406,7 +412,10 @@ impl State { height: self.current_height, round: self.current_round, address: self.address(), + public_key: *self.identity.public_key(), proposer: self.current_proposer, + // elapsed() is always <= time since epoch, so this won't underflow + #[allow(clippy::arithmetic_side_effects)] height_start_time: SystemTime::now() - self.stats.height_started().elapsed(), prev_payload_hash: self.previous_block.map(|b| b.block_hash), db_latest_height: self @@ -424,6 +433,7 @@ impl State { undecided_blocks_count, pending_proposal_parts, validator_set: self.validator_set().to_owned(), + sync_state: self.sync_state, }) } @@ -467,7 +477,11 @@ impl State { pub fn next_stream_id(&mut self) -> StreamId { let nonce = self.stream_nonce; - self.stream_nonce += 1; + // Stream nonce is reset each height; cannot realistically reach u32::MAX + #[allow(clippy::arithmetic_side_effects)] + { + self.stream_nonce += 1; + } streaming::new_stream_id(self.current_height, self.current_round, nonce) } diff --git a/crates/malachite-app/src/streaming.rs b/crates/malachite-app/src/streaming.rs index 7e9abd6..c0b19b6 100644 --- a/crates/malachite-app/src/streaming.rs +++ b/crates/malachite-app/src/streaming.rs @@ -20,13 +20,12 @@ use std::mem::size_of; use std::time::{Duration, Instant}; use bytes::Bytes; +use schnellru::{ByLength, LruMap}; +use tracing::{error, warn}; use arc_consensus_types::{Height, ProposalPart, ProposalParts, Round}; - use malachitebft_app_channel::app::streaming::{Sequence, StreamId, StreamMessage}; use malachitebft_app_channel::app::types::PeerId; -use schnellru::{ByLength, LruMap}; -use tracing::{error, warn}; /// Maximum number of messages allowed per stream /// @@ -35,19 +34,15 @@ use tracing::{error, warn}; /// = 128 * 128 KiB = 16 MiB const MAX_MESSAGES_PER_STREAM: usize = 128; -/// Maximum number of concurrent streams allowed per peer +/// Maximum number of concurrent streams allowed per peer. /// -/// Maximum memory per peer (if all streams at full capacity) -/// = MAX_STREAMS_PER_PEER * MAX_MESSAGES_PER_STREAM * CHUNK_SIZE -/// = 64 * 128 * 128 KiB = 1024 MiB -const MAX_STREAMS_PER_PEER: usize = 64; - -/// Maximum total number of concurrent streams across all peers +/// A proposer needs ~2 streams per round (one for the proposal, one potential +/// retry). A value of 4 gives headroom for a couple of in-flight rounds while +/// keeping the per-peer memory footprint bounded: /// -/// Maximum total memory across all peers (if all streams at full capacity) -/// = MAX_TOTAL_STREAMS * MAX_MESSAGES_PER_STREAM * CHUNK_SIZE -/// = 100 * 128 * 128 KiB = 1600 MiB total memory -const MAX_TOTAL_STREAMS: usize = 100; +/// = MAX_STREAMS_PER_PEER * MAX_MESSAGES_PER_STREAM * CHUNK_SIZE +/// = 4 * 128 * 128 KiB = 64 MiB +const MAX_STREAMS_PER_PEER: usize = 4; /// Size of chunks in which proposal data is split for streaming pub(crate) const CHUNK_SIZE: usize = 128 * 1024; @@ -61,6 +56,18 @@ const MAX_EVICTED_STREAMS: usize = 10_000; /// Stream IDs are exactly 16 bytes: u64 height + u32 round + u32 nonce. pub(crate) const STREAM_ID_LEN: usize = size_of::() + size_of::() + size_of::(); +/// Compute the global stream cap for a given validator set size. +/// +/// Sized as `MAX_STREAMS_PER_PEER * num_validators` so every validator can fill +/// its per-peer quota without triggering global eviction. Floored at +/// [`MAX_STREAMS_PER_PEER`] so the cap is non-zero before the validator set is +/// configured at startup (when `num_validators` is still 0). +fn max_total_streams(num_validators: usize) -> usize { + MAX_STREAMS_PER_PEER + .saturating_mul(num_validators) + .max(MAX_STREAMS_PER_PEER) +} + /// Outcome of [`PartStreamsMap::insert`]. #[derive(Debug)] pub enum InsertResult { @@ -214,7 +221,11 @@ impl StreamState { } // Increment message count - self.message_count += 1; + // Bounded by MAX_MESSAGES_PER_STREAM check above + #[allow(clippy::arithmetic_side_effects)] + { + self.message_count += 1; + } // This is the `Init` message. if let Some(init) = msg.content.as_data().and_then(|part| part.as_init()) { @@ -228,7 +239,12 @@ impl StreamState { // If we have received the fin message, we can determine when we will be done. // We are done if we have already received all messages from 0 to fin.sequence, // included. That is to say, if we have received `fin.sequence + 1` messages. - self.expected_messages = msg.sequence as usize + 1; + // Sequence is a u64 protocol field; on 64-bit targets usize == u64. + // The +1 cannot overflow because MAX_MESSAGES_PER_STREAM << u64::MAX. + #[allow(clippy::cast_possible_truncation, clippy::arithmetic_side_effects)] + { + self.expected_messages = msg.sequence as usize + 1; + } } // Add the message to the buffer. @@ -252,39 +268,65 @@ impl StreamState { } } -/// Map to track active proposal part streams from peers +/// Map to track active proposal part streams from peers. /// /// Enforces the following limits: /// - [`MAX_STREAMS_PER_PEER`] streams per peer /// - [`MAX_MESSAGES_PER_STREAM`] messages per stream /// - [`CHUNK_SIZE`] per data chunk -/// - [`MAX_TOTAL_STREAMS`] total concurrent streams +/// - `max_total_streams` total concurrent streams (= `MAX_STREAMS_PER_PEER * num_validators`) /// - Evict streams older than [`MAX_STREAM_AGE`] /// - Immediately evict streams that exceed message or size limits /// - Immediately evict streams from previous heights +/// +/// Worst-case memory at full saturation: +/// = max_total_streams * MAX_MESSAGES_PER_STREAM * CHUNK_SIZE +/// = (MAX_STREAMS_PER_PEER * num_validators) * 128 * 128 KiB +/// = 64 MiB * num_validators pub struct PartStreamsMap { current_height: Height, + /// `MAX_STREAMS_PER_PEER * num_validators`, floored at + /// [`MAX_STREAMS_PER_PEER`] during the pre-validator-set startup window. + max_total_streams: usize, streams: BTreeMap<(PeerId, StreamId), StreamState>, evicted: LruMap<(PeerId, StreamId), ()>, last_eviction: Instant, } impl PartStreamsMap { - /// Create a new empty PartStreamsMap - pub fn new(current_height: Height) -> Self { + /// Create a new empty PartStreamsMap. + /// + /// `num_validators` sets the global stream cap to + /// `MAX_STREAMS_PER_PEER * num_validators`. + pub fn new(current_height: Height, num_validators: usize) -> Self { Self { streams: BTreeMap::new(), last_eviction: Instant::now(), + // MAX_EVICTED_STREAMS (10_000) fits in u32 + #[allow(clippy::cast_possible_truncation)] evicted: LruMap::new(ByLength::new(MAX_EVICTED_STREAMS as u32)), current_height, + max_total_streams: max_total_streams(num_validators), } } - /// Update the current height + /// Update the current height. pub fn set_current_height(&mut self, height: Height) { self.current_height = height; } + /// Update the global stream cap after a validator set change. + /// + /// If the new cap is below the current stream count, evict from the busiest + /// peer until the invariant `streams.len() <= max_total_streams` holds + /// again. + pub fn set_num_validators(&mut self, num_validators: usize) { + self.max_total_streams = max_total_streams(num_validators); + while self.streams.len() > self.max_total_streams { + self.evict_oldest_stream(); + } + } + /// Insert a new proposal part message into the map /// /// ## Parameters @@ -332,7 +374,7 @@ impl PartStreamsMap { } // Check if we've exceeded the total streams limit - if self.streams.len() >= MAX_TOTAL_STREAMS { + if self.streams.len() >= self.max_total_streams { self.evict_oldest_stream(); } } @@ -398,14 +440,6 @@ impl PartStreamsMap { } } - /// Count active streams for a given peer - fn peer_streams_count(&mut self, peer_id: PeerId) -> usize { - self.streams - .keys() - .filter(|(pid, _)| *pid == peer_id) - .count() - } - /// Evict a stream from the map and mark it as evicted. /// The evicted LRU map is bounded by [`MAX_EVICTED_STREAMS`]; the oldest /// entry is automatically dropped when capacity is exceeded. @@ -442,18 +476,56 @@ impl PartStreamsMap { } } - /// Evict the oldest stream to make room for a new one + /// Evict the oldest stream from the peer with the most active streams. + /// + /// Targets the busiest peer so no single peer can push others out via the + /// global cap. fn evict_oldest_stream(&mut self) { - if let Some((oldest_key, _)) = self - .streams - .iter() - .min_by_key(|(_, state)| state.created_at) - { - let ref oldest_key @ (ref peer_id, ref stream_id) = oldest_key.clone(); + let peer_id = self.busiest_peer(); + let Some(peer_id) = peer_id else { return }; - warn!(%peer_id, %stream_id, "Evicting oldest stream due to total streams limit"); - self.evict(oldest_key); + let key = self.oldest_stream_of(peer_id); + let Some(ref key @ (ref peer_id, ref stream_id)) = key else { + return; + }; + + warn!(%peer_id, %stream_id, "Evicting oldest stream from peer with most streams"); + self.evict(key); + } + + /// Return the peer with the most active streams, if any. + /// + /// Uses a [`BTreeMap`] so ties are broken deterministically by [`PeerId`] + /// ordering rather than by hash-map iteration order. + fn busiest_peer(&self) -> Option { + let mut counts: BTreeMap = BTreeMap::new(); + for (pid, _) in self.streams.keys() { + #[allow(clippy::arithmetic_side_effects)] + { + *counts.entry(*pid).or_default() += 1; + } } + counts + .into_iter() + .max_by_key(|&(_, c)| c) + .map(|(pid, _)| pid) + } + + /// Count active streams for a given peer + fn peer_streams_count(&self, peer_id: PeerId) -> usize { + self.streams + .keys() + .filter(|(pid, _)| *pid == peer_id) + .count() + } + + /// Return the key of the oldest stream belonging to `peer_id`, if any. + fn oldest_stream_of(&self, peer_id: PeerId) -> Option<(PeerId, StreamId)> { + self.streams + .iter() + .filter(|((pid, _), _)| *pid == peer_id) + .min_by_key(|(_, state)| state.created_at) + .map(|(key, _)| key.clone()) } } @@ -466,6 +538,11 @@ mod tests { use super::*; + /// Default validator count for tests. Large enough that the global cap + /// (`MAX_STREAMS_PER_PEER * NUM_VALIDATORS`) does not interfere with + /// per-peer or per-stream limit tests. + const NUM_VALIDATORS: usize = 100; + impl PartStreamsMap { /// Test-only wrapper that panics on [`InsertResult::Invalid`]. /// Returns `Some(parts)` on [`InsertResult::Complete`], `None` on [`InsertResult::Pending`]. @@ -555,7 +632,7 @@ mod tests { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let msg = make_message(&stream_1, 0, make_init_part()); let result = map.must_insert(peer_1, msg); @@ -572,7 +649,7 @@ mod tests { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let init_msg = make_message(&stream_1, 0, make_init_part()); let data_msg = make_message(&stream_1, 1, make_data_part(42)); let data_fin_msg = make_message(&stream_1, 2, make_fin_part()); @@ -620,7 +697,7 @@ mod tests { // Test all permutations of message order for perm in parts.iter().permutations(parts.len()) { - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Insert all but the last message for msg in &perm[..3] { @@ -647,7 +724,7 @@ mod tests { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let init_msg = make_message(&stream_1, 0, make_init_part()); let data_msg = make_message(&stream_1, 1, make_data_part(42)); let data_msg_duplicate = make_message(&stream_1, 1, make_data_part(99)); // Same seq @@ -679,7 +756,7 @@ mod tests { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let init_msg = make_message(&stream_1, 0, make_init_part()); // Sequence 1 is missing let fin_msg = make_message(&stream_1, 2, make_fin_part()); @@ -704,7 +781,7 @@ mod tests { let stream_1 = make_stream_id(101); let stream_2 = make_stream_id(202); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Messages for two different streams let s1_init = make_message(&stream_1, 0, make_init_part()); @@ -752,7 +829,7 @@ mod tests { #[test] fn test_per_peer_stream_limit() { let peer_1 = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Create MAX_STREAMS_PER_PEER streams for i in 0..MAX_STREAMS_PER_PEER { @@ -812,7 +889,7 @@ mod tests { fn test_per_stream_message_limit() { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Send Init let init_msg = make_message(&stream_1, 0, make_init_part()); @@ -850,7 +927,7 @@ mod tests { fn test_per_peer_limit_independent_across_peers() { let peer_1 = PeerId::random(); let peer_2 = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Peer 1 creates MAX_STREAMS_PER_PEER streams for i in 0..MAX_STREAMS_PER_PEER { @@ -870,7 +947,7 @@ mod tests { ); } - if MAX_STREAMS_PER_PEER * 2 <= MAX_TOTAL_STREAMS { + if MAX_STREAMS_PER_PEER * 2 <= max_total_streams(NUM_VALIDATORS) { // Both peers should have their streams accepted assert_eq!( map.streams.len(), @@ -881,8 +958,8 @@ mod tests { // Total streams limit should have been enforced assert_eq!( map.streams.len(), - MAX_TOTAL_STREAMS, - "Should have total streams limited to MAX_TOTAL_STREAMS" + max_total_streams(NUM_VALIDATORS), + "Should have total streams limited to max_total_streams(NUM_VALIDATORS)" ); } @@ -908,7 +985,7 @@ mod tests { let stream_2 = make_stream_id(102); let stream_3 = make_stream_id(103); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Create first stream let msg1 = make_message(&stream_1, 0, make_init_part()); @@ -958,64 +1035,196 @@ mod tests { #[test] fn test_total_streams_limit_eviction() { - let mut map = PartStreamsMap::new(Height::new(1)); - let mut peers = Vec::new(); - - // Create MAX_TOTAL_STREAMS streams from different peers - for _ in 0..MAX_TOTAL_STREAMS { - let peer = PeerId::random(); - peers.push(peer); - let stream = make_stream_id(0); + // Use a small validator count so we can fill the global cap easily. + let num_validators = 4; + let cap = max_total_streams(num_validators); + let mut map = PartStreamsMap::new(Height::new(1), num_validators); + + // One peer opens 2 streams (more than any other peer). + let busy_peer = PeerId::random(); + for i in 0..2u8 { + let stream = make_stream_id(i); let msg = make_message(&stream, 0, make_init_part()); - map.must_insert(peer, msg); + map.must_insert(busy_peer, msg); } - assert_eq!( - map.streams.len(), - MAX_TOTAL_STREAMS, - "Should have MAX_TOTAL_STREAMS streams" - ); - - // Get the oldest stream's creation time to verify it gets evicted - let oldest_peer = peers[0]; + // Make its first stream the oldest. let oldest_stream = make_stream_id(0); - let oldest_key = (oldest_peer, oldest_stream.clone()); + let oldest_key = (busy_peer, oldest_stream.clone()); + map.streams.get_mut(&oldest_key).unwrap().created_at = + Instant::now() - Duration::from_secs(100); - // Manually set the first stream to be the oldest - if let Some(state) = map.streams.get_mut(&oldest_key) { - state.created_at = Instant::now() - Duration::from_secs(100); + // Fill the remaining capacity with 1-stream peers. + #[allow(clippy::arithmetic_side_effects)] + for _ in 0..cap - 2 { + let peer = PeerId::random(); + let stream = make_stream_id(0); + let msg = make_message(&stream, 0, make_init_part()); + map.must_insert(peer, msg); } + assert_eq!(map.streams.len(), cap); - // Try to create one more stream from a new peer + // One more stream from a new peer triggers eviction. let new_peer = PeerId::random(); let new_stream = make_stream_id(0); let new_msg = make_message(&new_stream, 0, make_init_part()); map.must_insert(new_peer, new_msg); - // Should still have MAX_TOTAL_STREAMS (oldest evicted, new one added) - assert_eq!( - map.streams.len(), - MAX_TOTAL_STREAMS, - "Should still have MAX_TOTAL_STREAMS streams after eviction" - ); + // Cap preserved: evicted one, added one. + assert_eq!(map.streams.len(), cap); - // The oldest stream should have been evicted + // The busiest peer's oldest stream should have been evicted. assert!( !map.streams.contains_key(&oldest_key), - "Oldest stream should have been evicted" + "Oldest stream from the busiest peer should have been evicted" ); - // The new stream should be present + // The new stream should be present. assert!( map.streams.contains_key(&(new_peer, new_stream)), "New stream should be present" ); } + #[test] + fn test_eviction_targets_busiest_peer_not_globally_oldest() { + // 3 validators, cap = 3 * 4 = 12 streams. + let num_validators = 3; + let cap = max_total_streams(num_validators); + let mut map = PartStreamsMap::new(Height::new(1), num_validators); + + // Peer A opens 1 stream and we make it the globally oldest. + let peer_a = PeerId::random(); + let stream_a = make_stream_id(0); + let msg = make_message(&stream_a, 0, make_init_part()); + map.must_insert(peer_a, msg); + map.streams + .get_mut(&(peer_a, stream_a.clone())) + .unwrap() + .created_at = Instant::now() - Duration::from_secs(200); + + // Peer B opens 4 streams — the per-peer maximum. + let peer_b = PeerId::random(); + for i in 0..MAX_STREAMS_PER_PEER as u8 { + let stream = make_stream_id(i); + let msg = make_message(&stream, 0, make_init_part()); + map.must_insert(peer_b, msg); + } + + // Make peer B's first stream older than all of its other streams + // but still newer than peer A's stream. + let peer_b_oldest = make_stream_id(0); + map.streams + .get_mut(&(peer_b, peer_b_oldest.clone())) + .unwrap() + .created_at = Instant::now() - Duration::from_secs(100); + + // Fill the rest of the cap with single-stream peers. + #[allow(clippy::arithmetic_side_effects)] + let remaining = cap - 1 - MAX_STREAMS_PER_PEER; + for i in 0..remaining { + let peer = PeerId::random(); + let stream = make_stream_id(i as u8); + let msg = make_message(&stream, 0, make_init_part()); + map.must_insert(peer, msg); + } + assert_eq!(map.streams.len(), cap); + + // Trigger eviction by inserting one more stream. + let new_peer = PeerId::random(); + let new_stream = make_stream_id(0); + map.must_insert(new_peer, make_message(&new_stream, 0, make_init_part())); + + assert_eq!(map.streams.len(), cap); + + // Peer A's stream is the globally oldest, but peer B is the busiest. + // The eviction policy targets peer B's oldest stream, not peer A's. + assert!( + map.streams.contains_key(&(peer_a, stream_a)), + "Peer A's stream should be preserved despite being the globally oldest" + ); + assert!( + !map.streams.contains_key(&(peer_b, peer_b_oldest)), + "Peer B's oldest stream should have been evicted (busiest peer)" + ); + } + + #[test] + fn test_zero_validators_uses_per_peer_floor() { + // With zero validators the cap must floor at MAX_STREAMS_PER_PEER so the + // map is usable during the pre-validator-set startup window. + let map = PartStreamsMap::new(Height::new(1), 0); + assert_eq!(map.max_total_streams, MAX_STREAMS_PER_PEER); + assert_eq!(max_total_streams(0), MAX_STREAMS_PER_PEER); + } + + #[test] + fn test_set_num_validators_grows_cap() { + let mut map = PartStreamsMap::new(Height::new(1), 0); + assert_eq!(map.max_total_streams, MAX_STREAMS_PER_PEER); + + map.set_num_validators(25); + assert_eq!(map.max_total_streams, MAX_STREAMS_PER_PEER * 25); + } + + #[test] + fn test_set_num_validators_shrinks_cap_and_trims_over_cap_streams() { + // Start with a generous cap and populate it with streams from many peers. + let num_validators = 10; + let cap = max_total_streams(num_validators); + let mut map = PartStreamsMap::new(Height::new(1), num_validators); + + for i in 0..cap { + let peer = PeerId::random(); + let stream = make_stream_id_u16(i as u16); + let msg = make_message(&stream, 0, make_init_part()); + map.must_insert(peer, msg); + } + assert_eq!(map.streams.len(), cap); + + // Shrinking the validator set reduces the cap; existing streams above + // the new cap must be trimmed in place. + let new_num_validators = 3; + let new_cap = max_total_streams(new_num_validators); + map.set_num_validators(new_num_validators); + + assert_eq!(map.max_total_streams, new_cap); + assert_eq!( + map.streams.len(), + new_cap, + "streams.len() must be clamped to the new cap after shrinkage" + ); + } + + #[test] + fn test_busiest_peer_tie_breaking_is_deterministic() { + // Two peers with equal stream counts. The BTreeMap-based implementation + // must pick the larger PeerId (max_by_key returns the last tied element + // in sorted order) regardless of insertion order. + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let expected = peer_a.max(peer_b); + + for insertion_order in [[peer_a, peer_b], [peer_b, peer_a]] { + let mut map = PartStreamsMap::new(Height::new(1), 10); + for peer in insertion_order { + for i in 0..2u8 { + let stream = make_stream_id(i); + map.must_insert(peer, make_message(&stream, 0, make_init_part())); + } + } + assert_eq!( + map.busiest_peer(), + Some(expected), + "busiest_peer must return the larger PeerId when counts tie" + ); + } + } + #[test] fn test_completed_streams_dont_count_toward_limits() { let peer_1 = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Create and complete MAX_STREAMS_PER_PEER streams for i in 0..MAX_STREAMS_PER_PEER { @@ -1057,7 +1266,7 @@ mod tests { #[test] fn test_evict_old_streams_removes_all_expired() { - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let peer = PeerId::random(); // Create 3 streams, 2 old and 1 new @@ -1100,7 +1309,7 @@ mod tests { #[test] fn test_message_limit_independent_across_streams() { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Fill first stream to capacity let stream_1 = make_stream_id(1); @@ -1122,7 +1331,7 @@ mod tests { fn test_evicted_stream_rejects_new_messages() { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Send Init let init_msg = make_message(&stream_1, 0, make_init_part()); @@ -1160,7 +1369,7 @@ mod tests { fn test_stale_height_streams_evicted() { let peer_1 = PeerId::random(); let stream_1 = make_stream_id(101); - let mut map = PartStreamsMap::new(Height::new(5)); + let mut map = PartStreamsMap::new(Height::new(5), NUM_VALIDATORS); // Send Init message for old height (height 3) let mut init_part = make_init_part(); @@ -1184,7 +1393,7 @@ mod tests { #[test] fn test_evicted_set_retained_across_eviction_cycles() { - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let peer = PeerId::random(); // Create and evict a stream by exceeding message limit @@ -1214,7 +1423,7 @@ mod tests { fn test_oversized_chunk_rejected() { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let init_msg = make_message(&stream, 0, make_init_part()); map.insert(peer, init_msg); @@ -1242,7 +1451,7 @@ mod tests { fn test_normal_chunk_accepted() { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let init_msg = make_message(&stream, 0, make_init_part()); map.insert(peer, init_msg); @@ -1269,7 +1478,7 @@ mod tests { fn test_non_data_parts_not_subject_to_size_check() { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Init and Fin are not Data variants, so they bypass the byte limit let init_msg = make_message(&stream, 0, make_init_part()); @@ -1292,7 +1501,7 @@ mod tests { #[test] fn test_invalid_stream_id_length_rejected() { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Too short let short = StreamId::new(vec![0x01; 8].into()); @@ -1340,7 +1549,7 @@ mod tests { #[test] fn test_evicted_set_capped() { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(100)); + let mut map = PartStreamsMap::new(Height::new(100), NUM_VALIDATORS); // Send many Init messages for stale height with distinct stream IDs. // Each gets immediately evicted because height 1 < current height 100. @@ -1389,6 +1598,89 @@ mod tests { ); } + #[test] + fn test_global_eviction_spares_low_volume_peer_when_others_saturate_cap() { + // Small validator set so the global cap is easy to saturate and we + // can exercise the global-eviction path. + let num_validators = 3; + let cap = max_total_streams(num_validators); + let saturating_peers_num = 2; + let mut map = PartStreamsMap::new(Height::new(1), num_validators); + let mut stream_id: u16 = 0; + + // Low-volume peer starts a single stream. + let low_volume_peer = PeerId::random(); + let low_volume_stream = make_stream_id_u16(stream_id); + let msg_init = make_message(&low_volume_stream, 0, make_init_part()); + let msg_part = make_message(&low_volume_stream, 1, make_data_part(0x42)); + stream_id += 1; + assert!( + map.must_insert(low_volume_peer, msg_init).is_none(), + "Init message on the low-volume stream should be accepted" + ); + + // Make it the globally oldest stream so a naive FIFO policy would + // target it first. + let low_volume_key = (low_volume_peer, low_volume_stream.clone()); + map.streams.get_mut(&low_volume_key).unwrap().created_at = + Instant::now() - Duration::from_secs(100); + + // Saturating peers each try to open MAX_STREAMS_PER_PEER + 5 streams; + // the per-peer limit caps each at MAX_STREAMS_PER_PEER. + let mut saturating_peers = Vec::with_capacity(saturating_peers_num); + for _ in 0..saturating_peers_num { + let peer_id = PeerId::random(); + saturating_peers.push(peer_id); + #[allow(clippy::arithmetic_side_effects)] + for _ in 0..MAX_STREAMS_PER_PEER + 5 { + let stream = make_stream_id_u16(stream_id); + stream_id += 1; + let msg = make_message(&stream, 0, make_init_part()); + map.must_insert(peer_id, msg); + } + + assert_eq!(map.peer_streams_count(peer_id), MAX_STREAMS_PER_PEER); + } + + // Fill the remaining capacity with 1-stream peers so the pool is at + // the global cap and the next insert triggers global eviction. + #[allow(clippy::arithmetic_side_effects)] + let filler_needed = cap - map.streams.len(); + for _ in 0..filler_needed { + let filler_peer = PeerId::random(); + let stream = make_stream_id_u16(stream_id); + stream_id += 1; + let msg = make_message(&stream, 0, make_init_part()); + map.must_insert(filler_peer, msg); + } + assert_eq!(map.streams.len(), cap, "Pool should be at the global cap"); + + // A new peer's stream now triggers global eviction. The busiest-peer + // policy must evict from a saturating peer, never the single-stream + // low-volume peer (even though it owns the globally oldest stream). + let new_peer = PeerId::random(); + let new_stream = make_stream_id_u16(stream_id); + map.must_insert(new_peer, make_message(&new_stream, 0, make_init_part())); + + assert_eq!(map.streams.len(), cap, "Pool should still be at the cap"); + assert!( + map.streams.contains_key(&low_volume_key), + "The low-volume stream should be preserved despite being the globally oldest" + ); + for peer_id in &saturating_peers { + assert!( + map.peer_streams_count(*peer_id) <= MAX_STREAMS_PER_PEER, + "Saturating peers should remain bounded by MAX_STREAMS_PER_PEER" + ); + } + + // Follow-up messages on the low-volume stream should still be accepted. + assert!( + map.must_insert(low_volume_peer, msg_part).is_none(), + "Follow up message on the low-volume stream should be accepted" + ); + } + // --- Property-Based Tests --- proptest! { @@ -1397,7 +1689,7 @@ mod tests { stream_attempts in prop::collection::vec(any::(), 1..50) ) { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Try to create streams using different IDs for stream_id_byte in stream_attempts { @@ -1424,7 +1716,7 @@ mod tests { ) { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Try to send many messages to the same stream for i in 0..message_count { @@ -1446,9 +1738,13 @@ mod tests { #[test] fn prop_total_streams_limit_never_exceeded( peer_count in 1..50usize, - streams_per_peer in 1..20usize + streams_per_peer in 1..=MAX_STREAMS_PER_PEER ) { - let mut map = PartStreamsMap::new(Height::new(1)); + // Use a small validator count so that `peer_count * streams_per_peer` + // can exceed the global cap and actually exercise global eviction. + let num_validators = 10; + let cap = max_total_streams(num_validators); + let mut map = PartStreamsMap::new(Height::new(1), num_validators); let mut peers = Vec::new(); // Generate unique peers @@ -1465,10 +1761,10 @@ mod tests { // Total streams should never exceed the limit prop_assert!( - map.streams.len() <= MAX_TOTAL_STREAMS, + map.streams.len() <= cap, "Total stream count {} exceeded limit {}", map.streams.len(), - MAX_TOTAL_STREAMS + cap ); } } @@ -1476,10 +1772,10 @@ mod tests { #[test] fn prop_stream_age_eviction_works( - stream_count in 1..30usize + stream_count in 1..=MAX_STREAMS_PER_PEER ) { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Create streams for i in 0..stream_count { @@ -1517,7 +1813,7 @@ mod tests { completion_count in 1..20usize ) { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Complete multiple streams for i in 0..completion_count { @@ -1548,7 +1844,7 @@ mod tests { fn prop_limits_independent_across_peers( peer_count in 2..10usize ) { - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let mut peers = Vec::new(); // Generate unique peers @@ -1580,10 +1876,10 @@ mod tests { // Also verify total doesn't exceed global limit prop_assert!( - map.streams.len() <= MAX_TOTAL_STREAMS, + map.streams.len() <= max_total_streams(NUM_VALIDATORS), "Total stream count {} exceeded limit {}", map.streams.len(), - MAX_TOTAL_STREAMS + max_total_streams(NUM_VALIDATORS) ); } @@ -1593,7 +1889,7 @@ mod tests { ) { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Send incomplete stream (no Fin message) for i in 0..message_count { @@ -1624,7 +1920,7 @@ mod tests { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Create messages in order let init = make_message(&stream, 0, make_init_part()); @@ -1668,7 +1964,7 @@ mod tests { ) { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Send initial messages for i in 0..message_count { @@ -1703,7 +1999,7 @@ mod tests { ) { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Choose a missing index in the middle of data parts (not init, not fin_part, not fin) // For total_parts=5: seq 0=init, 1=data, 2=data, 3=fin_part, 4=fin @@ -1745,7 +2041,7 @@ mod tests { #[test] fn prop_multiple_interleaved_streams_independent( - stream_count in 2..8usize, + stream_count in 2..=MAX_STREAMS_PER_PEER, messages_per_stream in 2..10usize, seed in any::() ) { @@ -1753,7 +2049,7 @@ mod tests { use rand::rngs::StdRng; let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let mut rng = StdRng::seed_from_u64(seed); // Create all messages for all streams @@ -1824,7 +2120,7 @@ mod tests { ) { let peer = PeerId::random(); let stream = make_stream_id(1); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); let mut seq = 0u64; @@ -1871,7 +2167,7 @@ mod tests { stream2_message_count in 1..(MAX_MESSAGES_PER_STREAM / 2), ) { let peer = PeerId::random(); - let mut map = PartStreamsMap::new(Height::new(1)); + let mut map = PartStreamsMap::new(Height::new(1), NUM_VALIDATORS); // Fill first stream up to its message count let stream_1 = make_stream_id(1); diff --git a/crates/malachite-app/src/utils/sync_state.rs b/crates/malachite-app/src/utils/sync_state.rs index 5cca4cd..b081305 100644 --- a/crates/malachite-app/src/utils/sync_state.rs +++ b/crates/malachite-app/src/utils/sync_state.rs @@ -19,7 +19,7 @@ use std::time::Duration; use arc_consensus_types::BlockTimestamp; /// Represents the synchronization state of the node with the network. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] pub enum SyncState { /// The node is still catching up with the network and has not yet reached the latest blocks. CatchingUp, diff --git a/crates/malachite-app/tests/cli_db_migrate.rs b/crates/malachite-app/tests/cli_db_migrate.rs index d05d6cd..d40df89 100644 --- a/crates/malachite-app/tests/cli_db_migrate.rs +++ b/crates/malachite-app/tests/cli_db_migrate.rs @@ -157,7 +157,8 @@ fn test_migrate_command_dry_run() { ]) .assert() .success() - .stdout(predicate::str::contains("Dry-run mode")); + .stdout(predicate::str::contains("Dry-run mode")) + .stdout(predicate::str::contains("migration scan complete")); } #[test] diff --git a/crates/malachite-app/tests/rpc_integration.rs b/crates/malachite-app/tests/rpc_integration.rs index 59b238a..498ad5e 100644 --- a/crates/malachite-app/tests/rpc_integration.rs +++ b/crates/malachite-app/tests/rpc_integration.rs @@ -22,8 +22,9 @@ use std::time::Duration; -use arc_consensus_types::{Address, Height, Round, ValidatorSet}; +use arc_consensus_types::{signing::PrivateKey, Address, Height, Round, ValidatorSet}; use arc_node_consensus::request::{AppRequest, Status}; +use arc_node_consensus::utils::sync_state::SyncState; use malachitebft_app_channel::{ConsensusRequest, NetworkRequest}; mod common; @@ -112,10 +113,12 @@ async fn test_status_endpoint() { // Respond to status request server.expect_app_request(|req| match req { AppRequest::GetStatus(reply) => { + let public_key = PrivateKey::from([0x11; 32]).public_key(); let status = Status { height: Height::new(100), round: Round::new(1), address: Address::repeat_byte(1), + public_key, proposer: Some(Address::repeat_byte(2)), height_start_time: std::time::SystemTime::now(), prev_payload_hash: None, @@ -124,6 +127,7 @@ async fn test_status_endpoint() { validator_set: ValidatorSet::default(), undecided_blocks_count: 0, pending_proposal_parts: vec![], + sync_state: SyncState::InSync, }; reply.send(status).ok(); } @@ -145,6 +149,7 @@ async fn test_status_endpoint() { assert_eq!(body.get("height").unwrap(), 100); assert_eq!(body.get("round").unwrap(), 1); assert!(body.get("address").is_some()); + assert!(body.get("public_key").is_some()); assert!(body.get("validator_set").is_some()); } @@ -381,10 +386,12 @@ async fn test_status_response_structure() { // Respond to status request with real data server.expect_app_request(|req| match req { AppRequest::GetStatus(reply) => { + let public_key = PrivateKey::from([0xAB; 32]).public_key(); let status = Status { height: Height::new(12345), round: Round::new(7), address: Address::repeat_byte(0xAB), + public_key, proposer: Some(Address::repeat_byte(0xCD)), height_start_time: std::time::SystemTime::now(), prev_payload_hash: None, @@ -393,6 +400,7 @@ async fn test_status_response_structure() { validator_set: ValidatorSet::default(), undecided_blocks_count: 5, pending_proposal_parts: vec![(Height::new(100), 3)], + sync_state: SyncState::InSync, }; reply.send(status).ok(); } @@ -414,6 +422,7 @@ async fn test_status_response_structure() { assert_eq!(body.get("height").unwrap(), 12345); assert_eq!(body.get("round").unwrap(), 7); assert!(body.get("address").is_some()); + assert!(body.get("public_key").is_some()); assert!(body.get("proposer").is_some()); assert!(body.get("height_start_time").is_some()); assert!(body.get("db_latest_height").is_some()); diff --git a/crates/malachite-cli/src/args.rs b/crates/malachite-cli/src/args.rs index f793433..09eaa5f 100644 --- a/crates/malachite-cli/src/args.rs +++ b/crates/malachite-cli/src/args.rs @@ -53,13 +53,18 @@ pub struct Args { #[arg(long, global = true, value_name = "HOME_DIR")] pub home: Option, - /// Log level (default: `malachite=debug`) - #[arg(long, global = true, value_name = "LOG_LEVEL")] - pub log_level: Option, - - /// Log format (default: `plaintext`) - #[arg(long, global = true, value_name = "LOG_FORMAT")] - pub log_format: Option, + /// Log level + #[arg(long, global = true, value_name = "LOG_LEVEL", default_value = "info")] + pub log_level: LogLevel, + + /// Log format + #[arg( + long, + global = true, + value_name = "LOG_FORMAT", + default_value = "plaintext" + )] + pub log_format: LogFormat, #[command(subcommand)] pub command: Commands, @@ -131,11 +136,6 @@ impl Args { Ok(self.get_home_dir()?.join("store.db")) } - /// get_log_level_or_default returns the log level from the command-line or the default value. - pub fn get_log_level_or_default(&self) -> LogLevel { - self.log_level.unwrap_or_default() - } - /// get_priv_validator_key_file_path returns the private validator key file path based on the /// configuration folder. pub fn get_default_priv_validator_key_file_path(&self) -> Result { diff --git a/crates/malachite-cli/src/cmd/start.rs b/crates/malachite-cli/src/cmd/start.rs index 420113d..b4ca376 100644 --- a/crates/malachite-cli/src/cmd/start.rs +++ b/crates/malachite-cli/src/cmd/start.rs @@ -119,9 +119,20 @@ pub struct StartCmd { /// When set, the node only runs the synchronization protocol /// and does not subscribe to consensus-related gossip topics. /// Use for sync-only full nodes. - #[clap(long)] + #[clap(long, conflicts_with = "validator")] pub no_consensus: bool, + /// Run as a validator node. + /// + /// When set, the node loads its consensus signing key, + /// signs a validator proof (ADR-006), and advertises a + /// validator identity on the P2P network. + /// + /// Without this flag the node runs as a full node: it + /// participates in gossip but does not sign votes or proposals. + #[clap(long, conflicts_with_all = ["no_consensus", "follow"])] + pub validator: bool, + // ===== Value Sync ===== /// Enable value sync #[clap(long, default_value = "true")] @@ -331,8 +342,15 @@ pub struct StartCmd { /// /// If not provided, local signing will be used (default behavior). /// + /// Only meaningful together with `--validator`; for full and sync-only nodes + /// no consensus signing occurs. + /// /// Example: http://validator-signer-proxy:10340 - #[clap(long = "signing.remote", value_name = "ENDPOINT")] + #[clap( + long = "signing.remote", + value_name = "ENDPOINT", + requires = "validator" + )] pub signing_remote: Option, /// Path to TLS certificate for remote signing @@ -351,12 +369,15 @@ pub struct StartCmd { /// In RPC sync mode, the node fetches blocks from trusted RPC endpoints /// instead of participating in consensus. This is useful for running /// read-only nodes that sync from validators. - #[clap(long = "follow", requires = "follow_endpoints")] + /// + /// When no --follow.endpoint is provided, a default endpoint is resolved + /// from the chain ID at startup. + #[clap(long = "follow")] pub follow: bool, /// RPC endpoint to fetch blocks from in RPC sync mode. /// This flag can be repeated. - /// Required when --follow is set. + /// Optional when --follow is set; defaults are resolved from the chain ID. /// /// Format: /// [,=] @@ -394,6 +415,7 @@ impl Default for StartCmd { discovery_num_outbound_peers: 20, discovery_num_inbound_peers: 20, no_consensus: false, + validator: false, value_sync: true, eth_socket: None, execution_socket: None, @@ -428,7 +450,6 @@ impl StartCmd { /// /// This method ensures that users don't specify both IPC and RPC options /// at the same time, as they represent different communication methods. - /// It also ensures that RPC sync mode has at least one endpoint configured. pub fn validate(&self) -> eyre::Result<()> { // Check if both IPC and RPC options are provided let has_ipc_options = self.eth_socket.is_some() || self.execution_socket.is_some(); @@ -445,14 +466,6 @@ impl StartCmd { )); } - // Validate RPC sync/follow configuration - if self.follow && self.follow_endpoints.is_empty() { - return Err(eyre::eyre!( - "Follow mode enabled but no endpoints provided.\n\ - Use --follow.endpoint to specify at least one endpoint." - )); - } - // Validate persistent-peers-only configuration if self.p2p_persistent_peers_only && self.p2p_persistent_peers.is_empty() { return Err(eyre::eyre!( @@ -801,6 +814,7 @@ mod tests { assert_eq!(cmd.discovery_num_inbound_peers, 20); assert!(cmd.value_sync); assert!(!cmd.discovery); + assert!(!cmd.validator); } #[test] @@ -872,6 +886,7 @@ mod tests { "test", "--p2p.addr", "/ip4/127.0.0.1/tcp/27000", + "--validator", "--signing.remote", "http://signer:10340", ]; @@ -888,6 +903,7 @@ mod tests { "test", "--p2p.addr", "/ip4/127.0.0.1/tcp/27000", + "--validator", "--signing.remote", "http://signer:10340", "--signing.tls-cert-path", @@ -901,6 +917,20 @@ mod tests { ); } + #[test] + fn signing_remote_without_validator_fails() { + let args = vec![ + "arc-node-consensus", + "--moniker", + "test", + "--p2p.addr", + "/ip4/127.0.0.1/tcp/27000", + "--signing.remote", + "http://signer:10340", + ]; + assert!(StartCmd::try_parse_from(args).is_err()); + } + #[test] fn signing_tls_cert_path_without_remote_fails() { let args = vec![ @@ -1295,4 +1325,55 @@ mod tests { assert!(cmd.gossipsub_mesh_prioritization); assert_eq!(cmd.gossipsub_load.as_deref(), Some("high")); } + + // Validator flag tests + #[test] + fn validator_defaults_to_false() { + let cmd = StartCmd::default(); + assert!(!cmd.validator); + } + + #[test] + fn validator_flag_parsing() { + let args = vec![ + "arc-node-consensus", + "--moniker", + "test", + "--p2p.addr", + "/ip4/127.0.0.1/tcp/27000", + "--validator", + ]; + let cmd = StartCmd::try_parse_from(args).unwrap(); + assert!(cmd.validator); + } + + #[test] + fn validator_conflicts_with_no_consensus() { + let args = vec![ + "arc-node-consensus", + "--moniker", + "test", + "--p2p.addr", + "/ip4/127.0.0.1/tcp/27000", + "--validator", + "--no-consensus", + ]; + assert!(StartCmd::try_parse_from(args).is_err()); + } + + #[test] + fn validator_conflicts_with_follow() { + let args = vec![ + "arc-node-consensus", + "--moniker", + "test", + "--p2p.addr", + "/ip4/127.0.0.1/tcp/27000", + "--validator", + "--follow", + "--follow.endpoint", + "http://localhost:8545", + ]; + assert!(StartCmd::try_parse_from(args).is_err()); + } } diff --git a/crates/malachite-cli/src/logging.rs b/crates/malachite-cli/src/logging.rs index b015fd7..870dfd1 100644 --- a/crates/malachite-cli/src/logging.rs +++ b/crates/malachite-cli/src/logging.rs @@ -64,7 +64,7 @@ pub fn enable_ansi() -> bool { } /// Common prefixes of the crates targeted by the default log level. -const TARGET_CRATES: &[&str] = &["informalsystems_malachitebft", "arc", "circle"]; +const TARGET_CRATES: &[&str] = &["arc", "circle"]; /// Build a tracing directive setting the log level for the /// crates to the given `log_level`. diff --git a/crates/malachite-cli/src/new.rs b/crates/malachite-cli/src/new.rs index 338f8e6..68e88b6 100644 --- a/crates/malachite-cli/src/new.rs +++ b/crates/malachite-cli/src/new.rs @@ -47,7 +47,11 @@ fn derive_bip39_child_sk_bytes(start_index: usize, count: usize) -> Result Result<()> { show_mesh: true, show_peers: cli.peers || cli.peers_full, show_peers_full: cli.peers_full, + show_duplicates: cli.duplicates, }; print!("{}", format_report(&analysis, &options)); diff --git a/crates/mesh-analysis/src/parse.rs b/crates/mesh-analysis/src/parse.rs index afad06e..973521d 100644 --- a/crates/mesh-analysis/src/parse.rs +++ b/crates/mesh-analysis/src/parse.rs @@ -18,13 +18,14 @@ use std::collections::{BTreeMap, HashSet}; use prometheus_parse::{Sample, Scrape, Value}; -use super::types::{DiscoveredPeer, NodeMetricsData, NodeType, TOPICS}; +use super::types::{DiscoveredPeer, MessageCounts, NodeMetricsData, NodeType, TOPICS}; /// Metric name prefixes we actually use. Lines not matching any of these /// (and not starting with `#`) are dropped before parsing, which avoids /// allocating `Sample` objects for the hundreds of metrics we don't need. const METRIC_PREFIXES: &[&str] = &[ "malachitebft_network_gossipsub_mesh_peer_counts", + "malachitebft_network_gossipsub_topic_msg_recv_counts", "malachitebft_network_peer_mesh_membership", "malachitebft_network_explicit_peers", "malachitebft_network_discovered_peers", @@ -137,6 +138,27 @@ fn extract_explicit_peers(samples: &[Sample]) -> Vec { v } +/// Sum message counts across all topics for duplicate analysis. +fn extract_message_counts(samples: &[Sample]) -> MessageCounts { + let mut unfiltered = 0u64; + let mut filtered = 0u64; + + for sample in samples { + let value = value_as_f64(&sample.value).unwrap_or(0.0) as u64; + if sample.metric == "malachitebft_network_gossipsub_topic_msg_recv_counts_unfiltered_total" + { + unfiltered += value; + } else if sample.metric == "malachitebft_network_gossipsub_topic_msg_recv_counts_total" { + filtered += value; + } + } + + MessageCounts { + unfiltered, + filtered, + } +} + /// Extract per-peer detail from `malachitebft_network_discovered_peers` for this node. fn extract_discovered_peers(samples: &[Sample]) -> BTreeMap { let mut peers = BTreeMap::new(); @@ -248,6 +270,7 @@ pub fn parse_all_metrics(raw_metrics: &[(String, String)]) -> Vec Vec S format_explicit_peering(&mut out, analysis); } + // -- Duplicates ----------------------------------------------------------- + if options.show_duplicates { + let has_any = analysis + .nodes + .iter() + .any(|n| n.message_counts.unfiltered > 0); + if has_any { + let _ = write!(out, "\n{}\n", "=".repeat(80)); + let _ = writeln!(out, "Gossipsub Duplicate Message Rates"); + let _ = write!(out, "{}\n\n", "=".repeat(80)); + format_duplicates(&mut out, &analysis.nodes); + } + } + // -- Peers detail -------------------------------------------------------- if options.show_peers { let _ = write!(out, "\n{}\n", "=".repeat(80)); @@ -457,6 +471,60 @@ fn format_validator_connectivity(out: &mut String, vc: &ValidatorConnectivity) { } } +fn format_duplicates(out: &mut String, nodes: &[NodeMetricsData]) { + let nodes_with_counts: Vec<&NodeMetricsData> = nodes + .iter() + .filter(|n| n.message_counts.unfiltered > 0) + .collect(); + + if nodes_with_counts.is_empty() { + let _ = writeln!(out, " No duplicate metrics available."); + return; + } + + let max_name = nodes_with_counts + .iter() + .map(|n| n.moniker.len()) + .max() + .unwrap_or(4) + .max(4); + + let _ = writeln!( + out, + "{:12} {:>10} {:>10} {:>8}", + "Node", + "Unfiltered", + "Filtered", + "Dups", + "Dup%", + mw = max_name, + ); + let _ = writeln!( + out, + "{:-12} {:->10} {:->10} {:->8}", + "", + "", + "", + "", + "", + mw = max_name, + ); + + for node in &nodes_with_counts { + let mc = &node.message_counts; + let _ = writeln!( + out, + "{:12} {:>10} {:>10} {:>7.1}%", + node.moniker, + mc.unfiltered, + mc.filtered, + mc.duplicates(), + mc.duplicate_pct(), + mw = max_name, + ); + } +} + fn format_explicit_peering(out: &mut String, analysis: &MeshAnalysis) { let validators: Vec<&NodeMetricsData> = analysis .nodes diff --git a/crates/mesh-analysis/src/types.rs b/crates/mesh-analysis/src/types.rs index ac35ec5..a2504ee 100644 --- a/crates/mesh-analysis/src/types.rs +++ b/crates/mesh-analysis/src/types.rs @@ -19,6 +19,29 @@ use std::fmt; pub(super) const TOPICS: [&str; 3] = ["/consensus", "/proposal_parts", "/liveness"]; +/// Aggregated message receive counts across all topics (total and after dedup). +#[derive(Debug, Clone, Default)] +pub struct MessageCounts { + /// Total messages received (including duplicates), summed across all topics. + pub unfiltered: u64, + /// Messages after deduplication, summed across all topics. + pub filtered: u64, +} + +impl MessageCounts { + pub fn duplicates(&self) -> u64 { + self.unfiltered.saturating_sub(self.filtered) + } + + /// Duplicate percentage (0.0–100.0). Returns 0.0 when no messages received. + pub fn duplicate_pct(&self) -> f64 { + if self.unfiltered == 0 { + return 0.0; + } + (self.duplicates() as f64 / self.unfiltered as f64) * 100.0 + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum NodeType { FullNode, @@ -54,6 +77,9 @@ pub struct NodeMetricsData { /// (moniker -> discovered peer info as seen by this node) pub discovered_peers: BTreeMap, + /// Gossipsub message counts (aggregate across all topics) for duplicate analysis. + pub message_counts: MessageCounts, + // connection counts pub connected_peers: i64, pub inbound_peers: i64, @@ -112,4 +138,5 @@ pub struct MeshDisplayOptions { pub show_mesh: bool, pub show_peers: bool, pub show_peers_full: bool, + pub show_duplicates: bool, } diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 12e9550..cb67b9d 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -104,7 +104,7 @@ fn follow_url_for_consensus( let chain_id = builder.config().chain.chain().id(); let url = if follow_url.is_empty() || follow_url == "auto" { - follow::url_for_chain_id(chain_id)? + follow::ws_url_for_chain_id(chain_id)? } else { follow_url.to_string() }; diff --git a/crates/node/tests/common.rs b/crates/node/tests/common.rs index 9576989..230fc46 100644 --- a/crates/node/tests/common.rs +++ b/crates/node/tests/common.rs @@ -14,7 +14,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code, clippy::unwrap_used)] +#![allow( + dead_code, + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] use alloy_genesis::Genesis; use alloy_primitives::KECCAK256_EMPTY; diff --git a/crates/node/tests/native_transfer.rs b/crates/node/tests/native_transfer.rs index 23ed56f..4ed8a1e 100644 --- a/crates/node/tests/native_transfer.rs +++ b/crates/node/tests/native_transfer.rs @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)] + mod common; use common::{setup_evm, setup_evm_with_inspector, WALLET_RECEIVER_INDEX, WALLET_SENDER_INDEX}; diff --git a/crates/precompiles/src/call_from.rs b/crates/precompiles/src/call_from.rs index 556709b..b220370 100644 --- a/crates/precompiles/src/call_from.rs +++ b/crates/precompiles/src/call_from.rs @@ -100,9 +100,15 @@ fn decode_child_call(inputs: &CallInputs) -> Result<(CallInputs, u64), SubcallEr let overhead = abi_decode_gas(calldata.len()); // EIP-150: deduct init_subcall overhead, then forward 63/64ths to child. + // Note: the EVM layer (`ArcEvm::init_subcall`) recalculates child_gas_limit to + // include EIP-2929 account access costs. This calculation serves as a fast-fail + // OOG check for the ABI decode overhead alone. let available = inputs.gas_limit.checked_sub(overhead).ok_or_else(|| { SubcallError::InsufficientGas("gas limit below ABI decode overhead".into()) })?; + // EIP-150: forward 63/64ths of available gas to child. + // available / 64 <= available, so the subtraction cannot underflow. + #[allow(clippy::arithmetic_side_effects)] let child_gas_limit = available - (available / 64); let child_inputs = CallInputs { diff --git a/crates/precompiles/src/helpers.rs b/crates/precompiles/src/helpers.rs index 6a3a7b6..5df5c51 100644 --- a/crates/precompiles/src/helpers.rs +++ b/crates/precompiles/src/helpers.rs @@ -63,7 +63,7 @@ pub const ERR_SELFDESTRUCTED_BALANCE_INCREASED: &str = /// - ABI-encoded string value of the error message. pub fn revert_message_to_bytes(msg: &str) -> Bytes { let encoded = msg.abi_encode(); - let mut result = Vec::with_capacity(4 + encoded.len()); + let mut result = Vec::with_capacity(REVERT_SELECTOR.len().saturating_add(encoded.len())); result.extend_from_slice(&REVERT_SELECTOR); result.extend_from_slice(&encoded); Bytes::from(result) @@ -240,13 +240,21 @@ pub(crate) fn write( if vals.is_original_zero() { 20000 // SSTORE_SET } else { - 5000 - revm_interpreter::gas::COLD_SLOAD_COST // WARM_SSTORE_RESET + // WARM_SSTORE_RESET: 5000 - COLD_SLOAD_COST (2,100) = 2,900 + #[allow(clippy::arithmetic_side_effects)] + { + 5000 - revm_interpreter::gas::COLD_SLOAD_COST + } } } else { revm_interpreter::gas::WARM_STORAGE_READ_COST }; if sstore_result.is_cold { - base_cost + revm_interpreter::gas::COLD_SLOAD_COST + // base_cost <= 20,000; + COLD_SLOAD_COST (2,100) fits in u64 + #[allow(clippy::arithmetic_side_effects)] + { + base_cost + revm_interpreter::gas::COLD_SLOAD_COST + } } else { base_cost } @@ -467,12 +475,12 @@ pub(crate) fn emit_event( ) -> Result<(), PrecompileErrorOrRevert> { let data = event.encode_log_data(); - record_cost_or_out_of_gas( - gas_counter, - LOG_BASE_COST - + LOG_TOPIC_COST * data.topics().len() as u64 - + LOG_DATA_COST * data.data.len() as u64, - )?; + let topic_gas = LOG_TOPIC_COST.saturating_mul(data.topics().len() as u64); + let data_gas = LOG_DATA_COST.saturating_mul(data.data.len() as u64); + let log_gas = LOG_BASE_COST + .saturating_add(topic_gas) + .saturating_add(data_gas); + record_cost_or_out_of_gas(gas_counter, log_gas)?; let log = revm::primitives::Log { address, data }; diff --git a/crates/precompiles/src/pq.rs b/crates/precompiles/src/pq.rs index 0351aca..6f12ab5 100644 --- a/crates/precompiles/src/pq.rs +++ b/crates/precompiles/src/pq.rs @@ -70,6 +70,8 @@ stateful!(run_pq, precompile_input, hardfork_flags; { // Charge base gas, then per-word message gas, then validate inputs record_cost_or_out_of_gas(&mut gas_counter, VERIFY_BASE_GAS)?; + // GAS_PER_MSG_WORD (6) < 32, so the product cannot exceed u64::MAX + #[allow(clippy::arithmetic_side_effects)] let msg_word_gas = (args.msg.len() as u64).div_ceil(32) * GAS_PER_MSG_WORD; record_cost_or_out_of_gas(&mut gas_counter, msg_word_gas)?; diff --git a/crates/precompiles/src/subcall.rs b/crates/precompiles/src/subcall.rs index 9a847e9..edf1fa9 100644 --- a/crates/precompiles/src/subcall.rs +++ b/crates/precompiles/src/subcall.rs @@ -65,6 +65,13 @@ pub trait SubcallPrecompile: Send + Sync { /// Called during `frame_init` when the EVM encounters a call to this precompile's address. /// Returns either a subcall request (with continuation data for `complete_subcall`) or an /// error. + /// + /// # Sender constraint + /// + /// The framework enforces that `child_inputs.caller` must equal `call_inputs.caller` + /// (the address that called this precompile) or `tx.origin` (the signing EOA). + /// Setting `child_inputs.caller` to any other address will cause the framework to + /// revert with "sender spoofing requires tx.origin as sender". fn init_subcall(&self, inputs: &CallInputs) -> Result; /// Finalize the precompile result after the child call completes. diff --git a/crates/quake/README.md b/crates/quake/README.md index 58946a7..202afc4 100644 --- a/crates/quake/README.md +++ b/crates/quake/README.md @@ -1572,6 +1572,31 @@ Send transaction load: ``` Under the hood, this commands calls Spammer from CC. All Spammer options are supported. +Download diagnostic artifacts from the remote testnet: +```bash +# Download all Prometheus metrics (covers the current head block, ~2h by default) +./quake remote download metrics + +# Download metrics for a specific time range +./quake remote download metrics --from 2024-01-15T10:30:00Z --to 2024-01-15T12:00:00Z + +# Download specific metrics only (metric names go after --) +./quake remote download metrics -- reth_db_size_bytes go_goroutines + +# Download node databases (both execution and consensus layers, all nodes) +./quake remote download db + +# Download execution layer only, from specific nodes +./quake remote download db --execution-only -- validator1 validator2 + +# Save to a custom output path +./quake remote download metrics -o /tmp/my-metrics.tar.gz +./quake remote download db -o /tmp/my-db.tar.gz +``` + +Both `download` subcommands output a `.tar.gz` archive named `quake-metrics-.tar.gz` / +`quake-db-.tar.gz` unless overridden with `-o`. + Once finished with your tests, remember to destroy the remote infrastructure! ```bash ./quake remote destroy diff --git a/crates/quake/scenarios/localdev.toml b/crates/quake/scenarios/localdev.toml index 754effa..06777a5 100644 --- a/crates/quake/scenarios/localdev.toml +++ b/crates/quake/scenarios/localdev.toml @@ -2,10 +2,21 @@ # enabled by default. cl.config.execution.persistence_backpressure = true +# All 5 validators use 0x65E0a200006D4FF91bD59F9694220dafc49dbBC1 (LOCALDEV_FEE_RECIPIENT) as +# cl_suggested_fee_recipient so tests can hard-code a single known beneficiary address. [nodes.validator1] +cl_suggested_fee_recipient = "0x65E0a200006D4FF91bD59F9694220dafc49dbBC1" + [nodes.validator2] +cl_suggested_fee_recipient = "0x65E0a200006D4FF91bD59F9694220dafc49dbBC1" + [nodes.validator3] +cl_suggested_fee_recipient = "0x65E0a200006D4FF91bD59F9694220dafc49dbBC1" + [nodes.validator4] +cl_suggested_fee_recipient = "0x65E0a200006D4FF91bD59F9694220dafc49dbBC1" + [nodes.validator5] +cl_suggested_fee_recipient = "0x65E0a200006D4FF91bD59F9694220dafc49dbBC1" [nodes.full1] diff --git a/crates/quake/src/infra/export.rs b/crates/quake/src/infra/export.rs index 194a81a..553ebb5 100644 --- a/crates/quake/src/infra/export.rs +++ b/crates/quake/src/infra/export.rs @@ -25,7 +25,7 @@ use tracing::{info, warn}; use crate::infra::remote::CC_INSTANCE; use crate::infra::terraform::TERRAFORM_STATE_FILENAME; -use crate::infra::{InfraData, INFRA_DATA_FILENAME}; +use crate::infra::{ssm, InfraData, INFRA_DATA_FILENAME}; use crate::testnet::{LAST_MANIFEST_FILENAME, QUAKE_DIR}; pub(crate) const SSH_KEY_FILENAME: &str = "ssh-private-key.pem"; @@ -174,6 +174,7 @@ pub fn import_shared_testnet(path: &Path) -> Result<()> { let testnet_dir = quake_dir.join(bundle.testnet_name.replace('_', "-")); fs::create_dir_all(&testnet_dir) .wrap_err_with(|| format!("Failed to create {testnet_dir:?}"))?; + ssm::ensure_owner_id(&testnet_dir).wrap_err("Failed to create local SSM owner ID")?; // Write the manifest and infra-data let manifest_path = quake_dir.join(format!("{}.toml", bundle.testnet_name)); diff --git a/crates/quake/src/infra/mod.rs b/crates/quake/src/infra/mod.rs index 1acc47f..706994e 100644 --- a/crates/quake/src/infra/mod.rs +++ b/crates/quake/src/infra/mod.rs @@ -38,22 +38,22 @@ pub(crate) const COMPOSE_PROJECT_NAME: &str = "arc_testnet"; pub(crate) const INFRA_DATA_FILENAME: &str = "infra-data.json"; -const PROMETHEUS_PORT: usize = 9090; -const GRAFANA_PORT: usize = 3000; -const BLOCKSCOUT_PORT: usize = 80; +const PROMETHEUS_PORT: u16 = 9090; +const GRAFANA_PORT: u16 = 3000; +const BLOCKSCOUT_PORT: u16 = 80; // Use ports >1024: AWS SSM can only bind to ports <1024 when the client process runs as root. -const PROMETHEUS_SSM_PORT: usize = 19090; -const GRAFANA_SSM_PORT: usize = 13000; -const BLOCKSCOUT_SSM_PORT: usize = 8000; +const PROMETHEUS_SSM_PORT: u16 = 19090; +const GRAFANA_SSM_PORT: u16 = 13000; +const BLOCKSCOUT_SSM_PORT: u16 = 8000; // RPC Proxy port on CC for routing RPC requests to nodes -pub(crate) const RPC_PROXY_PORT: usize = 8080; -pub(crate) const RPC_PROXY_SSM_PORT: usize = 18080; +pub(crate) const RPC_PROXY_PORT: u16 = 8080; +pub(crate) const RPC_PROXY_SSM_PORT: u16 = 18080; // Pprof Proxy port on CC for routing pprof requests to nodes -const PPROF_PROXY_PORT: usize = 6060; -pub(crate) const PPROF_PROXY_SSM_PORT: usize = 16060; +const PPROF_PROXY_PORT: u16 = 6060; +pub(crate) const PPROF_PROXY_SSM_PORT: u16 = 16060; #[derive(Debug, Default, Copy, Clone, ValueEnum)] pub(crate) enum BuildProfile { @@ -258,7 +258,9 @@ impl InfraData { } } - pub fn monitoring_ports(&self) -> (usize, usize, usize) { + /// Local testnets expose these services directly. Remote testnets expose + /// them through the Control Center SSM forwards instead. + pub fn monitoring_ports(&self) -> (u16, u16, u16) { match self.infra_type { InfraType::Local => (PROMETHEUS_PORT, GRAFANA_PORT, BLOCKSCOUT_PORT), InfraType::Remote => (PROMETHEUS_SSM_PORT, GRAFANA_SSM_PORT, BLOCKSCOUT_SSM_PORT), @@ -272,7 +274,7 @@ pub(crate) struct NodeInfraData { pub public_ip: IpAddress, pub instance_id: Option, // Mapping of remote to local ports for SSM tunnels (remote mode only). - pub ssm_tunnel_ports: Option>, + pub ssm_tunnel_ports: Option>, /// Map of subnet name to private IP for that subnet. /// Bridge nodes have multiple entries (one per ENI/network). /// Guaranteed non-empty by the custom deserializer. diff --git a/crates/quake/src/infra/remote.rs b/crates/quake/src/infra/remote.rs index 81b1307..be21901 100644 --- a/crates/quake/src/infra/remote.rs +++ b/crates/quake/src/infra/remote.rs @@ -528,6 +528,173 @@ impl RemoteInfra { ) .wrap_err("Failed to remove monitoring data on CC") } + + fn abs_local_path(&self, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + self.root_dir.join(path) + } + } + + /// Copy a file from the CC home directory to a local path. + fn scp_from_cc(&self, remote_source: &str, local_dest: &Path) -> Result<()> { + shell::scp_from( + &self.instance_id(CC_INSTANCE)?, + USER_NAME, + &self.private_key_path(), + &self.root_dir, + remote_source, + local_dest, + ) + } + + /// Download Prometheus metrics from CC via the query_range REST API. + /// + /// Runs `download-metrics.sh` on CC, then SCPs the resulting archive to `local_dest`. + /// `metric_names` filters which metrics are fetched; empty means all. + /// `from`/`to` are Unix timestamps; when omitted the script defaults to epoch→now. + pub fn download_metrics( + &self, + metric_names: &[String], + from: Option, + to: Option, + step: Option<&str>, + local_dest: &Path, + ) -> Result<()> { + let mut cmd = String::from("./download-metrics.sh"); + if let Some(start) = from { + cmd.push_str(&format!(" -s {start}")); + } + if let Some(end) = to { + cmd.push_str(&format!(" -e {end}")); + } + if let Some(s) = step { + cmd.push_str(&format!(" -t {s}")); + } + for name in metric_names { + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':') + { + return Err(color_eyre::eyre::eyre!( + "Invalid metric name '{name}': must match [a-zA-Z0-9_:]+" + )); + } + cmd.push_str(&format!(" {name}")); + } + + info!("📊 Querying Prometheus metrics on CC..."); + let output = self + .ssh_cc_with_output(&cmd) + .wrap_err("Failed to collect metrics on CC")?; + let last_line = output.lines().last().unwrap_or_default().trim(); + let result: serde_json::Value = serde_json::from_str(last_line) + .wrap_err("Failed to parse download-metrics.sh output")?; + let archive = result["archive"] + .as_str() + .ok_or_else(|| eyre!("missing 'archive' field in download-metrics.sh output"))?; + if archive.is_empty() { + let errors = result["errors"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|e| e.as_str()) + .collect::>() + .join("; ") + }) + .unwrap_or_default(); + return Err(eyre!("download-metrics.sh failed: {errors}")); + } + if let Some(errors) = result["errors"].as_array() { + for err in errors { + warn!("metric query failed: {}", err.as_str().unwrap_or("unknown")); + } + } + + let local_dest_abs = self.abs_local_path(local_dest); + info!("⬇️ Downloading metrics archive..."); + self.scp_from_cc(archive, &local_dest_abs) + .wrap_err("Failed to download metrics archive")?; + + if let Err(err) = self.ssh_cc_with_output(&format!("rm ~/{archive}")) { + warn!("⚠️ Failed to clean up temp archive on CC: {err:#}"); + } + + info!(path=%local_dest_abs.display(), "✅ Metrics downloaded"); + Ok(()) + } + + /// Download node databases from one or more nodes via CC. + /// + /// Runs `download-db.sh` on CC, which archives each node's data in parallel and + /// bundles them into a single archive. Empty `nodes` slice means all nodes. + pub fn download_node_db( + &self, + nodes: &[NodeName], + execution_only: bool, + consensus_only: bool, + local_dest: &Path, + ) -> Result<()> { + let target_nodes: Vec = if nodes.is_empty() { + self.infra_data + .node_names() + .into_iter() + .map(|s| s.to_owned()) + .collect() + } else { + nodes.to_vec() + }; + + let node_ips: Vec = target_nodes + .iter() + .map(|n| self.node_private_ip(n).cloned()) + .collect::>()?; + + let mut cmd = String::from("./download-db.sh"); + if execution_only { + cmd.push_str(" -x"); + } else if consensus_only { + cmd.push_str(" -c"); + } + for ip in &node_ips { + cmd.push_str(&format!(" {ip}")); + } + + info!( + "📦 Archiving node data from {} node(s)...", + target_nodes.len() + ); + let output = self + .ssh_cc_with_output(&cmd) + .wrap_err("Failed to archive node data")?; + let last_line = output.lines().last().unwrap_or_default().trim(); + let result: serde_json::Value = + serde_json::from_str(last_line).wrap_err("Failed to parse download-db.sh output")?; + let archive = result["archive"] + .as_str() + .ok_or_else(|| eyre!("missing 'archive' field in download-db.sh output"))?; + if let Some(errors) = result["errors"].as_array() { + for err in errors { + warn!( + "node db download error: {}", + err.as_str().unwrap_or("unknown") + ); + } + } + + let local_dest_abs = self.abs_local_path(local_dest); + info!("⬇️ Downloading db archive..."); + self.scp_from_cc(archive, &local_dest_abs) + .wrap_err("Failed to download db archive")?; + + if let Err(err) = self.ssh_cc_with_output(&format!("rm ~/{archive}")) { + warn!("⚠️ Failed to clean up temp archive on CC: {err:#}"); + } + + info!(path=%local_dest_abs.display(), "✅ Database downloaded"); + Ok(()) + } } impl InfraProvider for RemoteInfra { diff --git a/crates/quake/src/infra/ssm.rs b/crates/quake/src/infra/ssm.rs index 560dab5..5e6b9ae 100644 --- a/crates/quake/src/infra/ssm.rs +++ b/crates/quake/src/infra/ssm.rs @@ -14,10 +14,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use color_eyre::eyre::{eyre, Context, Ok, Result}; -use std::collections::HashMap; +use std::collections::HashSet; +use std::fmt; +use std::fs; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4, TcpStream}; use std::path::Path; +use std::sync::Arc; use std::time::Duration; + +use color_eyre::eyre::{eyre, Context, Result}; +use rand::random; use tokio::time::Instant; use tracing::{debug, info, warn}; @@ -25,32 +31,84 @@ use crate::infra::{remote, NodeInfraData}; use crate::shell; use crate::util::in_parallel; -/// Manage SSM sessions to forward local ports to remote ports in the Control -/// Center server. +/// Local file that stores the SSM owner ID for one imported remote testnet. +pub(crate) const OWNER_ID_FILENAME: &str = ".ssm-owner-id"; + +/// Each developer needs a stable local owner ID so Quake only matches +/// and terminates the SSM tunnels created by this machine. +pub(crate) fn ensure_owner_id(testnet_dir: &Path) -> Result { + let owner_id_path = testnet_dir.join(OWNER_ID_FILENAME); + if owner_id_path.exists() { + return load_owner_id(testnet_dir); + } + + let owner_id = format!("{:016x}", random::()); + fs::write(&owner_id_path, &owner_id).with_context(|| { + format!( + "Failed to write local SSM owner ID to {}", + owner_id_path.display() + ) + })?; + Ok(owner_id) +} + +/// Quake must reuse the same owner ID on one machine, otherwise it +/// loses track of its own tunnels and treats them as foreign. +pub(crate) fn load_owner_id(testnet_dir: &Path) -> Result { + let owner_id_path = testnet_dir.join(OWNER_ID_FILENAME); + let owner_id = fs::read_to_string(&owner_id_path).with_context(|| { + format!( + "Failed to read local SSM owner ID from {}", + owner_id_path.display() + ) + })?; + validate_owner_id(owner_id.trim()) + .with_context(|| format!("Invalid SSM owner ID in {}", owner_id_path.display())) +} + +/// Ensures `owner_id` is exactly 16 lowercase hex characters. +fn validate_owner_id(owner_id: &str) -> Result { + let is_lower_hex = |byte: u8| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte); + if owner_id.len() != 16 || !owner_id.bytes().all(is_lower_hex) { + return Err(eyre!("expected a 16-character lowercase hex owner ID")); + } + + Ok(owner_id.to_string()) +} + +/// [`Ssm`] manages the local SSM tunnels to Control Center. /// -/// SSM sessions are port-forwarding tunnels using StartPortForwardingSession: -/// only the handshake phase is throttled; once established, tunnels remain -/// active in the background. +/// Remote Quake commands expect a small set of localhost ports to forward into +/// Control Center. If those local forwards disappear, an imported testnet +/// loses access to Grafana, Prometheus, and the other Control Center +/// endpoints. /// -/// Note: SSH/SCP commands to nodes are routed through CC (Control Center) using -/// nodes' private IPs, avoiding additional SSM sessions. Only direct SSH to CC -/// uses StartSSHSession. +/// This type exists to keep those tunnels in one place. Other code can ask it +/// to start them, stop them, or print their current state. #[derive(Clone)] -pub(crate) struct Ssm(Vec); +pub(crate) struct Ssm { + sessions: Vec, + backend: Arc, +} impl Ssm { - /// Initialize SSM sessions to the Control Center server, if present. - pub fn new(cc: Option<&NodeInfraData>) -> Result { + /// `cc` is optional because Quake can build remote testnet state before + /// Terraform has created the Control Center. + pub fn new(owner_id: String, cc: Option<&NodeInfraData>) -> Result { let mut ssm_sessions = Vec::new(); - // Create (not started yet) SSM sessions to CC + // `None` means remote infra has not been provisioned yet, so there is + // no Control Center and no tunnels to build. if let Some(cc) = cc { + let owner_id = validate_owner_id(&owner_id) + .wrap_err("Invalid local SSM owner ID for SSM tunnels")?; if let Some(ssm_tunnel_ports) = cc.ssm_tunnel_ports.as_ref() { let instance_id = cc.instance_id().wrap_err("Instance ID not found for CC")?; for (remote_port, local_port) in ssm_tunnel_ports.iter() { - ssm_sessions.push(SSMSession::new( + ssm_sessions.push(SsmSession::new( remote::CC_INSTANCE.to_string(), instance_id.clone(), + owner_id.clone(), *local_port, *remote_port, )); @@ -58,209 +116,339 @@ impl Ssm { } } - Ok(Self(ssm_sessions)) + Ok(Self { + sessions: ssm_sessions, + backend: Arc::new(AwsCliBackend), + }) } - /// Start all inactive SSM sessions in parallel + /// A tunnel is only usable when the local port is listening *and* AWS + /// has the matching session. If another local process owns one of + /// Quake's expected ports, this fails rather than treating it as + /// healthy. pub async fn start(&self) -> Result<()> { debug!("Starting SSM sessions"); - let (_, inactive_sessions) = self.sessions_partitioned_active()?; - let reasons: Vec<_> = inactive_sessions.iter().map(|s| s.reason()).collect(); - - let start_results = - in_parallel(&inactive_sessions, |s| async move { s.start().await }).await; - for (session, result) in inactive_sessions.iter().zip(start_results) { - if let Err(e) = result { - return Err(eyre!( - "Failed to start SSM tunnel {}: {e}", - session.reason() - )); + let aws_sessions = self.managed_aws_sessions()?; + let mut conflicting_sessions = Vec::new(); + let mut stale_sessions = Vec::new(); + let mut sessions_to_start = Vec::new(); + + for session in &self.sessions { + let aws_sessions_for_tunnel = self.aws_sessions_for_tunnel(session, &aws_sessions); + let local_listener = self.backend.is_local_port_listening(session.local_port); + let status = self.tunnel_status(local_listener, aws_sessions_for_tunnel.len()); + match status { + TunnelStatus::Usable => continue, + TunnelStatus::Conflict => { + conflicting_sessions.push(session.label()); + } + TunnelStatus::StaleAws => { + stale_sessions.extend(aws_sessions_for_tunnel.into_iter().cloned()); + sessions_to_start.push(session); + } + TunnelStatus::Missing => sessions_to_start.push(session), } } - // Wait for the sessions to be active. The threads running `start-session` - // will terminate when their parent thread stops. Once all connections are - // established, the SSM tunnels stay active in the background. - debug!("Waiting for started SSM sessions to be ready",); - self.wait_for_connections(&reasons, Duration::from_secs(60)) + if !conflicting_sessions.is_empty() { + let conflicts = conflicting_sessions.join(", "); + return Err(eyre!( + "Quake SSM localhost ports are already occupied by another local process: {conflicts}" + )); + } + + self.terminate_sessions(&stale_sessions).await?; + self.start_sessions(&sessions_to_start).await?; + self.wait_for_local_listeners(&sessions_to_start, Duration::from_secs(60)) .await?; - info!("✅ SSM sessions started"); + info!("✅ SSM sessions ready"); Ok(()) } - /// Stop all active SSM sessions in parallel + /// Without cleanup, AWS can still show this machine's old sessions for + /// tunnels that no longer help the user. pub async fn stop(&self) -> Result<()> { debug!("Closing SSM sessions"); - let (active_sessions, _) = self.sessions_partitioned_active()?; - let stop_results = in_parallel(&active_sessions, |s| async move { s.stop().await }).await; - for (session, result) in active_sessions.iter().zip(stop_results) { - if let Err(e) = result { - return Err(eyre!("Failed to stop SSM tunnel {}: {e}", session.reason())); - } - } + let aws_sessions = self.managed_aws_sessions()?; + self.terminate_sessions(&aws_sessions).await?; info!("✅ SSM sessions terminated"); Ok(()) } - /// List all active SSM sessions + /// Shows Quake's local view of each tunnel and AWS's view of this + /// machine's matching sessions. pub async fn list(&self) -> Result<()> { println!("{}", self.list_formatted()?); Ok(()) } - /// Return a formatted string of all active SSM sessions. + /// Builds the text for `quake remote ssm list`. pub fn list_formatted(&self) -> Result { - let active_sessions_map = self.active_sessions_map()?; + if self.sessions.is_empty() { + return Ok("SSM tunnels:\n - No configured SSM tunnels\n".to_string()); + } - let mut result = active_sessions_map + let aws_sessions = self.managed_aws_sessions()?; + let lines = self + .sessions .iter() - .map(|(reason, (session_id, start_date, status))| { - format!(" - {session_id:>42} {start_date:>24} {status:>10} {reason}") + .map(|ssm_session| { + let aws_sessions_for_tunnel = + self.aws_sessions_for_tunnel(ssm_session, &aws_sessions); + let local_listener = self.backend.is_local_port_listening(ssm_session.local_port); + let status = self.tunnel_status(local_listener, aws_sessions_for_tunnel.len()); + + let aws_details = Self::format_matching_aws_sessions(&aws_sessions_for_tunnel); + + let listener_str = if local_listener { "up" } else { "down" }; + format!( + " - localhost:{} -> {}:{} status: {} listener: {} aws_sessions: {} aws: {}", + ssm_session.local_port, + ssm_session.instance_name, + ssm_session.remote_port, + status, + listener_str, + aws_sessions_for_tunnel.len(), + aws_details, + ) }) .collect::>() .join("\n"); - if result.is_empty() { - result.push_str(" - No active SSM sessions"); - } - result.push('\n'); - - let num_disconnected = self.0.len().saturating_sub(active_sessions_map.len()); - if num_disconnected > 0 { - result.push_str(&format!(" - {num_disconnected} SSM tunnels are disconnected: run `setup` or `remote ssm start` to restart the sessions (times out after 20 minutes of inactivity).")); - } - Ok(format!( - "Active SSM tunnels (session ID, start date, status, reason):\n{result}" + "SSM tunnels (local -> remote, status, local listener, \ +matching AWS sessions, AWS details):\n{lines}\n" )) } - /// Get a map of active sessions (Reason -> (SessionId, StartDate, Status)) - fn active_sessions_map(&self) -> Result> { - let filter = self - .0 + /// Filters out foreign sessions (other developers) and unrelated + /// sessions (e.g. direct SSH to Control Center). + fn managed_aws_sessions(&self) -> Result> { + let expected_reasons = self + .sessions .iter() - .map(|session| format!("Target==`{}`", session.instance_id)) - .collect::>() - .join(" || "); - let query = format!("Sessions[?{filter}] | sort_by([], &to_string(Reason))[*] | [].[SessionId, StartDate, Status, Reason]"); - #[rustfmt::skip] - let args = vec![ - "ssm", "describe-sessions", - "--state", "Active", - "--output", "text", - "--query", &query, - ]; + .map(SsmSession::reason) + .collect::>(); - let result = shell::exec_with_output("aws", args, Path::new(".")) - .wrap_err("Failed to query active SSM sessions")?; + Ok(self + .backend + .aws_sessions(&self.sessions)? + .into_iter() + .filter(|session| expected_reasons.contains(&session.reason)) + .collect()) + } - // Parse the output into a map of session reasons to session data - let mut map = HashMap::new(); - for line in result.lines() { - let parts: Vec<&str> = line.split_whitespace().collect(); - // Expected output: SessionId StartDate Status Reason - if parts.len() >= 4 { - let session_id = parts[0].to_string(); - let start_date = parts[1].to_string(); - let status = parts[2].to_string(); - let reason = parts[3].to_string(); - map.insert(reason, (session_id, start_date, status)); + /// Matches AWS sessions to one expected tunnel by `reason` string. + fn aws_sessions_for_tunnel<'a>( + &self, + session: &SsmSession, + aws_sessions: &'a [AwsSession], + ) -> Vec<&'a AwsSession> { + aws_sessions + .iter() + .filter(|active| active.reason == session.reason()) + .collect() + } + + /// Formats AWS session IDs and statuses for `quake remote ssm list`. + fn format_matching_aws_sessions(sessions: &[&AwsSession]) -> String { + if sessions.is_empty() { + return "[]".to_string(); + } + + let mut details = sessions + .iter() + .map(|session| format!("{}:{}", session.session_id, session.status)) + .collect::>(); + details.sort(); + + format!("[{}]", details.join(", ")) + } + + /// Derives the tunnel health from the local port check and matching + /// AWS session count. + fn tunnel_status(&self, local_listener: bool, aws_sessions_count: usize) -> TunnelStatus { + match (local_listener, aws_sessions_count > 0) { + (true, true) => TunnelStatus::Usable, + (true, false) => TunnelStatus::Conflict, + (false, true) => TunnelStatus::StaleAws, + (false, false) => TunnelStatus::Missing, + } + } + + /// Starts missing tunnels in parallel. + async fn start_sessions(&self, sessions: &[&SsmSession]) -> Result<()> { + if sessions.is_empty() { + return Ok(()); + } + + let start_results = in_parallel(sessions, move |session| { + let backend = Arc::clone(&self.backend); + async move { backend.start_session(&session) } + }) + .await; + + for (session, result) in sessions.iter().zip(start_results) { + if let Err(e) = result { + return Err(eyre!("Failed to start SSM tunnel {}: {e}", session.label())); } } - Ok(map) + Ok(()) } - /// Partition the sessions into active and inactive. - fn sessions_partitioned_active(&self) -> Result<(Vec<&SSMSession>, Vec<&SSMSession>)> { - let active_sessions_map = self.active_sessions_map()?; - Ok(self - .0 - .iter() - .partition(|s| active_sessions_map.contains_key(&s.reason()))) + /// Removes stale AWS sessions before starting fresh tunnels. + async fn terminate_sessions(&self, sessions: &[AwsSession]) -> Result<()> { + if sessions.is_empty() { + return Ok(()); + } + + let session_refs = sessions.iter().collect::>(); + let stop_results = in_parallel(&session_refs, move |session| { + let backend = Arc::clone(&self.backend); + async move { backend.terminate_session(&session.session_id) } + }) + .await; + + for (session, result) in sessions.iter().zip(stop_results) { + if let Err(e) = result { + return Err(eyre!( + "Failed to stop SSM session {}: {e}", + session.session_id + )); + } + } + Ok(()) } - /// Wait for all given sessions to be ready to use - async fn wait_for_connections( + /// Polls until all newly started local ports accept TCP connections. + async fn wait_for_local_listeners( &self, - sessions_reasons: &[String], + sessions: &[&SsmSession], timeout: Duration, ) -> Result<()> { let start_time = Instant::now(); while start_time.elapsed() < timeout { - if self.all_sessions_active(sessions_reasons)? { + if self.all_local_ports_listening(sessions) { return Ok(()); } tokio::time::sleep(Duration::from_millis(500)).await; } - Err(eyre!("Timeout waiting for SSM tunnels to be active")) + Err(eyre!( + "Timeout waiting for SSM tunnels to open local listeners" + )) } - /// Check if all given sessions are active - fn all_sessions_active(&self, sessions_reasons: &[String]) -> Result { - let active_sessions_map = self.active_sessions_map()?; - let active_session_reasons = active_sessions_map.keys().cloned().collect::>(); - for reason in sessions_reasons.iter() { - if !active_session_reasons.contains(reason) { - return Ok(false); - } - } - Ok(true) + /// checks whether every expected local forwarded port is accepting connections. + fn all_local_ports_listening(&self, sessions: &[&SsmSession]) -> bool { + sessions + .iter() + .all(|session| self.backend.is_local_port_listening(session.local_port)) } } -/// A single SSM tunnel from a local port to a remote port in an EC2 instance. +/// The pieces of an AWS SSM session that Quake needs for matching, +/// cleanup, and reporting. #[derive(Clone)] -pub(crate) struct SSMSession { - instance_name: String, - instance_id: String, - local_port: usize, - remote_port: usize, +struct AwsSession { + session_id: String, + /// AWS status word (e.g. `Connected`), shown in `ssm list`. + status: String, + /// Quake's stable tunnel name, used to match AWS sessions back to + /// expected tunnels across runs. + reason: String, } -impl SSMSession { - pub fn new( - instance_name: String, - instance_id: String, - local_port: usize, - remote_port: usize, - ) -> Self { - Self { - instance_name, - instance_id, - local_port, - remote_port, - } +/// Quake's tunnel-health classification. +enum TunnelStatus { + /// Local port listening, AWS session present. + Usable, + /// Local port listening, but no matching AWS session — another process + /// owns the port. + Conflict, + /// AWS session present, but local port not listening — stale record. + StaleAws, + /// Neither local port nor AWS session exists. + Missing, +} + +impl fmt::Display for TunnelStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let label = match self { + Self::Usable => "usable", + Self::Conflict => "conflict", + Self::StaleAws => "stale_aws", + Self::Missing => "missing", + }; + + f.write_str(label) } +} - /// String that uniquely identifies the session. - /// - /// We use the `reason` field in the SSM commands to uniquely identify - /// sessions. The start-session command run in interactive mode, so we can't - /// easily retrieve the session ID. This is needed to terminate the session. - /// - /// Instance ID and local port are enough to uniquely identify the session, - /// but we include other data for debugging. - fn reason(&self) -> String { - format!( - "quake-{}-{}-{}-{}", - self.instance_name, self.instance_id, self.local_port, self.remote_port - ) +/// Seam between [`Ssm`] logic and external side effects (AWS CLI, TCP +/// probes). Tests replace the real backend with a mock. +trait SsmBackend: Send + Sync { + /// Lists active AWS SSM sessions for the given expected tunnels. + fn aws_sessions(&self, sessions: &[SsmSession]) -> Result>; + /// Returns `true` if `local_port` on localhost accepts a TCP connection. + fn is_local_port_listening(&self, local_port: u16) -> bool; + /// Starts one SSM port-forwarding tunnel in the background. + fn start_session(&self, session: &SsmSession) -> Result<()>; + /// Terminates one AWS SSM session by ID. + fn terminate_session(&self, session_id: &str) -> Result<()>; +} + +/// Real [`SsmBackend`] that shells out to the AWS CLI. +struct AwsCliBackend; + +impl SsmBackend for AwsCliBackend { + fn aws_sessions(&self, sessions: &[SsmSession]) -> Result> { + if sessions.is_empty() { + return Ok(Vec::new()); + } + + // currently all sessions share the same instance ID, so we can just use the + // first. If that ever changes, we'll need to query sessions for each + // instance separately. + let instance_id = sessions[0].instance_id.as_str(); + let query = format!( + "Sessions[?Target==`{instance_id}`] | sort_by([], \ +&to_string(Reason))[*] | [].[SessionId, Status, Reason]" + ); + + #[rustfmt::skip] + let args = vec![ + "ssm", "describe-sessions", + "--state", "Active", + "--output", "text", + "--query", &query, + ]; + + let result = shell::exec_with_output("aws", args, Path::new(".")) + .wrap_err("Failed to query active SSM sessions")?; + + Ok(parse_aws_sessions_output(&result)) } - /// Start SSM session in the background - pub async fn start(&self) -> Result<()> { - debug!(instance_id=%self.instance_id, local_port=%self.local_port, "Starting SSM tunnel"); + fn is_local_port_listening(&self, local_port: u16) -> bool { + is_local_port_listening(local_port) + } - // Spawn task to run the SSM tunnel in the background - let reason = self.reason(); - let instance_id = self.instance_id.to_owned(); - let local_port = self.local_port; - let remote_port = self.remote_port; + fn start_session(&self, session: &SsmSession) -> Result<()> { + debug!( + instance_id = %session.instance_id, + local_port = session.local_port, + remote_port = session.remote_port, + "Starting SSM tunnel" + ); + + let reason = session.reason(); + let instance_id = session.instance_id.to_owned(); + let local_port = session.local_port; + let remote_port = session.remote_port; tokio::spawn(async move { #[rustfmt::skip] let args = [ @@ -268,7 +456,10 @@ impl SSMSession { "--target", &instance_id, "--reason", &reason, "--document-name", "AWS-StartPortForwardingSession", - "--parameters", &format!("{{\"portNumber\":[\"{remote_port}\"],\"localPortNumber\":[\"{local_port}\"]}}"), + "--parameters", &format!( + "{{\"portNumber\":[\"{remote_port}\"],\ +\"localPortNumber\":[\"{local_port}\"]}}" + ), ]; if let Err(e) = shell::exec("aws", args.to_vec(), Path::new("."), None, true) { warn!( @@ -282,32 +473,481 @@ impl SSMSession { Ok(()) } - /// Stop the SSM tunnel - pub async fn stop(&self) -> Result<()> { - debug!(instance_id=%self.instance_id, local_port=%self.local_port, "Stopping SSM tunnel"); + fn terminate_session(&self, session_id: &str) -> Result<()> { + let args = ["ssm", "terminate-session", "--session-id", session_id]; + shell::exec("aws", args.to_vec(), Path::new("."), None, false) + } +} - let Some(session_id) = self.get_session_id()? else { - warn!(%self.instance_id, "No active SSM session found"); - return Ok(()); - }; +/// One expected tunnel: instance name + ID, local port, remote port. +#[derive(Clone)] +pub(crate) struct SsmSession { + /// The short name Quake shows for the target machine. + instance_name: String, + /// The EC2 instance that the tunnel connects to. + instance_id: String, + /// Isolates this machine's tunnels from other developers'. + owner_id: String, + local_port: u16, + remote_port: u16, +} - // Terminate session - let args = ["ssm", "terminate-session", "--session-id", &session_id]; - shell::exec("aws", args.to_vec(), Path::new("."), None, false) +impl SsmSession { + pub fn new( + instance_name: String, + instance_id: String, + owner_id: String, + local_port: u16, + remote_port: u16, + ) -> Self { + Self { + instance_name, + instance_id, + owner_id, + local_port, + remote_port, + } } - /// Retrieve the session ID of the active SSM tunnel - fn get_session_id(&self) -> Result> { - let query = format!("Sessions[?Reason==`{}`].SessionId", self.reason()); + /// Stable name written into AWS `reason` so later runs on this machine can + /// match sessions back to expected tunnels. + fn reason(&self) -> String { + format!( + "quake-{}-{}-owner{}-{}-{}", + self.instance_name, self.instance_id, self.owner_id, self.local_port, self.remote_port + ) + } - #[rustfmt::skip] - let args = vec![ - "ssm", "describe-sessions", - "--state", "Active", - "--output", "text", - "--query", &query, - ]; - let session_id = shell::exec_with_output("aws", args, Path::new("."))?; - Ok((!session_id.is_empty()).then_some(session_id)) + fn label(&self) -> String { + format!( + "localhost:{} -> {}:{}", + self.local_port, self.instance_name, self.remote_port + ) + } +} + +/// Parses `aws ssm describe-sessions --output text` into [`AwsSession`]s. +fn parse_aws_sessions_output(output: &str) -> Vec { + output + .lines() + .filter_map(|line| { + let parts = line.split_whitespace().collect::>(); + if parts.len() < 3 { + return None; + } + + Some(AwsSession { + session_id: parts[0].to_string(), + status: parts[1].to_string(), + reason: parts[2].to_string(), + }) + }) + .collect() +} + +/// Checks whether a localhost port accepts a TCP connection. +fn is_local_port_listening(local_port: u16) -> bool { + let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, local_port)); + TcpStream::connect_timeout(&addr, Duration::from_millis(50)).is_ok() +} + +#[cfg(test)] +impl Ssm { + /// Builds an [`Ssm`] with a mock backend for tests. + fn with_backend(sessions: Vec, backend: Arc) -> Self { + Self { sessions, backend } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + use std::net::TcpListener; + use std::sync::Mutex; + use tempfile::tempdir; + + const TEST_OWNER_ID: &str = "8f3c12a1b7d94e6c"; + const FOREIGN_OWNER_ID: &str = "71de44b9538a0c2f"; + + #[derive(Default)] + struct MockSsmBackend { + aws_sessions: Mutex>, + listening_ports: Mutex>, + started_reasons: Mutex>, + terminated_sessions: Mutex>, + /// `next_id` keeps the counter used to mint fake AWS session IDs. + next_id: Mutex, + } + + impl MockSsmBackend { + /// Tests use this to create stale-session cases, where AWS still + /// shows a session after the local tunnel is gone. + fn add_aws_session(&self, session_id: &str, reason: String) { + self.aws_sessions + .lock() + .expect("aws_sessions mutex poisoned") + .push(AwsSession { + session_id: session_id.to_string(), + status: "Connected".to_string(), + reason, + }); + } + + /// Tests use this to say, "this tunnel is healthy," without starting + /// a real background session. + fn listen_on(&self, local_port: u16) { + self.listening_ports + .lock() + .expect("listening_ports mutex poisoned") + .insert(local_port); + } + + /// The tests check the `reason` strings because that is how Quake + /// names tunnels across runs. + fn started_reasons(&self) -> Vec { + self.started_reasons + .lock() + .expect("started_reasons mutex poisoned") + .clone() + } + + /// The tests use this to verify that stale AWS sessions are cleaned up + /// before new tunnels are started. + fn terminated_sessions(&self) -> Vec { + self.terminated_sessions + .lock() + .expect("terminated_sessions mutex poisoned") + .clone() + } + } + + impl SsmBackend for MockSsmBackend { + fn aws_sessions(&self, _sessions: &[SsmSession]) -> Result> { + Ok(self + .aws_sessions + .lock() + .expect("aws_sessions mutex poisoned") + .clone()) + } + + fn is_local_port_listening(&self, local_port: u16) -> bool { + self.listening_ports + .lock() + .expect("listening_ports mutex poisoned") + .contains(&local_port) + } + + fn start_session(&self, session: &SsmSession) -> Result<()> { + self.started_reasons + .lock() + .expect("started_reasons mutex poisoned") + .push(session.reason()); + self.listen_on(session.local_port); + + let mut next_id = self.next_id.lock().expect("next_id mutex poisoned"); + *next_id += 1; + self.add_aws_session(&format!("started-{}", *next_id), session.reason()); + Ok(()) + } + + fn terminate_session(&self, session_id: &str) -> Result<()> { + self.terminated_sessions + .lock() + .expect("terminated_sessions mutex poisoned") + .push(session_id.to_string()); + self.aws_sessions + .lock() + .expect("aws_sessions mutex poisoned") + .retain(|session| session.session_id != session_id); + Ok(()) + } + } + + fn test_session(local_port: u16, remote_port: u16) -> SsmSession { + SsmSession::new( + remote::CC_INSTANCE.to_string(), + "i-1234567890".to_string(), + TEST_OWNER_ID.to_string(), + local_port, + remote_port, + ) + } + + fn test_session_with_owner(owner_id: &str, local_port: u16, remote_port: u16) -> SsmSession { + SsmSession::new( + remote::CC_INSTANCE.to_string(), + "i-1234567890".to_string(), + owner_id.to_string(), + local_port, + remote_port, + ) + } + + #[test] + fn local_port_probe_detects_live_listener() { + let listener = + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("bind ephemeral listener"); + let local_port = listener.local_addr().expect("listener local addr").port(); + + assert!(is_local_port_listening(local_port)); + } + + #[test] + fn parse_aws_sessions_output_parses_valid_rows() { + let output = "\ +session-1 Connected quake-cc-i-123-13000-3000 +session-2 Terminated quake-cc-i-123-19090-9090 +"; + + let sessions = parse_aws_sessions_output(output); + + assert_eq!(sessions.len(), 2); + assert_eq!(sessions[0].session_id, "session-1"); + assert_eq!(sessions[0].status, "Connected"); + assert_eq!(sessions[0].reason, "quake-cc-i-123-13000-3000"); + assert_eq!(sessions[1].session_id, "session-2"); + assert_eq!(sessions[1].status, "Terminated"); + assert_eq!(sessions[1].reason, "quake-cc-i-123-19090-9090"); + } + + #[test] + fn parse_aws_sessions_output_skips_short_rows() { + let output = "\ +session-1 Connected quake-cc-i-123-13000-3000 + +too-short +session-2 Connected +session-3 Failed quake-cc-i-123-8000-80 +"; + + let sessions = parse_aws_sessions_output(output); + + assert_eq!(sessions.len(), 2); + assert_eq!(sessions[0].session_id, "session-1"); + assert_eq!(sessions[0].status, "Connected"); + assert_eq!(sessions[0].reason, "quake-cc-i-123-13000-3000"); + assert_eq!(sessions[1].session_id, "session-3"); + assert_eq!(sessions[1].status, "Failed"); + assert_eq!(sessions[1].reason, "quake-cc-i-123-8000-80"); + } + + #[test] + fn ensure_owner_id_creates_and_reuses_local_owner_id() { + let dir = tempdir().expect("create tempdir"); + + let first = ensure_owner_id(dir.path()).expect("owner ID created"); + let second = ensure_owner_id(dir.path()).expect("owner ID reused"); + + assert_eq!(first, second); + assert_eq!(first.len(), 16); + assert!(first + .bytes() + .all(|byte| { byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte) })); + let stored = + fs::read_to_string(dir.path().join(OWNER_ID_FILENAME)).expect("owner ID file exists"); + assert_eq!(stored, first); + } + + #[test] + fn load_owner_id_rejects_malformed_owner_id() { + let dir = tempdir().expect("create tempdir"); + fs::write(dir.path().join(OWNER_ID_FILENAME), "not-a-valid-owner-id") + .expect("write owner ID"); + + let err = load_owner_id(dir.path()).expect_err("invalid owner ID should fail"); + + assert!(err.to_string().contains("Invalid SSM owner ID")); + } + + #[tokio::test] + async fn start_skips_ports_that_are_already_listening() { + let session = test_session(13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.listen_on(session.local_port); + backend.add_aws_session("grafana-1", session.reason()); + let ssm = Ssm::with_backend(vec![session], backend.clone()); + + ssm.start().await.expect("start succeeds"); + + assert!(backend.started_reasons().is_empty()); + assert!(backend.terminated_sessions().is_empty()); + } + + /// Also proves `start` does not mutate state once it sees a conflict. + #[tokio::test] + async fn start_errors_when_listener_exists_without_matching_aws_session() { + let conflict = test_session(13000, 3000); + let stale = test_session(19090, 9090); + let backend = Arc::new(MockSsmBackend::default()); + backend.listen_on(conflict.local_port); + backend.add_aws_session("prom-1", stale.reason()); + let ssm = Ssm::with_backend(vec![conflict, stale], backend.clone()); + + let err = ssm + .start() + .await + .expect_err("start should fail on conflict"); + let err_text = err.to_string(); + + assert!(err_text.contains("localhost:13000 -> cc:3000")); + assert!(err_text.contains("already occupied by another local process")); + assert!(backend.started_reasons().is_empty()); + assert!(backend.terminated_sessions().is_empty()); + } + + /// AWS can show multiple sessions with the same `reason` after earlier + /// failures — all must be terminated before recreating. + #[tokio::test] + async fn start_recreates_tunnel_after_terminating_all_stale_aws_sessions() { + let session = test_session(13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("stale-1", session.reason()); + backend.add_aws_session("stale-2", session.reason()); + let ssm = Ssm::with_backend(vec![session.clone()], backend.clone()); + + ssm.start().await.expect("start succeeds"); + + assert_eq!( + backend.terminated_sessions(), + vec!["stale-1".to_string(), "stale-2".to_string()] + ); + assert_eq!(backend.started_reasons(), vec![session.reason()]); + assert!(backend.is_local_port_listening(session.local_port)); + } + + #[tokio::test] + async fn start_ignores_foreign_aws_sessions() { + let session = test_session(13000, 3000); + let foreign = test_session_with_owner(FOREIGN_OWNER_ID, 13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("foreign-1", foreign.reason()); + let ssm = Ssm::with_backend(vec![session.clone()], backend.clone()); + + ssm.start().await.expect("start succeeds"); + + assert!(backend.terminated_sessions().is_empty()); + assert_eq!(backend.started_reasons(), vec![session.reason()]); + assert!(backend + .aws_sessions + .lock() + .expect("aws_sessions mutex poisoned") + .iter() + .any(|aws_session| aws_session.session_id == "foreign-1")); + } + + /// Unrelated SSH sessions must be left alone. + #[tokio::test] + async fn stop_terminates_all_matching_sessions() { + let session_a = test_session(13000, 3000); + let session_b = test_session(19090, 9090); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("grafana-1", session_a.reason()); + backend.add_aws_session("grafana-2", session_a.reason()); + backend.add_aws_session("prom-1", session_b.reason()); + backend.add_aws_session("ssh-1", "quake-ssh-i-1234567890".to_string()); + let ssm = Ssm::with_backend(vec![session_a, session_b], backend.clone()); + + ssm.stop().await.expect("stop succeeds"); + + assert_eq!( + backend.terminated_sessions(), + vec![ + "grafana-1".to_string(), + "grafana-2".to_string(), + "prom-1".to_string(), + ] + ); + } + + #[tokio::test] + async fn stop_ignores_foreign_aws_sessions() { + let session = test_session(13000, 3000); + let foreign = test_session_with_owner(FOREIGN_OWNER_ID, 13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("local-1", session.reason()); + backend.add_aws_session("foreign-1", foreign.reason()); + let ssm = Ssm::with_backend(vec![session], backend.clone()); + + ssm.stop().await.expect("stop succeeds"); + + assert_eq!(backend.terminated_sessions(), vec!["local-1".to_string()]); + assert!(backend + .aws_sessions + .lock() + .expect("aws_sessions mutex poisoned") + .iter() + .any(|aws_session| aws_session.session_id == "foreign-1")); + } + + #[test] + fn list_formatted_reports_local_and_aws_state_separately() { + let session_a = test_session(13000, 3000); + let session_b = test_session(19090, 9090); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("grafana-1", session_a.reason()); + backend.add_aws_session("prom-1", session_b.reason()); + backend.listen_on(session_b.local_port); + let ssm = Ssm::with_backend(vec![session_a, session_b], backend); + + let rendered = ssm.list_formatted().expect("list formatting succeeds"); + + assert!(rendered.contains("localhost:13000 -> cc:3000")); + assert!(rendered.contains("status: stale_aws")); + assert!(rendered.contains("listener: down aws_sessions: 1")); + assert!(rendered.contains("aws: [grafana-1:Connected]")); + assert!(rendered.contains("localhost:19090 -> cc:9090")); + assert!(rendered.contains("status: usable")); + assert!(rendered.contains("listener: up aws_sessions: 1")); + assert!(rendered.contains("aws: [prom-1:Connected]")); + } + + #[test] + fn list_formatted_reports_conflicting_local_listener() { + let session = test_session(13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.listen_on(session.local_port); + let ssm = Ssm::with_backend(vec![session], backend); + + let rendered = ssm.list_formatted().expect("list formatting succeeds"); + + assert!(rendered.contains("localhost:13000 -> cc:3000")); + assert!(rendered.contains("status: conflict")); + assert!(rendered.contains("listener: up aws_sessions: 0")); + assert!(rendered.contains("aws: []")); + } + + /// Two AWS sessions with the same `reason` — both must appear in output. + #[test] + fn list_formatted_reports_all_matching_aws_sessions_for_one_tunnel() { + let session = test_session(13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("grafana-1", session.reason()); + backend.add_aws_session("grafana-2", session.reason()); + let ssm = Ssm::with_backend(vec![session], backend); + + let rendered = ssm.list_formatted().expect("list formatting succeeds"); + + assert!(rendered.contains("localhost:13000 -> cc:3000")); + assert!(rendered.contains("status: stale_aws")); + assert!(rendered.contains("listener: down aws_sessions: 2")); + assert!(rendered.contains("aws: [grafana-1:Connected, grafana-2:Connected]")); + } + + #[test] + fn list_formatted_hides_foreign_aws_sessions() { + let session = test_session(13000, 3000); + let foreign = test_session_with_owner(FOREIGN_OWNER_ID, 13000, 3000); + let backend = Arc::new(MockSsmBackend::default()); + backend.add_aws_session("foreign-1", foreign.reason()); + let ssm = Ssm::with_backend(vec![session], backend); + + let rendered = ssm.list_formatted().expect("list formatting succeeds"); + + assert!(rendered.contains("localhost:13000 -> cc:3000")); + assert!(rendered.contains("status: missing")); + assert!(rendered.contains("listener: down aws_sessions: 0")); + assert!(rendered.contains("aws: []")); + assert!(!rendered.contains("foreign-1")); } } diff --git a/crates/quake/src/main.rs b/crates/quake/src/main.rs index f6f9696..f53ad83 100644 --- a/crates/quake/src/main.rs +++ b/crates/quake/src/main.rs @@ -14,8 +14,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] +use chrono::{NaiveDate, NaiveDateTime, TimeZone, Utc}; use clap::{Args, Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity}; use color_eyre::eyre::{self, bail, Context, Result}; @@ -309,6 +314,9 @@ struct StartArgs { /// Create the testnet in remote infrastructure and start it immediately (no confirmation asked) #[clap(long, default_value = "false")] remote: bool, + /// Whether to start monitoring services (Prometheus, Grafana, cAdvisor, Blockscout) + #[clap(short = 'm', long, num_args = 0..=1, default_value_t = true, default_missing_value = "true")] + monitoring: bool, #[command(flatten)] setup_args: SetupArgs, #[command(flatten)] @@ -446,6 +454,9 @@ pub(crate) enum InfoSubcommand { /// Show full peer detail including peer types and scores #[clap(long, default_value = "false")] peers_full: bool, + /// Show duplicate message rates + #[clap(long, default_value = "false")] + duplicates: bool, }, /// Show performance metrics: block latency and throughput Perf { @@ -455,6 +466,15 @@ pub(crate) enum InfoSubcommand { /// Show only throughput metrics (txs/block, block size, gas/block) #[clap(long, default_value = "false")] throughput_only: bool, + /// Use two scrapes and show histogram deltas for the observation window only + #[clap(long, default_value = "false")] + interval: bool, + /// Seconds to wait before the first scrape (interval mode only) + #[clap(long, default_value = "30")] + warmup_seconds: u64, + /// Seconds between first and second scrape (interval mode only) + #[clap(long, default_value = "60")] + observation_seconds: u64, }, /// Show Malachite CL store.db table statistics (record counts, height ranges) Store { @@ -558,6 +578,93 @@ pub(crate) enum RemoteSubcommand { /// Path to the JSON file created by `quake remote export` path: PathBuf, }, + /// Download metrics or database data from remote infrastructure + Download { + #[clap(subcommand)] + command: DownloadSubcommand, + }, +} + +/// A datetime accepted by `--from`/`--to` flags, converted to a Unix timestamp. +/// +/// Accepted formats (timezone-naive values are treated as UTC): +/// `2024-01-15T10:30:00Z`, `2024-01-15T10:30:00+05:00` (RFC 3339) +/// `2024-01-15T10:30:00`, `2024-01-15 10:30:00` (naive datetime, UTC assumed) +/// `2024-01-15` (date only, start of day UTC) +#[derive(Debug, Clone, Copy)] +pub(crate) struct CliTimestamp(i64); + +impl CliTimestamp { + pub(crate) fn unix_secs(self) -> i64 { + self.0 + } +} + +impl std::str::FromStr for CliTimestamp { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { + return Ok(CliTimestamp(dt.timestamp())); + } + for fmt in &["%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"] { + if let Ok(ndt) = NaiveDateTime::parse_from_str(s, fmt) { + return Ok(CliTimestamp(Utc.from_utc_datetime(&ndt).timestamp())); + } + } + if let Ok(nd) = NaiveDate::parse_from_str(s, "%Y-%m-%d") { + let ndt = nd.and_hms_opt(0, 0, 0).expect("valid HMS"); + return Ok(CliTimestamp(Utc.from_utc_datetime(&ndt).timestamp())); + } + Err(format!( + "invalid datetime '{s}'; expected RFC 3339 or one of: \ + YYYY-MM-DDTHH:MM:SS, YYYY-MM-DD HH:MM:SS, YYYY-MM-DD" + )) + } +} + +#[derive(Debug, Subcommand)] +pub(crate) enum DownloadSubcommand { + /// Download Prometheus metrics from the Control Center. + /// + /// SSHes to CC and queries the Prometheus query_range API — no local SSM tunnel required. + /// Downloads all metrics by default; pass metric names after -- to filter. + /// Without --from/--to, start defaults to headStats.minTime (Prometheus head block only, ~2h max). + Metrics { + /// Start of the time range (default: headStats.minTime from Prometheus, covering the current head block; e.g. 2024-01-15T10:30:00Z or 2024-01-15) + #[clap(long)] + from: Option, + /// End of the time range (default: now; e.g. 2024-01-15T10:30:00Z or 2024-01-15) + #[clap(long)] + to: Option, + /// Query resolution — interval between data points (e.g. 30s, 1m, 5m). + /// Defaults to ceil((end-start)/10000) to stay within Prometheus' 11,000-point limit. + #[clap(long)] + step: Option, + /// Metric names to download (all metrics if not specified) + #[clap(last = true)] + metric_names: Vec, + /// Output file path (default: ./quake-metrics-.tar.gz) + #[clap(short = 'o', long)] + output: Option, + }, + /// Download node databases from one or more remote validators. + /// + /// Defaults to all nodes in the manifest. Pass node names after -- to restrict. + Db { + /// Node names to download from (default: all nodes in manifest) + #[clap(last = true)] + nodes: Vec, + /// Download only execution layer (Reth) data + #[clap(long, conflicts_with = "consensus_only")] + execution_only: bool, + /// Download only consensus layer (Malachite) data + #[clap(long)] + consensus_only: bool, + /// Output file path (default: ./quake-db-.tar.gz) + #[clap(short = 'o', long)] + output: Option, + }, } #[derive(Debug, Subcommand, serde::Deserialize, schemars::JsonSchema)] @@ -681,7 +788,9 @@ async fn main() -> Result<()> { rpc_manifest, ) .await?; - testnet.start(start_args.nodes_or_containers).await? + testnet + .start(start_args.nodes_or_containers, start_args.monitoring) + .await? } Commands::Stop { nodes_or_containers, @@ -706,7 +815,9 @@ async fn main() -> Result<()> { rpc_manifest, ) .await?; - testnet.start(start_args.nodes_or_containers).await?; + testnet + .start(start_args.nodes_or_containers, start_args.monitoring) + .await?; } Commands::Perturb { action, diff --git a/crates/quake/src/manifest.rs b/crates/quake/src/manifest.rs index 307d1b6..bad872e 100644 --- a/crates/quake/src/manifest.rs +++ b/crates/quake/src/manifest.rs @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use alloy_primitives::Address; use color_eyre::eyre::{bail, Context, Result}; use indexmap::IndexMap; use once_cell::sync::Lazy; @@ -560,6 +561,10 @@ pub struct Node { #[serde(default)] pub cl_prune_preset: Option, + /// Address to receive transaction fees and block rewards (--suggested-fee-recipient). + #[serde(default)] + pub cl_suggested_fee_recipient: Option
, + /// Mark this node as external (operated by a third party). /// External validators are expected to be multi-hop in mesh health checks /// rather than fully-connected. Also applies to their dedicated sentries. diff --git a/crates/quake/src/manifest/raw.rs b/crates/quake/src/manifest/raw.rs index a485d96..5d15ffb 100644 --- a/crates/quake/src/manifest/raw.rs +++ b/crates/quake/src/manifest/raw.rs @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use alloy_primitives::Address; use arc_consensus_types::Config as ClConfigOverride; use color_eyre::eyre::{bail, Result}; use indexmap::{IndexMap, IndexSet}; @@ -148,6 +149,10 @@ pub struct RawNode { #[serde(default, skip_serializing_if = "Option::is_none")] cl_voting_power: Option, + /// Address to receive transaction fees and block rewards (--suggested-fee-recipient). + #[serde(default, skip_serializing_if = "Option::is_none")] + cl_suggested_fee_recipient: Option
, + /// Mark this node as external (operated by a third party). /// External validators are expected to be multi-hop in mesh health checks /// rather than fully-connected. Also applies to their dedicated sentries. @@ -444,6 +449,7 @@ impl TryFrom for Manifest { follow_endpoints: raw_node.follow_endpoints, cl_voting_power: raw_node.cl_voting_power, cl_prune_preset: raw_node.cl_prune_preset, + cl_suggested_fee_recipient: raw_node.cl_suggested_fee_recipient, external: raw_node.external, }, ); @@ -548,6 +554,7 @@ impl RawNode { follow: node.follow, follow_endpoints: node.follow_endpoints, cl_voting_power: node.cl_voting_power, + cl_suggested_fee_recipient: node.cl_suggested_fee_recipient, external: node.external, }) } diff --git a/crates/quake/src/mcp.rs b/crates/quake/src/mcp.rs index c03ab5c..2af5e4c 100644 --- a/crates/quake/src/mcp.rs +++ b/crates/quake/src/mcp.rs @@ -331,6 +331,7 @@ impl QuakeMcpServer { show_mesh: true, show_peers: false, show_peers_full: false, + show_duplicates: false, }; let report = crate::mesh::format_report(&analysis, &options); Ok(CallToolResult::success(vec![Content::text(report)])) @@ -346,7 +347,8 @@ impl QuakeMcpServer { let testnet = self.testnet.read().await; let metrics_urls = testnet.nodes_metadata.all_consensus_metrics_urls(); let raw_metrics = arc_checks::fetch_all_metrics(&metrics_urls).await; - let mut nodes = arc_checks::parse_perf_metrics(&raw_metrics); + let nodes = + crate::util::parse_perf_metrics_with_groups(&raw_metrics, &testnet.manifest.nodes); if nodes.is_empty() { return Ok(CallToolResult::success(vec![Content::text( @@ -354,13 +356,12 @@ impl QuakeMcpServer { )])); } - crate::util::assign_node_groups( - nodes.iter_mut().map(|n| (n.name.as_str(), &mut n.group)), - &testnet.manifest.nodes, - ); - let options = arc_checks::PerfDisplayOptions::default(); - let report = arc_checks::format_perf_report(&nodes, &options); + let report = arc_checks::format_perf_report( + &nodes, + &options, + arc_checks::PerfReportKind::CumulativeSinceStart, + ); Ok(CallToolResult::success(vec![Content::text(report)])) } @@ -403,17 +404,18 @@ impl QuakeMcpServer { )] async fn start_nodes( &self, - params: Parameters, + params: Parameters, ) -> Result { self.ensure_ssm_tunnels().await?; let names = params.0.nodes.unwrap_or_default(); + let monitoring = params.0.monitoring.unwrap_or(true); let label = if names.is_empty() { "all nodes".to_string() } else { names.join(", ") }; let testnet = self.testnet.read().await; - testnet.start(names).await.map_err(|e| { + testnet.start(names, monitoring).await.map_err(|e| { rmcp::ErrorData::internal_error(format!("Failed to start nodes: {e}"), None) })?; Ok(CallToolResult::success(vec![Content::text(format!( @@ -502,10 +504,11 @@ impl QuakeMcpServer { )] async fn restart_testnet( &self, - params: Parameters, + params: Parameters, ) -> Result { self.ensure_ssm_tunnels().await?; let names = params.0.nodes.unwrap_or_default(); + let monitoring = params.0.monitoring.unwrap_or(true); let label = if names.is_empty() { "all nodes".to_string() } else { @@ -519,7 +522,7 @@ impl QuakeMcpServer { } { let testnet = self.testnet.read().await; - testnet.start(names).await.map_err(|e| { + testnet.start(names, monitoring).await.map_err(|e| { rmcp::ErrorData::internal_error(format!("Failed to start nodes: {e}"), None) })?; } @@ -869,7 +872,8 @@ impl QuakeMcpServer { /// Only available for remote testnets. /// /// Actions: - /// "start" — opens inactive tunnels (idempotent) + /// "start" — ensures tunnels are usable and recreates stale ones if + /// needed /// "stop" — closes all active tunnels /// "list" — shows active tunnel status #[tool( @@ -970,6 +974,15 @@ struct NodeNamesParams { nodes: Option>, } +/// Parameters for start/restart tools that accept node names and monitoring control. +#[derive(Debug, Deserialize, JsonSchema)] +struct StartNodeParams { + /// Optional list of node or container names. If empty or omitted, applies to all nodes. + nodes: Option>, + /// Start monitoring services (Prometheus, Grafana, cAdvisor, Blockscout). Defaults to true. + monitoring: Option, +} + /// Parameters for the clean_testnet tool. #[derive(Debug, Deserialize, JsonSchema)] struct CleanParams { @@ -1099,7 +1112,10 @@ impl QuakeMcpServer { /// Ensure SSM tunnels are active before performing remote operations. /// /// For local testnets this is a no-op. For remote testnets it calls the - /// idempotent `ssm_tunnels.start()` which only opens inactive sessions. + /// idempotent `ssm_tunnels.start()` which ensures the expected tunnels are + /// usable, recreates stale AWS sessions when needed, and fails if some + /// other local process is already listening on one of Quake's expected + /// localhost ports. async fn ensure_ssm_tunnels(&self) -> Result<(), rmcp::ErrorData> { let ssm = { let testnet = self.testnet.read().await; diff --git a/crates/quake/src/setup.rs b/crates/quake/src/setup.rs index 8a137fb..6103e80 100644 --- a/crates/quake/src/setup.rs +++ b/crates/quake/src/setup.rs @@ -19,6 +19,7 @@ use std::os::unix::fs::PermissionsExt; use std::time::Duration; use std::{fs, path::Path}; +use alloy_primitives::Address; use arc_consensus_types::{ Config, LoggingConfig, MetricsConfig, PruningConfig, RemoteSigningConfig, RpcConfig, RuntimeConfig, SigningConfig, @@ -751,6 +752,7 @@ pub(crate) fn generate_node_cli_flags( peers_ips: &[String], image_tag: Option<&str>, follow_endpoint_urls: &[String], + suggested_fee_recipient: Option
, ) -> Vec { // For older versions (< v0.5.0), skip CLI flags as they use config.toml if !supports_cli_flags(image_tag) { @@ -797,7 +799,17 @@ pub(crate) fn generate_node_cli_flags( // Enable value sync by default flags.push("--value-sync".to_string()); + if let Some(addr) = suggested_fee_recipient { + flags.push(format!("--suggested-fee-recipient={addr}")); + } + if let Some(node) = node { + let cl = &node.cl_config; + + // Always set log level and format to ensure consistent logs for testing, debugging, and log collection. + flags.push(format!("--log-level={}", cl.logging.log_level)); + flags.push(format!("--log-format={}", cl.logging.log_format)); + // Add persistent-peers-only if enabled if node.cl_persistent_peers_only { flags.push("--p2p.persistent-peers-only".to_string()); @@ -817,7 +829,6 @@ pub(crate) fn generate_node_cli_flags( // Translate cl.config.* typed fields to CLI flags. // Defaults come from the CL CLI itself (StartCmd) so we only emit // flags whose values differ from what the binary would use anyway. - let cl = &node.cl_config; let cli_defaults = StartCmd::default(); let discovery = &cl.consensus.p2p.discovery; @@ -839,11 +850,8 @@ pub(crate) fn generate_node_cli_flags( if !cl.consensus.enabled { flags.push("--no-consensus".to_string()); - } - - let default_log_level = arc_consensus_types::LogLevel::default(); - if cl.logging.log_level != default_log_level { - flags.push(format!("--log-level={}", cl.logging.log_level)); + } else if node.node_type == manifest::NodeType::Validator { + flags.push("--validator".to_string()); } if node.remote_signer.is_some() { @@ -883,6 +891,9 @@ pub(crate) fn generate_node_cli_flags( flags.push(format!("--follow.endpoint={url}")); } } + } else { + flags.push("--log-level=debug".to_string()); + flags.push("--log-format=plaintext".to_string()); } flags @@ -1398,7 +1409,8 @@ mod tests { #[test] fn generate_node_cli_flags_includes_required_flags() { - let flags = generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); assert!(flags_str.contains("--moniker")); @@ -1410,7 +1422,8 @@ mod tests { #[test] fn generate_node_cli_flags_with_cl_persistent_peers() { let peers = vec!["172.19.0.6".to_string(), "172.19.0.7".to_string()]; - let flags = generate_node_cli_flags("validator-1", None, "172.19.0.5", &peers, None, &[]); + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &peers, None, &[], None); let flags_str = flags.join(" "); assert!(flags_str.contains("--p2p.persistent-peers")); @@ -1420,7 +1433,8 @@ mod tests { #[test] fn generate_node_cli_flags_without_persistent_peers() { - let flags = generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); // Should not contain persistent peers flag when empty @@ -1429,7 +1443,8 @@ mod tests { #[test] fn generate_node_cli_flags_includes_metrics() { - let flags = generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); assert!(flags_str.contains("--metrics")); @@ -1438,7 +1453,8 @@ mod tests { #[test] fn generate_node_cli_flags_includes_rpc() { - let flags = generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); assert!(flags_str.contains("--rpc.addr")); @@ -1447,12 +1463,71 @@ mod tests { #[test] fn generate_node_cli_flags_includes_value_sync() { - let flags = generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); assert!(flags_str.contains("--value-sync")); } + #[test] + fn generate_node_cli_flags_logging_defaults_when_no_node() { + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); + + let flags_str = flags.join(" "); + assert!(flags_str.contains("--log-level=debug")); + assert!(flags_str.contains("--log-format=plaintext")); + } + + #[test] + fn generate_node_cli_flags_logging_from_node_config() { + use malachitebft_config::{LogFormat, LogLevel}; + + let mut node = manifest::Node::default(); + node.cl_config.logging.log_level = LogLevel::Info; + node.cl_config.logging.log_format = LogFormat::Json; + + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); + + let flags_str = flags.join(" "); + assert!(flags_str.contains("--log-level=info")); + assert!(flags_str.contains("--log-format=json")); + } + + #[test] + fn generate_node_cli_flags_emits_suggested_fee_recipient() { + use alloy_primitives::address; + let recipient = address!("0x98e503f35D0a019cB0a251aD243a4cCFCF371F46"); + let flags = generate_node_cli_flags( + "validator-1", + None, + "172.19.0.5", + &[], + None, + &[], + Some(recipient), + ); + let flags_str = flags.join(" "); + assert!(flags_str.contains(&format!("--suggested-fee-recipient={recipient}"))); + } + + #[test] + fn generate_node_cli_flags_omits_suggested_fee_recipient_when_none() { + let flags = + generate_node_cli_flags("validator-1", None, "172.19.0.5", &[], None, &[], None); + let flags_str = flags.join(" "); + assert!(!flags_str.contains("--suggested-fee-recipient")); + } + #[test] fn generate_node_cli_flags_with_remote_signer() { let node = manifest::Node { @@ -1461,8 +1536,15 @@ mod tests { ..Default::default() }; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(flags_str.contains("--signing.remote")); @@ -1472,8 +1554,15 @@ mod tests { #[test] fn generate_node_cli_flags_without_remote_signer() { let node = manifest::Node::default(); - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(!flags_str.contains("--signing.remote")); } @@ -1485,8 +1574,15 @@ mod tests { ..Default::default() }; let peers = vec!["172.19.0.6".to_string()]; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &peers, None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &peers, + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(flags_str.contains("--p2p.persistent-peers-only")); } @@ -1494,8 +1590,15 @@ mod tests { #[test] fn generate_node_cli_flags_without_persistent_peers_only() { let node = manifest::Node::default(); - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(!flags_str.contains("--p2p.persistent-peers-only")); } @@ -1509,8 +1612,15 @@ mod tests { }, ..Default::default() }; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(flags_str.contains("--gossipsub.explicit-peering")); } @@ -1524,8 +1634,15 @@ mod tests { }, ..Default::default() }; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(flags_str.contains("--gossipsub.mesh-prioritization")); } @@ -1539,8 +1656,15 @@ mod tests { }, ..Default::default() }; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(flags_str.contains("--gossipsub.load=high")); } @@ -1548,8 +1672,15 @@ mod tests { #[test] fn generate_node_cli_flags_without_gossipsub_overrides() { let node = manifest::Node::default(); - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!(!flags_str.contains("--gossipsub.explicit-peering")); assert!(!flags_str.contains("--gossipsub.mesh-prioritization")); @@ -1664,8 +1795,15 @@ mod tests { fn generate_node_cli_flags_includes_pruning_distance() { let mut node = manifest::Node::default(); node.cl_config.prune.certificates_distance = 500; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!( flags_str.contains("--prune.certificates.distance=500"), @@ -1681,8 +1819,15 @@ mod tests { fn generate_node_cli_flags_includes_pruning_before() { let mut node = manifest::Node::default(); node.cl_config.prune.certificates_before = arc_consensus_types::Height::new(100); - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!( flags_str.contains("--prune.certificates.before=100"), @@ -1704,8 +1849,15 @@ mod tests { node_type, ..Default::default() }; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!( !flags_str.contains("--prune.certificates.distance"), @@ -1730,8 +1882,15 @@ mod tests { fn generate_node_cli_flags_prune_distance_emitted() { let mut node = manifest::Node::default(); node.cl_config.prune.certificates_distance = 500; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!( flags_str.contains("--prune.certificates.distance=500"), @@ -1748,6 +1907,7 @@ mod tests { &[], Some("arc_consensus:v0.4.0"), &[], + None, ); assert!(flags.is_empty()); @@ -1762,6 +1922,7 @@ mod tests { &[], Some("arc_consensus:v0.5.0"), &[], + None, ); assert!(!flags.is_empty()); @@ -1778,8 +1939,15 @@ mod tests { "http://validator-1_el:8545".to_string(), "http://validator-2_el:8545".to_string(), ]; - let flags = - generate_node_cli_flags("rpc-1", Some(&node), "172.19.0.5", &[], None, &endpoints); + let flags = generate_node_cli_flags( + "rpc-1", + Some(&node), + "172.19.0.5", + &[], + None, + &endpoints, + None, + ); let flags_str = flags.join(" "); assert!( flags_str.contains("--follow"), @@ -1799,8 +1967,15 @@ mod tests { fn generate_node_cli_flags_no_follow_when_not_enabled() { let node = manifest::Node::default(); let endpoints = vec!["http://validator-1_el:8545".to_string()]; - let flags = - generate_node_cli_flags("rpc-1", Some(&node), "172.19.0.5", &[], None, &endpoints); + let flags = generate_node_cli_flags( + "rpc-1", + Some(&node), + "172.19.0.5", + &[], + None, + &endpoints, + None, + ); let flags_str = flags.join(" "); assert!( !flags_str.contains("--follow"), @@ -1812,7 +1987,8 @@ mod tests { fn generate_node_cli_flags_includes_no_consensus() { let mut node = manifest::Node::default(); node.cl_config.consensus.enabled = false; - let flags = generate_node_cli_flags("rpc-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("rpc-1", Some(&node), "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); assert!( flags_str.contains("--no-consensus"), @@ -1820,13 +1996,44 @@ mod tests { ); } + #[test] + fn generate_node_cli_flags_includes_validator_for_validator_nodes() { + let node = manifest::Node { + node_type: manifest::NodeType::Validator, + ..Default::default() + }; + let flags = + generate_node_cli_flags("val-1", Some(&node), "172.19.0.5", &[], None, &[], None); + let flags_str = flags.join(" "); + assert!( + flags_str.contains("--validator"), + "missing --validator: {flags_str}" + ); + } + + #[test] + fn generate_node_cli_flags_omits_validator_for_non_validator_nodes() { + let node = manifest::Node { + node_type: manifest::NodeType::NonValidator, + ..Default::default() + }; + let flags = + generate_node_cli_flags("fn-1", Some(&node), "172.19.0.5", &[], None, &[], None); + let flags_str = flags.join(" "); + assert!( + !flags_str.contains("--validator"), + "should not contain --validator: {flags_str}" + ); + } + #[test] fn generate_node_cli_flags_includes_cl_prune_preset() { let node = manifest::Node { cl_prune_preset: Some(manifest::ClPruningPreset::Full), ..Default::default() }; - let flags = generate_node_cli_flags("val-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = + generate_node_cli_flags("val-1", Some(&node), "172.19.0.5", &[], None, &[], None); let flags_str = flags.join(" "); assert!( flags_str.contains("--full"), @@ -1841,8 +2048,15 @@ mod tests { ..Default::default() }; node.cl_config.prune.certificates_distance = 500; - let flags = - generate_node_cli_flags("validator-1", Some(&node), "172.19.0.5", &[], None, &[]); + let flags = generate_node_cli_flags( + "validator-1", + Some(&node), + "172.19.0.5", + &[], + None, + &[], + None, + ); let flags_str = flags.join(" "); assert!( flags_str.contains("--prune.certificates.distance=500"), diff --git a/crates/quake/src/shell.rs b/crates/quake/src/shell.rs index 144edb9..49bb3e3 100644 --- a/crates/quake/src/shell.rs +++ b/crates/quake/src/shell.rs @@ -202,3 +202,26 @@ pub(crate) fn scp( exec("scp", args, dir, None, false).wrap_err_with(|| format!("Failed to copy files to {host}")) } + +/// Copy a file from a remote server to a local path. +/// +/// `remote_source` is relative to `/home/{user_name}/` on the remote server. +pub(crate) fn scp_from( + host: &str, + user_name: &str, + private_key_path: &str, + dir: &Path, + remote_source: &str, + local_dest: &Path, +) -> Result<()> { + let opts = ssh_opts(host); + let mut args: Vec<&str> = opts.iter().map(|s| s.trim_matches('"')).collect::>(); + args.extend(vec!["-i", private_key_path]); + args.push("-C"); // compress + let source = format!("{user_name}@{host}:/home/{user_name}/{remote_source}"); + let dest = local_dest.to_string_lossy().into_owned(); + args.push(&source); + args.push(&dest); + exec("scp", args, dir, None, false) + .wrap_err_with(|| format!("Failed to copy files from {host}")) +} diff --git a/crates/quake/src/testnet.rs b/crates/quake/src/testnet.rs index 7d542c1..0b1b6f1 100644 --- a/crates/quake/src/testnet.rs +++ b/crates/quake/src/testnet.rs @@ -39,7 +39,7 @@ use crate::rpc::{ControllerInfo, Controllers}; use crate::valset::ValidatorPowerUpdate; use crate::wait::{check_ws_connectable, wait_for_nodes, wait_for_nodes_sync, wait_for_rounds}; use crate::{build, genesis, info as info_mod, latency, monitor, setup, shell}; -use crate::{InfoSubcommand, RemoteSubcommand, SSMSubcommand}; +use crate::{DownloadSubcommand, InfoSubcommand, RemoteSubcommand, SSMSubcommand}; pub(crate) const QUAKE_DIR: &str = ".quake"; pub(crate) const LAST_MANIFEST_FILENAME: &str = ".last_manifest"; @@ -186,6 +186,12 @@ impl Testnet { ) } InfraType::Remote => { + let owner_id = if infra_data.control_center.is_some() { + infra::ssm::ensure_owner_id(&dir) + .wrap_err("Failed to initialize local SSM owner ID")? + } else { + String::new() + }; let terraform = Terraform::new( &repo_root_dir.join("crates").join("quake").join("terraform"), &relative_dir, @@ -194,8 +200,9 @@ impl Testnet { node_names, manifest.build_network_topology(), )?; - let ssm_tunnels = infra::ssm::Ssm::new(infra_data.control_center.as_ref()) - .wrap_err("Failed to initialize SSM tunnels")?; + let ssm_tunnels = + infra::ssm::Ssm::new(owner_id, infra_data.control_center.as_ref()) + .wrap_err("Failed to initialize SSM tunnels")?; Arc::new( RemoteInfra::new( &repo_root_dir, @@ -370,6 +377,9 @@ impl Testnet { setup::generate_prometheus_config(&path, self.nodes_metadata.values())?; } InfraType::Remote => { + infra::ssm::ensure_owner_id(&self.dir) + .wrap_err("Failed to create local SSM owner ID")?; + // Get consensus container IPs for all nodes (needed for persistent peers) let consensus_addresses_map = self.nodes_metadata.consensus_ip_addresses_map(); @@ -434,6 +444,7 @@ impl Testnet { &peers_ips, Some(self.images.cl.as_str()), &follow_endpoint_urls, + None, ); let compose_data = setup::ComposeTemplateDataRemote { @@ -535,7 +546,7 @@ impl Testnet { } /// Start testnet containers using Docker Compose - pub async fn start(&self, names: Vec) -> Result<()> { + pub async fn start(&self, names: Vec, monitoring: bool) -> Result<()> { // In remote mode, open long-lived SSM tunnels to the Control Center // server ports (required for RPC proxy and monitoring services) if let Ok(remote_infra) = self.remote_infra() { @@ -551,7 +562,16 @@ impl Testnet { self.infra.start(&containers)?; } else { // Start the testnet following the starting heights in the manifest - self.start_from_manifest().await?; + self.start_from_manifest(monitoring).await?; + } + + if monitoring { + if let Ok(remote_infra) = self.remote_infra() { + match remote_infra.start_monitoring() { + Ok(output) => info!(%output, "✅ Monitoring started on CC"), + Err(err) => warn!("⚠️ Failed to start monitoring on CC: {err:#}"), + } + } } if let Ok(remote_infra) = self.remote_infra() { @@ -563,12 +583,14 @@ impl Testnet { info!(dir=%self.dir.display(), "✅ Testnet started"); println!("📁 Testnet files: {}", self.dir.display()); - self.print_monitoring_info(); + if monitoring { + self.print_monitoring_info(); + } Ok(()) } /// Start nodes in the testnet following their starting heights in the manifest - async fn start_from_manifest(&self) -> Result<()> { + async fn start_from_manifest(&self, monitoring: bool) -> Result<()> { // Group nodes by starting height, then sort groups by height let nodes_by_height = self .manifest @@ -612,12 +634,14 @@ impl Testnet { // Start containers associated with the node group let mut containers: Vec<_> = nodes.iter().flat_map(|n| n.container_names()).collect(); // In local mode, start monitoring services with the first group of nodes - if let Ok(local_infra) = self.local_infra() { - if started_nodes.is_empty() { - containers.extend(BLOCKSCOUT_CONTAINERS.map(String::from)); + if monitoring { + if let Ok(local_infra) = self.local_infra() { + if started_nodes.is_empty() { + containers.extend(BLOCKSCOUT_CONTAINERS.map(String::from)); - let monitoring = local_infra.monitoring.clone(); - tokio::task::spawn_blocking(move || monitoring.start()).await??; + let monitoring = local_infra.monitoring.clone(); + tokio::task::spawn_blocking(move || monitoring.start()).await??; + } } } @@ -897,6 +921,7 @@ impl Testnet { mesh_only, peers, peers_full, + duplicates, }) => { let metrics_urls = self.nodes_metadata.all_consensus_metrics_urls(); let raw_metrics = crate::mesh::fetch_all_metrics(&metrics_urls).await; @@ -911,6 +936,7 @@ impl Testnet { show_mesh: true, show_peers: peers || peers_full, show_peers_full: peers_full, + show_duplicates: duplicates, }; print!("{}", crate::mesh::format_report(&analysis, &options)); } @@ -918,25 +944,68 @@ impl Testnet { Some(InfoSubcommand::Perf { latency_only, throughput_only, + interval, + warmup_seconds, + observation_seconds, }) => { let metrics_urls = self.nodes_metadata.all_consensus_metrics_urls(); - let raw_metrics = arc_checks::fetch_all_metrics(&metrics_urls).await; - let mut nodes = arc_checks::parse_perf_metrics(&raw_metrics); + let options = arc_checks::PerfDisplayOptions { + show_latency: !throughput_only, + show_throughput: !latency_only, + show_summary: !latency_only && !throughput_only, + }; - crate::util::assign_node_groups( - nodes.iter_mut().map(|n| (n.name.as_str(), &mut n.group)), - &self.manifest.nodes, - ); + if interval { + if warmup_seconds > 0 { + println!("Warming up ({warmup_seconds}s) before first scrape..."); + tokio::time::sleep(std::time::Duration::from_secs(warmup_seconds)).await; + } + let raw_before = arc_checks::fetch_all_metrics(&metrics_urls).await; + println!("Observing ({observation_seconds}s) before second scrape..."); + tokio::time::sleep(std::time::Duration::from_secs(observation_seconds)).await; + let raw_after = arc_checks::fetch_all_metrics(&metrics_urls).await; + + let nodes = crate::util::parse_perf_metrics_delta_with_groups( + &raw_before, + &raw_after, + &self.manifest.nodes, + ); - if nodes.is_empty() { - println!("No nodes responded to metrics requests. Is the testnet running?"); + if nodes.is_empty() { + println!( + "No interval perf data (no nodes with metrics in both scrapes). Is the testnet running?" + ); + } else { + print!( + "{}", + arc_checks::format_perf_report( + &nodes, + &options, + arc_checks::PerfReportKind::Interval { + observation_secs: observation_seconds, + }, + ) + ); + } } else { - let options = arc_checks::PerfDisplayOptions { - show_latency: !throughput_only, - show_throughput: !latency_only, - show_summary: !latency_only && !throughput_only, - }; - print!("{}", arc_checks::format_perf_report(&nodes, &options)); + let raw_metrics = arc_checks::fetch_all_metrics(&metrics_urls).await; + let nodes = crate::util::parse_perf_metrics_with_groups( + &raw_metrics, + &self.manifest.nodes, + ); + + if nodes.is_empty() { + println!("No nodes responded to metrics requests. Is the testnet running?"); + } else { + print!( + "{}", + arc_checks::format_perf_report( + &nodes, + &options, + arc_checks::PerfReportKind::CumulativeSinceStart, + ) + ); + } } } Some(InfoSubcommand::Store { nodes }) => { @@ -1089,6 +1158,44 @@ impl Testnet { } // File import handled in main(); start SSM tunnels so quake commands work immediately RemoteSubcommand::Import { .. } => infra.ssm_tunnels.start().await, + RemoteSubcommand::Download { command } => { + let ts = chrono::Utc::now().format("%Y%m%d-%H%M%S"); + let resolve = |output: Option, prefix: &str| -> PathBuf { + let default = PathBuf::from(format!("{prefix}-{ts}.tar.gz")); + match output { + None => default, + Some(p) if p.is_dir() => p.join(default), + Some(p) => p, + } + }; + match command { + DownloadSubcommand::Metrics { + from, + to, + step, + metric_names, + output, + } => { + let dest = resolve(output, "quake-metrics"); + infra.download_metrics( + &metric_names, + from.map(|dt| dt.unix_secs()), + to.map(|dt| dt.unix_secs()), + step.as_deref(), + &dest, + ) + } + DownloadSubcommand::Db { + nodes, + execution_only, + consensus_only, + output, + } => { + let dest = resolve(output, "quake-db"); + infra.download_node_db(&nodes, execution_only, consensus_only, &dest) + } + } + } } } @@ -1230,6 +1337,8 @@ impl Testnet { }) .unwrap_or_default(); + let fee_recipient = node_config.and_then(|nc| nc.cl_suggested_fee_recipient); + // Generate CLI flags for the consensus layer let cli_flags = setup::generate_node_cli_flags( name, @@ -1238,6 +1347,7 @@ impl Testnet { &peers_ips, Some(self.images.cl.as_str()), &follow_endpoint_urls, + fee_recipient, ); node_metadata.consensus.set_cli_flags(cli_flags); @@ -1249,6 +1359,7 @@ impl Testnet { &peers_ips, self.images.cl_upgrade.as_deref(), &follow_endpoint_urls, + fee_recipient, ); node_metadata.consensus.set_cli_flags_upgraded(cli_flags); } diff --git a/crates/quake/src/tests/mesh.rs b/crates/quake/src/tests/mesh.rs index eb8776c..a00b6aa 100644 --- a/crates/quake/src/tests/mesh.rs +++ b/crates/quake/src/tests/mesh.rs @@ -170,6 +170,8 @@ pub(super) fn check_strict( } } +const MAX_DUPLICATE_PCT: f64 = 98.0; + /// Run mesh analysis and optionally enforce strict tier expectations. /// /// Fetches gossipsub metrics, analyzes topology, classifies and categorizes @@ -200,6 +202,7 @@ pub(super) async fn run_mesh_checks( show_mesh: true, show_peers: false, show_peers_full: false, + show_duplicates: true, }; println!(); print!("{}", format_report(&analysis, &options)); @@ -269,6 +272,31 @@ pub(super) async fn run_mesh_checks( )), } } + + for node in &nodes_data { + let mc = &node.message_counts; + if mc.unfiltered == 0 { + continue; + } + let pct = mc.duplicate_pct(); + let passed = pct <= MAX_DUPLICATE_PCT; + let check_name = if label.is_empty() { + format!("dup:{}", node.moniker) + } else { + format!("{label}:dup:{}", node.moniker) + }; + let message = format!( + "{pct:.1}% duplicates ({} / {} unfiltered, threshold {MAX_DUPLICATE_PCT:.1}%)", + mc.duplicates(), + mc.unfiltered, + ); + if passed { + checks.push(CheckResult::success(check_name, message)); + } else { + checks.push(CheckResult::failure(check_name, message)); + } + } + Ok(checks) } @@ -282,6 +310,7 @@ pub(super) async fn run_mesh_checks( /// - External validators (behind sentries): must not be isolated /// - Sentries and full nodes (consensus enabled): must not be isolated /// - Nodes with consensus disabled or follow mode: skipped +/// - Duplicate rate must stay under 98% #[quake_test(group = "mesh", name = "health")] fn health_test<'a>( testnet: &'a Testnet, diff --git a/crates/quake/src/tests/sanity.rs b/crates/quake/src/tests/sanity.rs index f1b3725..e0e86f2 100644 --- a/crates/quake/src/tests/sanity.rs +++ b/crates/quake/src/tests/sanity.rs @@ -20,7 +20,12 @@ //! //! Runs mesh connectivity checks, block-time performance assertions, and //! consensus health checks in a single pass. Both performance and health use a -//! two-scrape delta approach, measuring only the observation window. +//! two-scrape delta approach, measuring only the observation window. After health, +//! it prints a full interval performance report (same delta as the block-time checks) +//! before the short per-node pass/fail lines. For mesh, by default it also prints the +//! same detailed topology report as `quake info mesh` before the compact mesh check +//! (`mesh_verbose=false` to skip). Health already prints per-node delta lines; no separate +//! `quake info health` is needed. //! Transaction load is configurable (default: 50 TPS of native transfers for 60 seconds). //! //! Works on both local (Docker Compose) and remote (AWS EC2) testnets. @@ -40,6 +45,7 @@ //! | `load_targets` | `""` (all nodes) | Comma-separated node names to send load to | //! | `load_mix` | `transfer=100` | Tx type mix (`--mix` format; `erc20`/`guzzler` need contracts in genesis) | //! | `strict_mesh` | `true` | Enforce mesh tier expectations | +//! | `mesh_verbose` | `true` | Print full `quake info mesh`-style report before mesh checks | //! | `block_time_p50_ms` | `550` | Fail if any node's p50 block time exceeds this | //! | `block_time_p95_ms` | `1000` | Fail if any node's p95 block time exceeds this | //! @@ -50,6 +56,7 @@ //! quake test validation:basic --set load_rate=0 # no load (baseline) //! quake test validation:basic --set load_mix=transfer=70,erc20=30 //! quake test validation:basic --set load_rate=500 --set duration_s=120 --set load_targets=rpc1,rpc2 +//! quake test validation:basic --set mesh_verbose=false # shorter logs (skip full mesh topology table) //! ``` use std::time::Duration; @@ -215,6 +222,7 @@ fn basic_test<'a>( let load_targets_str = params.get_or("load_targets", ""); let load_mix = params.get_or("load_mix", DEFAULT_LOAD_MIX); let strict_mesh = params.get_or("strict_mesh", "true") == "true"; + let mesh_verbose = params.get_or("mesh_verbose", "true") == "true"; let p50_ms: u64 = params .get_or("block_time_p50_ms", &DEFAULT_P50_MS.to_string()) .parse() @@ -248,10 +256,12 @@ fn basic_test<'a>( println!(" warmup: {warmup_s}s"); println!(" duration: {duration_s}s"); println!(" load: {load_desc}"); - if !load_targets.is_empty() { + if load_rate > 0 && !load_targets.is_empty() { println!(" targets: {}", load_targets.join(", ")); + } else if load_rate > 0 { + println!(" targets: all nodes"); } - println!(" mesh: strict={strict_mesh}"); + println!(" mesh: strict={strict_mesh}, full_report={mesh_verbose}"); println!(" perf: p50 < {p50_ms}ms, p95 < {p95_ms}ms"); println!("─────────────────────────────────────────────────────\n"); @@ -322,6 +332,31 @@ fn basic_test<'a>( health_checks.push(check.into()); } + // ── Performance (full interval tables, same delta as checks) ── + let perf_interval_nodes = crate::util::parse_perf_metrics_delta_with_groups( + &raw_before, + &raw_after, + &testnet.manifest.nodes, + ); + if !perf_interval_nodes.is_empty() { + let perf_display = arc_checks::PerfDisplayOptions { + show_latency: true, + show_throughput: true, + show_summary: true, + }; + println!("\n── Performance (observation window) ─────────────────"); + print!( + "{}", + arc_checks::format_perf_report( + &perf_interval_nodes, + &perf_display, + arc_checks::PerfReportKind::Interval { + observation_secs: duration_s, + }, + ) + ); + } + // ── Performance (delta between scrapes) ──────────────────── let perf_report = arc_checks::check_block_time_delta(&raw_before, &raw_after, p50_ms, p95_ms); @@ -335,8 +370,8 @@ fn basic_test<'a>( perf_checks.push(check.into()); } - // ── Mesh ─────────────────────────────────────────────────── - let mesh_checks = run_mesh_checks(testnet, strict_mesh, "mesh", false).await?; + // ── Mesh (optional full topology table like `quake info mesh`, then checks) ── + let mesh_checks = run_mesh_checks(testnet, strict_mesh, "mesh", mesh_verbose).await?; // ── Summary ──────────────────────────────────────────────── println!("\n── Summary ──────────────────────────────────────────"); diff --git a/crates/quake/src/tests/snapshot.rs b/crates/quake/src/tests/snapshot.rs index a69a397..95c95a5 100644 --- a/crates/quake/src/tests/snapshot.rs +++ b/crates/quake/src/tests/snapshot.rs @@ -67,7 +67,7 @@ pub(crate) async fn create_snapshot( info!("🔄 Restarting {node}"); testnet - .start(vec![node.to_string()]) + .start(vec![node.to_string()], false) .await .wrap_err_with(|| format!("Failed to restart {node}"))?; @@ -179,7 +179,7 @@ pub(crate) async fn restore_from_snapshot( info!("🚀 Starting {target_node}"); testnet - .start(vec![target_node.to_string()]) + .start(vec![target_node.to_string()], false) .await .wrap_err_with(|| format!("Failed to start {target_node}"))?; diff --git a/crates/quake/src/tests/sync.rs b/crates/quake/src/tests/sync.rs index d1a107d..934efbb 100644 --- a/crates/quake/src/tests/sync.rs +++ b/crates/quake/src/tests/sync.rs @@ -92,7 +92,7 @@ fn speed_test<'a>( info!("Waiting {downtime_s}s for the network to advance..."); tokio::time::sleep(Duration::from_secs(downtime_s)).await; info!("Starting {node}"); - testnet.start(vec![node.clone()]).await?; + testnet.start(vec![node.clone()], false).await?; } let config = arc_checks::SyncSpeedConfig { diff --git a/crates/quake/src/util.rs b/crates/quake/src/util.rs index 8ef8702..da90dfc 100644 --- a/crates/quake/src/util.rs +++ b/crates/quake/src/util.rs @@ -67,6 +67,36 @@ pub fn assign_node_groups<'a>( } } +/// Parse perf metrics and assign Validator / Non-Validator groups from the manifest. +/// +/// Combines [`arc_checks::parse_perf_metrics`] with [`assign_node_groups`] (same idea as +/// [`crate::mesh::parse_and_classify_metrics`] for mesh). +pub fn parse_perf_metrics_with_groups( + raw_metrics: &[(String, String)], + manifest_nodes: &IndexMap, +) -> Vec { + let mut nodes = arc_checks::parse_perf_metrics(raw_metrics); + assign_node_groups( + nodes.iter_mut().map(|n| (n.name.as_str(), &mut n.group)), + manifest_nodes, + ); + nodes +} + +/// Parse perf metrics from the delta between two scrapes and assign groups from the manifest. +pub fn parse_perf_metrics_delta_with_groups( + raw_before: &[(String, String)], + raw_after: &[(String, String)], + manifest_nodes: &IndexMap, +) -> Vec { + let mut nodes = arc_checks::parse_perf_metrics_delta(raw_before, raw_after); + assign_node_groups( + nodes.iter_mut().map(|n| (n.name.as_str(), &mut n.group)), + manifest_nodes, + ); + nodes +} + /// Override mesh-analysis node types using the manifest's authoritative NodeType. /// /// The mesh-analysis crate infers node types from Prometheus `peer_type` labels, diff --git a/crates/quake/terraform/cc-data.yaml b/crates/quake/terraform/cc-data.yaml index 6817f74..7dd8c54 100644 --- a/crates/quake/terraform/cc-data.yaml +++ b/crates/quake/terraform/cc-data.yaml @@ -56,6 +56,135 @@ write_files: [ -f "$f" ] && [ "$(cat "$f")" != "0" ] && FAILED=1 done exit $FAILED + - path: /home/${username}/download-metrics.sh + owner: root:root + permissions: "0755" + content: | + #!/bin/bash + # Usage: download-metrics.sh [-s ] [-e ] [-t ] [metric1 metric2 ...] + # Queries Prometheus query_range and saves each metric to a JSON file, + # then creates ~/quake-metrics-tmp-.tar.gz. + # Outputs a single JSON line: {"archive":"","errors":[": ",...]} + # Per-metric failures are collected and reported; the archive is always created. + # -t sets the query resolution (interval between data points, e.g. 30s, 1m, 5m). + # Defaults to ceil((end-start)/10000) to stay within Prometheus' 11,000-point limit. + set -uo pipefail + START="" + END="" + STEP="" + while getopts "s:e:t:" OPT; do + case "$OPT" in + s) START="$OPTARG" ;; + e) END="$OPTARG" ;; + t) STEP="$OPTARG" ;; + *) exit 1 ;; + esac + done + shift "$((OPTIND-1))" + NOW=$(date +%s) + if [ -z "$START" ]; then + START=$(curl -sf http://localhost:9090/api/v1/status/tsdb | jq -r '.data.headStats.minTime / 1000 | floor' 2>/dev/null) || START=0 + fi + [ -z "$END" ] && END=$NOW + if [ -z "$STEP" ]; then + STEP=$(( (END - START + 9999) / 10000 )) + [ "$STEP" -lt 1 ] && STEP=1 + STEP="$STEP"s + fi + PARAMS="start=$START&end=$END&step=$STEP" + TMP_DIR=$(mktemp -d) + trap "rm -rf $TMP_DIR" EXIT + if [ $# -eq 0 ]; then + METRICS_RESPONSE=$(curl -sf http://localhost:9090/api/v1/label/__name__/values) + CURL_EXIT=$? + if [ $CURL_EXIT -ne 0 ]; then + printf '{"archive":"","errors":["failed to list Prometheus metric names (curl exited with code %s)"]}\n' "$CURL_EXIT" + exit 0 + fi + METRICS=$(printf '%s' "$METRICS_RESPONSE" | jq -r '.data[]') + else + METRICS=$(printf '%s\n' "$@") + fi + ARCHIVE="quake-metrics-tmp-$$.tar.gz" + ERRORS=() + while IFS= read -r M; do + if RESPONSE=$(curl -s "http://localhost:9090/api/v1/query_range?query=$M&$PARAMS" 2>/dev/null); then + STATUS=$(printf '%s' "$RESPONSE" | jq -r '.status // "error"' 2>/dev/null) || STATUS="error" + if [ "$STATUS" = "success" ]; then + printf '%s' "$RESPONSE" > "$TMP_DIR/$M.json" + else + ERR=$(printf '%s' "$RESPONSE" | jq -r '.error // "unknown error"' 2>/dev/null || echo "unknown error") + ERRORS+=("$M: $ERR") + fi + else + ERRORS+=("$M: curl failed") + fi + done <<< "$METRICS" + tar czf ~/$ARCHIVE -C "$TMP_DIR" . + if [ $${#ERRORS[@]} -eq 0 ]; then + ERRORS_JSON="[]" + else + ERRORS_JSON=$(printf '%s\n' "$${ERRORS[@]}" | jq -R . | jq -s .) + fi + printf '{"archive":"%s","errors":%s}\n' "$ARCHIVE" "$ERRORS_JSON" + - path: /home/${username}/download-db.sh + owner: root:root + permissions: "0755" + content: | + #!/bin/bash + # Usage: download-db.sh [-x] [-c] [ip2 ...] + # Archives node data from all specified nodes in parallel, then bundles + # everything into ~/quake-db-tmp-.tar.gz. + # Outputs a single JSON line: {"archive":"","errors":[": ",...]} + # Per-node failures are collected and reported; the archive is always created. + # -x: execution layer only (reth); -c: consensus layer only (malachite); default: both + set -uo pipefail + EXEC_ONLY=false + CONS_ONLY=false + while getopts "xc" OPT; do + case "$OPT" in + x) EXEC_ONLY=true ;; + c) CONS_ONLY=true ;; + *) exit 1 ;; + esac + done + shift "$((OPTIND-1))" + SSH_OPTS="-o StrictHostKeyChecking=accept-new -o LogLevel=ERROR -i ~/.ssh/id_rsa" + USER="${username}" + NODE_TMP=quake-db-node-tmp-$$.tar.gz + if [ "$EXEC_ONLY" = true ]; then + DIRS="~/data/reth" + elif [ "$CONS_ONLY" = true ]; then + DIRS="~/data/malachite" + else + DIRS="~/data/reth ~/data/malachite" + fi + ARCHIVE="quake-db-tmp-$$.tar.gz" + TMP_DIR=$(mktemp -d) + ALL_IPS="$*" + ERRORS=() + trap "rm -rf $TMP_DIR; ./pssh.sh 'rm -f ~/$NODE_TMP' $ALL_IPS 2>/dev/null || true" EXIT + PSSH_OUTPUT=$(./pssh.sh "sudo tar czf ~/$NODE_TMP $DIRS" $ALL_IPS 2>&1) + PSSH_EXIT=$? + if [ $PSSH_EXIT -ne 0 ]; then + PSSH_SUMMARY=$(printf '%s' "$PSSH_OUTPUT" | head -1) + ERRORS+=("failed to archive node data (exit $PSSH_EXIT): $PSSH_SUMMARY") + fi + for IP in $ALL_IPS; do + SCP_OUTPUT=$(scp $SSH_OPTS $USER@$IP:~/$NODE_TMP "$TMP_DIR/$IP.tar.gz" 2>&1) + SCP_EXIT=$? + if [ $SCP_EXIT -ne 0 ]; then + SCP_SUMMARY=$(printf '%s' "$SCP_OUTPUT" | head -1) + ERRORS+=("$IP: scp failed (exit $SCP_EXIT): $SCP_SUMMARY") + fi + done + tar czf ~/$ARCHIVE -C "$TMP_DIR" . + if [ $${#ERRORS[@]} -eq 0 ]; then + ERRORS_JSON="[]" + else + ERRORS_JSON=$(printf '%s\n' "$${ERRORS[@]}" | jq -R . | jq -s .) + fi + printf '{"archive":"%s","errors":%s}\n' "$ARCHIVE" "$ERRORS_JSON" runcmd: # Set up 4 GiB swap to prevent hard OOM kills (dd instead of fallocate for XFS compatibility) - dd if=/dev/zero of=/swapfile bs=1M count=4096 diff --git a/crates/remote-signer/Cargo.toml b/crates/remote-signer/Cargo.toml index 1d740d2..6ce4070 100644 --- a/crates/remote-signer/Cargo.toml +++ b/crates/remote-signer/Cargo.toml @@ -18,7 +18,7 @@ integration-remote-signer = [] arc-consensus-types = { workspace = true } async-trait = { workspace = true } backon = { workspace = true, features = ["tokio-sleep"] } -ed25519-dalek = { workspace = true } +base64 = { workspace = true } eyre = { workspace = true } hex = { workspace = true } malachitebft-core-types = { workspace = true } @@ -35,5 +35,8 @@ url = { workspace = true } protox = { workspace = true } tonic-build = { workspace = true } +[dev-dependencies] +rand = { workspace = true } + [lints] workspace = true diff --git a/crates/remote-signer/src/client.rs b/crates/remote-signer/src/client.rs index 4f76d71..f9dbd0a 100644 --- a/crates/remote-signer/src/client.rs +++ b/crates/remote-signer/src/client.rs @@ -181,13 +181,17 @@ impl RemoteSignerClient { (client, result) }; - let mut attempt = 0; + let mut attempt = 0u32; let (_client, result) = task .retry(config.retry_config) .context(grpc_client) .notify(|error, backoff| { - attempt += 1; + // Bounded by config.retry_config.max_retries + #[allow(clippy::arithmetic_side_effects)] + { + attempt += 1; + } metrics.inc_sign_request_retries(); warn!( diff --git a/crates/remote-signer/src/provider.rs b/crates/remote-signer/src/provider.rs index 9aadf4a..1460e06 100644 --- a/crates/remote-signer/src/provider.rs +++ b/crates/remote-signer/src/provider.rs @@ -19,7 +19,6 @@ use std::sync::Arc; use async_trait::async_trait; -use ed25519_dalek::{Signature as Ed25519Signature, VerifyingKey}; use eyre::eyre; use malachitebft_core_types::{Context, SignedExtension, SignedProposal, SignedVote}; @@ -42,64 +41,14 @@ pub struct RemoteSigningProvider { public_key_cache: Arc>>, } -/// Validate a signature using a specific public key (synchronous) -pub fn validate_signature_with_key( - message: &[u8], - signature: &[u8], - public_key: &[u8], -) -> Result { - // Ensure public key is 32 bytes - if public_key.len() != 32 { - return Err(SigningError::from_source(eyre!( - "Invalid public key length: expected 32 bytes, got {}", - public_key.len() - ))); - } - - // Ensure signature is 64 bytes - if signature.len() != 64 { - return Err(SigningError::from_source(eyre!( - "Invalid signature length: expected 64 bytes, got {}", - signature.len() - ))); - } - - // Convert to fixed-size arrays - let public_key_bytes: [u8; 32] = public_key.try_into().expect("Checked length above"); - let signature_bytes: [u8; 64] = signature.try_into().expect("Checked length above"); - - // Parse the public key - let verifying_key = VerifyingKey::from_bytes(&public_key_bytes) - .map_err(|_| SigningError::from_source(eyre!("Invalid public key")))?; - - // Parse the signature - let ed25519_signature = Ed25519Signature::from_bytes(&signature_bytes); - - // Verify the signature - let result = verifying_key - .verify_strict(message, &ed25519_signature) - .inspect_err(|e| { - tracing::error!( - signature = %hex::encode(signature), - public_key = %hex::encode(public_key), - "Signature verification failed: {e}" - ); - }) - .is_ok(); - - Ok(VerificationResult::from_bool(result)) -} - /// Convert raw signature bytes to Ed25519 consensus signature -pub fn bytes_to_signature(signature_bytes: &[u8]) -> Result -where -{ +pub fn bytes_to_signature(signature_bytes: &[u8]) -> Result { if signature_bytes.len() == 64 { let mut sig_array = [0u8; 64]; sig_array.copy_from_slice(signature_bytes); Ok(ConsensusSignature::from_bytes(sig_array)) } else { - Err(SigningError::from_source(eyre::eyre!( + Err(SigningError::from_source(eyre!( "Invalid signature length: expected 64 bytes, got {}", signature_bytes.len() ))) @@ -196,10 +145,20 @@ impl SigningProvider for RemoteSigningProvider { signature: &ConsensusSignature, public_key: &PublicKey, ) -> Result { - let signature_bytes = signature.to_bytes(); - let public_key_bytes = public_key.as_bytes(); - - validate_signature_with_key(bytes, &signature_bytes, public_key_bytes) + Ok(VerificationResult::from_bool( + public_key + .verify(bytes, signature) + .inspect_err(|e| { + use base64::Engine; + tracing::error!( + signature = + base64::engine::general_purpose::STANDARD.encode(signature.to_bytes()), + public_key = format!("0x{}", hex::encode(public_key.as_bytes())), + "Signature verification failed: {e}" + ); + }) + .is_ok(), + )) } async fn sign_vote(&self, vote: Vote) -> Result, SigningError> { @@ -258,61 +217,12 @@ impl SigningProvider for RemoteSigningProvider { #[cfg(test)] mod unit_tests { + use arc_consensus_types::signing::PrivateKey; use arc_consensus_types::{Address, BlockHash, Height, Round, Value, ValueId}; use malachitebft_core_types::{NilOrVal, VoteType}; use super::*; - #[test] - fn test_validate_signature_with_key_invalid_public_key_length() { - let message = b"test message"; - let signature = [0u8; 64]; - - // Test with public key too short - let short_key = [0u8; 31]; - assert!(validate_signature_with_key(message, &signature, &short_key).is_err()); - - // Test with public key too long - let long_key = [0u8; 33]; - assert!(validate_signature_with_key(message, &signature, &long_key).is_err()); - - // Test with empty public key - let empty_key = []; - assert!(validate_signature_with_key(message, &signature, &empty_key).is_err()); - } - - #[test] - fn test_validate_signature_with_key_invalid_signature_length() { - let message = b"test message"; - let public_key = [0u8; 32]; - - // Test with signature too short - let short_sig = [0u8; 63]; - assert!(validate_signature_with_key(message, &short_sig, &public_key).is_err()); - - // Test with signature too long - let long_sig = [0u8; 65]; - assert!(validate_signature_with_key(message, &long_sig, &public_key).is_err()); - - // Test with empty signature - let empty_sig = []; - assert!(validate_signature_with_key(message, &empty_sig, &public_key).is_err()); - } - - #[test] - fn test_validate_signature_with_key_invalid_public_key_parsing() { - let message = b"test message"; - let signature = [0u8; 64]; - - // Test with invalid public key bytes (all 0xFF which might be invalid for Ed25519) - let invalid_key = [0xFFu8; 32]; - // This should return false due to invalid public key parsing - let result = validate_signature_with_key(message, &signature, &invalid_key); - // Note: This test might pass or fail depending on ed25519-dalek's validation - // The important thing is that it doesn't panic - let _ = result; // Should not panic - } - #[test] fn test_bytes_to_signature_valid_length() { let valid_sig_bytes = [1u8; 64]; @@ -323,32 +233,20 @@ mod unit_tests { #[test] fn test_bytes_to_signature_invalid_length() { - // Test with signature too short let short_sig = [1u8; 32]; - let result = bytes_to_signature(&short_sig); - assert!(result.is_err()); // Should return error + assert!(bytes_to_signature(&short_sig).is_err()); - // Test with signature too long let long_sig = [1u8; 128]; - let result = bytes_to_signature(&long_sig); - assert!(result.is_err()); // Should return error + assert!(bytes_to_signature(&long_sig).is_err()); - // Test with empty signature let empty_sig = []; - let result = bytes_to_signature(&empty_sig); - assert!(result.is_err()); // Should return error + assert!(bytes_to_signature(&empty_sig).is_err()); } #[test] fn signing_provider_verify_flows() { - use ed25519_dalek::Signer; - use ed25519_dalek::SigningKey; - - // Deterministic keypair - let secret_bytes = [7u8; 32]; - let signing_key = SigningKey::from_bytes(&secret_bytes); - let verifying_key = signing_key.verifying_key(); - let public_key_bytes = verifying_key.to_bytes(); + let private_key = PrivateKey::generate(rand::thread_rng()); + let public_key = private_key.public_key(); // Vote let vote = Vote { @@ -360,11 +258,8 @@ mod unit_tests { extension: None, }; let vote_bytes = vote.to_sign_bytes(); - let vote_sig = signing_key.sign(&vote_bytes).to_bytes(); - assert!( - validate_signature_with_key(&vote_bytes, &vote_sig, &public_key_bytes) - .is_ok_and(|r| r.is_valid()) - ); + let vote_sig = private_key.sign(&vote_bytes); + assert!(public_key.verify(&vote_bytes, &vote_sig).is_ok()); // Proposal let proposal = Proposal { @@ -375,29 +270,14 @@ mod unit_tests { validator_address: Address::new([4u8; 20]), }; let proposal_bytes = proposal.to_sign_bytes(); - let proposal_sig = signing_key.sign(&proposal_bytes).to_bytes(); - assert!( - validate_signature_with_key(&proposal_bytes, &proposal_sig, &public_key_bytes) - .is_ok_and(|r| r.is_valid()) - ); - - // Vote extension - // NOTE: Disabled due to vote extensions not being supported in Arc at this time - // let ext = MockExtension; - // let ext_bytes = serialization::extension_to_bytes::(&ext); - // let ext_sig = signing_key.sign(&ext_bytes).to_bytes(); - // assert!( - // validate_signature_with_key(&ext_bytes, &ext_sig, &public_key_bytes) - // .is_ok_and(|r| r.is_valid()) - // ); + let proposal_sig = private_key.sign(&proposal_bytes); + assert!(public_key.verify(&proposal_bytes, &proposal_sig).is_ok()); // Negative case: flip a byte in the vote signature - let mut bad_sig = vote_sig; - bad_sig[0] ^= 0x01; - assert!( - validate_signature_with_key(&vote_bytes, &bad_sig, &public_key_bytes) - .is_ok_and(|r| r.is_invalid()) - ); + let mut bad_sig_bytes = vote_sig.to_bytes(); + bad_sig_bytes[0] ^= 0x01; + let bad_sig = ConsensusSignature::from_bytes(bad_sig_bytes); + assert!(public_key.verify(&vote_bytes, &bad_sig).is_err()); } } @@ -412,31 +292,6 @@ mod integration_tests { use super::*; - /// Validate a signature using the public key from the external service - /// This is a test helper method that fetches the public key and validates the signature - async fn validate_signature( - provider: &RemoteSigningProvider, - message: &[u8], - signature: &[u8], - ) -> Result { - // Get the public key for validation - let public_key = provider.public_key().await?; - - // Validate signature is 64 bytes - if signature.len() != 64 { - return Err(RemoteSigningError::InvalidResponse(format!( - "Invalid signature length: expected 64 bytes, got {}", - signature.len() - ))); - } - - // Use the synchronous validation method with raw bytes - let result = validate_signature_with_key(message, signature, public_key.as_bytes()) - .is_ok_and(|r| r.is_valid()); - - Ok(result) - } - async fn create_provider() -> Result { let config = RemoteSigningConfig::default(); RemoteSigningProvider::new(config).await @@ -534,18 +389,20 @@ mod integration_tests { #[tokio::test] async fn signature_validation_workflow() { let provider = create_provider().await.expect("Failed to create provider"); + let public_key = provider + .public_key() + .await + .expect("Failed to get public key"); let message = b"test message for validation"; - // Sign the message - let signature = provider.client.sign_message(message).await.unwrap(); + let signature_bytes = provider.client.sign_message(message).await.unwrap(); + let signature = bytes_to_signature(&signature_bytes).expect("Invalid signature"); - // Validate using the async method - let is_valid = validate_signature(&provider, message, &signature) - .await - .unwrap(); - - assert!(is_valid, "Self-signed signature should be valid"); + assert!( + public_key.verify(message.as_slice(), &signature).is_ok(), + "Self-signed signature should be valid" + ); } #[tokio::test] @@ -614,19 +471,21 @@ mod integration_tests { .block_on(async { RemoteSigningProvider::new(config).await }) .expect("Failed to create provider"); + let public_key = rt + .block_on(provider.public_key()) + .expect("Failed to get public key"); + let message = b"test message for validation"; - // Sign the message - let signature = rt + let signature_bytes = rt .block_on(async { provider.client.sign_message(message).await }) .expect("Failed to sign message"); - // Validate the signature using async method - let is_valid = rt - .block_on(async { validate_signature(&provider, message, &signature).await }) - .expect("Failed to validate signature"); - - assert!(is_valid, "Self-signed signature should be valid"); + let signature = bytes_to_signature(&signature_bytes).expect("Invalid signature"); + assert!( + public_key.verify(message.as_slice(), &signature).is_ok(), + "Self-signed signature should be valid" + ); } #[test] @@ -638,26 +497,23 @@ mod integration_tests { .block_on(RemoteSigningProvider::new(config)) .expect("Failed to create provider"); - // Get the public key let public_key = rt .block_on(provider.public_key()) .expect("Failed to get public key"); let message = b"test message"; - // Sign the message - let signature = rt + let signature_bytes = rt .block_on(async { provider.client.sign_message(message).await }) .expect("Failed to sign message"); - // Validate signature is 64 bytes - assert_eq!(signature.len(), 64, "Signature should be 64 bytes"); - - // Validate using the synchronous method (with raw bytes) - let is_valid = validate_signature_with_key(message, &signature, public_key.as_bytes()) - .is_ok_and(|r| r.is_valid()); + assert_eq!(signature_bytes.len(), 64, "Signature should be 64 bytes"); - assert!(is_valid, "Signature validation should succeed"); + let signature = bytes_to_signature(&signature_bytes).expect("Invalid signature"); + assert!( + public_key.verify(message.as_slice(), &signature).is_ok(), + "Signature validation should succeed" + ); } #[test] diff --git a/crates/shared/src/chain_ids.rs b/crates/shared/src/chain_ids.rs index b20f21d..8a66a18 100644 --- a/crates/shared/src/chain_ids.rs +++ b/crates/shared/src/chain_ids.rs @@ -19,7 +19,7 @@ //! Shared between execution-config and malachite-app to ensure consistency. /// Arc mainnet. -pub const MAINNET_CHAIN_ID: u64 = 5042000; +pub const MAINNET_CHAIN_ID: u64 = 5042; /// Arc devnet. pub const DEVNET_CHAIN_ID: u64 = 5042001; /// Arc testnet. diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 9dd7a64..8797b22 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -23,6 +23,7 @@ integration-remote-signer = ["integration", "arc-remote-signer/integration-remot arc-consensus-types = { workspace = true } arc-remote-signer = { workspace = true, optional = true } async-trait = { workspace = true } +base64 = { workspace = true } bytes = { workspace = true } hex = { workspace = true } malachitebft-signing-ed25519 = { workspace = true, optional = true } diff --git a/crates/signer/src/local.rs b/crates/signer/src/local.rs index fc978e2..ca92162 100644 --- a/crates/signer/src/local.rs +++ b/crates/signer/src/local.rs @@ -59,9 +59,11 @@ impl LocalSigningProvider { .inner() .verify(signature.inner(), data) .inspect_err(|e| { + use base64::Engine; tracing::error!( - signature = %hex::encode(signature.to_bytes()), - public_key = %hex::encode(public_key.as_bytes()), + signature = + base64::engine::general_purpose::STANDARD.encode(signature.to_bytes()), + public_key = format!("0x{}", hex::encode(public_key.as_bytes())), "Signature verification failed: {e}" ); }) diff --git a/crates/snapshots/src/download.rs b/crates/snapshots/src/download.rs index 704cf29..e3a2532 100644 --- a/crates/snapshots/src/download.rs +++ b/crates/snapshots/src/download.rs @@ -177,6 +177,7 @@ impl DownloadProgress { } } + #[allow(clippy::arithmetic_side_effects)] // f64 division and index bounded by BYTE_UNITS.len() fn format_size(size: u64) -> String { let mut size = size as f64; let mut unit_index = 0; @@ -187,6 +188,7 @@ impl DownloadProgress { format!("{:.2} {}", size, BYTE_UNITS[unit_index]) } + #[allow(clippy::arithmetic_side_effects)] // divisors are non-zero constants fn format_duration(duration: Duration) -> String { let secs = duration.as_secs(); if secs < 60 { @@ -198,8 +200,9 @@ impl DownloadProgress { } } + #[allow(clippy::arithmetic_side_effects)] // progress display math, total_size > 0 guarded fn update(&mut self, chunk_size: u64) -> Result<()> { - self.downloaded += chunk_size; + self.downloaded = self.downloaded.saturating_add(chunk_size); if self.total_size == 0 { return Ok(()); } diff --git a/crates/spammer/src/lib.rs b/crates/spammer/src/lib.rs index b265300..1abed06 100644 --- a/crates/spammer/src/lib.rs +++ b/crates/spammer/src/lib.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] mod accounts; mod cli; diff --git a/crates/spammer/src/main.rs b/crates/spammer/src/main.rs index 7314bc4..15bb1e8 100644 --- a/crates/spammer/src/main.rs +++ b/crates/spammer/src/main.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] use clap::{Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity}; diff --git a/crates/test/checks/src/lib.rs b/crates/test/checks/src/lib.rs index 544b784..e9a3cf6 100644 --- a/crates/test/checks/src/lib.rs +++ b/crates/test/checks/src/lib.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] pub mod fetch; pub mod health; @@ -34,12 +38,15 @@ pub use health::{ pub use mempool::check_mempool; pub use mesh::check_mesh; pub use mev::check_pending_state; -pub use perf::{check_block_time, check_block_time_delta, format_perf_report, parse_perf_metrics}; +pub use perf::{ + check_block_time, check_block_time_delta, format_perf_report, parse_perf_metrics, + parse_perf_metrics_delta, +}; pub use store::{check_store_pruning, collect_store_info, StoreInfo}; pub use sync_speed::{ check_sync_speed, collect_sync_speed, poll_height, SyncSpeedConfig, SyncSpeedResult, }; pub use types::{ CheckResult, HistogramStats, NodeHealthData, NodeHealthDelta, NodePerfData, PerfDisplayOptions, - Report, + PerfReportKind, Report, }; diff --git a/crates/test/checks/src/perf.rs b/crates/test/checks/src/perf.rs index 78f3c02..e5f2971 100644 --- a/crates/test/checks/src/perf.rs +++ b/crates/test/checks/src/perf.rs @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::fmt::Write; use color_eyre::eyre::Result; @@ -21,7 +22,9 @@ use prometheus_parse::{Sample, Scrape, Value}; use url::Url; use crate::fetch::fetch_all_metrics; -use crate::types::{CheckResult, HistogramStats, NodePerfData, PerfDisplayOptions, Report}; +use crate::types::{ + CheckResult, HistogramStats, NodePerfData, PerfDisplayOptions, PerfReportKind, Report, +}; /// Only lines starting with these prefixes (plus their `# TYPE`/`# HELP` /// comments) are kept from the raw scrape. Everything else is dropped before @@ -71,6 +74,15 @@ fn extract_moniker(samples: &[Sample]) -> Option { .find_map(|s| s.labels.get("moniker").map(|m| m.to_string())) } +/// Stable node id for perf: Prometheus `moniker` when present, else the +/// connection name from [`fetch_all_metrics`]. Must match everywhere we pair +/// scrapes (see [`parse_perf_metrics`], [`samples_by_moniker`]) and aligns +/// with manifest keys used by Quake's `assign_node_groups` when moniker +/// equals the manifest node name. +fn display_name_for_scrape(connection_name: &str, samples: &[Sample]) -> String { + extract_moniker(samples).unwrap_or_else(|| connection_name.to_string()) +} + /// Extract a gauge/counter value as f64. fn value_as_f64(v: &Value) -> Option { match v { @@ -267,7 +279,7 @@ pub fn parse_perf_metrics(raw_metrics: &[(String, String)]) -> Vec .filter(|(_, m)| !m.is_empty()) .map(|(name, raw)| { let samples = parse_metrics(raw); - let display_name = extract_moniker(&samples).unwrap_or_else(|| name.clone()); + let display_name = display_name_for_scrape(name, &samples); NodePerfData { name: display_name, @@ -293,6 +305,105 @@ pub fn parse_perf_metrics(raw_metrics: &[(String, String)]) -> Vec .collect() } +/// Map each node's Prometheus scrape to parsed samples keyed by +/// [`display_name_for_scrape`] (same rule as [`parse_perf_metrics`] per row). +/// +/// If two connections share the same display name, the last scrape wins — same +/// as collapsing duplicate keys; avoid duplicate monikers across endpoints. +fn samples_by_moniker(raw: &[(String, String)]) -> HashMap> { + let mut m = HashMap::new(); + for (name, raw_text) in raw { + if raw_text.is_empty() { + continue; + } + let samples = parse_metrics(raw_text); + let display_name = display_name_for_scrape(name, &samples); + m.insert(display_name, samples); + } + m +} + +fn delta_histogram_stats( + before_samples: &[Sample], + after_samples: &[Sample], + metric: &str, +) -> Option { + let b = extract_raw_histogram(before_samples, metric)?; + let a = extract_raw_histogram(after_samples, metric)?; + subtract_raw_histograms(&b, &a).and_then(|d| stats_from_raw(&d)) +} + +/// Parse performance metrics from the **delta** between two scrapes per node. +/// +/// Only nodes present in **both** scrapes are included (intersection by +/// [`display_name_for_scrape`], same pairing idea as [`crate::health::compute_health_deltas`]). +/// Histograms use the same metric names as [`parse_perf_metrics`]; percentiles apply to +/// observations recorded between the two scrapes. +pub fn parse_perf_metrics_delta( + raw_before: &[(String, String)], + raw_after: &[(String, String)], +) -> Vec { + let before_map = samples_by_moniker(raw_before); + let after_map = samples_by_moniker(raw_after); + + let mut names: Vec = after_map + .keys() + .filter(|n| before_map.contains_key(*n)) + .cloned() + .collect(); + names.sort(); + + let mut out = Vec::new(); + for name in names { + let (Some(before_samples), Some(after_samples)) = + (before_map.get(&name), after_map.get(&name)) + else { + continue; + }; + + out.push(NodePerfData { + name, + group: None, + block_time: delta_histogram_stats( + before_samples, + after_samples, + "arc_malachite_app_block_time", + ), + block_finalize_time: delta_histogram_stats( + before_samples, + after_samples, + "arc_malachite_app_block_finalize_time", + ), + block_build_time: delta_histogram_stats( + before_samples, + after_samples, + "arc_malachite_app_block_build_time", + ), + consensus_time: delta_histogram_stats( + before_samples, + after_samples, + "malachitebft_core_consensus_consensus_time", + ), + block_tx_count: delta_histogram_stats( + before_samples, + after_samples, + "arc_malachite_app_block_transactions_count", + ), + block_size: delta_histogram_stats( + before_samples, + after_samples, + "arc_malachite_app_block_size_bytes", + ), + block_gas_used: delta_histogram_stats( + before_samples, + after_samples, + "arc_malachite_app_block_gas_used", + ), + }); + } + out +} + // ── Formatting ─────────────────────────────────────────────────────────── /// Format a histogram stat value with the given decimal precision. @@ -469,7 +580,11 @@ fn write_table( } /// Build a formatted performance report from parsed node data. -pub fn format_perf_report(nodes: &[NodePerfData], options: &PerfDisplayOptions) -> String { +pub fn format_perf_report( + nodes: &[NodePerfData], + options: &PerfDisplayOptions, + kind: PerfReportKind, +) -> String { let mut out = String::new(); let sorted = sorted_by_group(nodes); let has_groups = nodes.iter().any(|n| n.group.is_some()); @@ -484,20 +599,42 @@ pub fn format_perf_report(nodes: &[NodePerfData], options: &PerfDisplayOptions) let _ = writeln!(out, "{}", "=".repeat(80)); let _ = writeln!(out, "Performance Metrics"); - if let Some(s) = max_block_stats { - let _ = writeln!( - out, - "({node_count} node{}, ~{} blocks over {}, cumulative since start)", - if node_count == 1 { "" } else { "s" }, - s.count, - fmt_duration(s.sum), - ); - } else { - let _ = writeln!( - out, - "({node_count} node{})", - if node_count == 1 { "" } else { "s" } - ); + match kind { + PerfReportKind::CumulativeSinceStart => { + if let Some(s) = max_block_stats { + let _ = writeln!( + out, + "({node_count} node{}, ~{} blocks over {}, cumulative since start)", + if node_count == 1 { "" } else { "s" }, + s.count, + fmt_duration(s.sum), + ); + } else { + let _ = writeln!( + out, + "({node_count} node{})", + if node_count == 1 { "" } else { "s" } + ); + } + } + PerfReportKind::Interval { observation_secs } => { + if let Some(s) = max_block_stats { + let _ = writeln!( + out, + "({node_count} node{}, ~{} blocks in window, observation {}s, delta between two scrapes)", + if node_count == 1 { "" } else { "s" }, + s.count, + observation_secs, + ); + } else { + let _ = writeln!( + out, + "({node_count} node{}, observation {}s, delta between two scrapes)", + if node_count == 1 { "" } else { "s" }, + observation_secs, + ); + } + } } let _ = writeln!(out, "{}", "=".repeat(80)); @@ -679,20 +816,6 @@ pub async fn check_block_time( Ok(Report { checks }) } -/// Parse raw Prometheus text into per-node raw block_time histograms. -fn parse_raw_block_time(raw_metrics: &[(String, String)]) -> Vec<(String, Option)> { - raw_metrics - .iter() - .filter(|(_, m)| !m.is_empty()) - .map(|(name, raw)| { - let samples = parse_metrics(raw); - let display_name = extract_moniker(&samples).unwrap_or_else(|| name.clone()); - let hist = extract_raw_histogram(&samples, "arc_malachite_app_block_time"); - (display_name, hist) - }) - .collect() -} - /// Assert block time p50/p95 thresholds using only the **delta** between two /// Prometheus scrapes, isolating the observation window. /// @@ -704,57 +827,44 @@ pub fn check_block_time_delta( p50_threshold_ms: u64, p95_threshold_ms: u64, ) -> Report { - let before_nodes = parse_raw_block_time(raw_before); - let after_nodes = parse_raw_block_time(raw_after); + // Single source of truth: same intersection and subtraction as the full + // perf delta report (avoids duplicate node names from Vec+HashMap pairing). + let nodes = parse_perf_metrics_delta(raw_before, raw_after); let p50_threshold_s = p50_threshold_ms as f64 / 1000.0; let p95_threshold_s = p95_threshold_ms as f64 / 1000.0; - let mut checks: Vec = after_nodes - .iter() - .filter_map(|(name, after_hist)| { - let before_hist = before_nodes - .iter() - .find(|(n, _)| n == name) - .and_then(|(_, h)| h.as_ref()); - - let delta_stats = match (before_hist, after_hist.as_ref()) { - (Some(before), Some(after)) => { - subtract_raw_histograms(before, after).and_then(|d| stats_from_raw(&d)) - } - _ => { - // Node not present in both scrapes — skip (not a perf concern) - return None; - } - }; - - Some(match delta_stats { - Some(stats) if stats.count > 0 => { - let p50_ok = stats.p50 <= p50_threshold_s; - let p95_ok = stats.p95 <= p95_threshold_s; + let mut checks: Vec = nodes + .into_iter() + .map(|node| { + let name = node.name; + match node.block_time { + Some(ref s) if s.count > 0 => { + let p50_ok = s.p50 <= p50_threshold_s; + let p95_ok = s.p95 <= p95_threshold_s; let passed = p50_ok && p95_ok; - let message = format!( - "block_time p50={:.3}s (limit {:.3}s{}) p95={:.3}s (limit {:.3}s{}) ({} blocks)", - stats.p50, - p50_threshold_s, - if p50_ok { "" } else { " EXCEEDED" }, - stats.p95, - p95_threshold_s, - if p95_ok { "" } else { " EXCEEDED" }, - stats.count, - ); CheckResult { - name: name.clone(), + name, passed, - message, + message: format!( + "block_time p50={:.3}s (limit {:.3}s{}) p95={:.3}s (limit {:.3}s{}) ({} blocks)", + s.p50, + p50_threshold_s, + if p50_ok { "" } else { " EXCEEDED" }, + s.p95, + p95_threshold_s, + if p95_ok { "" } else { " EXCEEDED" }, + s.count, + ), } } _ => CheckResult { - name: name.clone(), + name, passed: false, - message: "no block_time delta data (counter reset or no observations)".to_string(), + message: "no block_time delta data (counter reset or no observations)" + .to_string(), }, - }) + } }) .collect(); @@ -900,7 +1010,11 @@ arc_malachite_app_block_time_count{moniker="v1"} 100 fn format_report_produces_output() { let raw = vec![("node1".to_string(), sample_histogram_text())]; let nodes = parse_perf_metrics(&raw); - let report = format_perf_report(&nodes, &PerfDisplayOptions::default()); + let report = format_perf_report( + &nodes, + &PerfDisplayOptions::default(), + PerfReportKind::CumulativeSinceStart, + ); assert!(report.contains("Performance Metrics")); assert!(report.contains("Latency")); @@ -915,6 +1029,33 @@ arc_malachite_app_block_time_count{moniker="v1"} 100 assert!(nodes.is_empty()); } + #[test] + fn parse_perf_metrics_delta_block_time_count_matches_delta_check() { + let before_text = make_histogram_text("v1", &[0, 0, 0, 0, 50, 50, 50, 50], 15.0); + let after_text = make_histogram_text("v1", &[0, 0, 0, 0, 50, 90, 140, 150], 90.0); + let raw_before = vec![("node1".to_string(), before_text)]; + let raw_after = vec![("node1".to_string(), after_text)]; + let nodes = parse_perf_metrics_delta(&raw_before, &raw_after); + assert_eq!(nodes.len(), 1); + let bt = nodes[0].block_time.as_ref().expect("block_time delta"); + assert_eq!(bt.count, 100); + } + + #[test] + fn format_perf_report_interval_banner() { + let raw = vec![("n1".to_string(), sample_histogram_text())]; + let nodes = parse_perf_metrics(&raw); + let out = format_perf_report( + &nodes, + &PerfDisplayOptions::default(), + PerfReportKind::Interval { + observation_secs: 60, + }, + ); + assert!(out.contains("delta between two scrapes")); + assert!(out.contains("60")); + } + #[test] fn format_report_respects_display_options() { let raw = vec![("node1".to_string(), sample_histogram_text())]; @@ -925,7 +1066,8 @@ arc_malachite_app_block_time_count{moniker="v1"} 100 show_throughput: false, show_summary: false, }; - let report = format_perf_report(&nodes, &latency_only); + let report = + format_perf_report(&nodes, &latency_only, PerfReportKind::CumulativeSinceStart); assert!(report.contains("Latency"), "should contain Latency section"); assert!( !report.contains("Throughput"), @@ -938,7 +1080,11 @@ arc_malachite_app_block_time_count{moniker="v1"} 100 show_throughput: true, show_summary: false, }; - let report = format_perf_report(&nodes, &throughput_only); + let report = format_perf_report( + &nodes, + &throughput_only, + PerfReportKind::CumulativeSinceStart, + ); assert!(!report.contains("Latency"), "should not contain Latency"); assert!( report.contains("Throughput"), diff --git a/crates/test/checks/src/types.rs b/crates/test/checks/src/types.rs index 8c2aeb1..5b1b266 100644 --- a/crates/test/checks/src/types.rs +++ b/crates/test/checks/src/types.rs @@ -36,11 +36,21 @@ pub struct CheckResult { pub message: String, } +/// How to label performance report output (cumulative vs observation window). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PerfReportKind { + /// Histograms are cumulative since each node's process start. + CumulativeSinceStart, + /// Histograms are deltas between two scrapes (observation window). + Interval { observation_secs: u64 }, +} + /// Statistics extracted from a Prometheus histogram. /// /// Percentiles are estimated via linear interpolation between bucket /// boundaries (same algorithm as Prometheus `histogram_quantile()`). -/// Values are cumulative since process start, not windowed. +/// For cumulative parses, values cover process lifetime; for interval +/// deltas, they cover only the window between two scrapes. #[derive(Debug, Clone, Default)] pub struct HistogramStats { pub count: u64, diff --git a/crates/test/framework/src/lib.rs b/crates/test/framework/src/lib.rs index 144d925..959da7b 100644 --- a/crates/test/framework/src/lib.rs +++ b/crates/test/framework/src/lib.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] //! Arc Integration Test Framework //! diff --git a/crates/types/src/address.rs b/crates/types/src/address.rs index 5441dce..52a5ed0 100644 --- a/crates/types/src/address.rs +++ b/crates/types/src/address.rs @@ -85,10 +85,7 @@ impl Default for Address { impl fmt::Display for Address { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for byte in self.0.iter() { - write!(f, "{byte:02X}")?; - } - Ok(()) + write!(f, "{:#x}", self.0) } } @@ -177,7 +174,7 @@ mod tests { ]; let address = Address::new(address_bytes); let display_string = address.to_string(); - assert_eq!(display_string, "123456789ABCDEF0112233445566778899AABBCC"); + assert_eq!(display_string, "0x123456789abcdef0112233445566778899aabbcc"); } #[test] @@ -187,7 +184,7 @@ mod tests { let debug_string = format!("{:?}", address); assert_eq!( debug_string, - "Address(AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA)" + "Address(0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)" ); } @@ -237,6 +234,23 @@ mod tests { assert_eq!(deserialized, address); } + #[test] + fn test_address_serde_lowercase() { + let address = Address::new([ + 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, + 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, + ]); + + // Serde uses 0x + lowercase hex (same as Display) + let serialized = serde_json::to_string(&address).unwrap(); + assert_eq!(serialized, "\"0x123456789abcdef0112233445566778899aabbcc\""); + + // Deserialization is case-insensitive + let upper = "\"0x123456789ABCDEF0112233445566778899AABBCC\""; + let deserialized: Address = serde_json::from_str(upper).unwrap(); + assert_eq!(deserialized, address); + } + #[test] fn test_address_from_alloy_address() { let alloy_address = AlloyAddress::new([0x55; 20]); diff --git a/crates/types/src/codec/mod.rs b/crates/types/src/codec/mod.rs index 98fa78b..2489bdf 100644 --- a/crates/types/src/codec/mod.rs +++ b/crates/types/src/codec/mod.rs @@ -18,16 +18,8 @@ pub use malachitebft_codec::{Codec, HasEncodedLen}; /// Shared macro for implementing versioned codecs. /// -/// # Phased Rollout Strategy -/// -/// **Phase 1 (Current)**: Decoder supports both legacy and versioned messages, encoder uses legacy format -/// - Encoding: Uses legacy protobuf-only format (no version byte prefix) -/// - Decoding: Supports both legacy protobuf AND versioned messages (with version byte prefix) -/// - This ensures backward compatibility with existing nodes -/// -/// **Phase 2 (Future)**: Once all nodes are upgraded to Phase 1, enable versioned encoding -/// - Encoding: Add version byte prefix to all encoded messages -/// - Decoding: Continue supporting both formats for a transition period +/// Encoding: Adds a version byte prefix to all encoded messages. +/// Decoding: Reads the version byte prefix and decodes the rest as protobuf. /// /// # Parameters /// - `$codec_ty`: The codec type to implement for (e.g., `NetCodec`, `WalCodec`) @@ -46,7 +38,7 @@ macro_rules! impl_versioned_codec { return Err($crate::codec::error::CodecError::EmptyBytes); } - // XXX: remove after all nodes are upgraded to use versioning + // TODO: Phase 3: Remove after all nodes are upgraded to use versioning if let Ok(msg) = malachitebft_codec::Codec::decode( &$crate::codec::proto::ProtobufCodec, bytes.clone(), @@ -67,25 +59,19 @@ macro_rules! impl_versioned_codec { .map_err($crate::codec::error::CodecError::Protobuf) } - // TODO: Phase 2 - Enable versioned encoding once all nodes support decoding versioned messages - // fn encode(&self, msg: &$ty) -> Result { - // use bytes::BufMut; - // - // let encoded = - // malachitebft_codec::Codec::encode(&$crate::codec::proto::ProtobufCodec, msg) - // .map_err($crate::codec::error::CodecError::Protobuf)?; - // - // let mut result = bytes::BytesMut::with_capacity(1 + encoded.len()); - // result.put_u8($version_val as u8); - // result.put(encoded); - // - // Ok(result.freeze()) - // } - - // Phase 1: Use legacy protobuf encoding for backward compatibility fn encode(&self, msg: &$ty) -> Result { - malachitebft_codec::Codec::encode(&$crate::codec::proto::ProtobufCodec, msg) - .map_err($crate::codec::error::CodecError::Protobuf) + use bytes::BufMut; + + let encoded = + malachitebft_codec::Codec::encode(&$crate::codec::proto::ProtobufCodec, msg) + .map_err($crate::codec::error::CodecError::Protobuf)?; + + #[allow(clippy::arithmetic_side_effects)] // 1 + valid allocation length + let mut result = bytes::BytesMut::with_capacity(1 + encoded.len()); + result.put_u8($version_val as u8); + result.put(encoded); + + Ok(result.freeze()) } } }; diff --git a/crates/types/src/codec/network.rs b/crates/types/src/codec/network.rs index f6bdbe7..9b7b3d4 100644 --- a/crates/types/src/codec/network.rs +++ b/crates/types/src/codec/network.rs @@ -88,8 +88,12 @@ impl HasEncodedLen> for NetCodec { ProtobufCodec .encoded_len(response) .map_err(CodecError::Protobuf) - // TODO: Phase 2 - Add 1 for the version byte once versioned encoding is enabled - // .map(|len| len + 1) + // +1 version byte; encoded length fits in usize + .map(|len| { + #[allow(clippy::arithmetic_side_effects)] + let total = len + 1; + total + }) } } diff --git a/crates/types/src/codec/proto.rs b/crates/types/src/codec/proto.rs index 47cbdd9..c28a8ca 100644 --- a/crates/types/src/codec/proto.rs +++ b/crates/types/src/codec/proto.rs @@ -36,6 +36,13 @@ use crate::{ use super::{Codec, HasEncodedLen}; +/// Upper bound on signatures per certificate. Well above any realistic validator set size, +/// but prevents unbounded allocation from a malicious peer sending inflated repeated fields. +const MAX_SIGNATURES_PER_CERTIFICATE: usize = 1_000; + +/// Upper bound on values in a single sync response. +const MAX_SYNC_VALUES: usize = 1_000; + #[derive(Copy, Clone, Debug)] pub struct ProtobufCodec; @@ -178,6 +185,13 @@ pub fn encode_round_certificate( pub fn decode_round_certificate( certificate: proto::RoundCertificate, ) -> Result, ProtoError> { + if certificate.signatures.len() > MAX_SIGNATURES_PER_CERTIFICATE { + return Err(ProtoError::Other(format!( + "RoundCertificate signature count {} exceeds maximum {MAX_SIGNATURES_PER_CERTIFICATE}", + certificate.signatures.len(), + ))); + } + Ok(RoundCertificate { height: Height::new(certificate.height), round: Round::new(certificate.round), @@ -442,6 +456,12 @@ pub fn decode_sync_response( let response = match response { proto::sync_response::Response::ValueResponse(value_response) => { + if value_response.values.len() > MAX_SYNC_VALUES { + return Err(ProtoError::Other(format!( + "ValueResponse value count {} exceeds maximum {MAX_SYNC_VALUES}", + value_response.values.len(), + ))); + } sync::Response::ValueResponse(sync::ValueResponse::new( Height::new(value_response.start_height), value_response @@ -535,6 +555,13 @@ pub(crate) fn encode_polka_certificate( pub(crate) fn decode_polka_certificate( certificate: proto::PolkaCertificate, ) -> Result, ProtoError> { + if certificate.signatures.len() > MAX_SIGNATURES_PER_CERTIFICATE { + return Err(ProtoError::Other(format!( + "PolkaCertificate signature count {} exceeds maximum {MAX_SIGNATURES_PER_CERTIFICATE}", + certificate.signatures.len(), + ))); + } + let value_id = certificate .value_id .ok_or_else(|| ProtoError::missing_field::("value_id")) @@ -624,6 +651,13 @@ fn decode_commit_certificate_fields( value_id: Option, signatures: Vec, ) -> Result, ProtoError> { + if signatures.len() > MAX_SIGNATURES_PER_CERTIFICATE { + return Err(ProtoError::Other(format!( + "CommitCertificate signature count {} exceeds maximum {MAX_SIGNATURES_PER_CERTIFICATE}", + signatures.len(), + ))); + } + let value_id = value_id .ok_or_else(|| ProtoError::missing_field::("value_id")) .and_then(ValueId::from_proto)?; @@ -920,4 +954,217 @@ mod tests { "error should mention invalid peer ID, got: {err}" ); } + + #[test] + fn test_commit_certificate_rejects_excessive_signatures() { + let oversized: Vec = (0..MAX_SIGNATURES_PER_CERTIFICATE + 1) + .map(|_| proto::sync::CommitSignature { + validator_address: Some(proto::Address { + value: Bytes::from(vec![0u8; 20]), + }), + signature: Some(proto::Signature { + bytes: Bytes::from(vec![0u8; 64]), + }), + }) + .collect(); + + let result = decode_commit_certificate_fields( + 1, + 0, + Some(proto::ValueId { + block_hash: Bytes::from(vec![0u8; 32]), + }), + oversized, + ); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("exceeds maximum"),); + } + + #[test] + fn test_polka_certificate_rejects_excessive_signatures() { + let oversized: Vec = (0..MAX_SIGNATURES_PER_CERTIFICATE + 1) + .map(|_| proto::PolkaSignature { + validator_address: Some(proto::Address { + value: Bytes::from(vec![0u8; 20]), + }), + signature: Some(proto::Signature { + bytes: Bytes::from(vec![0u8; 64]), + }), + }) + .collect(); + + let cert = proto::PolkaCertificate { + height: 1, + round: 0, + value_id: Some(proto::ValueId { + block_hash: Bytes::from(vec![0u8; 32]), + }), + signatures: oversized, + }; + + let result = decode_polka_certificate(cert); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("exceeds maximum"),); + } + + #[test] + fn test_round_certificate_rejects_excessive_signatures() { + let oversized: Vec = (0..MAX_SIGNATURES_PER_CERTIFICATE + 1) + .map(|_| proto::RoundSignature { + vote_type: 0, + validator_address: Some(proto::Address { + value: Bytes::from(vec![0u8; 20]), + }), + signature: Some(proto::Signature { + bytes: Bytes::from(vec![0u8; 64]), + }), + value_id: None, + }) + .collect(); + + let cert = proto::RoundCertificate { + height: 1, + round: 0, + cert_type: 0, + signatures: oversized, + }; + + let result = decode_round_certificate(cert); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("exceeds maximum"),); + } + + #[test] + fn test_value_response_rejects_excessive_values() { + let oversized: Vec = (0..MAX_SYNC_VALUES + 1) + .map(|_| proto::SyncedValue { + value_bytes: Bytes::new(), + certificate: Some(proto::sync::CommitCertificate { + height: 1, + round: 0, + value_id: Some(proto::ValueId { + block_hash: Bytes::from(vec![0u8; 32]), + }), + signatures: vec![], + }), + }) + .collect(); + + let proto_response = proto::SyncResponse { + response: Some(proto::sync_response::Response::ValueResponse( + proto::ValueResponse { + start_height: 1, + values: oversized, + }, + )), + }; + + let result = decode_sync_response(proto_response); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("exceeds maximum"),); + } + + #[test] + fn test_commit_certificate_accepts_max_signatures() { + let at_limit: Vec = (0..MAX_SIGNATURES_PER_CERTIFICATE) + .map(|_| proto::sync::CommitSignature { + validator_address: Some(proto::Address { + value: Bytes::from(vec![0u8; 20]), + }), + signature: Some(proto::Signature { + bytes: Bytes::from(vec![0u8; 64]), + }), + }) + .collect(); + + let result = decode_commit_certificate_fields( + 1, + 0, + Some(proto::ValueId { + block_hash: Bytes::from(vec![0u8; 32]), + }), + at_limit, + ); + + assert!(result.is_ok()); + } + + #[test] + fn test_polka_certificate_accepts_max_signatures() { + let at_limit: Vec = (0..MAX_SIGNATURES_PER_CERTIFICATE) + .map(|_| proto::PolkaSignature { + validator_address: Some(proto::Address { + value: Bytes::from(vec![0u8; 20]), + }), + signature: Some(proto::Signature { + bytes: Bytes::from(vec![0u8; 64]), + }), + }) + .collect(); + + let cert = proto::PolkaCertificate { + height: 1, + round: 0, + value_id: Some(proto::ValueId { + block_hash: Bytes::from(vec![0u8; 32]), + }), + signatures: at_limit, + }; + + assert!(decode_polka_certificate(cert).is_ok()); + } + + #[test] + fn test_round_certificate_accepts_max_signatures() { + let at_limit: Vec = (0..MAX_SIGNATURES_PER_CERTIFICATE) + .map(|_| proto::RoundSignature { + vote_type: 0, + validator_address: Some(proto::Address { + value: Bytes::from(vec![0u8; 20]), + }), + signature: Some(proto::Signature { + bytes: Bytes::from(vec![0u8; 64]), + }), + value_id: None, + }) + .collect(); + + let cert = proto::RoundCertificate { + height: 1, + round: 0, + cert_type: 0, + signatures: at_limit, + }; + + assert!(decode_round_certificate(cert).is_ok()); + } + + #[test] + fn test_value_response_accepts_max_values() { + let at_limit: Vec = (0..MAX_SYNC_VALUES) + .map(|_| proto::SyncedValue { + value_bytes: Bytes::new(), + certificate: Some(proto::sync::CommitCertificate { + height: 1, + round: 0, + value_id: Some(proto::ValueId { + block_hash: Bytes::from(vec![0u8; 32]), + }), + signatures: vec![], + }), + }) + .collect(); + + let proto_response = proto::SyncResponse { + response: Some(proto::sync_response::Response::ValueResponse( + proto::ValueResponse { + start_height: 1, + values: at_limit, + }, + )), + }; + + assert!(decode_sync_response(proto_response).is_ok()); + } } diff --git a/crates/types/src/height.rs b/crates/types/src/height.rs index feaa1aa..104ff94 100644 --- a/crates/types/src/height.rs +++ b/crates/types/src/height.rs @@ -38,7 +38,7 @@ impl Height { } pub fn increment(&self) -> Self { - Self(self.0 + 1) + Self(self.0.checked_add(1).expect("block height overflow")) } pub fn decrement(&self) -> Option { @@ -73,7 +73,7 @@ impl malachitebft_core_types::Height for Height { const INITIAL: Self = Self(1); fn increment_by(&self, n: u64) -> Self { - Self(self.0 + n) + Self(self.0.checked_add(n).expect("block height overflow")) } fn decrement_by(&self, n: u64) -> Option { diff --git a/crates/types/src/proposal_part.rs b/crates/types/src/proposal_part.rs index d9378e8..c4c63a2 100644 --- a/crates/types/src/proposal_part.rs +++ b/crates/types/src/proposal_part.rs @@ -110,6 +110,7 @@ impl ProposalInit { } /// Approximate in-memory size, not wire size. + #[allow(clippy::arithmetic_side_effects)] // sum of fixed-size fields, cannot overflow pub fn size_bytes(&self) -> usize { std::mem::size_of_val(&self.height) + std::mem::size_of_val(&self.round) diff --git a/crates/types/src/proposer.rs b/crates/types/src/proposer.rs index 358b0f0..668a7c5 100644 --- a/crates/types/src/proposer.rs +++ b/crates/types/src/proposer.rs @@ -48,10 +48,16 @@ impl ProposerSelector for RoundRobin { assert!(round != Round::Nil && round.as_i64() >= 0); let proposer_index = { + // height >= 1 (genesis doesn't propose), round >= 0 (asserted above). + #[allow(clippy::cast_possible_truncation)] // u64 fits usize on 64-bit let height = height.as_u64() as usize; + #[allow(clippy::cast_possible_truncation)] // round asserted non-negative let round = round.as_i64() as usize; - (height - 1 + round) % validator_set.len() + #[allow(clippy::arithmetic_side_effects)] // preconditions guarantee no overflow + { + (height - 1 + round) % validator_set.len() + } }; validator_set diff --git a/crates/types/src/rpc_sync.rs b/crates/types/src/rpc_sync.rs index 02dd876..77a6101 100644 --- a/crates/types/src/rpc_sync.rs +++ b/crates/types/src/rpc_sync.rs @@ -66,7 +66,9 @@ impl SyncEndpointUrl { // If HTTP uses a non-default port, set WS port to HTTP port + 1 if let Some(http_port) = self.http.port() { - ws_url.set_port(Some(http_port + 1)).expect("valid port"); + ws_url + .set_port(Some(http_port.checked_add(1).expect("port overflow"))) + .expect("valid port"); } ws_url diff --git a/crates/types/src/validator_set.rs b/crates/types/src/validator_set.rs index 40509a7..1fdf92d 100644 --- a/crates/types/src/validator_set.rs +++ b/crates/types/src/validator_set.rs @@ -103,7 +103,10 @@ impl ValidatorSet { /// The total voting power of the validator set pub fn total_voting_power(&self) -> VotingPower { - self.validators.iter().map(|v| v.voting_power).sum() + self.validators + .iter() + .try_fold(0u64, |acc, v| acc.checked_add(v.voting_power)) + .expect("total voting power overflow") } /// Get a validator by its index @@ -181,4 +184,19 @@ mod tests { let vs = ValidatorSet::new(vec![v1, v2, v3]); assert_eq!(vs.total_voting_power(), 6); } + + #[test] + #[should_panic(expected = "total voting power overflow")] + fn total_voting_power_overflow_panics() { + let mut rng = StdRng::seed_from_u64(0x42); + + let sk1 = PrivateKey::generate(&mut rng); + let sk2 = PrivateKey::generate(&mut rng); + + let v1 = Validator::new(sk1.public_key(), u64::MAX); + let v2 = Validator::new(sk2.public_key(), 1); + + let vs = ValidatorSet::new(vec![v1, v2]); + let _ = vs.total_voting_power(); + } } diff --git a/crates/types/tests/unit/main.rs b/crates/types/tests/unit/main.rs index 04d292e..005e342 100644 --- a/crates/types/tests/unit/main.rs +++ b/crates/types/tests/unit/main.rs @@ -14,7 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::unwrap_used)] +#![allow( + clippy::arithmetic_side_effects, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] // adapted from https://github.com/informalsystems/malachite/tree/v0.4.0/code/crates/test mod certificates; diff --git a/crates/version/build.rs b/crates/version/build.rs index d749c57..d3a57fb 100644 --- a/crates/version/build.rs +++ b/crates/version/build.rs @@ -198,6 +198,7 @@ fn emit_long_version(idempotent: bool) { let version = extract_version_from_describe(&git_describe); // Extract commits since tag (e.g., "74" from "v0.2.0-rc1-74-g3ecc938-dirty") + #[allow(clippy::arithmetic_side_effects)] // build script, index after '-' char let commits_since_tag = if let Some(pos) = git_describe.find("-g") { if let Some(commits_pos) = git_describe[..pos].rfind('-') { &git_describe[commits_pos + 1..pos] diff --git a/deploy/helm/circle-chain-reth/Chart.yaml b/deploy/helm/circle-chain-reth/Chart.yaml new file mode 100644 index 0000000..54e2f09 --- /dev/null +++ b/deploy/helm/circle-chain-reth/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: circle-chain-reth +version: 0.1.0 +appVersion: "1.0" +description: EaaS deployment for arc-execution and engine-bench +dependencies: + - name: circle-helm-lib + repository: oci://ghcr.io/circlefin/circle-helm-charts + version: 4.0.0-f6299dfe diff --git a/deploy/helm/circle-chain-reth/values.yaml b/deploy/helm/circle-chain-reth/values.yaml new file mode 100644 index 0000000..02d7c6e --- /dev/null +++ b/deploy/helm/circle-chain-reth/values.yaml @@ -0,0 +1,14 @@ +# Placeholder values for EaaS engine-bench deployment. +# Actual values are provided by customized_helm_values/circle-chain-reth.yaml +# in terraform-aws-argocd-eaas. + +common: + name: circle-chain-reth + repo: 452601555275.dkr.ecr.us-east-1.amazonaws.com/loadtest/arc-execution + logTagSvc: circle-chain-reth + +deployments: + arc-execution: + +jobs: + engine-bench: diff --git a/deployments/Dockerfile.engine-bench b/deployments/Dockerfile.engine-bench new file mode 100644 index 0000000..30e0ad9 --- /dev/null +++ b/deployments/Dockerfile.engine-bench @@ -0,0 +1,81 @@ +############### +# cargo chef +############### +FROM public.ecr.aws/docker/library/rust:1.91.1-bookworm AS chef + +# Install any custom CA certificates from the certs named context. +RUN --mount=type=bind,from=certs,target=/usr/local/share/ca-certificates/ \ + update-ca-certificates -v + +COPY assets/apt/apt-retry.conf /etc/apt/apt.conf.d/apt-retry.conf +COPY assets/apt/debian.sources /etc/apt/sources.list.d/debian.sources +# Remove default sources.list to use only our fallback-enabled sources +RUN rm -f /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libclang-dev pkg-config && \ + rm -rf /var/lib/apt/lists/* +RUN cargo install cargo-chef@0.1.73 + +FROM chef AS planner + +WORKDIR /src +COPY . . + +RUN cargo chef prepare --bin arc-engine-bench --recipe-path arc-engine-bench-recipe.json + +############### +# builder image +############### +FROM chef AS builder + +RUN printf 'arc:x:999:999::/data:/bin/false\n' > /tmp/passwd && \ + printf 'arc:x:999:\n' > /tmp/group + +ARG BUILD_PROFILE=release +ARG RUSTFLAGS="" +ARG GIT_COMMIT_HASH="0000000000000000000000000000000000000000" +ARG GIT_VERSION="v0.0.0-unknown" +ARG GIT_SHORT_HASH="00000000" +ARG ARC_IDEMPOTENT_BUILD=false + +ENV CARGO_HOME=/usr/local/cargo +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_HTTP_CAINFO=/etc/ssl/certs/ca-certificates.crt +ENV RUSTFLAGS=$RUSTFLAGS +ENV VERGEN_GIT_SHA=$GIT_COMMIT_HASH +ENV VERGEN_GIT_DESCRIBE=$GIT_VERSION +ENV VERGEN_GIT_SHA_SHORT=$GIT_SHORT_HASH +ENV ARC_IDEMPOTENT_BUILD=$ARC_IDEMPOTENT_BUILD + +WORKDIR /app + +COPY --from=planner /src/arc-engine-bench-recipe.json arc-engine-bench-recipe.json +RUN --mount=type=cache,id=bench-cargo-registry,sharing=locked,target=/usr/local/cargo/registry \ + --mount=type=cache,id=bench-cargo-git,sharing=locked,target=/usr/local/cargo/git \ + --mount=type=cache,id=bench-cargo-target,sharing=locked,target=/app/target \ + cargo chef cook --bin arc-engine-bench --profile $BUILD_PROFILE --recipe-path arc-engine-bench-recipe.json + +COPY . . + +RUN --mount=type=cache,id=bench-cargo-registry,sharing=locked,target=/usr/local/cargo/registry \ + --mount=type=cache,id=bench-cargo-git,sharing=locked,target=/usr/local/cargo/git \ + --mount=type=cache,id=bench-cargo-target,sharing=locked,target=/app/target \ + cargo build --bin arc-engine-bench --profile $BUILD_PROFILE --locked && \ + if [ "$BUILD_PROFILE" = "dev" ]; then TARGET_DIR="debug"; else TARGET_DIR="$BUILD_PROFILE"; fi && \ + cp /app/target/$TARGET_DIR/arc-engine-bench /tmp/arc-engine-bench + +####################### +# dev runtime image +####################### +FROM gcr.io/distroless/cc-debian12:nonroot AS dev-runtime + +COPY --from=chef /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +COPY --from=builder /tmp/passwd /etc/passwd +COPY --from=builder /tmp/group /etc/group +COPY --from=builder /tmp/arc-engine-bench /usr/local/bin/arc-engine-bench + +USER arc + +ENTRYPOINT ["/usr/local/bin/arc-engine-bench"] +CMD ["--help"] diff --git a/deployments/monitoring/config-prometheus/prometheus.yml b/deployments/monitoring/config-prometheus/prometheus.yml index c4bcfd3..a7cc6bf 100644 --- a/deployments/monitoring/config-prometheus/prometheus.yml +++ b/deployments/monitoring/config-prometheus/prometheus.yml @@ -9,6 +9,14 @@ scrape_configs: labels: client_name: "reth" client_type: "execution" + - job_name: "arc_engine_bench_target" + metrics_path: "/" + scrape_interval: 1s + static_configs: + - targets: ['host.docker.internal:19001'] + labels: + client_name: "reth" + client_type: "execution" - job_name: "arc_execution0" metrics_path: "/" scrape_interval: 1s diff --git a/docker-bake.hcl b/docker-bake.hcl index 9f6a080..4f73036 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -9,6 +9,7 @@ variable "IMAGES" { default = [ { name = "execution" }, { name = "consensus" }, + { name = "engine-bench" }, ] } diff --git a/docs/installation.md b/docs/installation.md index 11322fc..ff4a5bd 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,11 +1,13 @@ # Install -The Arc node binaries can be installed by [building from source](#build-from-source). +The Arc node binaries can be installed in two ways: +downloading pre-built binaries via [`arcup`](#pre-built-binary), +or by building them from [source](#build-from-source). After the installation, refer to [Running an Arc Node](./running-an-arc-node.md) for how to run an Arc node. -> **Pre-built binaries** and **Docker images** are coming soon. +> **Docker:** Container images and Docker Compose instructions are coming soon. ## Versions @@ -16,6 +18,43 @@ Consult the table below to confirm which version to run for each network. |-------------|---------| | Arc Testnet | v0.6.0 | +## Pre-built Binary + +This repository includes `arcup`, a script that installs Arc node binaries +into `$ARC_BIN_DIR` directory, defaulting to `~/.arc/bin`: + +```sh +curl -L https://raw.githubusercontent.com/circlefin/arc-node/main/arcup/install | bash +``` + +More precisely, the [configured paths](./running-an-arc-node.md#configure-paths) +for Arc nodes are based on the `$ARC_HOME` variable, with `~/.arc` as default value. +If `$ARC_BIN_DIR` is not set, its default value is `$ARC_HOME/bin`, defaulting +to `~/.arc/bin`. +`$ARC_BIN_DIR` must be part of the system `PATH`. + +To be sure that the binaries installed under `$ARC_BIN_DIR` are available in +the `PATH`, load the produced environment file: + +```sh +source $ARC_HOME/env +``` + +Next, verify that the three Arc binaries are installed: + +```sh +arc-snapshots --version +arc-node-execution --version +arc-node-consensus --version +``` + +The `arcup` script should also be in the `PATH` +and can be used to update Arc binaries: + +```sh +arcup +``` + ## Build from Source The Arc node source code is available in the diff --git a/docs/running-an-arc-node.md b/docs/running-an-arc-node.md index cb8b7c0..b194d5e 100644 --- a/docs/running-an-arc-node.md +++ b/docs/running-an-arc-node.md @@ -36,18 +36,29 @@ In a simplified version, define `$ARC_HOME` and `$ARC_RUN` variables once, then use the derived variables in the remaining of this guide: ```sh +cat << "EOF" > ~/.arc_env # Base directory for Arc node data (default: ~/.arc) ARC_HOME="${ARC_HOME:-$HOME/.arc}" # Linux runtime directory: ARC_RUN="/run/arc" + # Mac OS runtime directory: #ARC_RUN="$ARC_HOME/run" ARC_EXECUTION=$ARC_HOME/execution ARC_CONSENSUS=$ARC_HOME/consensus +EOF +``` + +Source it to load these variables into your current shell session: + +```sh +source ~/.arc_env ``` +Or using the POSIX shorthand: `. ~/.arc_env` + ### Setup directories The standard installation sets up `$ARC_HOME=~/.arc` as base directory. @@ -157,7 +168,7 @@ arc-node-consensus start \ --follow \ --follow.endpoint https://rpc.drpc.testnet.arc.network,wss=rpc.drpc.testnet.arc.network \ --follow.endpoint https://rpc.quicknode.testnet.arc.network,wss=rpc.quicknode.testnet.arc.network \ - --follow.endpoint https://rpc.blockdaemon.testnet.arc.network,wss=rpc.blockdaemon.testnet.arc.network \ + --follow.endpoint https://rpc.blockdaemon.testnet.arc.network,wss=rpc.blockdaemon.testnet.arc.network/websocket \ --metrics 127.0.0.1:29000 ``` @@ -170,11 +181,6 @@ companion execution layer. The consensus layer operates in the **follow** mode. We provide three endpoints from which the node retrieves finalized blocks. -> **Note:** The Blockdaemon endpoint currently does not support WebSocket -> connections. The node will log retry warnings for this endpoint but still -> syncs correctly via the other two endpoints. HTTP block fetching from -> Blockdaemon works normally. - ### Verify operation After starting both the consensus and execution layer, wait about 30 seconds. @@ -273,15 +279,15 @@ address and port, exposing the _protected_ RPC endpoint. Check out [reth system requirements](https://reth.rs/run/system-requirements/) for more info on EL configuration. -**Note**: during periods of sustained high load, such as during startup or extended sync if the node is far behind, the execution layer memory may surge on some hardware. This should not be an issue if running with the suggested System Requirements. However, if you do observe this, you can enable backpressure to throttle the pace of execution according to the speed of disk writes, which will constrain memory growth. +**Note**: during periods of sustained high load, such as during startup or extended sync if the node is far behind, the execution layer memory may surge on some hardware. This should not be an issue if running with the suggested System Requirements. However, if you do observe this, you can enable backpressure to throttle the pace of execution according to the speed of disk writes, which will constrain memory growth. -To enable this, the `reth_` namespace should enabled on the **execution layer**: +To enable this, the `reth_` namespace should enabled on the **execution layer**: ```sh --http.api eth,net,web3,txpool,trace,debug,reth ``` -And on the **consensus layer** backpressure must be activated: +And on the **consensus layer** backpressure must be activated: ```sh --execution-persistence-backpressure \ @@ -366,7 +372,7 @@ ExecStart=/usr/local/bin/arc-node-consensus start \ --follow \ --follow.endpoint https://rpc.drpc.testnet.arc.network,wss=rpc.drpc.testnet.arc.network \ --follow.endpoint https://rpc.quicknode.testnet.arc.network,wss=rpc.quicknode.testnet.arc.network \ - --follow.endpoint https://rpc.blockdaemon.testnet.arc.network,wss=rpc.blockdaemon.testnet.arc.network \ + --follow.endpoint https://rpc.blockdaemon.testnet.arc.network,wss=rpc.blockdaemon.testnet.arc.network/websocket \ --metrics 127.0.0.1:29000 Restart=always diff --git a/scripts/genesis/genesis.ts b/scripts/genesis/genesis.ts index 45d0fed..0a1fc32 100644 --- a/scripts/genesis/genesis.ts +++ b/scripts/genesis/genesis.ts @@ -47,6 +47,11 @@ export const schemaGenesisConfig = z */ timestamp: schemaBigInt, + /** + * The coinbase of the genesis block. + */ + coinbase: schemaAddress, + /** * The prefund accounts. The balance of the account will be initialized to the specified value in the genesis block. */ @@ -137,6 +142,7 @@ export const buildGenesis = async (ctx: BuilderContext, config: GenesisConfig) = const parsed = schemaGenesisConfig.parse(config) const { timestamp, + coinbase, prefund, NativeFiatToken: nativeFiatToken, ProtocolConfig: protocolConfig, @@ -302,7 +308,7 @@ export const buildGenesis = async (ctx: BuilderContext, config: GenesisConfig) = gasLimit: toHex(protocolConfig.feeParams.blockGasLimit ?? 30_000_000n), difficulty: '0x0', mixHash: toBytes32(0n), - coinbase: protocolConfig.beneficiary, + coinbase: coinbase, number: '0x0', alloc: allocs, } diff --git a/scripts/localdev.mjs b/scripts/localdev.mjs index 7a71d7f..5626502 100755 --- a/scripts/localdev.mjs +++ b/scripts/localdev.mjs @@ -65,7 +65,8 @@ class ProcessManager { start = async (options = {}) => { this._check_not_running(options) && (await this._prepare?.(options)) return new Promise((resolve) => { - const child = spawn(this.command, this._getArgs(options), { stdio: 'inherit' }) + const command = options.bin ?? this.command + const child = spawn(command, this._getArgs(options), { stdio: 'inherit' }) child.on('spawn', () => fs.writeFileSync(this.pidfile(options), `${child.pid}`)) ;['exit', 'SIGINT', 'SIGTERM'].forEach((signal) => { process.on(signal, () => { @@ -92,16 +93,7 @@ const localdevManager = new ProcessManager({ const network = options.network ?? 'localdev' const chain_or_genesis = options.genesis ?? `arc-${network}` const blockTime = options.blockTime ?? '200ms' - const launchArgs = [ - 'run', - '--release', - ...(options.frozen ? ['--frozen'] : []), - ...(options.offline ? ['--offline'] : []), - '--package', - 'arc-node-execution', - '--bin', - 'arc-node-execution', - '--', + const nodeArgs = [ 'node', `--chain=${chain_or_genesis}`, `--config=${path.join(rootdir, `assets/localdev/reth.toml`)}`, @@ -118,7 +110,21 @@ const localdevManager = new ProcessManager({ '--invalid-tx-list-enable', '--arc.denylist.enabled', ] - return launchArgs + if (options.bin) { + return nodeArgs + } + return [ + 'run', + '--release', + ...(options.frozen ? ['--frozen'] : []), + ...(options.offline ? ['--offline'] : []), + '--package', + 'arc-node-execution', + '--bin', + 'arc-node-execution', + '--', + ...nodeArgs, + ] }, prepare: (options = {}) => { const network = options.network ?? 'localdev' @@ -146,6 +152,9 @@ const localdevManager = new ProcessManager({ if (options.genesis) { launchArgs.push(`--genesis=${options.genesis}`) } + if (options.bin) { + launchArgs.push(`--bin=${options.bin}`) + } const child = spawn(process.argv[0], launchArgs, { stdio: ['ignore', out, out], @@ -208,6 +217,9 @@ try { break case '--genesis': options.genesis = tokens[1] + break + case '--bin': + options.bin = tokens[1] } continue } diff --git a/tests/helpers/networks/index.ts b/tests/helpers/networks/index.ts index 5714a1d..98c416b 100644 --- a/tests/helpers/networks/index.ts +++ b/tests/helpers/networks/index.ts @@ -20,6 +20,8 @@ import hre from 'hardhat' import * as localdev from './localdev' import { schemaGenesisConfig } from '../../../scripts/genesis' +export { LOCALDEV_FEE_RECIPIENT } from './localdev' + /** * Get the clients for the current network * @returns The clients for the current network diff --git a/tests/helpers/networks/localdev.ts b/tests/helpers/networks/localdev.ts index a6b09d2..fdf949c 100644 --- a/tests/helpers/networks/localdev.ts +++ b/tests/helpers/networks/localdev.ts @@ -17,9 +17,14 @@ import hre from 'hardhat' import { createWalletClient, getChain } from '../../../scripts/hardhat/viem-helper' import { LocalDevAccountCreator } from '../../../scripts/genesis/AccountCreator' +import { Address } from 'viem' import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts' import { expect } from 'chai' +// Fee recipient used by all localdev validators via --suggested-fee-recipient; must match +// cl_suggested_fee_recipient in crates/quake/scenarios/localdev.toml. +export const LOCALDEV_FEE_RECIPIENT: Address = '0x65E0a200006D4FF91bD59F9694220dafc49dbBC1' + /** * Get the clients for the localdev network * @returns The clients for the localdev network diff --git a/tests/localdev/NativeFiatToken.test.ts b/tests/localdev/NativeFiatToken.test.ts index 2f55fe2..1de7722 100644 --- a/tests/localdev/NativeFiatToken.test.ts +++ b/tests/localdev/NativeFiatToken.test.ts @@ -15,7 +15,14 @@ // limitations under the License. import { expect } from 'chai' -import { balancesSnapshot, NativeCoinAuthority, NativeTransferHelper, ReceiptVerifier, getClients } from '../helpers' +import { + balancesSnapshot, + NativeCoinAuthority, + NativeTransferHelper, + ReceiptVerifier, + LOCALDEV_FEE_RECIPIENT, + getClients, +} from '../helpers' import { ProtocolConfig } from '../helpers/ProtocolConfig' import { signPermit, USDC } from '../helpers/FiatToken' import { NativeCoinControl, ERR_BLOCKED_ADDRESS } from '../helpers/NativeCoinControl' @@ -436,9 +443,10 @@ describe('NativeFiatToken', () => { const { client, operator, sender, createRandWallet } = await clients() const receiver = await createRandWallet().then((x) => x.account) - // Get beneficiary address from ProtocolConfig + // When rewardBeneficiary is zero, fees go to the genesis coinbase const protocolConfig = ProtocolConfig.attach(client) - const beneficiary = await protocolConfig.read.rewardBeneficiary() + const configBeneficiary = await protocolConfig.read.rewardBeneficiary() + const beneficiary = configBeneficiary === zeroAddress ? LOCALDEV_FEE_RECIPIENT : configBeneficiary // Blocklist the receiver address first await USDC.attach(operator).write.blacklist([receiver.address]).then(ReceiptVerifier.waitSuccess) @@ -561,7 +569,7 @@ describe('NativeFiatToken', () => { // Verify we have 1 transfer events for the chain // - Event 0: sender -> nativeTransferHelperA - receipt.verifyGasUsedApproximately(37760n).verifyEvents((ev) => { + receipt.verifyGasUsedApproximately(39918n).verifyEvents((ev) => { ev.expectCount(1).expectNativeTransfer({ from: sender, to: nativeTransferHelperA.address, amount }) }) @@ -645,7 +653,7 @@ describe('NativeFiatToken', () => { 60000n, // set gas manually ) .then(ReceiptVerifier.build) - receipt.isReverted().verifyNoEvents().verifyGasUsedApproximately(37849n) + receipt.isReverted().verifyNoEvents().verifyGasUsedApproximately(40007n) // Verify that no balance changes occurred await balances.decrease({ sender: receipt.totalFee() }).verify() @@ -663,7 +671,7 @@ describe('NativeFiatToken', () => { 120000n, // set gas manually ) .then(ReceiptVerifier.build) - receipt2.isReverted().verifyNoEvents().verifyGasUsedApproximately(37849n) + receipt2.isReverted().verifyNoEvents().verifyGasUsedApproximately(40007n) // Verify that no balance changes occurred await balances.decrease({ sender: receipt2.totalFee() }).verify() diff --git a/tests/localdev/ProtocolConfig.test.ts b/tests/localdev/ProtocolConfig.test.ts index 178e06d..a7b2913 100644 --- a/tests/localdev/ProtocolConfig.test.ts +++ b/tests/localdev/ProtocolConfig.test.ts @@ -20,6 +20,7 @@ import { privateKeyToAccount, generatePrivateKey } from 'viem/accounts' import { PublicClient, WalletClient } from '@nomicfoundation/hardhat-viem/types' import { balancesSnapshot, + LOCALDEV_FEE_RECIPIENT, ReceiptVerifier, expectAddressEq, ProtocolConfig, @@ -229,15 +230,11 @@ describe('ProtocolConfig Smoke Tests', function () { }) describe('Core Integration', function () { - it('should use ProtocolConfig beneficiary as block miner', async function () { - // Get current beneficiary from contract - const protocolConfig = ProtocolConfig.attach(publicClient) - const contractBeneficiary = await protocolConfig.read.rewardBeneficiary() + it('should use LOCALDEV_FEE_RECIPIENT as block miner', async function () { + // The mock CL propagates LOCALDEV_FEE_RECIPIENT as block.miner + const { block } = await sendTransactionAndGetBlock(parseEther('0.01'), 1000000000000n, 100000000n) - // Send transaction and verify miner and balances match contract beneficiary - await sendTransactionAndVerifyBalances({ - beneficiary: contractBeneficiary, - }) + expectAddressEq(block.miner, LOCALDEV_FEE_RECIPIENT, 'Block miner should be LOCALDEV_FEE_RECIPIENT') }) it('should reflect beneficiary changes in block mining', async function () { diff --git a/tests/localdev/genesis.test.ts b/tests/localdev/genesis.test.ts index 64f47d7..6f4727e 100644 --- a/tests/localdev/genesis.test.ts +++ b/tests/localdev/genesis.test.ts @@ -16,7 +16,17 @@ import hre from 'hardhat' import { expect } from 'chai' -import { createWalletClient, encodeDeployData, Hex, http, keccak256, parseAbi, parseGwei, toHex } from 'viem' +import { + createWalletClient, + encodeDeployData, + Hex, + http, + keccak256, + parseAbi, + parseGwei, + toHex, + zeroAddress, +} from 'viem' import { privateKeyToAccount } from 'viem/accounts' import { Denylist, @@ -182,7 +192,8 @@ describe('genesis', () => { expect(owner.toLowerCase()).to.be.eq(expectAddr.admin) expect(controller.toLowerCase()).to.be.eq(expectAddr.admin) expect(pauser.toLowerCase()).to.be.eq(expectAddr.admin) - expect(beneficiary.toLowerCase()).to.be.eq(expectAddr.proxyAdmin) + // Zero sentinel: EL honors the CL-provided --suggested-fee-recipient per validator. + expect(beneficiary.toLowerCase()).to.be.eq(zeroAddress) }) it('fee params', async () => { diff --git a/tests/localdev/subcall.test.ts b/tests/localdev/subcall.test.ts index a157a7a..90bd12c 100644 --- a/tests/localdev/subcall.test.ts +++ b/tests/localdev/subcall.test.ts @@ -20,7 +20,7 @@ import { balancesSnapshot, NativeCoinAuthority, ReceiptVerifier, getClients } fr import { USDC } from '../helpers/FiatToken' import { CallHelper } from '../helpers/CallHelper' import { callFromAddress, memoAddress, multicall3FromAddress } from '../../scripts/genesis' -import { encodeFunctionData, erc20Abi, keccak256, pad, toHex, maxUint256, Address, parseAbi, parseEther } from 'viem' +import { encodeErrorResult, encodeFunctionData, erc20Abi, keccak256, pad, toHex, maxUint256, Address, parseAbi, parseEther } from 'viem' const memoArtifact = hre.artifacts.readArtifactSync('Memo') const multicall3FromArtifact = hre.artifacts.readArtifactSync('Multicall3From') @@ -152,50 +152,32 @@ describe('Memo', () => { .verify() }) - // Indirect call through CallHelper — callFrom spoofs CallHelper as sender, transfers from its balance. - // sender → CallHelper.execute → Memo → callFrom(CallHelper, USDC, transfer) - it('indirect call via CallHelper', async () => { + // sender → CallHelper.execute → Memo → callFrom(CallHelper, USDC, transfer) → REVERT + // CallHelper ≠ tx.origin, so the sender validation rejects the spoofed sender. + it('indirect call via CallHelper rejected as sender spoofing', async () => { const { client, sender, receiver } = await clients() - const memoIndex = await readMemoIndex() const amount = USDC.parseUnits('0.001') - const nativeAmount = USDC.toNative(amount) const memoId = keccak256(toHex('indirect-memo')) const memo = toHex('indirect hello') const transferData = encodeUSDCTransfer(receiver.account.address, amount) - const callDataHash = keccak256(transferData) const memoData = encodeMemo(USDC.address, transferData, memoId, memo) - const balances = await balancesSnapshot(client, { - sender, - receiver, - callHelper: callHelper.address, - }) + const balances = await balancesSnapshot(client, { sender, receiver }) const receipt = await CallHelper.attach({ wallet: sender, public: client }, callHelper.address) .write.execute([memoAddress, memoData, 0n]) .then(ReceiptVerifier.waitSuccess) receipt.verifyEvents((ev) => { - ev.expectBeforeMemo({ memoIndex }) - .expectNativeTransfer({ from: callHelper.address, to: receiver, amount: nativeAmount }) - .expectUSDCTransfer({ from: callHelper.address, to: receiver, value: amount }) - .expectMemo({ - sender: callHelper.address, - target: USDC.address, - callDataHash, - memoId, - memo, - memoIndex, - }) - .expectExecutionResult({ helper: callHelper.address, success: true, result: '0x' }) - .expectAllEventsMatched() + ev.expectExecutionResult({ + helper: callHelper.address, + success: false, + revertString: 'sender spoofing requires tx.origin as sender', + }).expectAllEventsMatched() }) - await balances - .increase({ receiver: nativeAmount }) - .decrease({ callHelper: nativeAmount, sender: receipt.totalFee() }) - .verify() + await balances.decrease({ sender: receipt.totalFee() }).verify() }) // 3-deep recursive memo nesting, innermost does transferFrom. @@ -305,10 +287,10 @@ describe('Memo', () => { await balances.decrease({ sender: receipt.totalFee() }).verify() }) - // Child reverts — journal rollback undoes memoIndex++ and events. Only ExecutionResult survives. - // sender → CallHelper.execute → Memo → callFrom → CallHelper.revertWithString → REVERT + // sender → Memo → callFrom(sender, callHelper, revertWithString) → REVERT + // Outer tx reverts; memoIndex increment is rolled back. it('child reverts — state and memoIndex rolled back', async () => { - const { client, sender, receiver } = await clients() + const { sender } = await clients() const memoIndexBefore = await readMemoIndex() const revertData = encodeFunctionData({ @@ -318,30 +300,17 @@ describe('Memo', () => { }) const memoData = encodeMemo(callHelper.address, revertData, keccak256(toHex('revert-test')), toHex('will revert')) - const balances = await balancesSnapshot(client, { sender, receiver }) - - const receipt = await CallHelper.attach({ wallet: sender, public: client }, callHelper.address) - .write.execute([memoAddress, memoData, 0n]) - .then(ReceiptVerifier.waitSuccess) - - // Only the CallHelper ExecutionResult event survives (Memo events rolled back). - expect(receipt.receipt.logs).to.have.lengthOf(1) - receipt.verifyEvents((ev) => { - const result = ev.getExecutionResult(0) - expect(result.slice(0, 10)).to.eq('0xed1966a2') // MemoFailed selector - ev.setLogIndex(1).expectAllEventsMatched() - }) + await expect( + sender.sendTransaction({ to: memoAddress, data: memoData }), + ).to.be.rejectedWith('execution reverted') // memoIndex unchanged — the increment was inside the reverted frame const memoIndexAfter = await readMemoIndex() expect(memoIndexAfter).to.eq(memoIndexBefore) - - // No balance changes except gas - await balances.decrease({ sender: receipt.totalFee() }).verify() }) - // Custom error propagation: ErrorMessage wraps inside MemoFailed through multiple layers. - // sender → CallHelper.execute → Memo → callFrom → CallHelper.revertWithError → REVERT + // sender → Memo → callFrom(sender, callHelper, revertWithError) → REVERT + // MemoFailed wraps the inner ErrorMessage. it('child reverts with custom error — error propagated through MemoFailed', async () => { const { client, sender } = await clients() @@ -352,64 +321,38 @@ describe('Memo', () => { }) const memoData = encodeMemo(callHelper.address, revertData, keccak256(toHex('error-prop')), toHex('error test')) - const receipt = await CallHelper.attach({ wallet: sender, public: client }, callHelper.address) - .write.execute([memoAddress, memoData, 0n]) - .then(ReceiptVerifier.waitSuccess) + // Build the expected nested error: MemoFailed(abi.encode(ErrorMessage('custom error message'))) + const innerError = encodeErrorResult({ abi: CallHelper.abi, errorName: 'ErrorMessage', args: ['custom error message'] }) + const expectedError = encodeErrorResult({ abi: memoArtifact.abi, errorName: 'MemoFailed', args: [innerError] }) - expect(receipt.receipt.logs).to.have.lengthOf(1) - receipt.verifyEvents((ev) => { - const result = ev.getExecutionResult(0) - // Outer: MemoFailed(bytes) selector = 0xed1966a2 - expect(result.slice(0, 10)).to.eq('0xed1966a2') - // The inner bytes contain ErrorMessage("custom error message") selector = 0xa183e9a5 - expect(result).to.contain('a183e9a5') - ev.setLogIndex(1).expectAllEventsMatched() - }) + // eth_call surfaces raw revert data (sendTransaction doesn't via viem) + interface ChainedError { data?: string; cause?: ChainedError } + const err = await client.call({ account: sender.account.address, to: memoAddress, data: memoData }).catch((e: unknown) => e as ChainedError) + const findData = (e?: ChainedError): string | undefined => e?.data?.startsWith('0x') ? e.data : e?.cause ? findData(e.cause) : undefined + const rawRevert = findData(err as ChainedError) + expect(rawRevert).to.equal(expectedError, 'revert data should be MemoFailed(ErrorMessage)') }) // Two sequential txs: CALL succeeds, then STATICCALL reverts. State from first persists. - // tx1: sender → CallHelper.execute → Memo → callFrom → USDC.transfer (success) - // tx2: sender → CallHelper.staticCall → Memo → REVERT (read-only) + // tx1: sender → Memo → callFrom(sender, USDC, transfer) (success) + // tx2: sender → CallHelper.staticCall → Memo → REVERT (read-only context) it('call then static call — first succeeds, second reverts', async () => { const { client, sender, receiver } = await clients() - const memoIndex = await readMemoIndex() const amount = USDC.parseUnits('0.001') const nativeAmount = USDC.toNative(amount) const memoIdCall = keccak256(toHex('call-then-static')) const memo = toHex('first call') const transferData = encodeUSDCTransfer(receiver.account.address, amount) - const callDataHash = keccak256(transferData) const memoData = encodeMemo(USDC.address, transferData, memoIdCall, memo) - const balances = await balancesSnapshot(client, { - sender, - receiver, - callHelper: callHelper.address, - }) - - // Transaction 1: execute (CALL) succeeds - const receipt1 = await CallHelper.attach({ wallet: sender, public: client }, callHelper.address) - .write.execute([memoAddress, memoData, 0n]) - .then(ReceiptVerifier.waitSuccess) + const balances = await balancesSnapshot(client, { sender, receiver }) - receipt1.verifyEvents((ev) => { - ev.expectBeforeMemo({ memoIndex }) - .expectNativeTransfer({ from: callHelper.address, to: receiver, amount: nativeAmount }) - .expectUSDCTransfer({ from: callHelper.address, to: receiver, value: amount }) - .expectMemo({ - sender: callHelper.address, - target: USDC.address, - callDataHash, - memoId: memoIdCall, - memo, - memoIndex, - }) - .expectExecutionResult({ helper: callHelper.address, success: true, result: '0x' }) - .expectAllEventsMatched() - }) + // Transaction 1: direct EOA → Memo (CALL) succeeds + // Event sequence (BeforeMemo → NativeTransfer → USDCTransfer → Memo) covered by "direct call to Memo" test above + const receipt1 = await sender.sendTransaction({ to: memoAddress, data: memoData }).then(ReceiptVerifier.waitSuccess) - // Transaction 2: staticCall should fail + // Transaction 2: staticCall should fail (static context rejected before sender check) const receipt2 = await CallHelper.attach({ wallet: sender, public: client }, callHelper.address) .write.staticCall([memoAddress, memoData]) .then(ReceiptVerifier.waitSuccess) @@ -421,10 +364,7 @@ describe('Memo', () => { // Only the first call transferred funds await balances .increase({ receiver: nativeAmount }) - .decrease({ - callHelper: nativeAmount, - sender: receipt1.totalFee() + receipt2.totalFee(), - }) + .decrease({ sender: nativeAmount + receipt1.totalFee() + receipt2.totalFee() }) .verify() }) @@ -741,77 +681,4 @@ describe('Memo + Multicall3From', () => { .verify() }) - // Two sequential memos in a single tx via executeBatch — tests callFrom depth-keyed - // continuation storage doesn't collide; memoIndex increments correctly across both. - // sender → CallHelper.executeBatch → Memo × 2 → callFrom(CallHelper, USDC, transfer) × 2 - it('batch sequential memo in one transaction', async () => { - const { client, sender, receiver } = await clients() - - const memoIndex = await readMemoIndex() - const amount1 = USDC.parseUnits('0.0001') - const amount2 = USDC.parseUnits('0.0002') - const nativeAmount1 = USDC.toNative(amount1) - const nativeAmount2 = USDC.toNative(amount2) - const batchMemoId1 = keccak256(toHex('batch-1')) - const batchMemoId2 = keccak256(toHex('batch-2')) - const batchMemo1 = toHex('first') - const batchMemo2 = toHex('second') - - const transferData1 = encodeUSDCTransfer(receiver.account.address, amount1) - const transferData2 = encodeUSDCTransfer(receiver.account.address, amount2) - const memoData1 = encodeMemo(USDC.address, transferData1, batchMemoId1, batchMemo1) - const memoData2 = encodeMemo(USDC.address, transferData2, batchMemoId2, batchMemo2) - - const balances = await balancesSnapshot(client, { - sender, - receiver, - callHelper: callHelper.address, - }) - - const receipt = await CallHelper.attach({ wallet: sender, public: client }, callHelper.address) - .write.executeBatch([ - [ - { target: memoAddress, allowFailure: false, value: 0n, callData: memoData1 }, - { target: memoAddress, allowFailure: false, value: 0n, callData: memoData2 }, - ], - ]) - .then(ReceiptVerifier.waitSuccess) - - receipt.verifyEvents((ev) => { - // First call - ev.expectBeforeMemo({ memoIndex }) - .expectNativeTransfer({ from: callHelper.address, to: receiver, amount: nativeAmount1 }) - .expectUSDCTransfer({ from: callHelper.address, to: receiver, value: amount1 }) - .expectMemo({ - sender: callHelper.address, - target: USDC.address, - callDataHash: keccak256(transferData1), - memoId: batchMemoId1, - memo: batchMemo1, - memoIndex, - }) - .expectExecutionResult({ helper: callHelper.address, success: true, result: '0x' }) - // Second call - .expectBeforeMemo({ memoIndex: memoIndex + 1n }) - .expectNativeTransfer({ from: callHelper.address, to: receiver, amount: nativeAmount2 }) - .expectUSDCTransfer({ from: callHelper.address, to: receiver, value: amount2 }) - .expectMemo({ - sender: callHelper.address, - target: USDC.address, - callDataHash: keccak256(transferData2), - memoId: batchMemoId2, - memo: batchMemo2, - memoIndex: memoIndex + 1n, - }) - .expectExecutionResult({ helper: callHelper.address, success: true, result: '0x' }) - .expectAllEventsMatched() - }) - - // Both transfers from callHelper - const totalNative = nativeAmount1 + nativeAmount2 - await balances - .increase({ receiver: totalNative }) - .decrease({ callHelper: totalNative, sender: receipt.totalFee() }) - .verify() - }) }) diff --git a/tests/simulation/ProtocolConfig.test.ts b/tests/simulation/ProtocolConfig.test.ts index 46d586f..6a1b3f6 100644 --- a/tests/simulation/ProtocolConfig.test.ts +++ b/tests/simulation/ProtocolConfig.test.ts @@ -17,7 +17,13 @@ import { expect } from 'chai' import hre from 'hardhat' import { getChain } from '../../scripts/hardhat/viem-helper' -import { ProtocolConfig, loadGenesisConfig, type FeeParams, type ConsensusParams } from '../helpers' +import { + ProtocolConfig, + LOCALDEV_FEE_RECIPIENT, + loadGenesisConfig, + type FeeParams, + type ConsensusParams, +} from '../helpers' import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts' import { Address, decodeFunctionResult, encodeFunctionData, parseAbi, zeroAddress } from 'viem' import { multicall3Address } from '../../scripts/genesis' @@ -62,11 +68,14 @@ describe('ProtocolConfig simulation', () => { }) describe('ProtocolConfig Network Parameter Validation', () => { - it('should verify network block miner matches ProtocolConfig beneficiary', async () => { + it('should use LOCALDEV_FEE_RECIPIENT as block miner when rewardBeneficiary is zero', async () => { const { client, randomWallet, protocolConfig } = await clients() - const beneficiary = await protocolConfig.read.rewardBeneficiary() + const protocolConfigBeneficiary = await protocolConfig.read.rewardBeneficiary() + expect(protocolConfigBeneficiary).to.addressEqual( + zeroAddress, + 'rewardBeneficiary should be zero address in localdev genesis', + ) const controller = await protocolConfig.read.controller() - // Simulate transactions without changing state const result = await client.simulateBlocks({ blocks: [ @@ -105,8 +114,8 @@ describe('ProtocolConfig simulation', () => { expect(block.miner).to.not.be.undefined expect(block.miner).to.addressEqual( - beneficiary, - `Simulated block miner (${block.miner}) should use original beneficiary (${beneficiary})`, + LOCALDEV_FEE_RECIPIENT, + `Simulated block miner (${block.miner}) should use LOCALDEV_FEE_RECIPIENT`, ) // verify the beneficiary read call returns the updated value diff --git a/tests/simulation/native_transfer.test.ts b/tests/simulation/native_transfer.test.ts index d01f88f..ce83f95 100644 --- a/tests/simulation/native_transfer.test.ts +++ b/tests/simulation/native_transfer.test.ts @@ -16,10 +16,12 @@ import hre from 'hardhat' import { expect } from 'chai' -import { encodeFunctionData, parseEther, parseEventLogs } from 'viem' +import { erc20Abi, encodeFunctionData, parseEther, parseEventLogs } from 'viem' import { getChain } from '../../scripts/hardhat/viem-helper' import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts' -import { NativeCoinAuthority } from '../helpers' + +// EIP-7708 system address +const EIP7708_SYSTEM_ADDRESS = '0xfffffffffffffffffffffffffffffffffffffffe' /** * Tests for native token transfers @@ -75,14 +77,14 @@ describe('Native Transfer Tests', () => { expect(calls[0].logs).to.have.lengthOf(1, 'One native transfer event from the EOA to the contract') const events = parseEventLogs({ - abi: NativeCoinAuthority.abi, - eventName: 'NativeCoinTransferred', + abi: erc20Abi, + eventName: 'Transfer', logs: calls[0].logs, - }) + }).filter((x) => x.address.toLowerCase() === EIP7708_SYSTEM_ADDRESS) expect(events).to.have.lengthOf(1) expect(events[0].args.from).to.equal(caller) expect(events[0].args.to).to.equal(nativeTransferHelperAddress) - expect(events[0].args.amount).to.equal(1n) + expect(events[0].args.value).to.equal(1n) }) it('CREATE + call value that overflows recipient balance does not emit an event', async () => { @@ -135,14 +137,14 @@ describe('Native Transfer Tests', () => { expect(calls[0].logs).to.have.lengthOf(1, 'One native transfer event from the EOA to the contract') const events = parseEventLogs({ - abi: NativeCoinAuthority.abi, - eventName: 'NativeCoinTransferred', + abi: erc20Abi, + eventName: 'Transfer', logs: calls[0].logs, - }) + }).filter((x) => x.address.toLowerCase() === EIP7708_SYSTEM_ADDRESS) expect(events).to.have.lengthOf(1) expect(events[0].args.from).to.equal(caller) expect(events[0].args.to).to.equal(nativeTransferHelperAddress) - expect(events[0].args.amount).to.equal(1n) + expect(events[0].args.value).to.equal(1n) }) it('CALL with value that overflows recipient balance does not emit an event', async () => { @@ -192,13 +194,13 @@ describe('Native Transfer Tests', () => { expect(calls[0].status).to.equal('success', 'Overall txn should succeed') const events = parseEventLogs({ - abi: NativeCoinAuthority.abi, - eventName: 'NativeCoinTransferred', + abi: erc20Abi, + eventName: 'Transfer', logs: calls[0].logs, - }) + }).filter((x) => x.address.toLowerCase() === EIP7708_SYSTEM_ADDRESS) expect(events).to.have.lengthOf(1) expect(events[0].args.from).to.equal(caller) expect(events[0].args.to).to.equal(callHelperAddress) - expect(events[0].args.amount).to.equal(1n) + expect(events[0].args.value).to.equal(1n) }) })