diff --git a/.github/workflows/build-nethermind-packages.yml b/.github/workflows/build-nethermind-packages.yml index 4155421f3cc..2fd74db605e 100644 --- a/.github/workflows/build-nethermind-packages.yml +++ b/.github/workflows/build-nethermind-packages.yml @@ -22,7 +22,7 @@ jobs: run: | sudo apt-get update && sudo apt-get install xmlstarlet -y --no-install-recommends version_prefix=$(xmlstarlet sel -t -v "//Project/PropertyGroup/VersionPrefix" Directory.Build.props) - version_suffix=$(xmlstarlet sel -t -v "//Project/PropertyGroup/VersionSuffix" Directory.Build.props) + version_suffix=$(xmlstarlet sel -t -v "//Project/PropertyGroup/VersionSuffix" Directory.Build.props 2>/dev/null || echo "") version=$([[ -n "$version_suffix" ]] && echo "$version_prefix-$version_suffix" || echo "$version_prefix") echo "Detected version $version" echo "PACKAGE_PREFIX=nethermind-$version-${GITHUB_SHA:0:8}" >> $GITHUB_ENV diff --git a/Dockerfile b/Dockerfile index 8b491797c68..fad4df00175 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0.306-noble@sha256:953b8dd2d8e25c934579905b00d7077c5622632ff617f471a211ce9b72013205 AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0.306-noble@sha256:d88e637d15248531967111fba05c6725ad45ea1dace3d35d8cfe2c4d4094e25d AS build ARG BUILD_CONFIG=release ARG CI=true @@ -25,7 +25,7 @@ RUN arch=$([ "$TARGETARCH" = "amd64" ] && echo "x64" || echo "$TARGETARCH") && \ # A temporary symlink to support the old executable name RUN ln -sr /publish/nethermind /publish/Nethermind.Runner -FROM mcr.microsoft.com/dotnet/aspnet:9.0.10-noble@sha256:d3c20e8e331018eb5e7402066fde168304b7c605ecde4dbadb40872dc8bf28db +FROM mcr.microsoft.com/dotnet/aspnet:9.0.10-noble@sha256:e183ff1306983bdb53592c3a76503c1c003461e2acd8fa7c62e04dff35819ee8 WORKDIR /nethermind diff --git a/Dockerfile.chiseled b/Dockerfile.chiseled index f0db4430f7a..f5b27248434 100644 --- a/Dockerfile.chiseled +++ b/Dockerfile.chiseled @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0.306-noble@sha256:953b8dd2d8e25c934579905b00d7077c5622632ff617f471a211ce9b72013205 AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0.306-noble@sha256:d88e637d15248531967111fba05c6725ad45ea1dace3d35d8cfe2c4d4094e25d AS build ARG BUILD_CONFIG=release ARG CI=true diff --git a/scripts/build/Dockerfile b/scripts/build/Dockerfile index d08a89349ec..49bbc7ab59b 100644 --- a/scripts/build/Dockerfile +++ b/scripts/build/Dockerfile @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM mcr.microsoft.com/dotnet/sdk:9.0.306-noble@sha256:953b8dd2d8e25c934579905b00d7077c5622632ff617f471a211ce9b72013205 +FROM mcr.microsoft.com/dotnet/sdk:9.0.306-noble@sha256:d88e637d15248531967111fba05c6725ad45ea1dace3d35d8cfe2c4d4094e25d ARG COMMIT_HASH ARG SOURCE_DATE_EPOCH diff --git a/src/Nethermind/Chains/foundation.json b/src/Nethermind/Chains/foundation.json index 554155c6879..30a9ec37e5e 100644 --- a/src/Nethermind/Chains/foundation.json +++ b/src/Nethermind/Chains/foundation.json @@ -199,6 +199,14 @@ "eip7251TransitionTimestamp": "0x681b3057", "eip7702TransitionTimestamp": "0x681b3057", "eip7623TransitionTimestamp": "0x681b3057", + "eip7594TransitionTimestamp": "0x6930b057", + "eip7823TransitionTimestamp": "0x6930b057", + "eip7825TransitionTimestamp": "0x6930b057", + "eip7883TransitionTimestamp": "0x6930b057", + "eip7918TransitionTimestamp": "0x6930b057", + "eip7934TransitionTimestamp": "0x6930b057", + "eip7939TransitionTimestamp": "0x6930b057", + "eip7951TransitionTimestamp": "0x6930b057", "depositContractAddress": "0x00000000219ab540356cbb839cbe05303d7705fa", "terminalTotalDifficulty": "C70D808A128D7380000", "blobSchedule": [ @@ -208,6 +216,20 @@ "target": 6, "max": 9, "baseFeeUpdateFraction": "0x4c6964" + }, + { + "name": "bpo1", + "timestamp": "0x69383057", + "target": 10, + "max": 15, + "baseFeeUpdateFraction": "0x7f5a51" + }, + { + "name": "bpo2", + "timestamp": "0x695db057", + "target": 14, + "max": 21, + "baseFeeUpdateFraction": "0xb24b3f" } ] }, diff --git a/src/Nethermind/Directory.Build.props b/src/Nethermind/Directory.Build.props index 4f9ae5b1a3b..ccd398530a8 100644 --- a/src/Nethermind/Directory.Build.props +++ b/src/Nethermind/Directory.Build.props @@ -5,7 +5,7 @@ $(SOURCE_DATE_EPOCH) $([System.DateTimeOffset]::UtcNow.ToUnixTimeSeconds()) - 1.35.0 + 1.35.1 diff --git a/src/Nethermind/Nethermind.Benchmark/Evm/MemoryCostBenchmark.cs b/src/Nethermind/Nethermind.Benchmark/Evm/MemoryCostBenchmark.cs index a2f6ac7b7fe..54092dc5652 100644 --- a/src/Nethermind/Nethermind.Benchmark/Evm/MemoryCostBenchmark.cs +++ b/src/Nethermind/Nethermind.Benchmark/Evm/MemoryCostBenchmark.cs @@ -9,8 +9,7 @@ namespace Nethermind.Benchmarks.Evm { public class MemoryCostBenchmark { - private IEvmMemory _current = new EvmPooledMemory(); - private IEvmMemory _improved = new EvmPooledMemory(); + private readonly IEvmMemory _current = new EvmPooledMemory(); private UInt256 _location; private UInt256 _length; diff --git a/src/Nethermind/Nethermind.Blockchain.Test/BlockhashProviderTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/BlockhashProviderTests.cs index 0a04c6bce7f..5020c48f7ef 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/BlockhashProviderTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/BlockhashProviderTests.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using FluentAssertions; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; @@ -9,12 +10,11 @@ using Nethermind.Core.Specs; using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; +using Nethermind.Evm.State; using Nethermind.Logging; using Nethermind.Specs; using Nethermind.Specs.Forks; using Nethermind.Specs.Test; -using Nethermind.Evm.State; -using Nethermind.State; using NUnit.Framework; namespace Nethermind.Blockchain.Test; @@ -80,7 +80,8 @@ public void Can_lookup_correctly_on_disconnected_main_chain() BlockhashProvider provider = CreateBlockHashProvider(tree, Frontier.Instance); BlockHeader notCanonParent = tree.FindHeader(chainLength - 4, BlockTreeLookupOptions.None)!; - BlockHeader expectedHeader = tree.FindHeader(chainLength - 3, BlockTreeLookupOptions.None)!; + Block expected = tree.FindBlock(chainLength - 3, BlockTreeLookupOptions.None)!; + Block headParent = tree.FindBlock(chainLength - 2, BlockTreeLookupOptions.None)!; Block head = tree.FindBlock(chainLength - 1, BlockTreeLookupOptions.None)!; @@ -88,12 +89,12 @@ public void Can_lookup_correctly_on_disconnected_main_chain() tree.Insert(branch, BlockTreeInsertBlockOptions.SaveHeader).Should().Be(AddBlockResult.Added); tree.UpdateMainChain(branch); // Update branch - tree.UpdateMainChain([headParent, head], true); // Update back to original again, but skipping the branch block. + tree.UpdateMainChain([expected, headParent, head], true); // Update back to original again, but skipping the branch block. Block current = Build.A.Block.WithParent(head).TestObject; // At chainLength Hash256? result = provider.GetBlockhash(current.Header, chainLength - 3); - Assert.That(result, Is.EqualTo(expectedHeader.Hash)); + Assert.That(result, Is.EqualTo(expected.Header.Hash)); } [Test, MaxTime(Timeout.MaxTestTime)] diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/HeaderStoreTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/HeaderStoreTests.cs index bc1d967eb9e..c918b6a3a84 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Blocks/HeaderStoreTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Blocks/HeaderStoreTests.cs @@ -17,7 +17,7 @@ public class HeaderStoreTests [Test] public void TestCanStoreAndGetHeader() { - HeaderStore store = new(new MemDb(), new MemDb()); + IHeaderStore store = new HeaderStore(new MemDb(), new MemDb()); BlockHeader header = Build.A.BlockHeader.WithNumber(100).TestObject; BlockHeader header2 = Build.A.BlockHeader.WithNumber(102).TestObject; @@ -39,7 +39,7 @@ public void TestCanStoreAndGetHeader() public void TestCanReadHeaderStoredWithHash() { IDb headerDb = new MemDb(); - HeaderStore store = new(headerDb, new MemDb()); + IHeaderStore store = new HeaderStore(headerDb, new MemDb()); BlockHeader header = Build.A.BlockHeader.WithNumber(100).TestObject; headerDb.Set(header.Hash!, new HeaderDecoder().Encode(header).Bytes); @@ -50,7 +50,7 @@ public void TestCanReadHeaderStoredWithHash() [Test] public void TestCanReadCacheHeader() { - HeaderStore store = new(new MemDb(), new MemDb()); + IHeaderStore store = new HeaderStore(new MemDb(), new MemDb()); BlockHeader header = Build.A.BlockHeader.WithNumber(100).TestObject; store.Cache(header); @@ -60,7 +60,7 @@ public void TestCanReadCacheHeader() [Test] public void TestCanDeleteHeader() { - HeaderStore store = new(new MemDb(), new MemDb()); + IHeaderStore store = new HeaderStore(new MemDb(), new MemDb()); BlockHeader header = Build.A.BlockHeader.WithNumber(100).TestObject; store.Insert(header); store.Delete(header.Hash!); @@ -72,7 +72,7 @@ public void TestCanDeleteHeader() public void TestCanDeleteHeaderStoredWithHash() { IDb headerDb = new MemDb(); - HeaderStore store = new(headerDb, new MemDb()); + IHeaderStore store = new HeaderStore(headerDb, new MemDb()); BlockHeader header = Build.A.BlockHeader.WithNumber(100).TestObject; headerDb.Set(header.Hash!, new HeaderDecoder().Encode(header).Bytes); diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs index c17cd8becb1..cb4505b4478 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs @@ -554,7 +554,15 @@ public AddBlockResult SuggestBlock(Block block, BlockTreeSuggestOptions options return blockHash is null ? null : FindHeader(blockHash, options, blockNumber: number); } - public Hash256? FindBlockHash(long blockNumber) => GetBlockHashOnMainOrBestDifficultyHash(blockNumber); + public Hash256? FindBlockHash(long blockNumber) + { + Hash256? blockHash = _headerStore.GetBlockHash(blockNumber); + if (blockHash is not null) + { + return blockHash; + } + return GetBlockHashOnMainOrBestDifficultyHash(blockNumber); + } public bool HasBlock(long blockNumber, Hash256 blockHash) => _blockStore.HasBlock(blockNumber, blockHash); @@ -566,7 +574,17 @@ public AddBlockResult SuggestBlock(Block block, BlockTreeSuggestOptions options return null; } - BlockHeader? header = _headerStore.Get(blockHash, shouldCache: false, blockNumber: blockNumber); + BlockHeader? header = null; + if ((options & BlockTreeLookupOptions.RequireCanonical) == 0) + { + header = _headerStore.GetFromCache(blockHash); + if (header is not null && (!blockNumber.HasValue || header.Number == blockNumber.Value)) + { + return header; + } + } + + header = _headerStore.Get(blockHash, out bool fromCache, shouldCache: false, blockNumber: blockNumber); if (header is null) { bool allowInvalid = (options & BlockTreeLookupOptions.AllowInvalid) == BlockTreeLookupOptions.AllowInvalid; @@ -583,6 +601,7 @@ public AddBlockResult SuggestBlock(Block block, BlockTreeSuggestOptions options bool createLevelIfMissing = (options & BlockTreeLookupOptions.DoNotCreateLevelIfMissing) == BlockTreeLookupOptions.None; bool requiresCanonical = (options & BlockTreeLookupOptions.RequireCanonical) == BlockTreeLookupOptions.RequireCanonical; + bool isMainChain = false; if ((totalDifficultyNeeded && header.TotalDifficulty is null) || requiresCanonical) { (BlockInfo blockInfo, ChainLevelInfo level) = LoadInfo(header.Number, header.Hash, true); @@ -609,16 +628,16 @@ public AddBlockResult SuggestBlock(Block block, BlockTreeSuggestOptions options SetTotalDifficultyFromBlockInfo(header, blockInfo); } + isMainChain = level?.MainChainBlock?.BlockHash?.Equals(blockHash) == true; if (requiresCanonical) { - bool isMain = level.MainChainBlock?.BlockHash?.Equals(blockHash) == true; - header = isMain ? header : null; + header = isMainChain ? header : null; } } - if (header is not null && ShouldCache(header.Number)) + if (header is not null && !fromCache && ShouldCache(header.Number)) { - _headerStore.Cache(header); + _headerStore.Cache(header, isMainChain); } return header; @@ -645,11 +664,6 @@ public AddBlockResult SuggestBlock(Block block, BlockTreeSuggestOptions options return null; } - public Hash256? FindHash(long number) - { - return GetBlockHashOnMainOrBestDifficultyHash(number); - } - public IOwnedReadOnlyList FindHeaders(Hash256? blockHash, int numberOfBlocks, int skip, bool reverse) { if (numberOfBlocks == 0) @@ -755,7 +769,9 @@ as it does not require the step of resolving number -> hash */ if (level.HasBlockOnMainChain) { - return level.BlockInfos[0].BlockHash; + Hash256 blockHash = level.BlockInfos[0].BlockHash; + _headerStore.CacheBlockHash(blockNumber, blockHash); + return blockHash; } UInt256 bestDifficultySoFar = UInt256.Zero; @@ -939,11 +955,9 @@ public void MarkChainAsProcessed(IReadOnlyList blocks) for (int i = 0; i < blocks.Count; i++) { Block block = blocks[i]; - if (ShouldCache(block.Number)) - { - _blockStore.Cache(block); - _headerStore.Cache(block.Header); - } + // Header also updates the number <-> hash mapping so always update + _blockStore.Cache(block); + _headerStore.Cache(block.Header, isMainChain: true); ChainLevelInfo? level = LoadLevel(block.Number); int? index = (level?.FindIndex(block.Hash)) ?? throw new InvalidOperationException($"Cannot mark unknown block {block.ToString(Block.Format.FullHashAndNumber)} as processed"); @@ -1008,12 +1022,9 @@ public void UpdateMainChain(IReadOnlyList blocks, bool wereProcessed, boo for (int i = 0; i < blocks.Count; i++) { Block block = blocks[i]; - if (ShouldCache(block.Number)) - { - _blockStore.Cache(block); - _headerStore.Cache(block.Header); - } - + _blockStore.Cache(block); + // Header also updates the number <-> hash mapping so always update + _headerStore.Cache(block.Header, wereProcessed); // we only force update head block for last block in processed blocks bool lastProcessedBlock = i == blocks.Count - 1; @@ -1430,12 +1441,23 @@ private bool ShouldCache(long number) } Block? block = null; + if ((options & BlockTreeLookupOptions.RequireCanonical) == 0) + { + block = _blockStore.GetFromCache(blockHash); + if (block is not null && block.TotalDifficulty.HasValue && (!blockNumber.HasValue || block.Number == blockNumber.Value)) + { + return block; + } + } + + bool fromCache = false; blockNumber ??= _headerStore.GetBlockNumber(blockHash); if (blockNumber is not null) { block = _blockStore.Get( blockNumber.Value, blockHash, + out fromCache, (options & BlockTreeLookupOptions.ExcludeTxHashes) != 0 ? RlpBehaviors.ExcludeHashes : RlpBehaviors.None, shouldCache: false); } @@ -1458,6 +1480,7 @@ private bool ShouldCache(long number) bool requiresCanonical = (options & BlockTreeLookupOptions.RequireCanonical) == BlockTreeLookupOptions.RequireCanonical; + bool isMainChain = false; if ((totalDifficultyNeeded && block.TotalDifficulty is null) || requiresCanonical) { (BlockInfo blockInfo, ChainLevelInfo level) = LoadInfo(block.Number, block.Hash, true); @@ -1484,17 +1507,17 @@ private bool ShouldCache(long number) SetTotalDifficultyFromBlockInfo(block.Header, blockInfo); } + isMainChain = level?.MainChainBlock?.BlockHash.Equals(blockHash) == true; if (requiresCanonical) { - bool isMain = level.MainChainBlock?.BlockHash.Equals(blockHash) == true; - block = isMain ? block : null; + block = isMainChain ? block : null; } } - if (block is not null && ShouldCache(block.Number)) + if (block is not null && !fromCache && ShouldCache(block.Number)) { _blockStore.Cache(block); - _headerStore.Cache(block.Header); + _headerStore.Cache(block.Header, isMainChain); } return block; diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs b/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs index 4e2a519c14f..12d710f96f6 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs @@ -108,7 +108,7 @@ public void UpdateMainChain(IReadOnlyList blocks, bool wereProcessed, boo public BlockInfo FindCanonicalBlockInfo(long blockNumber) => _overlayTree.FindCanonicalBlockInfo(blockNumber) ?? _baseTree.FindCanonicalBlockInfo(blockNumber); - public Hash256 FindHash(long blockNumber) => _overlayTree.FindHash(blockNumber) ?? _baseTree.FindHash(blockNumber); + public Hash256 FindBlockHash(long blockNumber) => _overlayTree.FindBlockHash(blockNumber) ?? _baseTree.FindBlockHash(blockNumber); public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse) { @@ -268,9 +268,6 @@ public bool HasBlock(long blockNumber, Hash256 blockHash) => public BlockHeader? FindHeader(long blockNumber, BlockTreeLookupOptions options) => _overlayTree.FindHeader(blockNumber, options) ?? _baseTree.FindHeader(blockNumber, options); - public Hash256? FindBlockHash(long blockNumber) => - _overlayTree.FindBlockHash(blockNumber) ?? _baseTree.FindBlockHash(blockNumber); - public bool IsMainChain(BlockHeader blockHeader) => _baseTree.IsMainChain(blockHeader) || _overlayTree.IsMainChain(blockHeader); diff --git a/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs b/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs index 52bee65baf3..f92fc45ac18 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockhashProvider.cs @@ -46,6 +46,13 @@ public BlockhashProvider(IBlockFinder blockTree, ISpecProvider specProvider, IWo return null; } + return (currentBlock.ParentHash == _blockTree.HeadHash || currentBlock.ParentHash == _blockTree.Head?.ParentHash) ? + _blockTree.FindBlockHash(number) : + GetBlockHashFromNonHeadParent(currentBlock, number); + } + + private Hash256 GetBlockHashFromNonHeadParent(BlockHeader currentBlock, long number) + { BlockHeader header = _blockTree.FindParentHeader(currentBlock, BlockTreeLookupOptions.TotalDifficultyNotNeeded) ?? throw new InvalidDataException("Parent header cannot be found when executing BLOCKHASH operation"); diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 3cb168ba391..1fe38c8532f 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -18,8 +18,7 @@ public class BlockStore([KeyFilter(DbNames.Blocks)] IDb blockDb) : IBlockStore private readonly BlockDecoder _blockDecoder = new(); public const int CacheSize = 128 + 32; - private readonly ClockCache - _blockCache = new(CacheSize); + private readonly ClockCache _blockCache = new(CacheSize); public void SetMetadata(byte[] key, byte[] value) { @@ -67,9 +66,16 @@ public void Delete(long blockNumber, Hash256 blockHash) public Block? Get(long blockNumber, Hash256 blockHash, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = false) { - Block? b = blockDb.Get(blockNumber, blockHash, _blockDecoder, _blockCache, rlpBehaviors, shouldCache); + Block? b = blockDb.Get(blockNumber, blockHash, _blockDecoder, out _, _blockCache, rlpBehaviors, shouldCache); if (b is not null) return b; - return blockDb.Get(blockHash, _blockDecoder, _blockCache, rlpBehaviors, shouldCache); + return blockDb.Get(blockHash, _blockDecoder, out _, _blockCache, rlpBehaviors, shouldCache); + } + + public Block? Get(long blockNumber, Hash256 blockHash, out bool fromCache, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = false) + { + Block? b = blockDb.Get(blockNumber, blockHash, _blockDecoder, out fromCache, _blockCache, rlpBehaviors, shouldCache); + if (b is not null) return b; + return blockDb.Get(blockHash, _blockDecoder, out fromCache, _blockCache, rlpBehaviors, shouldCache); } public byte[]? GetRlp(long blockNumber, Hash256 blockHash) @@ -96,4 +102,7 @@ public void Cache(Block block) { _blockCache.Set(block.Hash, block); } + + public Block? GetFromCache(Hash256 blockHash) + => _blockCache.Get(blockHash); } diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs index eca635bc9d2..b5e62629eb7 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs @@ -15,9 +15,12 @@ public interface IBlockStore { void Insert(Block block, WriteFlags writeFlags = WriteFlags.None); void Delete(long blockNumber, Hash256 blockHash); - Block? Get(long blockNumber, Hash256 blockHash, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true); + Block? Get(long blockNumber, Hash256 blockHash, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) + => Get(blockNumber, blockHash, out _, rlpBehaviors, shouldCache); + Block? Get(long blockNumber, Hash256 blockHash, out bool fromCache, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true); byte[]? GetRlp(long blockNumber, Hash256 blockHash); ReceiptRecoveryBlock? GetReceiptRecoveryBlock(long blockNumber, Hash256 blockHash); void Cache(Block block); + Block? GetFromCache(Hash256 blockHash); bool HasBlock(long blockNumber, Hash256 blockHash); } diff --git a/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs b/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs index 425d56d1955..f6426eb9e90 100644 --- a/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs @@ -19,12 +19,15 @@ public class HeaderStore : IHeaderStore { // SyncProgressResolver MaxLookupBack is 256, add 16 wiggle room public const int CacheSize = 256 + 16; + // Go a bit further back for numbers as smaller + public const int NumberCacheSize = CacheSize * 4; private readonly IDb _headerDb; private readonly IDb _blockNumberDb; private readonly HeaderDecoder _headerDecoder = new(); - private readonly ClockCache _headerCache = - new(CacheSize); + private readonly ClockCache _headerCache = new(CacheSize); + private readonly ClockCache _numberCache = new(NumberCacheSize); + private readonly ClockCache _hashCache = new(NumberCacheSize); public HeaderStore([KeyFilter(DbNames.Headers)] IDb headerDb, [KeyFilter(DbNames.BlockNumbers)] IDb blockNumberDb) { @@ -45,40 +48,58 @@ public void BulkInsert(IReadOnlyList headers) using IWriteBatch blockNumberWriteBatch = _blockNumberDb.StartWriteBatch(); Span blockNumberSpan = stackalloc byte[8]; - foreach (var header in headers) + foreach (BlockHeader header in headers) { using NettyRlpStream newRlp = _headerDecoder.EncodeToNewNettyStream(header); headerWriteBatch.Set(header.Number, header.Hash, newRlp.AsSpan()); header.Number.WriteBigEndian(blockNumberSpan); blockNumberWriteBatch.Set(header.Hash, blockNumberSpan); + CacheNumber(header.Hash, header.Number, isMainChain: false); } } - public BlockHeader? Get(Hash256 blockHash, bool shouldCache = false, long? blockNumber = null) + public BlockHeader? Get(Hash256 blockHash, out bool fromCache, bool shouldCache = false, long? blockNumber = null) { blockNumber ??= GetBlockNumberFromBlockNumberDb(blockHash); - BlockHeader? header = null; + BlockHeader? header; if (blockNumber is not null) { - header = _headerDb.Get(blockNumber.Value, blockHash, _headerDecoder, _headerCache, shouldCache: shouldCache); + header = _headerDb.Get(blockNumber.Value, blockHash, _headerDecoder, out fromCache, _headerCache, shouldCache: shouldCache); + if (header is not null) + { + return header; + } } - return header ?? _headerDb.Get(blockHash, _headerDecoder, _headerCache, shouldCache: shouldCache); + + return _headerDb.Get(blockHash, _headerDecoder, out fromCache, _headerCache, shouldCache: shouldCache); } - public void Cache(BlockHeader header) + public void CacheBlockHash(long blockNumber, Hash256 blockHash) + => CacheNumber(blockHash, blockNumber, isMainChain: true); + + public void Cache(BlockHeader header, bool isMainChain) { + CacheNumber(header.Hash, header.Number, isMainChain); _headerCache.Set(header.Hash, header); } + public BlockHeader? GetFromCache(Hash256 blockHash) + => _headerCache.Get(blockHash); + public void Delete(Hash256 blockHash) { - long? blockNumber = GetBlockNumberFromBlockNumberDb(blockHash); - if (blockNumber is not null) _headerDb.Delete(blockNumber.Value, blockHash); + long? blockNumber = GetBlockNumber(blockHash); + if (blockNumber is not null) + { + _headerDb.Delete(blockNumber.Value, blockHash); + _hashCache.Delete(blockNumber.Value); + } _blockNumberDb.Delete(blockHash); _headerDb.Delete(blockHash); _headerCache.Delete(blockHash); + _numberCache.Delete(blockHash); } public void InsertBlockNumber(Hash256 blockHash, long blockNumber) @@ -86,19 +107,66 @@ public void InsertBlockNumber(Hash256 blockHash, long blockNumber) Span blockNumberSpan = stackalloc byte[8]; blockNumber.WriteBigEndian(blockNumberSpan); _blockNumberDb.Set(blockHash, blockNumberSpan); + CacheNumber(blockHash, blockNumber, isMainChain: false); + } + + private void CacheNumber(Hash256 blockHash, long blockNumber, bool isMainChain) + { + _numberCache.Set(blockHash, blockNumber); + if (isMainChain) + { + _hashCache.Set(blockNumber, blockHash); + } + } + + public Hash256? GetBlockHash(long blockNumber) + { + if (_hashCache.TryGet(blockNumber, out Hash256? hash)) + { + return hash; + } + + return null; } public long? GetBlockNumber(Hash256 blockHash) { + if (_numberCache.TryGet(blockHash, out long number)) + { + return number; + } + + return GetBlockNumberThroughHeaderCache(blockHash); + } + + private long? GetBlockNumberThroughHeaderCache(Hash256 blockHash) + { + if (_headerCache.TryGet(blockHash, out BlockHeader? header)) + { + CacheNumber(blockHash, header.Number, isMainChain: false); + return header.Number; + } + long? blockNumber = GetBlockNumberFromBlockNumberDb(blockHash); if (blockNumber is not null) return blockNumber.Value; // Probably still hash based - return Get(blockHash)?.Number; + blockNumber = Get(blockHash, out _)?.Number; + if (blockNumber.HasValue) + { + CacheNumber(blockHash, blockNumber.Value, isMainChain: false); + } + return blockNumber; } private long? GetBlockNumberFromBlockNumberDb(Hash256 blockHash) { + // Double check cache as we have done a fair amount of checks + // since the cache check and something else might have populated it. + if (_numberCache.TryGet(blockHash, out long number)) + { + return number; + } Span numberSpan = _blockNumberDb.GetSpan(blockHash); if (numberSpan.IsNullOrEmpty()) return null; try @@ -108,7 +176,9 @@ public void InsertBlockNumber(Hash256 blockHash, long blockNumber) throw new InvalidDataException($"Unexpected number span length: {numberSpan.Length}"); } - return BinaryPrimitives.ReadInt64BigEndian(numberSpan); + number = BinaryPrimitives.ReadInt64BigEndian(numberSpan); + CacheNumber(blockHash, number, isMainChain: false); + return number; } finally { diff --git a/src/Nethermind/Nethermind.Blockchain/Headers/IHeaderStore.cs b/src/Nethermind/Nethermind.Blockchain/Headers/IHeaderStore.cs index dc56e2d9116..66f57b98cdc 100644 --- a/src/Nethermind/Nethermind.Blockchain/Headers/IHeaderStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Headers/IHeaderStore.cs @@ -12,9 +12,14 @@ public interface IHeaderStore { void Insert(BlockHeader header); void BulkInsert(IReadOnlyList headers); - BlockHeader? Get(Hash256 blockHash, bool shouldCache, long? blockNumber = null); - void Cache(BlockHeader header); + BlockHeader? Get(Hash256 blockHash, bool shouldCache = true, long? blockNumber = null) + => Get(blockHash, out _, shouldCache, blockNumber); + BlockHeader? Get(Hash256 blockHash, out bool fromCache, bool shouldCache = true, long? blockNumber = null); + void Cache(BlockHeader header, bool isMainChain = false); void Delete(Hash256 blockHash); void InsertBlockNumber(Hash256 blockHash, long blockNumber); long? GetBlockNumber(Hash256 blockHash); + Hash256? GetBlockHash(long blockNumber); + void CacheBlockHash(long blockNumber, Hash256 blockHash); + BlockHeader? GetFromCache(Hash256 blockHash); } diff --git a/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs b/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs index 22bd63cbded..1c01cc0fb03 100644 --- a/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs @@ -155,8 +155,6 @@ AddBlockResult Insert(Block block, BlockTreeInsertBlockOptions insertBlockOption BlockInfo FindCanonicalBlockInfo(long blockNumber); - Hash256? FindHash(long blockNumber); - IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse); void DeleteInvalidBlock(Block invalidBlock); diff --git a/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs b/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs index b6635141370..51143c0198a 100644 --- a/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs @@ -99,8 +99,6 @@ public void UpdateHeadBlock(Hash256 blockHash) public bool IsMainChain(BlockHeader blockHeader) => _wrapped.IsMainChain(blockHeader); - public Hash256 FindHash(long blockNumber) => _wrapped.FindHash(blockNumber); - public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse) => _wrapped.FindHeaders(hash, numberOfBlocks, skip, reverse); public Block FindBlock(long blockNumber, BlockTreeLookupOptions options) => _wrapped.FindBlock(blockNumber, options); diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Validators/ValidatorInfoDecoder.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Validators/ValidatorInfoDecoder.cs index 462db51a196..71b9c3e5a65 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Validators/ValidatorInfoDecoder.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Validators/ValidatorInfoDecoder.cs @@ -23,7 +23,9 @@ internal class ValidatorInfoDecoder : IRlpStreamDecoder, IRlpObje int addressesSequenceLength = rlpStream.ReadSequenceLength(); int addressesCheck = rlpStream.Position + addressesSequenceLength; - Address[] addresses = new Address[addressesSequenceLength / Rlp.LengthOfAddressRlp]; + var count = addressesSequenceLength / Rlp.LengthOfAddressRlp; + rlpStream.GuardLimit(count); + Address[] addresses = new Address[count]; int i = 0; while (rlpStream.Position < addressesCheck) { diff --git a/src/Nethermind/Nethermind.Consensus.Clique/SnapshotDecoder.cs b/src/Nethermind/Nethermind.Consensus.Clique/SnapshotDecoder.cs index 436256334cb..8bf9756d81d 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/SnapshotDecoder.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/SnapshotDecoder.cs @@ -68,8 +68,9 @@ private static (int contentLength, int signersLength, int votesLength, int tally private static SortedList DecodeSigners(RlpStream rlpStream) { rlpStream.ReadSequenceLength(); - SortedList signers = new SortedList(AddressComparer.Instance); int length = rlpStream.DecodeInt(); + rlpStream.GuardLimit(length); + SortedList signers = new(AddressComparer.Instance); for (int i = 0; i < length; i++) { Address signer = rlpStream.DecodeAddress(); @@ -83,8 +84,9 @@ private static SortedList DecodeSigners(RlpStream rlpStream) private static List DecodeVotes(RlpStream rlpStream) { rlpStream.ReadSequenceLength(); - List votes = new List(); int length = rlpStream.DecodeInt(); + rlpStream.GuardLimit(length); + List votes = new(length); for (int i = 0; i < length; i++) { Address signer = rlpStream.DecodeAddress(); @@ -100,8 +102,9 @@ private static List DecodeVotes(RlpStream rlpStream) private static Dictionary DecodeTally(RlpStream rlpStream) { rlpStream.ReadSequenceLength(); - Dictionary tally = new Dictionary(); int length = rlpStream.DecodeInt(); + rlpStream.GuardLimit(length); + Dictionary tally = new(length); for (int i = 0; i < length; i++) { Address address = rlpStream.DecodeAddress(); diff --git a/src/Nethermind/Nethermind.Consensus.Test/GenesisLoaderTests.cs b/src/Nethermind/Nethermind.Consensus.Test/GenesisLoaderTests.cs new file mode 100644 index 00000000000..9c41299d4ec --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus.Test/GenesisLoaderTests.cs @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using Nethermind.Evm.State; +using Nethermind.Logging; +using Nethermind.State; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Consensus.Test; + +[TestFixture] +public class GenesisLoaderTests +{ + [Test] + public void Load_ShouldFlushCacheAfterSuccessfulGenesisProcessing() + { + // Arrange + Block genesisBlock = Build.A.Block.Genesis.TestObject; + + IGenesisBuilder genesisBuilder = Substitute.For(); + genesisBuilder.Build().Returns(genesisBlock); + + IStateReader stateReader = Substitute.For(); + + IBlockTree blockTree = Substitute.For(); + blockTree.When(x => x.SuggestBlock(Arg.Any())).Do(_ => + { + // Simulate block processing by triggering NewHeadBlock event + blockTree.NewHeadBlock += Raise.EventWith(blockTree, new BlockEventArgs(genesisBlock)); + }); + + IWorldState worldState = Substitute.For(); + IDisposable scopeDisposable = Substitute.For(); + worldState.BeginScope(IWorldState.PreGenesis).Returns(scopeDisposable); + + IWorldStateManager worldStateManager = Substitute.For(); + + IBlockchainProcessor blockchainProcessor = Substitute.For(); + + GenesisLoader.Config config = new(null, TimeSpan.FromSeconds(10)); + ILogManager logManager = LimboLogs.Instance; + + GenesisLoader loader = new( + genesisBuilder, + stateReader, + blockTree, + worldState, + worldStateManager, + blockchainProcessor, + config, + logManager + ); + + // Act + loader.Load(); + + // Assert - verify FlushCache was called + worldStateManager.Received(1).FlushCache(Arg.Any()); + } + + [Test] + public void Load_ShouldNotFlushCache_WhenGenesisProcessingTimesOut() + { + // Arrange + Block genesisBlock = Build.A.Block.Genesis.TestObject; + + IGenesisBuilder genesisBuilder = Substitute.For(); + genesisBuilder.Build().Returns(genesisBlock); + + IStateReader stateReader = Substitute.For(); + + IBlockTree blockTree = Substitute.For(); + blockTree.When(x => x.SuggestBlock(Arg.Any())).Do(_ => + { + // Do nothing - simulate timeout + }); + + IWorldState worldState = Substitute.For(); + IDisposable scopeDisposable = Substitute.For(); + worldState.BeginScope(IWorldState.PreGenesis).Returns(scopeDisposable); + + IWorldStateManager worldStateManager = Substitute.For(); + + IBlockchainProcessor blockchainProcessor = Substitute.For(); + + GenesisLoader.Config config = new(null, TimeSpan.FromMilliseconds(100)); + ILogManager logManager = LimboLogs.Instance; + + GenesisLoader loader = new( + genesisBuilder, + stateReader, + blockTree, + worldState, + worldStateManager, + blockchainProcessor, + config, + logManager + ); + + // Act & Assert - expect timeout exception + Assert.Throws(() => loader.Load()); + + // Verify FlushCache was NOT called since genesis processing failed + worldStateManager.DidNotReceive().FlushCache(Arg.Any()); + } + + [Test] + public void Load_ShouldNotFlushCache_WhenGenesisBlockIsInvalid() + { + // Arrange + Block genesisBlock = Build.A.Block.Genesis.TestObject; + + IGenesisBuilder genesisBuilder = Substitute.For(); + genesisBuilder.Build().Returns(genesisBlock); + + IStateReader stateReader = Substitute.For(); + + IBlockTree blockTree = Substitute.For(); + + IWorldState worldState = Substitute.For(); + IDisposable scopeDisposable = Substitute.For(); + worldState.BeginScope(IWorldState.PreGenesis).Returns(scopeDisposable); + + IWorldStateManager worldStateManager = Substitute.For(); + + IBlockchainProcessor blockchainProcessor = Substitute.For(); + blockTree.When(x => x.SuggestBlock(Arg.Any())).Do(_ => + { + // Simulate invalid block by triggering InvalidBlock event + blockchainProcessor.InvalidBlock += Raise.EventWith( + blockchainProcessor, + new IBlockchainProcessor.InvalidBlockEventArgs { InvalidBlock = genesisBlock }); + }); + + GenesisLoader.Config config = new(null, TimeSpan.FromSeconds(10)); + ILogManager logManager = LimboLogs.Instance; + + GenesisLoader loader = new( + genesisBuilder, + stateReader, + blockTree, + worldState, + worldStateManager, + blockchainProcessor, + config, + logManager + ); + + // Act & Assert - expect InvalidBlockException + Assert.Throws(() => loader.Load()); + + // Verify FlushCache was NOT called since genesis was invalid + worldStateManager.DidNotReceive().FlushCache(Arg.Any()); + } + + [Test] + public void Load_ShouldFlushCacheAfterScopeExit() + { + // Arrange + Block genesisBlock = Build.A.Block.Genesis.TestObject; + + IGenesisBuilder genesisBuilder = Substitute.For(); + genesisBuilder.Build().Returns(genesisBlock); + + IStateReader stateReader = Substitute.For(); + + IBlockTree blockTree = Substitute.For(); + bool scopeExited = false; + blockTree.When(x => x.SuggestBlock(Arg.Any())).Do(_ => + { + // Simulate block processing by triggering NewHeadBlock event + blockTree.NewHeadBlock += Raise.EventWith(blockTree, new BlockEventArgs(genesisBlock)); + }); + + IWorldState worldState = Substitute.For(); + IDisposable scopeDisposable = Substitute.For(); + scopeDisposable.When(x => x.Dispose()).Do(_ => scopeExited = true); + worldState.BeginScope(IWorldState.PreGenesis).Returns(scopeDisposable); + + IWorldStateManager worldStateManager = Substitute.For(); + worldStateManager.When(x => x.FlushCache(Arg.Any())).Do(_ => + { + // Verify that scope was exited before FlushCache was called + scopeExited.Should().BeTrue("FlushCache should be called after scope exit"); + }); + + IBlockchainProcessor blockchainProcessor = Substitute.For(); + + GenesisLoader.Config config = new(null, TimeSpan.FromSeconds(10)); + ILogManager logManager = LimboLogs.Instance; + + GenesisLoader loader = new( + genesisBuilder, + stateReader, + blockTree, + worldState, + worldStateManager, + blockchainProcessor, + config, + logManager + ); + + // Act + loader.Load(); + + // Assert - verify FlushCache was called after scope exit + worldStateManager.Received(1).FlushCache(Arg.Any()); + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/GenesisLoader.cs b/src/Nethermind/Nethermind.Consensus/Processing/GenesisLoader.cs index c743aefa99e..fd369d528e8 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/GenesisLoader.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/GenesisLoader.cs @@ -18,6 +18,7 @@ public class GenesisLoader( IStateReader stateReader, IBlockTree blockTree, IWorldState worldState, + IWorldStateManager worldStateManager, IBlockchainProcessor blockchainProcessor, GenesisLoader.Config genesisConfig, ILogManager logManager @@ -29,6 +30,12 @@ public record Config(Hash256? ExpectedGenesisHash, TimeSpan GenesisTimeout); ILogger _logger = logManager.GetClassLogger(); public void Load() + { + DoLoad(); + worldStateManager.FlushCache(CancellationToken.None); + } + + private void DoLoad() { using var _ = worldState.BeginScope(IWorldState.PreGenesis); diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockTree.cs b/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockTree.cs index 68dd42dd8df..c19570b472b 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockTree.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/StatelessBlockTree.cs @@ -143,9 +143,6 @@ public Task Accept(IBlockTreeVisitor blockTreeVisitor, CancellationToken cancell public BlockInfo FindCanonicalBlockInfo(long blockNumber) => throw new NotSupportedException(); - public Hash256 FindHash(long blockNumber) - => throw new NotSupportedException(); - public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse) => throw new NotSupportedException(); diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs index 676d91b3646..11283658999 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs @@ -33,6 +33,7 @@ public class BlockTreeBuilder : BuilderBase private IReceiptStorage? _receiptStorage; private IEthereumEcdsa? _ecdsa; private Hash256? _stateRoot; + private Func? _stateRootGen; private Func>? _logCreationFunction; private bool _onlyHeaders; @@ -200,6 +201,12 @@ public BlockTreeBuilder WithStateRoot(Hash256 stateRoot) return this; } + public BlockTreeBuilder WithStateRoot(Func stateRootGen) + { + _stateRootGen = stateRootGen; + return this; + } + public BlockTreeBuilder OfChainLength(int chainLength, int splitVariant = 0, int splitFrom = 0, bool withWithdrawals = false, params Address[] blockBeneficiaries) { OfChainLength(out _, chainLength, splitVariant, splitFrom, withWithdrawals, blockBeneficiaries); @@ -324,6 +331,10 @@ private Block CreateBlock(int splitVariant, int splitFrom, int blockIndex, Block .TestObject; } + if (_stateRootGen is not null) + { + currentBlock.Header.StateRoot = _stateRootGen(currentBlock); + } currentBlock.Header.AuRaStep = blockIndex; return currentBlock; diff --git a/src/Nethermind/Nethermind.Core.Test/RlpTests.cs b/src/Nethermind/Nethermind.Core.Test/RlpTests.cs index b45b0c29fdf..49e9a776687 100644 --- a/src/Nethermind/Nethermind.Core.Test/RlpTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/RlpTests.cs @@ -272,5 +272,41 @@ public void RlpContextWithSliceMemory_shouldNotCopyUnderlyingData(bool sliceValu isACopy.Should().NotBe(sliceValue); } } + + [TestCase(50)] + [TestCase(100)] + public void Over_limit_throws(int limit) + { + RlpStream stream = Prepare100BytesStream(); + RlpLimit rlpLimit = new(limit); + if (limit < 100) + { + Assert.Throws(() => stream.DecodeByteArray(rlpLimit)); + } + else + { + Assert.DoesNotThrow(() => stream.DecodeByteArray(rlpLimit)); + } + } + + [Test] + public void Not_enough_bytes_throws() + { + RlpStream stream = Prepare100BytesStream(); + stream.Data[1] = 101; // tamper with length, it is more than available bytes + Assert.Throws(() => stream.DecodeByteArray()); + } + + private static RlpStream Prepare100BytesStream() + { + byte[] randomBytes = new byte[100]; + Random.Shared.NextBytes(randomBytes); + + int requiredLength = Rlp.LengthOf(randomBytes); + RlpStream stream = new(requiredLength); + stream.Encode(randomBytes); + stream.Reset(); + return stream; + } } } diff --git a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs index 50412982399..8b93eb27754 100644 --- a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs +++ b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs @@ -113,12 +113,10 @@ public readonly Span AsSpan(int start, int length) return AsSpan().ToArray(); } - public override string? ToString() - { - return typeof(T) == typeof(byte) ? - SpanExtensions.ToHexString(MemoryMarshal.AsBytes(AsSpan()), withZeroX: true) : + public override string? ToString() => + typeof(T) == typeof(byte) ? + MemoryMarshal.AsBytes(AsSpan()).ToHexString(withZeroX: true) : base.ToString(); - } public readonly ArraySegment AsArraySegment() { diff --git a/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs b/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs index a13c6eaef1f..ed7e2e98db3 100644 --- a/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/ClockCache.cs @@ -58,8 +58,12 @@ public bool Set(TKey key, TValue val) if (_cacheMap.TryGetValue(key, out LruCacheItem ov)) { - // Fast path: atomic update using TryUpdate - if (_cacheMap.TryUpdate(key, new(val, ov.Offset), comparisonValue: ov)) + bool needsUpdate = !(typeof(TValue).IsValueType ? + EqualityComparer.Default.Equals(ov.Value, val) : + ReferenceEquals(ov.Value, val)); + + // Fast path: no update or atomic update using TryUpdate + if (!needsUpdate || _cacheMap.TryUpdate(key, new(val, ov.Offset), comparisonValue: ov)) { MarkAccessed(ov.Offset); return false; diff --git a/src/Nethermind/Nethermind.Core/KeyValueStoreExtensions.cs b/src/Nethermind/Nethermind.Core/KeyValueStoreExtensions.cs index 906b39acf45..bea10d063f4 100644 --- a/src/Nethermind/Nethermind.Core/KeyValueStoreExtensions.cs +++ b/src/Nethermind/Nethermind.Core/KeyValueStoreExtensions.cs @@ -22,8 +22,6 @@ public static IWriteBatch LikeABatch(this IWriteOnlyKeyValueStore keyValueStore, return new FakeWriteBatch(keyValueStore, onDispose); } - #region Getters - public static byte[]? Get(this IReadOnlyKeyValueStore db, Hash256 key) { #if DEBUG @@ -88,12 +86,6 @@ public static bool KeyExists(this IReadOnlyKeyValueStore db, long key) return span.IsNullOrEmpty() ? null : new DbSpanMemoryManager(db, span); } - - #endregion - - - #region Setters - public static void Set(this IWriteOnlyKeyValueStore db, Hash256 key, byte[] value, WriteFlags writeFlags = WriteFlags.None) { if (db.PreferWriteByArray) @@ -115,14 +107,15 @@ public static void Set(this IWriteOnlyKeyValueStore db, Hash256 key, in CappedAr db.PutSpan(key.Bytes, value.AsSpan(), writeFlags); } + [SkipLocalsInit] public static void Set(this IWriteOnlyKeyValueStore db, long blockNumber, Hash256 key, ReadOnlySpan value, WriteFlags writeFlags = WriteFlags.None) { Span blockNumberPrefixedKey = stackalloc byte[40]; - GetBlockNumPrefixedKey(blockNumber, key, blockNumberPrefixedKey); + GetBlockNumPrefixedKey(blockNumber, in key.ValueHash256, blockNumberPrefixedKey); db.PutSpan(blockNumberPrefixedKey, value, writeFlags); } - public static void GetBlockNumPrefixedKey(long blockNumber, ValueHash256 blockHash, Span output) + public static void GetBlockNumPrefixedKey(long blockNumber, in ValueHash256 blockHash, Span output) { blockNumber.WriteBigEndian(output); blockHash!.Bytes.CopyTo(output[8..]); @@ -147,7 +140,7 @@ public static void Delete(this IWriteOnlyKeyValueStore db, long key) public static void Delete(this IWriteOnlyKeyValueStore db, long blockNumber, Hash256 hash) { Span key = stackalloc byte[40]; - GetBlockNumPrefixedKey(blockNumber, hash, key); + GetBlockNumPrefixedKey(blockNumber, in hash.ValueHash256, key); db.Remove(key); } @@ -155,7 +148,5 @@ public static void Set(this IWriteOnlyKeyValueStore db, long key, byte[] value) { db[key.ToBigEndianSpanWithoutLeadingZeros(out _)] = value; } - - #endregion } } diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 7069f962465..f5fb43269de 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -217,7 +217,7 @@ internal static long[] PopulateJumpDestinationBitmap_Vector512(long[] bitmap, Re int overflow = size - inThis; // Clear bits [lane .. lane+inThis-1] from both masks - ulong clearThis = (Bmi1.X64.IsSupported ? + ulong clearThis = (Bmi2.X64.IsSupported ? Bmi2.X64.ZeroHighBits(ulong.MaxValue, (uint)inThis) : ((1UL << inThis) - 1UL)) << lane; @@ -228,7 +228,7 @@ internal static long[] PopulateJumpDestinationBitmap_Vector512(long[] bitmap, Re // If it spilled, mark those lanes for the next chunk if (overflow > 0) { - carryMask = (Bmi1.X64.IsSupported ? + carryMask = (Bmi2.X64.IsSupported ? Bmi2.X64.ZeroHighBits(ulong.MaxValue, (uint)overflow) : ((1UL << overflow) - 1UL)); break; diff --git a/src/Nethermind/Nethermind.Facade/Eth/RpcTransaction/AccessListForRpc.cs b/src/Nethermind/Nethermind.Facade/Eth/RpcTransaction/AccessListForRpc.cs index 7527ee93a49..d29ca507813 100644 --- a/src/Nethermind/Nethermind.Facade/Eth/RpcTransaction/AccessListForRpc.cs +++ b/src/Nethermind/Nethermind.Facade/Eth/RpcTransaction/AccessListForRpc.cs @@ -36,7 +36,7 @@ private class Item [JsonConstructor] public Item() { } - public Item(Address address, IEnumerable storageKeys) + public Item(Address address, IEnumerable? storageKeys) { Address = address; StorageKeys = storageKeys; @@ -67,10 +67,98 @@ public class JsonConverter : JsonConverter { public override AccessListForRpc? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - List? list = JsonSerializer.Deserialize>(ref reader, options); - return list is null ? null : new AccessListForRpc(list); + if (reader.TokenType != JsonTokenType.StartArray) + throw new JsonException("Expected start of array"); + + const int maxItems = 1000; + const int maxStorageKeysPerItem = 1000; + const int maxStorageKeys = 10000; + + var items = new List(); + int itemCount = 0; + int totalItemStorageItemsCount = 0; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + + if (itemCount >= maxItems) + throw new JsonException($"Access list cannot have more than {maxItems} items."); + + if (reader.TokenType != JsonTokenType.StartObject) + throw new JsonException("Expected start of item object"); + + Address? address = null; + List? storageKeys = null; + + // Read Item properties + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("Expected property name"); + + string propName = reader.GetString() ?? throw new JsonException("Property name cannot be null"); + reader.Read(); // move to property value + + if (string.Equals(propName, nameof(Item.Address), StringComparison.OrdinalIgnoreCase)) + { + address = JsonSerializer.Deserialize
(ref reader, options); + } + else if (string.Equals(propName, nameof(Item.StorageKeys), StringComparison.OrdinalIgnoreCase)) + { + if (reader.TokenType == JsonTokenType.Null) + { + storageKeys = null; + } + else if (reader.TokenType == JsonTokenType.StartArray) + { + storageKeys = new List(); + int currentItemStorageItemsCount = 0; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + + if (currentItemStorageItemsCount >= maxStorageKeysPerItem) + throw new JsonException($"An item cannot have more than {maxStorageKeysPerItem} storage keys."); + + if (totalItemStorageItemsCount >= maxStorageKeys) + throw new JsonException($"Access List cannot have more than {maxStorageKeys} storage keys."); + + UInt256 key = JsonSerializer.Deserialize(ref reader, options); + storageKeys.Add(key); + currentItemStorageItemsCount++; + totalItemStorageItemsCount++; + } + } + else + { + throw new JsonException("Expected array or null for StorageKeys"); + } + } + else + { + // Skip unknown properties + reader.Skip(); + } + } + + if (address is null) + throw new JsonException("Item missing required Address"); + + items.Add(new Item(address, storageKeys)); + itemCount++; + } + + return new AccessListForRpc(items); } + public override void Write(Utf8JsonWriter writer, AccessListForRpc value, JsonSerializerOptions options) { JsonSerializer.Serialize(writer, value._items, options); diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs index ea2f6f219b1..a10917a84b6 100644 --- a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs @@ -28,13 +28,15 @@ public void Delete(long blockNumber, Hash256 blockHash) _blockNumDict.Remove(blockNumber); } - public Block? Get(long blockNumber, Hash256 blockHash, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) + public Block? Get(long blockNumber, Hash256 blockHash, out bool fromCache, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) { if (_blockNumDict.TryGetValue(blockNumber, out Block block)) { + fromCache = true; return block; } + fromCache = false; block = readonlyBaseBlockStore.Get(blockNumber, blockHash, rlpBehaviors, false); if (block is not null && shouldCache) { @@ -67,6 +69,8 @@ public void Delete(long blockNumber, Hash256 blockHash) public void Cache(Block block) => Insert(block); + public Block? GetFromCache(Hash256 blockHash) => null; + public bool HasBlock(long blockNumber, Hash256 blockHash) => _blockNumDict.ContainsKey(blockNumber); } diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs index bcc2487ed0a..95bb5f3e45f 100644 --- a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs @@ -34,19 +34,21 @@ public void BulkInsert(IReadOnlyList headers) } } - public BlockHeader? Get(Hash256 blockHash, bool shouldCache = false, long? blockNumber = null) + public BlockHeader? Get(Hash256 blockHash, out bool fromCache, bool shouldCache = false, long? blockNumber = null) { blockNumber ??= GetBlockNumber(blockHash); if (blockNumber.HasValue && _headerDict.TryGetValue(blockHash, out BlockHeader? header)) { + fromCache = true; if (shouldCache) { - Cache(header); + InsertBlockNumber(header.Hash, header.Number); } return header; } + fromCache = false; header = readonlyBaseHeaderStore.Get(blockHash, false, blockNumber); if (header is not null && shouldCache) { @@ -55,7 +57,7 @@ public void BulkInsert(IReadOnlyList headers) return header; } - public void Cache(BlockHeader header) + public void Cache(BlockHeader header, bool isCanonical = false) { Insert(header); } @@ -75,4 +77,10 @@ public void InsertBlockNumber(Hash256 blockHash, long blockNumber) { return _blockNumberDict.TryGetValue(blockHash, out var blockNumber) ? blockNumber : readonlyBaseHeaderStore.GetBlockNumber(blockHash); } + + public Hash256? GetBlockHash(long blockNumber) => null; + + public BlockHeader? GetFromCache(Hash256 blockHash) => null; + + public void CacheBlockHash(long blockNumber, Hash256 blockHash) { } } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index d5bd3ef2df0..191592ca38b 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -292,7 +292,6 @@ private async Task InitPeer() _forkInfo, _api.GossipPolicy, _api.WorldStateManager!, - _api.BlockTree, _api.LogManager, _api.TxGossipPolicy); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs index 484cfe255e8..6007f2194ee 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs @@ -309,6 +309,14 @@ public async Task Eth_get_filter_changes_missing() Assert.That(serialized2, Is.EqualTo("{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32000,\"message\":\"Filter not found\"},\"id\":67}")); } + [Test] + public async Task Eth_get_filter_changes_too_big() + { + using Context ctx = await Context.Create(); + string serialized2 = await ctx.Test.TestEthRpc("eth_getFilterChanges", ((UInt256)uint.MaxValue + 1).ToHexString(true)); + Assert.That(serialized2, Is.EqualTo("{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32000,\"message\":\"Filter not found\"},\"id\":67}")); + } + [Test] public async Task Eth_uninstall_filter() { diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs index 9e994e034d9..1a43ecd6fa4 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs @@ -227,7 +227,6 @@ await base.Build(builder => Container.Resolve(), Substitute.For(), WorldStateManager, - Substitute.For(), LimboLogs.Instance, Substitute.For() ); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs index 65cdf2cd9bf..3937a3f6565 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs @@ -171,7 +171,7 @@ public IReadOnlyCollection GetBlockTrace(Block block, Cancellat { if (parameter.BlockNumber is long number) { - Hash256? hash = _blockTree.FindHash(number); + Hash256? hash = _blockTree.FindBlockHash(number); if (hash is null) return null; return _blockStore.GetRlp(number, hash); } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs index 6fd03117474..6adb304f637 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs @@ -527,7 +527,7 @@ public ResultWrapper eth_getTransactionByBlockNumberAndIndex( public ResultWrapper> eth_getFilterChanges(UInt256 filterId) { - int id = (int)filterId; + int id = filterId <= int.MaxValue ? (int)filterId : -1; FilterType filterType = _blockchainBridge.GetFilterType(id); switch (filterType) { @@ -535,19 +535,19 @@ public ResultWrapper> eth_getFilterChanges(UInt256 filterId) { return _blockchainBridge.FilterExists(id) ? ResultWrapper>.Success(_blockchainBridge.GetBlockFilterChanges(id)) - : ResultWrapper>.Fail($"Filter not found", ErrorCodes.InvalidInput); + : ResultWrapper>.Fail("Filter not found", ErrorCodes.InvalidInput); } case FilterType.PendingTransactionFilter: { return _blockchainBridge.FilterExists(id) ? ResultWrapper>.Success(_blockchainBridge.GetPendingTransactionFilterChanges(id)) - : ResultWrapper>.Fail($"Filter not found", ErrorCodes.InvalidInput); + : ResultWrapper>.Fail("Filter not found", ErrorCodes.InvalidInput); } case FilterType.LogFilter: { return _blockchainBridge.FilterExists(id) ? ResultWrapper>.Success(_blockchainBridge.GetLogFilterChanges(id).ToArray()) - : ResultWrapper>.Fail($"Filter not found", ErrorCodes.InvalidInput); + : ResultWrapper>.Fail("Filter not found", ErrorCodes.InvalidInput); } default: { diff --git a/src/Nethermind/Nethermind.Logging.NLog/NLogManager.cs b/src/Nethermind/Nethermind.Logging.NLog/NLogManager.cs index 830e25fd2ab..4497f7cf048 100644 --- a/src/Nethermind/Nethermind.Logging.NLog/NLogManager.cs +++ b/src/Nethermind/Nethermind.Logging.NLog/NLogManager.cs @@ -31,6 +31,7 @@ public NLogManager(string logFileName, string logDirectory = null, string logRul // Required since 'NLog.config' could change during runtime, we need to re-apply the configuration _logManagerOnConfigurationChanged = (sender, args) => Setup(logFileName, logDirectory, logRules); LogManager.ConfigurationChanged += _logManagerOnConfigurationChanged; + Static.LogManager = this; } private static void Setup(string logFileName, string logDirectory = null, string logRules = null) diff --git a/src/Nethermind/Nethermind.Logging/StaticLoggerFactory.cs b/src/Nethermind/Nethermind.Logging/StaticLoggerFactory.cs new file mode 100644 index 00000000000..c38d81648b9 --- /dev/null +++ b/src/Nethermind/Nethermind.Logging/StaticLoggerFactory.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Logging; + +public static class Static +{ + public static ILogManager LogManager { get; set; } = LimboLogs.Instance; +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs index 8ee688de990..fd73ea05442 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs @@ -140,7 +140,7 @@ public async Task can_parse_forkchoiceUpdated_with_implicit_null_payloadAttribut public void ForkchoiceV1_ToString_returns_correct_results() { ForkchoiceStateV1 forkchoiceState = new(TestItem.KeccakA, TestItem.KeccakF, TestItem.KeccakC); - forkchoiceState.ToString().Should().Be("ForkChoice: 0x03783f...35b760, Safe: 0x017e66...b18f72, Finalized: 0xe61d9a...97c37a"); + forkchoiceState.ToString().Should().Be("ForkChoice: 0x03783fac2efed8fbc9ad443e592ee30e61d65f471140c10ca155e937b435b760, Safe: 0x017e667f4b8c174291d1543c466717566e206df1bfd6f30271055ddafdb18f72, Finalized: 0xe61d9a3d3848fb2cdd9a2ab61e2f21a10ea431275aed628a0557f9dee697c37a"); } [Test] @@ -1551,6 +1551,9 @@ public void Should_return_expected_capabilities_for_mainnet() nameof(IEngineRpcModule.engine_getPayloadV4), nameof(IEngineRpcModule.engine_newPayloadV4), + + nameof(IEngineRpcModule.engine_getPayloadV5), + nameof(IEngineRpcModule.engine_getBlobsV2) }; Assert.That(result, Is.EquivalentTo(expectedMethods)); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/ForkchoiceStateV1.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/ForkchoiceStateV1.cs index 546e2fe4df9..1eb47a3018f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/ForkchoiceStateV1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/ForkchoiceStateV1.cs @@ -10,33 +10,26 @@ namespace Nethermind.Merge.Plugin.Data; /// /// /// -public class ForkchoiceStateV1 +public class ForkchoiceStateV1(Hash256 headBlockHash, Hash256 finalizedBlockHash, Hash256 safeBlockHash) { - public ForkchoiceStateV1(Hash256 headBlockHash, Hash256 finalizedBlockHash, Hash256 safeBlockHash) - { - HeadBlockHash = headBlockHash; - FinalizedBlockHash = finalizedBlockHash; - SafeBlockHash = safeBlockHash; - } - /// /// Hash of the head of the canonical chain. /// - public Hash256 HeadBlockHash { get; set; } + public Hash256 HeadBlockHash { get; set; } = headBlockHash; /// /// Safe block hash of the canonical chain under certain synchrony and honesty assumptions. This value MUST be either equal to or an ancestor of headBlockHash. /// /// Can be when transition block is not finalized yet. - public Hash256 SafeBlockHash { get; set; } + public Hash256 SafeBlockHash { get; set; } = safeBlockHash; /// /// Hash of the most recent finalized block /// /// Can be when transition block is not finalized yet. - public Hash256 FinalizedBlockHash { get; set; } + public Hash256 FinalizedBlockHash { get; set; } = finalizedBlockHash; - public override string ToString() => $"ForkChoice: {HeadBlockHash.ToShortString()}, Safe: {SafeBlockHash.ToShortString()}, Finalized: {FinalizedBlockHash.ToShortString()}"; + public override string ToString() => $"ForkChoice: {HeadBlockHash}, Safe: {SafeBlockHash}, Finalized: {FinalizedBlockHash}"; public string ToString(long? headNumber, long? safeNumber, long? finalizedNumber) => headNumber is null || safeNumber is null || finalizedNumber is null ? ToString() diff --git a/src/Nethermind/Nethermind.Network.Benchmark/InFlowBenchmarks.cs b/src/Nethermind/Nethermind.Network.Benchmark/InFlowBenchmarks.cs index 1a06dbf9cfc..e9a29e1c87c 100644 --- a/src/Nethermind/Nethermind.Network.Benchmark/InFlowBenchmarks.cs +++ b/src/Nethermind/Nethermind.Network.Benchmark/InFlowBenchmarks.cs @@ -19,9 +19,9 @@ namespace Nethermind.Network.Benchmarks { public class InFlowBenchmarks { - private static byte[] _input = Bytes.FromHexString("96cb6a910a279cab5bf0e0cd0432c7d1d28f76b794402bd97ba5a7dfa1b0e163fc75cd604bbc43d3c888618db2453158b06548d9bdb1b8208b70e0075d675bbe80268afff07570005454467c0c3e85a3468e3225015521fecb9a0c1a2092fd445f14552e33d4ebb4d7b533606989210b4c702662b4d2417f293b2701ee4336624916cddefb4c3777d1a144e1df66162b60bcaed3319becbbe19062f7e3715505dc84fa23a531d7dfd35b9ad95042a85faa643a40365d52d0b57e67cc428676cc364567a6c8cf7fda48bd4c54fd2c8fad1e504f3c2451f538793c022fcdfaef9d8fdafae2c989a7e61dba88879cc9062b774bc3f999b0cdece9aab3d34fafc1513e2de18e6c297d07a5992cc68adb27ce3de39c290884e2f6f0f8b24645c622b2cd9d396498e689a44dc775c22f4baa6b5c1685c19b020110c5788b0e61ebf1794f05df9237d61ca95f49f00fa0217c6a0935f5674c32f8eb7b9c5fe351d429ba0383761d980b53b1187a9a367f80075c020fcc71a75689a4a9e74b8c6137ab267ce6e697008ff5f8c29af53fd4f97018da197f990181f112be738fd0f57bcfc72a78dda731b64aecdd3f83586f1fcfdbdf2c58eff42b38fed19730a1fd0dd21d248d78a396689a508561590543dbe62b44475bec724f2c13678dc877d6cc15099634c10ee4206b339e45c4829c8d81537c3b57fab792929f79a391c29d4f10f6db8318d9f2bffffdafb3038f005ab5e9ce4bf46960f7cf078cebc4287da0330f6a02555912ce7db52a893e4739af43dd64eb9a70e8b4460da9955c39e5c24d2d35b29999b4c9979ba5acd16abd09427b57ad78db5076b0671fa4bf90a48b863db8a372c99e28a21213246f15cf36470f3a397b9e3cd1b0e4ce4e9ff1f95c5bc4c63c691a960d12e45f452adc7961eaca8dd24a3f51b16c6b15c930500b3c9b9e99638eedbf18332de4f5fa3e948872f46bbe2594de8bf32a1ca36a80a284dc1e055379543bded00cf110bdf5da82f0dc5e5a410c5aa925dd4a7dd2408796f05d3f1ab43c5b30a17e7e392212b0a61eacdfb200a1788e99f85a896e77fe7a86a17e61fdf00da6d9b2aac1935068bb6271a265e6b41aca2662b9beca7005bbbcbb371bb7daf60bf3be282e67ba17cb7d120015a8f7867391197c457e913a9558382f4dfbdd1e8693babf5835707c0142be013ec92869415c772dfc4581c5bb0aaaf0f7dabee7d488ea7e8a8954507e7f04f942429be3dda2317f719f8cf8e68720a91cce20d2406f35347cadb24e4ce3024a70c7938fa14179a9434996f73d666bd9fcb9b0dc02492fdef10a52c39a976058d611d854e6cad01cf85ff380499909638f7ca8f7397cb52b7d3cd9401b5905838b6134e6dbf4a03dff05551043532dbe9833502bba97839964806d55655804eef2317f68caf724f28625bcae7b761fac920d256c78e3843529e4ce7d25861db489128112002df79ee13cdcf5776e0416bdb36c022deba8d5329900da5c9a3bac6d39284691ca2ee684ee7fc9335c5183542b3751199587e9a0a0cad9b7e5365703ceda47d5959548a6a097fe8ff0f1fe8c59ad377db05f11002bacf555d14e99d7ae8cfd7960967384d912ecc3e11cc24cf9849d2592b834fa04ce12e0cb75eb4668ad6d3e43f16cf0dc2c9f54d77877014859b3b74b2bf9489b06682e0bd8f868a7fc34d8e7ff7d2c03435c5a15cc61a0a0791db38a9cc205e9f18e28e1445bfe982e25aeaa196d8dd14d25b65add73760726e15a1f54a414fd9377d2f87d33f87e117731fa6d04377686f33578c8b5481580fd81a74a17e4fb011bdcd9a10ee65ca37ae9eb5e606fde06d95d50880f4732ea6cb81f3d9adc88d43aa15aeadd4e790973f88b0dcdbe18cab73a9309f9e151c21d4025aa1af0fb37fe31ce25ce39c51fdb4e932b45c168111f54398fea71dd3e82ab01499db695e36308bc800246f17393d218a241191086fa02a4d70514e5bb3c5ff229a8320e53e0cd2b605c674dd6006b74a394fcfcf4d8fe0b61c3768b9e53b6601854ebe5dcd210b3db16adf68065a4f60a7dbd3c79776a21f6440cdc263aaa423b11954795d9424e3543b27efbe8dd76e002c30889e872c6d12d82c48933c382347ee89e69cf5d0120b342969312f98a19a8e972a8ef4f78a23701ac80deaf90cbc6b13fddea674790d6e9fe46d054f900b569e624bd50f8c8b20d3cdb29117f0cd28cb3a15c8530fb4827ebd3af739e872b59e6566c928933"); + private static readonly byte[] _input = Bytes.FromHexString("96cb6a910a279cab5bf0e0cd0432c7d1d28f76b794402bd97ba5a7dfa1b0e163fc75cd604bbc43d3c888618db2453158b06548d9bdb1b8208b70e0075d675bbe80268afff07570005454467c0c3e85a3468e3225015521fecb9a0c1a2092fd445f14552e33d4ebb4d7b533606989210b4c702662b4d2417f293b2701ee4336624916cddefb4c3777d1a144e1df66162b60bcaed3319becbbe19062f7e3715505dc84fa23a531d7dfd35b9ad95042a85faa643a40365d52d0b57e67cc428676cc364567a6c8cf7fda48bd4c54fd2c8fad1e504f3c2451f538793c022fcdfaef9d8fdafae2c989a7e61dba88879cc9062b774bc3f999b0cdece9aab3d34fafc1513e2de18e6c297d07a5992cc68adb27ce3de39c290884e2f6f0f8b24645c622b2cd9d396498e689a44dc775c22f4baa6b5c1685c19b020110c5788b0e61ebf1794f05df9237d61ca95f49f00fa0217c6a0935f5674c32f8eb7b9c5fe351d429ba0383761d980b53b1187a9a367f80075c020fcc71a75689a4a9e74b8c6137ab267ce6e697008ff5f8c29af53fd4f97018da197f990181f112be738fd0f57bcfc72a78dda731b64aecdd3f83586f1fcfdbdf2c58eff42b38fed19730a1fd0dd21d248d78a396689a508561590543dbe62b44475bec724f2c13678dc877d6cc15099634c10ee4206b339e45c4829c8d81537c3b57fab792929f79a391c29d4f10f6db8318d9f2bffffdafb3038f005ab5e9ce4bf46960f7cf078cebc4287da0330f6a02555912ce7db52a893e4739af43dd64eb9a70e8b4460da9955c39e5c24d2d35b29999b4c9979ba5acd16abd09427b57ad78db5076b0671fa4bf90a48b863db8a372c99e28a21213246f15cf36470f3a397b9e3cd1b0e4ce4e9ff1f95c5bc4c63c691a960d12e45f452adc7961eaca8dd24a3f51b16c6b15c930500b3c9b9e99638eedbf18332de4f5fa3e948872f46bbe2594de8bf32a1ca36a80a284dc1e055379543bded00cf110bdf5da82f0dc5e5a410c5aa925dd4a7dd2408796f05d3f1ab43c5b30a17e7e392212b0a61eacdfb200a1788e99f85a896e77fe7a86a17e61fdf00da6d9b2aac1935068bb6271a265e6b41aca2662b9beca7005bbbcbb371bb7daf60bf3be282e67ba17cb7d120015a8f7867391197c457e913a9558382f4dfbdd1e8693babf5835707c0142be013ec92869415c772dfc4581c5bb0aaaf0f7dabee7d488ea7e8a8954507e7f04f942429be3dda2317f719f8cf8e68720a91cce20d2406f35347cadb24e4ce3024a70c7938fa14179a9434996f73d666bd9fcb9b0dc02492fdef10a52c39a976058d611d854e6cad01cf85ff380499909638f7ca8f7397cb52b7d3cd9401b5905838b6134e6dbf4a03dff05551043532dbe9833502bba97839964806d55655804eef2317f68caf724f28625bcae7b761fac920d256c78e3843529e4ce7d25861db489128112002df79ee13cdcf5776e0416bdb36c022deba8d5329900da5c9a3bac6d39284691ca2ee684ee7fc9335c5183542b3751199587e9a0a0cad9b7e5365703ceda47d5959548a6a097fe8ff0f1fe8c59ad377db05f11002bacf555d14e99d7ae8cfd7960967384d912ecc3e11cc24cf9849d2592b834fa04ce12e0cb75eb4668ad6d3e43f16cf0dc2c9f54d77877014859b3b74b2bf9489b06682e0bd8f868a7fc34d8e7ff7d2c03435c5a15cc61a0a0791db38a9cc205e9f18e28e1445bfe982e25aeaa196d8dd14d25b65add73760726e15a1f54a414fd9377d2f87d33f87e117731fa6d04377686f33578c8b5481580fd81a74a17e4fb011bdcd9a10ee65ca37ae9eb5e606fde06d95d50880f4732ea6cb81f3d9adc88d43aa15aeadd4e790973f88b0dcdbe18cab73a9309f9e151c21d4025aa1af0fb37fe31ce25ce39c51fdb4e932b45c168111f54398fea71dd3e82ab01499db695e36308bc800246f17393d218a241191086fa02a4d70514e5bb3c5ff229a8320e53e0cd2b605c674dd6006b74a394fcfcf4d8fe0b61c3768b9e53b6601854ebe5dcd210b3db16adf68065a4f60a7dbd3c79776a21f6440cdc263aaa423b11954795d9424e3543b27efbe8dd76e002c30889e872c6d12d82c48933c382347ee89e69cf5d0120b342969312f98a19a8e972a8ef4f78a23701ac80deaf90cbc6b13fddea674790d6e9fe46d054f900b569e624bd50f8c8b20d3cdb29117f0cd28cb3a15c8530fb4827ebd3af739e872b59e6566c928933"); - private IByteBuffer _decoderBuffer = PooledByteBufferAllocator.Default.Buffer(1024 * 1024); + private readonly IByteBuffer _decoderBuffer = PooledByteBufferAllocator.Default.Buffer(1024 * 1024); private NewBlockMessage _outputMessage; private NewBlockMessageSerializer _newBlockMessageSerializer; @@ -57,8 +57,8 @@ public void Setup() public void IterationSetup() { _secrets = NetTestVectors.GetSecretsPair(); - FrameCipher frameCipher = new FrameCipher(_secrets.B.AesSecret); - FrameMacProcessor frameMacProcessor = new FrameMacProcessor(TestItem.IgnoredPublicKey, _secrets.B); + FrameCipher frameCipher = new(_secrets.B.AesSecret); + FrameMacProcessor frameMacProcessor = new(TestItem.IgnoredPublicKey, _secrets.B); _zeroDecoder = new TestZeroDecoder(frameCipher, frameMacProcessor); } @@ -79,7 +79,8 @@ private void SetupAll(bool useLimboOutput = false) ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Paranoid; } - private class TestZeroDecoder : ZeroFrameDecoder + private class TestZeroDecoder(IFrameCipher frameCipher, FrameMacProcessor frameMacProcessor) + : ZeroFrameDecoder(frameCipher, frameMacProcessor) { public IByteBuffer Decode(IByteBuffer input) { @@ -87,20 +88,10 @@ public IByteBuffer Decode(IByteBuffer input) base.Decode(null, input, result); return (IByteBuffer)result[0]; } - - public TestZeroDecoder(IFrameCipher frameCipher, FrameMacProcessor frameMacProcessor) - : base(frameCipher, frameMacProcessor, LimboLogs.Instance) - { - } } - private class TestZeroMerger : Rlpx.ZeroFrameMerger + private class TestZeroMerger() : ZeroFrameMerger(LimboLogs.Instance) { - public TestZeroMerger() - : base(LimboLogs.Instance) - { - } - public IByteBuffer Decode(IByteBuffer input) { var result = new List(); diff --git a/src/Nethermind/Nethermind.Network.Benchmark/OutFlowBenchmarks.cs b/src/Nethermind/Nethermind.Network.Benchmark/OutFlowBenchmarks.cs index 220e1601ff9..1188d0d8a9a 100644 --- a/src/Nethermind/Nethermind.Network.Benchmark/OutFlowBenchmarks.cs +++ b/src/Nethermind/Nethermind.Network.Benchmark/OutFlowBenchmarks.cs @@ -68,26 +68,17 @@ private void SetupAll(bool useLimboOutput = false) ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Paranoid; } - private class TestZeroEncoder : ZeroFrameEncoder + private class TestZeroEncoder(IFrameCipher frameCipher, IFrameMacProcessor frameMacProcessor) + : ZeroFrameEncoder(frameCipher, frameMacProcessor) { public void Encode(IByteBuffer message, IByteBuffer buffer) { base.Encode(null, message, buffer); } - - public TestZeroEncoder(IFrameCipher frameCipher, IFrameMacProcessor frameMacProcessor) - : base(frameCipher, frameMacProcessor, LimboLogs.Instance) - { - } } private class TestZeroSplitter : ZeroPacketSplitter { - public TestZeroSplitter() - : base(LimboLogs.Instance) - { - } - public void Encode(IByteBuffer input, IByteBuffer output) { base.Encode(null, input, output); diff --git a/src/Nethermind/Nethermind.Network.Discovery/NettyDiscoveryBaseHandler.cs b/src/Nethermind/Nethermind.Network.Discovery/NettyDiscoveryBaseHandler.cs index 41b7f0de984..17a3c34b12d 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/NettyDiscoveryBaseHandler.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/NettyDiscoveryBaseHandler.cs @@ -7,19 +7,14 @@ namespace Nethermind.Network.Discovery; -public abstract class NettyDiscoveryBaseHandler : SimpleChannelInboundHandler +public abstract class NettyDiscoveryBaseHandler(ILogManager? logManager) : SimpleChannelInboundHandler { - private readonly ILogger _logger; + private readonly ILogger _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); // https://github.com/ethereum/devp2p/blob/master/discv4.md#wire-protocol // https://github.com/ethereum/devp2p/blob/master/discv5/discv5-wire.md#udp-communication protected const int MaxPacketSize = 1280; - protected NettyDiscoveryBaseHandler(ILogManager? logManager) - { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - public override void ChannelRead(IChannelHandlerContext ctx, object msg) { if (msg is DatagramPacket packet && AcceptInboundMessage(packet) && !ValidatePacket(packet)) diff --git a/src/Nethermind/Nethermind.Network.Discovery/Serializers/NeighborsMsgSerializer.cs b/src/Nethermind/Nethermind.Network.Discovery/Serializers/NeighborsMsgSerializer.cs index 2fac4ab1431..46c95bedcc2 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/Serializers/NeighborsMsgSerializer.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/Serializers/NeighborsMsgSerializer.cs @@ -12,13 +12,17 @@ namespace Nethermind.Network.Discovery.Serializers; -public class NeighborsMsgSerializer : DiscoveryMsgSerializerBase, IZeroInnerMessageSerializer +public class NeighborsMsgSerializer( + IEcdsa ecdsa, + [KeyFilter(IProtectedPrivateKey.NodeKey)] + IPrivateKeyGenerator nodeKey, + INodeIdResolver nodeIdResolver) + : DiscoveryMsgSerializerBase(ecdsa, nodeKey, nodeIdResolver), IZeroInnerMessageSerializer { private static readonly Func _decodeItem = static ctx => { int lastPosition = ctx.ReadSequenceLength() + ctx.Position; int count = ctx.PeekNumberOfItemsRemaining(lastPosition); - ReadOnlySpan ip = ctx.DecodeByteArraySpan(); IPEndPoint address = GetAddress(ip, ctx.DecodeInt()); if (count > 3) @@ -30,12 +34,6 @@ public class NeighborsMsgSerializer : DiscoveryMsgSerializerBase, IZeroInnerMess return new Node(new PublicKey(id), address); }; - public NeighborsMsgSerializer(IEcdsa ecdsa, - [KeyFilter(IProtectedPrivateKey.NodeKey)] IPrivateKeyGenerator nodeKey, - INodeIdResolver nodeIdResolver) : base(ecdsa, nodeKey, nodeIdResolver) - { - } - public void Serialize(IByteBuffer byteBuffer, NeighborsMsg msg) { (int totalLength, int contentLength, int nodesContentLength) = GetLength(msg); diff --git a/src/Nethermind/Nethermind.Network.Discovery/Serializers/PongMsgSerializer.cs b/src/Nethermind/Nethermind.Network.Discovery/Serializers/PongMsgSerializer.cs index 3b6f5817424..4bc929cee9c 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/Serializers/PongMsgSerializer.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/Serializers/PongMsgSerializer.cs @@ -40,9 +40,9 @@ public void Serialize(IByteBuffer byteBuffer, PongMsg msg) public PongMsg Deserialize(IByteBuffer msgBytes) { - (PublicKey FarPublicKey, _, IByteBuffer Data) = PrepareForDeserialization(msgBytes); + (PublicKey farPublicKey, _, IByteBuffer data) = PrepareForDeserialization(msgBytes); - NettyRlpStream rlp = new(Data); + NettyRlpStream rlp = new(data); rlp.ReadSequenceLength(); rlp.ReadSequenceLength(); @@ -54,7 +54,7 @@ public PongMsg Deserialize(IByteBuffer msgBytes) byte[] token = rlp.DecodeByteArray(); long expirationTime = rlp.DecodeLong(); - PongMsg msg = new(FarPublicKey, expirationTime, token); + PongMsg msg = new(farPublicKey, expirationTime, token); return msg; } diff --git a/src/Nethermind/Nethermind.Network.Enr/EthEntry.cs b/src/Nethermind/Nethermind.Network.Enr/EthEntry.cs index a3c1e80976a..6cf94537d1c 100644 --- a/src/Nethermind/Nethermind.Network.Enr/EthEntry.cs +++ b/src/Nethermind/Nethermind.Network.Enr/EthEntry.cs @@ -8,10 +8,8 @@ namespace Nethermind.Network.Enr; /// /// https://github.com/ethereum/devp2p/blob/master/enr-entries/eth.md /// -public class EthEntry : EnrContentEntry +public class EthEntry(byte[] forkHash, long nextBlock) : EnrContentEntry(new ForkId(forkHash, nextBlock)) { - public EthEntry(byte[] forkHash, long nextBlock) : base(new ForkId(forkHash, nextBlock)) { } - public override string Key => EnrContentKey.Eth; protected override int GetRlpLengthOfValue() diff --git a/src/Nethermind/Nethermind.Network.Enr/NodeRecordSigner.cs b/src/Nethermind/Nethermind.Network.Enr/NodeRecordSigner.cs index 94840f38ec1..b5610d3d000 100644 --- a/src/Nethermind/Nethermind.Network.Enr/NodeRecordSigner.cs +++ b/src/Nethermind/Nethermind.Network.Enr/NodeRecordSigner.cs @@ -45,7 +45,7 @@ public NodeRecord Deserialize(RlpStream rlpStream) throw new NetworkingException("RLP received for ENR is bigger than 300 bytes", NetworkExceptionType.Discovery); NodeRecord nodeRecord = new(); - ReadOnlySpan sigBytes = rlpStream.DecodeByteArraySpan(); + ReadOnlySpan sigBytes = rlpStream.DecodeByteArraySpan(RlpLimit.L65); Signature signature = new(sigBytes, 0); bool canVerify = true; diff --git a/src/Nethermind/Nethermind.Network.Stats/Model/DisconnectReason.cs b/src/Nethermind/Nethermind.Network.Stats/Model/DisconnectReason.cs index 345eb95dd78..10b773a3b29 100644 --- a/src/Nethermind/Nethermind.Network.Stats/Model/DisconnectReason.cs +++ b/src/Nethermind/Nethermind.Network.Stats/Model/DisconnectReason.cs @@ -53,8 +53,9 @@ public enum DisconnectReason : byte InvalidReceiptRoot, EthSyncException, InvalidBlockRangeUpdate, + MessageLimitsBreached, - // These are from EthDisconnectReason which does not necessarily used in Nethermind. + // These are from EthDisconnectReason that does not necessarily use in Nethermind. EthDisconnectRequested, TcpSubSystemError, BreachOfProtocol, @@ -67,7 +68,7 @@ public enum DisconnectReason : byte ReceiveMessageTimeout, MultipleHeaderDependencies, - // Try not to use this. Instead create a new one. + // Try not to use this. Instead, create a new one. Other, } @@ -95,6 +96,7 @@ public static EthDisconnectReason ToEthDisconnectReason(this DisconnectReason di DisconnectReason.EthDisconnectRequested => EthDisconnectReason.DisconnectRequested, DisconnectReason.TcpSubSystemError => EthDisconnectReason.TcpSubSystemError, DisconnectReason.BreachOfProtocol => EthDisconnectReason.BreachOfProtocol, + DisconnectReason.MessageLimitsBreached => EthDisconnectReason.BreachOfProtocol, DisconnectReason.UselessPeer => EthDisconnectReason.UselessPeer, DisconnectReason.AlreadyConnected => EthDisconnectReason.AlreadyConnected, DisconnectReason.NullNodeIdentityReceived => EthDisconnectReason.NullNodeIdentityReceived, diff --git a/src/Nethermind/Nethermind.Network.Stats/StatsParameters.cs b/src/Nethermind/Nethermind.Network.Stats/StatsParameters.cs index f88948362ed..001cae24f65 100644 --- a/src/Nethermind/Nethermind.Network.Stats/StatsParameters.cs +++ b/src/Nethermind/Nethermind.Network.Stats/StatsParameters.cs @@ -35,7 +35,8 @@ private StatsParameters() { DisconnectReason.UnexpectedIdentity, (TimeSpan.FromMinutes(15), -10000) }, { DisconnectReason.IncompatibleP2PVersion, (TimeSpan.FromMinutes(15), -10000) }, { DisconnectReason.UselessPeer, (TimeSpan.FromMinutes(15), -10000) }, - { DisconnectReason.BreachOfProtocol, (TimeSpan.FromMinutes(15), -10000) } + { DisconnectReason.BreachOfProtocol, (TimeSpan.FromMinutes(15), -10000) }, + { DisconnectReason.MessageLimitsBreached, (TimeSpan.FromMinutes(15), -10000) } }; public Dictionary RemoteDisconnectParams { get; } = new() diff --git a/src/Nethermind/Nethermind.Network.Stats/SyncLimits/NethermindSyncLimits.cs b/src/Nethermind/Nethermind.Network.Stats/SyncLimits/NethermindSyncLimits.cs index 9e60940887d..3edae4794a0 100644 --- a/src/Nethermind/Nethermind.Network.Stats/SyncLimits/NethermindSyncLimits.cs +++ b/src/Nethermind/Nethermind.Network.Stats/SyncLimits/NethermindSyncLimits.cs @@ -5,9 +5,10 @@ namespace Nethermind.Stats.SyncLimits { public static class NethermindSyncLimits { - public const int MaxHeaderFetch = 512; // Amount of block headers to be fetched per retrieval request - public const int MaxBodyFetch = 256; // Amount of block bodies to be fetched per retrieval request - public const int MaxReceiptFetch = 256; // Amount of transaction receipts to allow fetching per request - public const int MaxCodeFetch = 1024; // Amount of contract codes to allow fetching per request + public const int MaxHeaderFetch = 1024; // Number of block headers to be fetched per retrieval request + public const int MaxBodyFetch = 512; // Number of block bodies to be fetched per retrieval request + public const int MaxReceiptFetch = 512; // Number of transaction receipts to allow fetching per request + public const int MaxCodeFetch = 1024; // Number of contract codes to allow fetching per request + public const int MaxHashesFetch = 10000; // Number of hashes to allow fetching per request } } diff --git a/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs b/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs index f377009b851..7f6debcb360 100644 --- a/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs @@ -56,8 +56,14 @@ public class ForkInfoTests [TestCase(15_051_000, 1_710_338_134ul, "0xdce96c2d", 1_710_338_135ul, "Future Shanghai timestamp")] [TestCase(15_051_000, 1_710_338_135ul, "0x9f3d2254", 1_746_612_311ul, "First Cancun timestamp")] [TestCase(15_051_000, 1_746_612_310ul, "0x9f3d2254", 1_746_612_311ul, "Future Cancun timestamp")] - [TestCase(15_051_000, 1_746_612_311ul, "0xc376cf8b", 0ul, "First Prague timestamp")] - [TestCase(15_051_000, 1_846_612_311ul, "0xc376cf8b", 0ul, "Future Prague timestamp")] + [TestCase(15_051_000, 1_746_612_311ul, "0xc376cf8b", 1_764_798_551ul, "First Prague timestamp")] + [TestCase(15_051_000, 1_764_798_550ul, "0xc376cf8b", 1_764_798_551ul, "Future Prague timestamp")] + [TestCase(15_051_000, 1_764_798_551ul, "0x5167e2a6", 1_765_290_071ul, "First Osaka timestamp")] + [TestCase(15_051_000, 1_765_290_070ul, "0x5167e2a6", 1_765_290_071ul, "Future Osaka timestamp")] + [TestCase(15_051_000, 1_765_290_071ul, "0xcba2a1c0", 1_767_747_671ul, "First BPO1 timestamp")] + [TestCase(15_051_000, 1_767_747_670ul, "0xcba2a1c0", 1_767_747_671ul, "Future BPO1 timestamp")] + [TestCase(15_051_000, 1_767_747_671ul, "0x07c9462e", 0ul, "First BPO2 timestamp")] + [TestCase(15_051_000, 1_867_747_671ul, "0x07c9462e", 0ul, "Future BPO2 timestamp")] public void Fork_id_and_hash_as_expected(long head, ulong headTimestamp, string forkHashHex, ulong next, string description) { Test(head, headTimestamp, KnownHashes.MainnetGenesis, forkHashHex, next, description, MainnetSpecProvider.Instance, "foundation.json"); diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs index ba1cd186deb..9e8e255ab08 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs @@ -5,6 +5,7 @@ using System.Text.RegularExpressions; using DotNetty.Buffers; using FluentAssertions; +using Nethermind.Consensus.Scheduler; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Test.Builders; @@ -68,6 +69,7 @@ private P2PProtocolHandler CreateSession() TestItem.PublicKeyA, _nodeStatsManager, _serializer, + Substitute.For(), LimboLogs.Instance); } diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/SessionTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/SessionTests.cs index f5609b63b6a..0515797be47 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/SessionTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/SessionTests.cs @@ -35,7 +35,7 @@ public void SetUp() _pipeline = Substitute.For(); _channelHandlerContext.Channel.Returns(_channel); _channel.Pipeline.Returns(_pipeline); - _pipeline.Get().Returns(new ZeroPacketSplitter(LimboLogs.Instance)); + _pipeline.Get().Returns(new ZeroPacketSplitter()); _packetSender = Substitute.For(); } diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandlerTests.cs index e61dc1266b9..c956e27817a 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandlerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandlerTests.cs @@ -20,9 +20,11 @@ using Nethermind.Network.P2P.Subprotocols.Eth.V63; using Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages; using Nethermind.Network.Rlpx; +using Nethermind.Serialization.Rlp; using Nethermind.Specs; using Nethermind.Stats; using Nethermind.Stats.Model; +using Nethermind.Stats.SyncLimits; using Nethermind.Synchronization; using Nethermind.TxPool; using NSubstitute; @@ -54,27 +56,30 @@ public void TearDown() [Test] public async Task Can_request_and_handle_receipts() { + const int count = NethermindSyncLimits.MaxReceiptFetch; TxReceipt[][]? receipts = Enumerable.Repeat( Enumerable.Repeat(Build.A.Receipt.WithAllFieldsFilled.TestObject, 100).ToArray(), - 1000).ToArray(); // TxReceipt[1000][100] + count).ToArray(); // TxReceipt[1000][100] using ReceiptsMessage receiptsMsg = new(receipts.ToPooledList()); Packet receiptsPacket = new("eth", Eth63MessageCode.Receipts, _ctx._receiptMessageSerializer.Serialize(receiptsMsg)); Task> task = _ctx.ProtocolHandler.GetReceipts( - Enumerable.Repeat(Keccak.Zero, 1000).ToArray(), + Enumerable.Repeat(Keccak.Zero, count).ToArray(), CancellationToken.None); _ctx.ProtocolHandler.HandleMessage(receiptsPacket); using var result = await task; - result.Should().HaveCount(1000); + result.Should().HaveCount(count); } [Test] public async Task Limit_receipt_request() { + const int count = NethermindSyncLimits.MaxReceiptFetch; + TxReceipt[] oneBlockReceipt = Enumerable.Repeat(Build.A.Receipt.WithAllFieldsFilled.TestObject, 100).ToArray(); Packet smallReceiptsPacket = new("eth", Eth63MessageCode.Receipts, _ctx._receiptMessageSerializer.Serialize( @@ -82,7 +87,7 @@ public async Task Limit_receipt_request() )); Packet largeReceiptsPacket = new("eth", Eth63MessageCode.Receipts, _ctx._receiptMessageSerializer.Serialize( - new(RepeatPooled(oneBlockReceipt, 1000)) + new(RepeatPooled(oneBlockReceipt, count)) )); GetReceiptsMessage? receiptsMessage = null; @@ -92,7 +97,7 @@ public async Task Limit_receipt_request() .Do(info => receiptsMessage = (GetReceiptsMessage)info[0]); Task> receiptsTask = _ctx.ProtocolHandler - .GetReceipts(RepeatPooled(Keccak.Zero, 1000), CancellationToken.None) + .GetReceipts(RepeatPooled(Keccak.Zero, count), CancellationToken.None) .AddResultTo(_disposables); _ctx.ProtocolHandler.HandleMessage(smallReceiptsPacket); @@ -101,7 +106,7 @@ public async Task Limit_receipt_request() Assert.That(receiptsMessage?.Hashes?.Count, Is.EqualTo(8)); receiptsTask = _ctx.ProtocolHandler - .GetReceipts(RepeatPooled(Keccak.Zero, 1000), CancellationToken.None) + .GetReceipts(RepeatPooled(Keccak.Zero, count), CancellationToken.None) .AddResultTo(_disposables); _ctx.ProtocolHandler.HandleMessage(largeReceiptsPacket); @@ -111,7 +116,7 @@ public async Task Limit_receipt_request() // Back to 10 receiptsTask = _ctx.ProtocolHandler - .GetReceipts(RepeatPooled(Keccak.Zero, 1000), CancellationToken.None) + .GetReceipts(RepeatPooled(Keccak.Zero, count), CancellationToken.None) .AddResultTo(_disposables); _ctx.ProtocolHandler.HandleMessage(smallReceiptsPacket); @@ -127,12 +132,11 @@ private ArrayPoolList RepeatPooled(T txReceipts, int count) => [Test] public void Will_not_serve_receipts_requests_above_512() { - using GetReceiptsMessage getReceiptsMessage = new( - RepeatPooled(Keccak.Zero, 513)); + using GetReceiptsMessage getReceiptsMessage = new(RepeatPooled(Keccak.Zero, NethermindSyncLimits.MaxReceiptFetch + 1)); Packet getReceiptsPacket = new("eth", Eth63MessageCode.GetReceipts, _ctx._getReceiptMessageSerializer.Serialize(getReceiptsMessage)); - _ctx.ProtocolHandler.HandleMessage(getReceiptsPacket); + Assert.Throws(() => _ctx.ProtocolHandler.HandleMessage(getReceiptsPacket)); _ctx.Session.Received().InitiateDisconnect(Arg.Any(), Arg.Any()); } diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandlerTests.cs index 934aa5c2e6b..00c374ba0c9 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandlerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandlerTests.cs @@ -54,7 +54,6 @@ public class Eth69ProtocolHandlerTests private Eth69ProtocolHandler _handler = null!; private ITxGossipPolicy _txGossipPolicy = null!; private ITimerFactory _timerFactory = null!; - private IBlockFinder _blockFinder = null!; [SetUp] public void Setup() @@ -72,13 +71,12 @@ public void Setup() _genesisBlock = Build.A.Block.Genesis.TestObject; _syncManager.Head.Returns(_genesisBlock.Header); _syncManager.Genesis.Returns(_genesisBlock.Header); + _syncManager.LowestBlock.Returns(0); _timerFactory = Substitute.For(); _txGossipPolicy = Substitute.For(); _txGossipPolicy.ShouldListenToGossipedTransactions.Returns(true); _txGossipPolicy.ShouldGossipTransaction(Arg.Any()).Returns(true); _svc = Build.A.SerializationService().WithEth69(_specProvider).TestObject; - _blockFinder = Substitute.For(); - _blockFinder.GetLowestBlock().Returns(3); _handler = new Eth69ProtocolHandler( _session, _svc, @@ -89,7 +87,6 @@ public void Setup() _pooledTxsRequestor, _gossipPolicy, new ForkInfo(_specProvider, _syncManager), - _blockFinder, LimboLogs.Instance, _txGossipPolicy); _handler.Init(); @@ -280,7 +277,7 @@ public void On_init_sends_a_status_message() && m.Protocol == Protocol.Eth && m.GenesisHash == _genesisBlock.Hash && m.LatestBlockHash == _genesisBlock.Hash - && m.EarliestBlock == 3)); + && m.EarliestBlock == 0)); } private void HandleIncomingStatusMessage() diff --git a/src/Nethermind/Nethermind.Network.Test/ProtocolsManagerTests.cs b/src/Nethermind/Nethermind.Network.Test/ProtocolsManagerTests.cs index feb5c3bf5e8..42d92a65360 100644 --- a/src/Nethermind/Nethermind.Network.Test/ProtocolsManagerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/ProtocolsManagerTests.cs @@ -82,7 +82,7 @@ public Context() _pipeline = Substitute.For(); _channelHandlerContext.Channel.Returns(_channel); _channel.Pipeline.Returns(_pipeline); - _pipeline.Get().Returns(new ZeroPacketSplitter(LimboLogs.Instance)); + _pipeline.Get().Returns(new ZeroPacketSplitter()); _packetSender = Substitute.For(); _syncServer = Substitute.For(); _syncServer = Substitute.For(); @@ -129,7 +129,6 @@ public Context() forkInfo, _gossipPolicy, Substitute.For(), - _blockTree, LimboLogs.Instance); } diff --git a/src/Nethermind/Nethermind.Network.Test/Rlpx/HobbitTests.cs b/src/Nethermind/Nethermind.Network.Test/Rlpx/HobbitTests.cs index c4fb7335fea..99907c20b1b 100644 --- a/src/Nethermind/Nethermind.Network.Test/Rlpx/HobbitTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/Rlpx/HobbitTests.cs @@ -214,10 +214,10 @@ private EmbeddedChannel BuildEmbeddedChannel(StackType inbound, StackType outbou throw new NotSupportedException(); } - IChannelHandler decoder = new ZeroFrameDecoder(_frameCipherB, _macProcessorB, LimboLogs.Instance); + IChannelHandler decoder = new ZeroFrameDecoder(_frameCipherB, _macProcessorB); IChannelHandler merger = new ZeroFrameMerger(LimboLogs.Instance); - IChannelHandler encoder = new ZeroFrameEncoder(_frameCipherA, _macProcessorA, LimboLogs.Instance); - IFramingAware splitter = new ZeroPacketSplitter(LimboLogs.Instance); + IChannelHandler encoder = new ZeroFrameEncoder(_frameCipherA, _macProcessorA); + IFramingAware splitter = new ZeroPacketSplitter(); Assert.That(splitter.MaxFrameSize, Is.EqualTo(Frame.DefaultMaxFrameSize), "default max frame size"); diff --git a/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameDecoderTestWrapper.cs b/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameDecoderTestWrapper.cs index e5080230985..8cd56825966 100644 --- a/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameDecoderTestWrapper.cs +++ b/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameDecoderTestWrapper.cs @@ -5,7 +5,6 @@ using DotNetty.Buffers; using DotNetty.Codecs; using DotNetty.Transport.Channels; -using Nethermind.Logging; using Nethermind.Network.Rlpx; using NSubstitute; @@ -15,7 +14,7 @@ internal class ZeroFrameDecoderTestWrapper : ZeroFrameDecoder { private readonly IChannelHandlerContext _context; - public ZeroFrameDecoderTestWrapper(IFrameCipher frameCipher, FrameMacProcessor frameMacProcessor) : base(frameCipher, frameMacProcessor, LimboLogs.Instance) + public ZeroFrameDecoderTestWrapper(IFrameCipher frameCipher, FrameMacProcessor frameMacProcessor) : base(frameCipher, frameMacProcessor) { _context = Substitute.For(); _context.Allocator.Returns(PooledByteBufferAllocator.Default); diff --git a/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameEncoderTestWrapper.cs b/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameEncoderTestWrapper.cs index f11176d2391..e2de7444707 100644 --- a/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameEncoderTestWrapper.cs +++ b/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroFrameEncoderTestWrapper.cs @@ -3,7 +3,6 @@ using DotNetty.Buffers; using DotNetty.Transport.Channels; -using Nethermind.Logging; using Nethermind.Network.Rlpx; using NSubstitute; @@ -13,7 +12,7 @@ internal class ZeroFrameEncoderTestWrapper : ZeroFrameEncoder { private readonly IChannelHandlerContext _context; - public ZeroFrameEncoderTestWrapper(IFrameCipher frameCipher, IFrameMacProcessor frameMacProcessor) : base(frameCipher, frameMacProcessor, LimboLogs.Instance) + public ZeroFrameEncoderTestWrapper(IFrameCipher frameCipher, IFrameMacProcessor frameMacProcessor) : base(frameCipher, frameMacProcessor) { _context = Substitute.For(); } diff --git a/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroPacketSplitterTestWrapper.cs b/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroPacketSplitterTestWrapper.cs index ea4766ad39f..5049c93037c 100644 --- a/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroPacketSplitterTestWrapper.cs +++ b/src/Nethermind/Nethermind.Network.Test/Rlpx/TestWrappers/ZeroPacketSplitterTestWrapper.cs @@ -3,7 +3,6 @@ using DotNetty.Buffers; using DotNetty.Transport.Channels; -using Nethermind.Logging; using Nethermind.Network.Rlpx; using NSubstitute; @@ -24,7 +23,7 @@ public IByteBuffer Encode(IByteBuffer input) return result; } - public ZeroPacketSplitterTestWrapper() : base(LimboLogs.Instance) + public ZeroPacketSplitterTestWrapper() : base() { _context.Allocator.Returns(PooledByteBufferAllocator.Default); } diff --git a/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameEncodeDecodeTests.cs b/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameEncodeDecodeTests.cs index 6304be58efa..bd779c7c5e6 100644 --- a/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameEncodeDecodeTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameEncodeDecodeTests.cs @@ -40,10 +40,10 @@ public async Task TwoWayConcurrentEncodeDecodeTests() private async Task RunStreamTests(FrameCipher frameCipher, FrameMacProcessor macProcessor, FrameCipher frameCipher2, FrameMacProcessor macProcessor2) { - ZeroPacketSplitter splitter = new(LimboLogs.Instance); - ZeroFrameEncoder encoder = new(frameCipher, macProcessor, LimboLogs.Instance); + ZeroPacketSplitter splitter = new(); + ZeroFrameEncoder encoder = new(frameCipher, macProcessor); - ZeroFrameDecoder decoder = new(frameCipher2, macProcessor2, LimboLogs.Instance); + ZeroFrameDecoder decoder = new(frameCipher2, macProcessor2); ZeroFrameMerger frameMerger = new(LimboLogs.Instance); IByteBuffer reDecoded = null; diff --git a/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameMergerTests.cs b/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameMergerTests.cs index 0d060f279e4..c6caabd1278 100644 --- a/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameMergerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/Rlpx/ZeroNettyFrameMergerTests.cs @@ -4,7 +4,6 @@ using DotNetty.Buffers; using DotNetty.Transport.Channels; using Nethermind.Core.Extensions; -using Nethermind.Logging; using Nethermind.Network.P2P.Messages; using Nethermind.Network.Rlpx; using Nethermind.Network.Test.Rlpx.TestWrappers; @@ -25,7 +24,7 @@ public void Encode(IByteBuffer input, IByteBuffer output) base.Encode(_context, input, output); } - public TestFrameHelper() : base(LimboLogs.Instance) + public TestFrameHelper() : base() { } } diff --git a/src/Nethermind/Nethermind.Network/NetworkNodeDecoder.cs b/src/Nethermind/Nethermind.Network/NetworkNodeDecoder.cs index 641d0657188..c9654dbd9f8 100644 --- a/src/Nethermind/Nethermind.Network/NetworkNodeDecoder.cs +++ b/src/Nethermind/Nethermind.Network/NetworkNodeDecoder.cs @@ -12,6 +12,8 @@ namespace Nethermind.Network { public class NetworkNodeDecoder : IRlpStreamDecoder, IRlpObjectDecoder { + private static readonly RlpLimit RlpLimit = RlpLimit.For((int)1.KiB(), nameof(NetworkNode.HostIp)); + static NetworkNodeDecoder() { Rlp.RegisterDecoder(typeof(NetworkNode), new NetworkNodeDecoder()); @@ -21,9 +23,9 @@ public NetworkNode Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBe { rlpStream.ReadSequenceLength(); - PublicKey publicKey = new(rlpStream.DecodeByteArraySpan()); - string ip = rlpStream.DecodeString(); - int port = (int)rlpStream.DecodeByteArraySpan().ReadEthUInt64(); + PublicKey publicKey = new(rlpStream.DecodeByteArraySpan(RlpLimit.L64)); + string ip = rlpStream.DecodeString(RlpLimit); + int port = (int)rlpStream.DecodeByteArraySpan(RlpLimit.L8).ReadEthUInt64(); rlpStream.SkipItem(); long reputation = 0L; try diff --git a/src/Nethermind/Nethermind.Network/P2P/ISession.cs b/src/Nethermind/Nethermind.Network/P2P/ISession.cs index 69d1ca2ae61..f8616a02fc3 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ISession.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ISession.cs @@ -29,7 +29,6 @@ public interface ISession : IDisposable Node Node { get; } DateTime LastPingUtc { get; set; } DateTime LastPongUtc { get; set; } - void ReceiveMessage(Packet packet); void ReceiveMessage(ZeroPacket zeroPacket); int DeliverMessage(T message) where T : P2PMessage; void EnableSnappy(); diff --git a/src/Nethermind/Nethermind.Network/P2P/Messages/AddCapabilityMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Messages/AddCapabilityMessageSerializer.cs index 4a798306823..1ab325c5940 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Messages/AddCapabilityMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Messages/AddCapabilityMessageSerializer.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using DotNetty.Buffers; +using Nethermind.Core.Extensions; using Nethermind.Serialization.Rlp; using Nethermind.Stats.Model; @@ -12,6 +13,8 @@ namespace Nethermind.Network.P2P.Messages /// public class AddCapabilityMessageSerializer : IZeroMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For((int)1.KiB(), nameof(Capability.ProtocolCode)); + public void Serialize(IByteBuffer byteBuffer, AddCapabilityMessage msg) { int totalLength = GetLength(msg, out int contentLength); @@ -27,7 +30,7 @@ public AddCapabilityMessage Deserialize(IByteBuffer byteBuffer) { NettyRlpStream context = new(byteBuffer); context.ReadSequenceLength(); - string protocolCode = context.DecodeString(); + string protocolCode = context.DecodeString(RlpLimit); byte version = context.DecodeByte(); return new AddCapabilityMessage(new Capability(protocolCode, version)); diff --git a/src/Nethermind/Nethermind.Network/P2P/Messages/DisconnectMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Messages/DisconnectMessageSerializer.cs index a8a53930f99..416b30f1322 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Messages/DisconnectMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Messages/DisconnectMessageSerializer.cs @@ -50,7 +50,7 @@ public DisconnectMessage Deserialize(IByteBuffer msgBytes) rlpStream.ReadSequenceLength(); int reason = rlpStream.DecodeInt(); - DisconnectMessage disconnectMessage = new DisconnectMessage(reason); + DisconnectMessage disconnectMessage = new(reason); return disconnectMessage; } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Messages/HelloMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Messages/HelloMessageSerializer.cs index b2ad3dea2a5..7ffa7ff586d 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Messages/HelloMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Messages/HelloMessageSerializer.cs @@ -70,7 +70,7 @@ public HelloMessage Deserialize(IByteBuffer msgBytes) helloMessage.ListenPort = rlpStream.DecodeInt(); - ReadOnlySpan publicKeyBytes = rlpStream.DecodeByteArraySpan(); + ReadOnlySpan publicKeyBytes = rlpStream.DecodeByteArraySpan(RlpLimit.L64); if (publicKeyBytes.Length != PublicKey.LengthInBytes && publicKeyBytes.Length != PublicKey.PrefixedLengthInBytes) { diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/P2PProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/P2PProtocolHandler.cs index 79ef7d152c7..16a824407e5 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/P2PProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/P2PProtocolHandler.cs @@ -10,6 +10,7 @@ using System.Threading; using System.Threading.Tasks; using FastEnumUtility; +using Nethermind.Consensus.Scheduler; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; @@ -29,8 +30,9 @@ public class P2PProtocolHandler( PublicKey localNodeId, INodeStatsManager nodeStatsManager, IMessageSerializationService serializer, + IBackgroundTaskScheduler backgroundTaskScheduler, ILogManager logManager) - : ProtocolHandlerBase(session, nodeStatsManager, serializer, logManager), IPingSender, IP2PProtocolHandler + : ProtocolHandlerBase(session, nodeStatsManager, serializer, backgroundTaskScheduler, logManager), IPingSender, IP2PProtocolHandler { private TaskCompletionSource _pongCompletionSource; private readonly INodeStatsManager _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ProtocolHandlerBase.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ProtocolHandlerBase.cs index 51eea57c190..a07b28f4fd5 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ProtocolHandlerBase.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ProtocolHandlerBase.cs @@ -2,54 +2,86 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; using DotNetty.Buffers; +using Nethermind.Consensus.Scheduler; using Nethermind.Core; using Nethermind.Logging; using Nethermind.Network.P2P.EventArg; using Nethermind.Network.P2P.Messages; +using Nethermind.Network.P2P.Utils; using Nethermind.Network.Rlpx; using Nethermind.Serialization.Rlp; using Nethermind.Stats; using Nethermind.Stats.Model; +using Nethermind.Synchronization; namespace Nethermind.Network.P2P.ProtocolHandlers { - public abstract class ProtocolHandlerBase( - ISession session, - INodeStatsManager nodeStats, - IMessageSerializationService serializer, - ILogManager logManager) - : IProtocolHandler + public abstract class ProtocolHandlerBase : IProtocolHandler { public abstract string Name { get; } public bool IsPriority { get; set; } - protected INodeStatsManager StatsManager { get; } = nodeStats ?? throw new ArgumentNullException(nameof(nodeStats)); - private readonly IMessageSerializationService _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); - protected internal ISession Session { get; } = session ?? throw new ArgumentNullException(nameof(session)); + protected INodeStatsManager StatsManager { get; } + private readonly IMessageSerializationService _serializer; + protected internal ISession Session { get; } protected long Counter; private readonly TaskCompletionSource _initCompletionSource = new(); - protected internal ILogger Logger { get; } = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + protected ProtocolHandlerBase(ISession session, + INodeStatsManager nodeStats, + IMessageSerializationService serializer, + IBackgroundTaskScheduler backgroundTaskScheduler, + ILogManager logManager) + { + StatsManager = nodeStats ?? throw new ArgumentNullException(nameof(nodeStats)); + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); + Session = session ?? throw new ArgumentNullException(nameof(session)); + Logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + BackgroundTaskScheduler = new BackgroundTaskSchedulerWrapper(this, backgroundTaskScheduler); + } + + protected internal ILogger Logger { get; } protected abstract TimeSpan InitTimeout { get; } + protected BackgroundTaskSchedulerWrapper BackgroundTaskScheduler { get; } + protected T Deserialize(byte[] data) where T : P2PMessage { + var size = data.Length; try { return _serializer.Deserialize(data); } + catch (RlpLimitException e) + { + HandleRlpLimitException(size, e); + throw; + } catch (RlpException e) { - if (Logger.IsDebug) Logger.Debug($"Failed to deserialize message {typeof(T).Name}, with exception {e}"); - ReportIn($"{typeof(T).Name} - Deserialization exception", data.Length); + HandleRlpException(size, e); throw; } } + private void HandleRlpException(int dataLength, RlpException e) where T : P2PMessage + { + if (Logger.IsDebug) Logger.Debug($"Failed to deserialize message {typeof(T).Name} on session {Session}, with exception {e}"); + ReportIn($"{typeof(T).Name} - Deserialization exception", dataLength); + } + + private void HandleRlpLimitException(int dataLength, RlpLimitException e) where T : P2PMessage + { + Session.InitiateDisconnect(DisconnectReason.MessageLimitsBreached, e.Message); + if (Logger.IsDebug) Logger.Debug($"Failed to deserialize message {typeof(T).Name} on session {Session} due to rlp limits, with exception {e}"); + ReportIn($"{typeof(T).Name} - Deserialization limit exception", dataLength); + } + protected T Deserialize(IByteBuffer data) where T : P2PMessage { int size = data.ReadableBytes; @@ -57,21 +89,27 @@ protected T Deserialize(IByteBuffer data) where T : P2PMessage { int originalReaderIndex = data.ReaderIndex; T result = _serializer.Deserialize(data); - if (data.IsReadable()) - { - throw new IncompleteDeserializationException( - $"Incomplete deserialization detected. Buffer is still readable. Read bytes: {data.ReaderIndex - originalReaderIndex}. Readable bytes: {data.ReadableBytes}"); - } + if (data.IsReadable()) ThrowIncompleteDeserializationException(data, originalReaderIndex); return result; } + catch (RlpLimitException e) + { + HandleRlpLimitException(size, e); + throw; + } catch (RlpException e) { - if (Logger.IsDebug) Logger.Debug($"Failed to deserialize message {typeof(T).Name}, with exception {e}"); - ReportIn($"{typeof(T).Name} - Deserialization exception", size); + HandleRlpException(size, e); throw; } } + [DoesNotReturn] + private static void ThrowIncompleteDeserializationException(IByteBuffer data, int originalReaderIndex) + { + throw new IncompleteDeserializationException($"Incomplete deserialization detected. Buffer is still readable. Read bytes: {data.ReaderIndex - originalReaderIndex}. Readable bytes: {data.ReadableBytes}"); + } + protected internal void Send(T message) where T : P2PMessage { Interlocked.Increment(ref Counter); @@ -126,7 +164,7 @@ protected void ReportIn(MessageBase msg, int size) { if (Logger.IsTrace || NetworkDiagTracer.IsEnabled) { - ReportIn(msg.ToString(), size); + ReportIn(msg.ToString() ?? "", size); } } @@ -139,6 +177,22 @@ protected void ReportIn(string messageInfo, int size) NetworkDiagTracer.ReportIncomingMessage(Session?.Node?.Address, Name, messageInfo, size); } + protected void HandleInBackground(ZeroPacket message, Func> handle) where TReq : P2PMessage where TRes : P2PMessage => + BackgroundTaskScheduler.ScheduleSyncServe(DeserializeAndReport(message), handle); + + protected void HandleInBackground(ZeroPacket message, Func> handle) where TReq : P2PMessage where TRes : P2PMessage => + BackgroundTaskScheduler.ScheduleSyncServe(DeserializeAndReport(message), handle); + + protected void HandleInBackground(ZeroPacket message, Func handle) where TReq : P2PMessage => + BackgroundTaskScheduler.ScheduleBackgroundTask(DeserializeAndReport(message), handle); + + private TReq DeserializeAndReport(ZeroPacket message) where TReq : P2PMessage + { + TReq messageObject = Deserialize(message.Content); + ReportIn(messageObject, message.Content.ReadableBytes); + return messageObject; + } + public abstract void Dispose(); public abstract byte ProtocolVersion { get; } @@ -158,10 +212,5 @@ protected void ReportIn(string messageInfo, int size) public abstract event EventHandler SubprotocolRequested; } - public class IncompleteDeserializationException : Exception - { - public IncompleteDeserializationException(string msg) : base(msg) - { - } - } + public class IncompleteDeserializationException(string msg) : Exception(msg); } diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs index c5c88d446f0..63efdeed3cb 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs @@ -17,10 +17,12 @@ using Nethermind.Core.Extensions; using Nethermind.Int256; using Nethermind.Logging; +using Nethermind.Network.P2P.Messages; using Nethermind.Network.P2P.Subprotocols.Eth.V62; using Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages; using Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages; using Nethermind.Network.P2P.Utils; +using Nethermind.Network.Rlpx; using Nethermind.Serialization.Rlp; using Nethermind.Stats; using Nethermind.Stats.Model; @@ -41,7 +43,6 @@ public abstract class SyncPeerProtocolHandlerBase : ZeroProtocolHandlerBase, ISy public virtual bool IncludeInTxPool => true; protected ISyncServer SyncServer { get; } - protected BackgroundTaskSchedulerWrapper BackgroundTaskScheduler { get; } public long HeadNumber { get; set; } public Hash256 HeadHash { get; set; } @@ -65,10 +66,9 @@ protected SyncPeerProtocolHandlerBase(ISession session, INodeStatsManager statsManager, ISyncServer syncServer, IBackgroundTaskScheduler backgroundTaskScheduler, - ILogManager logManager) : base(session, statsManager, serializer, logManager) + ILogManager logManager) : base(session, statsManager, serializer, backgroundTaskScheduler, logManager) { SyncServer = syncServer ?? throw new ArgumentNullException(nameof(syncServer)); - BackgroundTaskScheduler = new BackgroundTaskSchedulerWrapper(this, backgroundTaskScheduler ?? throw new ArgumentNullException(nameof(BackgroundTaskScheduler))); _timestamper = Timestamper.Default; _txDecoder = TxDecoder.Instance; _headersRequests = new MessageQueue>(Send); @@ -319,10 +319,7 @@ startingHash is null protected async Task Handle(GetBlockBodiesMessage request, CancellationToken cancellationToken) { using GetBlockBodiesMessage message = request; - if (Logger.IsTrace) - { - Logger.Trace($"Received bodies request of length {message.BlockHashes.Count} from {Session.Node:c}:"); - } + if (Logger.IsTrace) Logger.Trace($"Received bodies request of length {message.BlockHashes.Count} from {Session.Node:c}:"); long startTime = Stopwatch.GetTimestamp(); @@ -366,11 +363,6 @@ protected void HandleBodies(BlockBodiesMessage blockBodiesMessage, long size) protected async Task Handle(GetReceiptsMessage msg, CancellationToken cancellationToken) { using var message = msg; - if (message.Hashes.Count > 512) - { - throw new EthSyncException("Incoming receipts request for more than 512 blocks"); - } - long startTime = Stopwatch.GetTimestamp(); ReceiptsMessage resp = await FulfillReceiptsRequest(message, cancellationToken); if (Logger.IsTrace) Logger.Trace($"OUT {Counter:D5} Receipts to {Node:c} in {Stopwatch.GetElapsedTime(startTime).TotalMilliseconds:N0}ms"); diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroNettyP2PHandler.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroNettyP2PHandler.cs index e7e8f7a5914..436b97980b0 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroNettyP2PHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroNettyP2PHandler.cs @@ -15,19 +15,13 @@ namespace Nethermind.Network.P2P.ProtocolHandlers; -public class ZeroNettyP2PHandler : SimpleChannelInboundHandler +public class ZeroNettyP2PHandler(ISession session, ILogManager logManager) : SimpleChannelInboundHandler { - private readonly ISession _session; - private readonly ILogger _logger; + private readonly ISession _session = session ?? throw new ArgumentNullException(nameof(session)); + private readonly ILogger _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); public bool SnappyEnabled { get; private set; } - public ZeroNettyP2PHandler(ISession session, ILogManager logManager) - { - _session = session ?? throw new ArgumentNullException(nameof(session)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - public void Init(IPacketSender packetSender, IChannelHandlerContext context) { _session.Init(5, context, packetSender); diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroProtocolHandlerBase.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroProtocolHandlerBase.cs index 306044b52a4..6427402d7b6 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroProtocolHandlerBase.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/ZeroProtocolHandlerBase.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using DotNetty.Common.Utilities; +using Nethermind.Consensus.Scheduler; using Nethermind.Core.Extensions; using Nethermind.Logging; using Nethermind.Network.Rlpx; @@ -12,8 +13,13 @@ namespace Nethermind.Network.P2P.ProtocolHandlers { - public abstract class ZeroProtocolHandlerBase(ISession session, INodeStatsManager nodeStats, IMessageSerializationService serializer, ILogManager logManager) - : ProtocolHandlerBase(session, nodeStats, serializer, logManager), IZeroProtocolHandler + public abstract class ZeroProtocolHandlerBase( + ISession session, + INodeStatsManager nodeStats, + IMessageSerializationService serializer, + IBackgroundTaskScheduler backgroundTaskScheduler, + ILogManager logManager) + : ProtocolHandlerBase(session, nodeStats, serializer, backgroundTaskScheduler, logManager), IZeroProtocolHandler { protected readonly INodeStats _nodeStats = nodeStats.GetOrAdd(session.Node); diff --git a/src/Nethermind/Nethermind.Network/P2P/Session.cs b/src/Nethermind/Nethermind.Network/P2P/Session.cs index b58bbdf6be5..a9d07402c3f 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Session.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Session.cs @@ -245,52 +245,6 @@ public int DeliverMessage(T message) where T : P2PMessage } } - public void ReceiveMessage(Packet packet) - { - Interlocked.Add(ref Metrics.P2PBytesReceived, packet.Data.Length); - - lock (_sessionStateLock) - { - if (State < SessionState.Initialized) - { - throw new InvalidOperationException($"{nameof(ReceiveMessage)} called on {this}"); - } - - if (IsClosing) - { - return; - } - } - - int dynamicMessageCode = packet.PacketType; - (string protocol, int messageId) = _resolver.ResolveProtocol(packet.PacketType); - packet.Protocol = protocol; - - MsgReceived?.Invoke(this, new PeerEventArgs(_node, packet.Protocol, packet.PacketType, packet.Data.Length)); - - RecordIncomingMessageMetric(protocol, messageId, packet.Data.Length); - - if (_logger.IsTrace) - _logger.Trace($"{this} received a message of length {packet.Data.Length} " + - $"({dynamicMessageCode} => {protocol}.{messageId})"); - - if (protocol is null) - { - if (_logger.IsTrace) - _logger.Warn($"Received a message from node: {RemoteNodeId}, ({dynamicMessageCode} => {messageId}), " + - $"known protocols ({_protocols.Count}): " + - $"{string.Join(", ", _protocols.Select(static x => $"{x.Value.Name} {x.Value.MessageIdSpaceSize}"))}"); - return; - } - - packet.PacketType = messageId; - - if (State < SessionState.DisconnectingProtocols) - { - _protocols[protocol].HandleMessage(packet); - } - } - public bool TryGetProtocolHandler(string protocolCode, out IProtocolHandler handler) { return _protocols.TryGetValue(protocolCode, out handler); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessage.cs index f9cec26aefd..3c762d8e441 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessage.cs @@ -8,14 +8,9 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth { - public abstract class HashesMessage : P2PMessage + public abstract class HashesMessage(IOwnedReadOnlyList hashes) : P2PMessage { - protected HashesMessage(IOwnedReadOnlyList hashes) - { - Hashes = hashes ?? throw new ArgumentNullException(nameof(hashes)); - } - - public IOwnedReadOnlyList Hashes { get; } + public IOwnedReadOnlyList Hashes { get; } = hashes ?? throw new ArgumentNullException(nameof(hashes)); public override string ToString() { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessageSerializer.cs index 3e19519d884..d9bcdc3ad62 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/HashesMessageSerializer.cs @@ -16,21 +16,21 @@ protected Hash256[] DeserializeHashes(IByteBuffer byteBuffer) return DeserializeHashes(nettyRlpStream); } - protected static Hash256[] DeserializeHashes(RlpStream rlpStream) + protected static Hash256[] DeserializeHashes(RlpStream rlpStream, RlpLimit? limit = null) { - Hash256[] hashes = rlpStream.DecodeArray(static itemContext => itemContext.DecodeKeccak()); + Hash256[] hashes = rlpStream.DecodeArray(static itemContext => itemContext.DecodeKeccak(), limit: limit); return hashes; } - protected ArrayPoolList DeserializeHashesArrayPool(IByteBuffer byteBuffer) + protected ArrayPoolList DeserializeHashesArrayPool(IByteBuffer byteBuffer, RlpLimit? limit = null) { NettyRlpStream nettyRlpStream = new(byteBuffer); - return DeserializeHashesArrayPool(nettyRlpStream); + return DeserializeHashesArrayPool(nettyRlpStream, limit); } - protected static ArrayPoolList DeserializeHashesArrayPool(RlpStream rlpStream) + protected static ArrayPoolList DeserializeHashesArrayPool(RlpStream rlpStream, RlpLimit? limit = null) { - return rlpStream.DecodeArrayPoolList(static itemContext => itemContext.DecodeKeccak()); + return rlpStream.DecodeArrayPoolList(static itemContext => itemContext.DecodeKeccak(), limit: limit); } public void Serialize(IByteBuffer byteBuffer, T message) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs index 6fc37559591..dbd7bdda279 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs @@ -14,6 +14,7 @@ using Nethermind.Logging; using Nethermind.Network.Contract.P2P; using Nethermind.Network.P2P.EventArg; +using Nethermind.Network.P2P.Messages; using Nethermind.Network.P2P.ProtocolHandlers; using Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages; using Nethermind.Network.Rlpx; @@ -168,9 +169,7 @@ bool CanAcceptBlockGossip() break; case Eth62MessageCode.GetBlockHeaders: - GetBlockHeadersMessage getBlockHeadersMessage = Deserialize(message.Content); - ReportIn(getBlockHeadersMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getBlockHeadersMessage, Handle); + HandleInBackground(message, Handle); break; case Eth62MessageCode.BlockHeaders: BlockHeadersMessage headersMsg = Deserialize(message.Content); @@ -178,9 +177,7 @@ bool CanAcceptBlockGossip() Handle(headersMsg, size); break; case Eth62MessageCode.GetBlockBodies: - GetBlockBodiesMessage getBodiesMsg = Deserialize(message.Content); - ReportIn(getBodiesMsg, size); - BackgroundTaskScheduler.ScheduleSyncServe(getBodiesMsg, Handle); + HandleInBackground(message, Handle); break; case Eth62MessageCode.BlockBodies: BlockBodiesMessage bodiesMsg = Deserialize(message.Content); @@ -243,7 +240,6 @@ private void Handle(StatusMessage status) protected void Handle(TransactionsMessage msg) { IOwnedReadOnlyList iList = msg.Transactions; - BackgroundTaskScheduler.ScheduleBackgroundTask((iList, 0), _handleSlow); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessage.cs index 730e231b2fa..515765ec3c2 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessage.cs @@ -9,8 +9,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class BlockBodiesMessage : P2PMessage { - public override int PacketType { get; } = Eth62MessageCode.BlockBodies; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth62MessageCode.BlockBodies; + public override string Protocol => "eth"; public OwnedBlockBodies? Bodies { get; set; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs index 8ecf4388976..0e5a3eab323 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs @@ -6,11 +6,13 @@ using Nethermind.Core; using Nethermind.Core.Buffers; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class BlockBodiesMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxBodyFetch, nameof(BlockBodiesMessage.Bodies)); private readonly BlockBodyDecoder _blockBodyDecoder = BlockBodyDecoder.Instance; public void Serialize(IByteBuffer byteBuffer, BlockBodiesMessage message) @@ -54,7 +56,7 @@ public BlockBodiesMessage Deserialize(IByteBuffer byteBuffer) Rlp.ValueDecoderContext ctx = new(memoryOwner.Memory, true); int startingPosition = ctx.Position; - BlockBody[]? bodies = ctx.DecodeArray(_blockBodyDecoder, false); + BlockBody[]? bodies = ctx.DecodeArray(_blockBodyDecoder, false, limit: RlpLimit); byteBuffer.SetReaderIndex(byteBuffer.ReaderIndex + (ctx.Position - startingPosition)); return new() { Bodies = new(bodies, memoryOwner) }; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessage.cs index 6ddde2450d3..d29c0744f17 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessage.cs @@ -9,8 +9,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class BlockHeadersMessage : P2PMessage { - public override int PacketType { get; } = Eth62MessageCode.BlockHeaders; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth62MessageCode.BlockHeaders; + public override string Protocol => "eth"; public IOwnedReadOnlyList? BlockHeaders { get; set; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessageSerializer.cs index 30e182a4c9c..a230de2f1bc 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockHeadersMessageSerializer.cs @@ -4,11 +4,13 @@ using DotNetty.Buffers; using Nethermind.Core; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class BlockHeadersMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHeaderFetch, nameof(BlockHeadersMessage.BlockHeaders)); private readonly HeaderDecoder _headerDecoder = new(); public void Serialize(IByteBuffer byteBuffer, BlockHeadersMessage message) @@ -44,7 +46,7 @@ public int GetLength(BlockHeadersMessage message, out int contentLength) public static BlockHeadersMessage Deserialize(RlpStream rlpStream) { BlockHeadersMessage message = new(); - message.BlockHeaders = Rlp.DecodeArrayPool(rlpStream); + message.BlockHeaders = Rlp.DecodeArrayPool(rlpStream, limit: RlpLimit); return message; } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessage.cs index e0e925394f6..40a0147b23a 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessage.cs @@ -7,16 +7,11 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { - public class GetBlockBodiesMessage : P2PMessage + public class GetBlockBodiesMessage(IReadOnlyList blockHashes) : P2PMessage { - public IReadOnlyList BlockHashes { get; } - public override int PacketType { get; } = Eth62MessageCode.GetBlockBodies; - public override string Protocol { get; } = "eth"; - - public GetBlockBodiesMessage(IReadOnlyList blockHashes) - { - BlockHashes = blockHashes; - } + public IReadOnlyList BlockHashes { get; } = blockHashes; + public override int PacketType => Eth62MessageCode.GetBlockBodies; + public override string Protocol => "eth"; public GetBlockBodiesMessage(params Hash256[] blockHashes) : this((IReadOnlyList)blockHashes) { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessageSerializer.cs index 9aeed10d02b..366fb5baf0b 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockBodiesMessageSerializer.cs @@ -4,11 +4,14 @@ using DotNetty.Buffers; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class GetBlockBodiesMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxBodyFetch, nameof(GetBlockBodiesMessage.BlockHashes)); + public void Serialize(IByteBuffer byteBuffer, GetBlockBodiesMessage message) { int length = GetLength(message, out int contentLength); @@ -41,7 +44,7 @@ public int GetLength(GetBlockBodiesMessage message, out int contentLength) public static GetBlockBodiesMessage Deserialize(RlpStream rlpStream) { - Hash256[] hashes = rlpStream.DecodeArray(ctx => rlpStream.DecodeKeccak(), false); + Hash256[] hashes = rlpStream.DecodeArray(ctx => rlpStream.DecodeKeccak(), false, limit: RlpLimit); return new GetBlockBodiesMessage(hashes); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessage.cs index bc059be1ca8..8e178a1353e 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessage.cs @@ -10,8 +10,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages [DebuggerDisplay("{StartBlockHash} {MaxHeaders} {Skip} {Reverse}")] public class GetBlockHeadersMessage : P2PMessage { - public override int PacketType { get; } = Eth62MessageCode.GetBlockHeaders; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth62MessageCode.GetBlockHeaders; + public override string Protocol => "eth"; public long StartBlockNumber { get; set; } public Hash256? StartBlockHash { get; set; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessageSerializer.cs index ec44c9cbf13..76cdfec614c 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/GetBlockHeadersMessageSerializer.cs @@ -15,7 +15,7 @@ public static GetBlockHeadersMessage Deserialize(RlpStream rlpStream) GetBlockHeadersMessage message = new(); rlpStream.ReadSequenceLength(); byte[] startingBytes = rlpStream.DecodeByteArray(); - if (startingBytes.Length == 32) + if (startingBytes.Length == Hash256.Size) { message.StartBlockHash = new Hash256(startingBytes); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessage.cs index 524f4cef675..ce242a53bd9 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessage.cs @@ -6,17 +6,12 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { - public class NewBlockHashesMessage : P2PMessage + public class NewBlockHashesMessage(params (Hash256, long)[] blockHashes) : P2PMessage { - public override int PacketType { get; } = Eth62MessageCode.NewBlockHashes; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth62MessageCode.NewBlockHashes; + public override string Protocol => "eth"; - public (Hash256, long)[] BlockHashes { get; } - - public NewBlockHashesMessage(params (Hash256, long)[] blockHashes) - { - BlockHashes = blockHashes; - } + public (Hash256, long)[] BlockHashes { get; } = blockHashes; public override string ToString() { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessageSerializer.cs index 102ca84c9cc..13b51de6e9f 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockHashesMessageSerializer.cs @@ -4,11 +4,14 @@ using DotNetty.Buffers; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class NewBlockHashesMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NewBlockHashesMessage.BlockHashes)); + public void Serialize(IByteBuffer byteBuffer, NewBlockHashesMessage message) { int length = GetLength(message, out int contentLength); @@ -51,7 +54,7 @@ private static NewBlockHashesMessage Deserialize(RlpStream rlpStream) { ctx.ReadSequenceLength(); return (ctx.DecodeKeccak(), (long)ctx.DecodeUInt256()); - }, false); + }, false, limit: RlpLimit); return new NewBlockHashesMessage(blockHashes); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockMessage.cs index 2636fc3cd69..e02f72d5aab 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/NewBlockMessage.cs @@ -9,8 +9,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class NewBlockMessage : P2PMessage { - public override int PacketType { get; } = Eth62MessageCode.NewBlock; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth62MessageCode.NewBlock; + public override string Protocol => "eth"; public Block Block { get; set; } public UInt256 TotalDifficulty { get; set; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/StatusMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/StatusMessage.cs index 58dc1d7b699..d2b6f55a14a 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/StatusMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/StatusMessage.cs @@ -16,12 +16,9 @@ public class StatusMessage : P2PMessage public Hash256? GenesisHash { get; set; } public ForkId? ForkId { get; set; } - public override int PacketType { get; } = Eth62MessageCode.Status; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth62MessageCode.Status; + public override string Protocol => "eth"; - public override string ToString() - { - return $"{Protocol}.{ProtocolVersion} network: {NetworkId} | diff: {TotalDifficulty} | best: {BestHash?.ToShortString()} | genesis: {GenesisHash?.ToShortString()} | fork: {ForkId}"; - } + public override string ToString() => $"{Protocol}.{ProtocolVersion} network: {NetworkId} | diff: {TotalDifficulty} | best: {BestHash?.ToShortString()} | genesis: {GenesisHash?.ToShortString()} | fork: {ForkId}"; } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessage.cs index 6fb8c871841..51ca4867838 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessage.cs @@ -7,20 +7,15 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { - public class TransactionsMessage : P2PMessage + public class TransactionsMessage(IOwnedReadOnlyList transactions) : P2PMessage { // This is the target size for the packs of transactions. A pack can get larger than this if a single // transaction exceeds this size. This solution is similar to Geth one. public const int MaxPacketSize = 102400; - public override int PacketType { get; } = Eth62MessageCode.Transactions; - public override string Protocol { get; } = "eth"; - public IOwnedReadOnlyList Transactions { get; } - - public TransactionsMessage(IOwnedReadOnlyList transactions) - { - Transactions = transactions; - } + public override int PacketType => Eth62MessageCode.Transactions; + public override string Protocol => "eth"; + public IOwnedReadOnlyList Transactions { get; } = transactions; public override string ToString() => $"{nameof(TransactionsMessage)}({Transactions?.Count})"; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessageSerializer.cs index 33c633924d8..b35f064a929 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/TransactionsMessageSerializer.cs @@ -5,11 +5,13 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class TransactionsMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(TransactionsMessage.Transactions)); private readonly TxDecoder _decoder = TxDecoder.Instance; public void Serialize(IByteBuffer byteBuffer, TransactionsMessage message) @@ -45,7 +47,7 @@ public int GetLength(TransactionsMessage message, out int contentLength) public static IOwnedReadOnlyList DeserializeTxs(RlpStream rlpStream) { - return Rlp.DecodeArrayPool(rlpStream, RlpBehaviors.InMempoolForm); + return Rlp.DecodeArrayPool(rlpStream, RlpBehaviors.InMempoolForm, limit: RlpLimit); } } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/TxFloodController.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/TxFloodController.cs index 83a132a492a..704f834f56e 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/TxFloodController.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/TxFloodController.cs @@ -8,25 +8,17 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V62 { - internal class TxFloodController + internal class TxFloodController(Eth62ProtocolHandler protocolHandler, ITimestamper timestamper, ILogger logger) { private DateTime _checkpoint = DateTime.UtcNow; - private readonly Eth62ProtocolHandler _protocolHandler; - private readonly ITimestamper _timestamper; - private readonly ILogger _logger; + private readonly Eth62ProtocolHandler _protocolHandler = protocolHandler ?? throw new ArgumentNullException(nameof(protocolHandler)); + private readonly ITimestamper _timestamper = timestamper ?? throw new ArgumentNullException(nameof(timestamper)); private readonly TimeSpan _checkInterval = TimeSpan.FromSeconds(60); private long _notAcceptedSinceLastCheck; private readonly Random _random = new(); internal bool IsDowngraded { get; private set; } - public TxFloodController(Eth62ProtocolHandler protocolHandler, ITimestamper timestamper, ILogger logger) - { - _protocolHandler = protocolHandler ?? throw new ArgumentNullException(nameof(protocolHandler)); - _timestamper = timestamper ?? throw new ArgumentNullException(nameof(timestamper)); - _logger = logger; - } - public void Report(bool accepted) { TryReset(); @@ -36,12 +28,12 @@ public void Report(bool accepted) _notAcceptedSinceLastCheck++; if (!IsDowngraded && _notAcceptedSinceLastCheck / _checkInterval.TotalSeconds > 10) { - if (_logger.IsDebug) _logger.Debug($"Downgrading {_protocolHandler} due to tx flooding"); + if (logger.IsDebug) logger.Debug($"Downgrading {_protocolHandler} due to tx flooding"); IsDowngraded = true; } else if (_notAcceptedSinceLastCheck / _checkInterval.TotalSeconds > 100) { - if (_logger.IsDebug) _logger.Debug($"Disconnecting {_protocolHandler} due to tx flooding"); + if (logger.IsDebug) logger.Debug($"Disconnecting {_protocolHandler} due to tx flooding"); _protocolHandler.Disconnect( DisconnectReason.TxFlooding, $"tx flooding {_notAcceptedSinceLastCheck}/{_checkInterval.TotalSeconds > 100}"); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandler.cs index ac0b4c83278..2dc39f7e537 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Eth63ProtocolHandler.cs @@ -56,9 +56,7 @@ public override void HandleMessage(ZeroPacket message) switch (message.PacketType) { case Eth63MessageCode.GetReceipts: - GetReceiptsMessage getReceiptsMessage = Deserialize(message.Content); - ReportIn(getReceiptsMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getReceiptsMessage, Handle); + HandleInBackground(message, Handle); break; case Eth63MessageCode.Receipts: ReceiptsMessage receiptsMessage = Deserialize(message.Content); @@ -66,9 +64,7 @@ public override void HandleMessage(ZeroPacket message) Handle(receiptsMessage, size); break; case Eth63MessageCode.GetNodeData: - GetNodeDataMessage getNodeDataMessage = Deserialize(message.Content); - ReportIn(getNodeDataMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getNodeDataMessage, Handle); + HandleInBackground(message, Handle); break; case Eth63MessageCode.NodeData: NodeDataMessage nodeDataMessage = Deserialize(message.Content); @@ -98,13 +94,7 @@ private async Task Handle(GetNodeDataMessage msg, CancellationT protected Task FulfillNodeDataRequest(GetNodeDataMessage msg, CancellationToken cancellationToken) { - if (msg.Hashes.Count > 4096) - { - throw new EthSyncException("Incoming node data request for more than 4096 nodes"); - } - IOwnedReadOnlyList nodeData = SyncServer.GetNodeData(msg.Hashes, cancellationToken); - return Task.FromResult(new NodeDataMessage(nodeData)); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessage.cs index 8da22b5d4df..f72d7cd5a93 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessage.cs @@ -8,7 +8,7 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages { public class GetNodeDataMessage(IOwnedReadOnlyList keys) : HashesMessage(keys) { - public override int PacketType { get; } = Eth63MessageCode.GetNodeData; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth63MessageCode.GetNodeData; + public override string Protocol => "eth"; } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessageSerializer.cs index 336296fa3f3..0c9aea68c1e 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetNodeDataMessageSerializer.cs @@ -4,14 +4,18 @@ using DotNetty.Buffers; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages { public class GetNodeDataMessageSerializer : HashesMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(GetNodeDataMessage.Hashes)); + public override GetNodeDataMessage Deserialize(IByteBuffer byteBuffer) { - ArrayPoolList? keys = DeserializeHashesArrayPool(byteBuffer); + ArrayPoolList? keys = DeserializeHashesArrayPool(byteBuffer, RlpLimit); return new GetNodeDataMessage(keys); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessage.cs index bc87c59f009..05e01e49d22 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessage.cs @@ -8,7 +8,7 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages { public class GetReceiptsMessage(IOwnedReadOnlyList blockHashes) : HashesMessage(blockHashes) { - public override int PacketType { get; } = Eth63MessageCode.GetReceipts; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth63MessageCode.GetReceipts; + public override string Protocol => "eth"; } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessageSerializer.cs index 06b5743d470..7721683823c 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/GetReceiptsMessageSerializer.cs @@ -5,15 +5,18 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages { public class GetReceiptsMessageSerializer : HashesMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxReceiptFetch, nameof(GetReceiptsMessage.Hashes)); + public static GetReceiptsMessage Deserialize(byte[] bytes) { RlpStream rlpStream = bytes.AsRlpStream(); - ArrayPoolList? hashes = rlpStream.DecodeArrayPoolList(static itemContext => itemContext.DecodeKeccak()); + ArrayPoolList? hashes = rlpStream.DecodeArrayPoolList(static itemContext => itemContext.DecodeKeccak(), limit: RlpLimit); return new GetReceiptsMessage(hashes); } @@ -25,7 +28,7 @@ public override GetReceiptsMessage Deserialize(IByteBuffer byteBuffer) public static GetReceiptsMessage Deserialize(RlpStream rlpStream) { - ArrayPoolList? hashes = HashesMessageSerializer.DeserializeHashesArrayPool(rlpStream); + ArrayPoolList? hashes = DeserializeHashesArrayPool(rlpStream, RlpLimit); return new GetReceiptsMessage(hashes); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessage.cs index ab9c7b45c7d..8ad5343b927 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessage.cs @@ -9,8 +9,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages public class NodeDataMessage(IOwnedReadOnlyList? data) : P2PMessage { public IOwnedReadOnlyList Data { get; } = data ?? ArrayPoolList.Empty(); - public override int PacketType { get; } = Eth63MessageCode.NodeData; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth63MessageCode.NodeData; + public override string Protocol => "eth"; public override string ToString() => $"{nameof(NodeDataMessage)}({Data.Count})"; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessageSerializer.cs index e4519a03d8d..2663b042e12 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/NodeDataMessageSerializer.cs @@ -4,11 +4,14 @@ using DotNetty.Buffers; using Nethermind.Core.Collections; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages { public class NodeDataMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NodeDataMessage.Data)); + public void Serialize(IByteBuffer byteBuffer, NodeDataMessage message) { int length = GetLength(message, out int contentLength); @@ -25,7 +28,7 @@ public void Serialize(IByteBuffer byteBuffer, NodeDataMessage message) public NodeDataMessage Deserialize(IByteBuffer byteBuffer) { RlpStream rlpStream = new NettyRlpStream(byteBuffer); - ArrayPoolList result = rlpStream.DecodeArrayPoolList(static stream => stream.DecodeByteArray()); + ArrayPoolList result = rlpStream.DecodeArrayPoolList(static stream => stream.DecodeByteArray(), limit: RlpLimit); return new NodeDataMessage(result); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessage.cs index 289662c83db..d454590eb37 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessage.cs @@ -10,8 +10,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages public class ReceiptsMessage(IOwnedReadOnlyList txReceipts) : P2PMessage { public IOwnedReadOnlyList TxReceipts { get; } = txReceipts ?? ArrayPoolList.Empty(); - public override int PacketType { get; } = Eth63MessageCode.Receipts; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth63MessageCode.Receipts; + public override string Protocol => "eth"; private static ReceiptsMessage? _empty; public static ReceiptsMessage Empty => _empty ??= new ReceiptsMessage(null); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs index 86db5ff59c5..e59dceb9c21 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs @@ -8,11 +8,13 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages { public class ReceiptsMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxReceiptFetch * 4, nameof(ReceiptsMessage.TxReceipts)); private readonly ISpecProvider _specProvider; private readonly IRlpStreamDecoder _decoder; private readonly Func _decodeArrayFunc; @@ -23,7 +25,7 @@ protected ReceiptsMessageSerializer(ISpecProvider specProvider, IRlpStreamDecode { _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _decoder = decoder ?? throw new ArgumentNullException(nameof(decoder)); - _decodeArrayFunc = ctx => ctx.DecodeArray(nestedContext => _decoder.Decode(nestedContext)) ?? []; + _decodeArrayFunc = ctx => ctx.DecodeArray(nestedContext => _decoder.Decode(nestedContext), limit: RlpLimit) ?? []; } public void Serialize(IByteBuffer byteBuffer, ReceiptsMessage message) @@ -34,7 +36,7 @@ public void Serialize(IByteBuffer byteBuffer, ReceiptsMessage message) NettyRlpStream stream = new(byteBuffer); stream.StartSequence(contentLength); - // Track last‐seen block number & its RLP behavior + // Track the last ‐ seen block number & its RLP behavior long lastBlockNumber = -1; RlpBehaviors behaviors = RlpBehaviors.None; @@ -89,7 +91,7 @@ public ReceiptsMessage Deserialize(IByteBuffer byteBuffer) public ReceiptsMessage Deserialize(RlpStream rlpStream) { - ArrayPoolList data = rlpStream.DecodeArrayPoolList(_decodeArrayFunc, true); + ArrayPoolList data = rlpStream.DecodeArrayPoolList(_decodeArrayFunc, limit: RlpLimit); ReceiptsMessage message = new(data); return message; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs index 77566e0736c..d0e8872ce1a 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs @@ -66,9 +66,7 @@ public override void HandleMessage(ZeroPacket message) break; case Eth65MessageCode.GetPooledTransactions: - GetPooledTransactionsMessage getPooledTxMsg = Deserialize(message.Content); - ReportIn(getPooledTxMsg, size); - BackgroundTaskScheduler.ScheduleBackgroundTask(getPooledTxMsg, Handle); + HandleInBackground(message, Handle); break; case Eth65MessageCode.NewPooledTransactionHashes: if (CanReceiveTransactions) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessage.cs index 284eb9aaa3a..365a6a83451 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessage.cs @@ -8,8 +8,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V65.Messages { public class GetPooledTransactionsMessage(IOwnedReadOnlyList hashes) : HashesMessage(hashes) { - public override int PacketType { get; } = Eth65MessageCode.GetPooledTransactions; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth65MessageCode.GetPooledTransactions; + public override string Protocol => "eth"; public override string ToString() => $"{nameof(GetPooledTransactionsMessage)}({Hashes?.Count})"; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessageSerializer.cs index 22d16d8b79e..2f3e7c4b471 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/GetPooledTransactionsMessageSerializer.cs @@ -4,14 +4,18 @@ using DotNetty.Buffers; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V65.Messages { public class GetPooledTransactionsMessageSerializer : HashesMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(GetPooledTransactionsMessage.Hashes)); + public override GetPooledTransactionsMessage Deserialize(IByteBuffer byteBuffer) { - ArrayPoolList? hashes = DeserializeHashesArrayPool(byteBuffer); + ArrayPoolList? hashes = DeserializeHashesArrayPool(byteBuffer, RlpLimit); return new GetPooledTransactionsMessage(hashes); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessage.cs index f23bcd94d2b..f00fc49b385 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessage.cs @@ -8,12 +8,12 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V65.Messages { public class NewPooledTransactionHashesMessage(IOwnedReadOnlyList hashes) : HashesMessage(hashes) { - // we are able to safely send message with up to 3102 hashes to not exceed message size of 102400 bytes + // we are able to safely send a message with up to 3102 hashes to not exceed message size of 102400 bytes // which is used by Geth and us as max message size. (3102 items message has 102370 bytes) public const int MaxCount = 2048; - public override int PacketType { get; } = Eth65MessageCode.NewPooledTransactionHashes; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth65MessageCode.NewPooledTransactionHashes; + public override string Protocol => "eth"; public override string ToString() => $"{nameof(NewPooledTransactionHashesMessage)}({Hashes?.Count})"; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessageSerializer.cs index dd3580492e9..535551f2ddf 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/NewPooledTransactionHashesMessageSerializer.cs @@ -4,15 +4,19 @@ using DotNetty.Buffers; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V65.Messages { public class NewPooledTransactionHashesMessageSerializer : HashesMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NewPooledTransactionHashesMessage.Hashes)); + public override NewPooledTransactionHashesMessage Deserialize(IByteBuffer byteBuffer) { - ArrayPoolList? hashes = DeserializeHashesArrayPool(byteBuffer); + ArrayPoolList? hashes = DeserializeHashesArrayPool(byteBuffer, RlpLimit); return new NewPooledTransactionHashesMessage(hashes); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/PooledTransactionsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/PooledTransactionsMessage.cs index 887d4b17bd1..f8169bf5daa 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/PooledTransactionsMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Messages/PooledTransactionsMessage.cs @@ -7,15 +7,11 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V65.Messages { - public class PooledTransactionsMessage : TransactionsMessage + public class PooledTransactionsMessage(IOwnedReadOnlyList transactions) + : TransactionsMessage(transactions) { - public override int PacketType { get; } = Eth65MessageCode.PooledTransactions; - public override string Protocol { get; } = "eth"; - - public PooledTransactionsMessage(IOwnedReadOnlyList transactions) - : base(transactions) - { - } + public override int PacketType => Eth65MessageCode.PooledTransactions; + public override string Protocol => "eth"; public override string ToString() => $"{nameof(PooledTransactionsMessage)}({Transactions?.Count})"; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs index aa9c7d54ca9..d9192294a6c 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs @@ -68,9 +68,7 @@ public override void HandleMessage(ZeroPacket message) switch (message.PacketType) { case Eth66MessageCode.GetBlockHeaders: - GetBlockHeadersMessage getBlockHeadersMessage = Deserialize(message.Content); - ReportIn(getBlockHeadersMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getBlockHeadersMessage, Handle); + HandleInBackground(message, Handle); break; case Eth66MessageCode.BlockHeaders: BlockHeadersMessage headersMsg = Deserialize(message.Content); @@ -78,9 +76,7 @@ public override void HandleMessage(ZeroPacket message) Handle(headersMsg, size); break; case Eth66MessageCode.GetBlockBodies: - GetBlockBodiesMessage getBodiesMsg = Deserialize(message.Content); - ReportIn(getBodiesMsg, size); - BackgroundTaskScheduler.ScheduleSyncServe(getBodiesMsg, Handle); + HandleInBackground(message, Handle); break; case Eth66MessageCode.BlockBodies: BlockBodiesMessage bodiesMsg = Deserialize(message.Content); @@ -88,10 +84,7 @@ public override void HandleMessage(ZeroPacket message) HandleBodies(bodiesMsg, size); break; case Eth66MessageCode.GetPooledTransactions: - GetPooledTransactionsMessage getPooledTxMsg - = Deserialize(message.Content); - ReportIn(getPooledTxMsg, size); - BackgroundTaskScheduler.ScheduleSyncServe(getPooledTxMsg, Handle); + HandleInBackground(message, Handle); break; case Eth66MessageCode.PooledTransactions: if (CanReceiveTransactions) @@ -109,9 +102,7 @@ PooledTransactionsMessage pooledTxMsg break; case Eth66MessageCode.GetReceipts: - GetReceiptsMessage getReceiptsMessage = Deserialize(message.Content); - ReportIn(getReceiptsMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getReceiptsMessage, Handle); + HandleInBackground(message, Handle); break; case Eth66MessageCode.Receipts: ReceiptsMessage receiptsMessage = Deserialize(message.Content); @@ -119,9 +110,7 @@ PooledTransactionsMessage pooledTxMsg Handle(receiptsMessage, size); break; case Eth66MessageCode.GetNodeData: - GetNodeDataMessage getNodeDataMessage = Deserialize(message.Content); - ReportIn(getNodeDataMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getNodeDataMessage, Handle); + HandleInBackground(message, Handle); break; case Eth66MessageCode.NodeData: NodeDataMessage nodeDataMessage = Deserialize(message.Content); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockBodiesMessageSerializer.cs index c5d4e2ca982..dc7fc425bb6 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockBodiesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockBodiesMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class BlockBodiesMessageSerializer : Eth66MessageSerializer - { - public BlockBodiesMessageSerializer() : base(new V62.Messages.BlockBodiesMessageSerializer()) - { - } - } + public class BlockBodiesMessageSerializer() + : Eth66MessageSerializer(new V62.Messages.BlockBodiesMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockHeadersMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockHeadersMessageSerializer.cs index 3b15a9945d6..459d1566c0e 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockHeadersMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/BlockHeadersMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class BlockHeadersMessageSerializer : Eth66MessageSerializer - { - public BlockHeadersMessageSerializer() : base(new V62.Messages.BlockHeadersMessageSerializer()) - { - } - } + public class BlockHeadersMessageSerializer() + : Eth66MessageSerializer(new V62.Messages.BlockHeadersMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockBodiesMessageSerializer.cs index 7bf40351cae..f0ebeb4a9fc 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockBodiesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockBodiesMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class GetBlockBodiesMessageSerializer : Eth66MessageSerializer - { - public GetBlockBodiesMessageSerializer() : base(new V62.Messages.GetBlockBodiesMessageSerializer()) - { - } - } + public class GetBlockBodiesMessageSerializer() + : Eth66MessageSerializer(new V62.Messages.GetBlockBodiesMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockHeadersMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockHeadersMessageSerializer.cs index 6b476986446..b80f3dbc1bf 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockHeadersMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetBlockHeadersMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class GetBlockHeadersMessageSerializer : Eth66MessageSerializer - { - public GetBlockHeadersMessageSerializer() : base(new V62.Messages.GetBlockHeadersMessageSerializer()) - { - } - } + public class GetBlockHeadersMessageSerializer() + : Eth66MessageSerializer(new V62.Messages.GetBlockHeadersMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetNodeDataMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetNodeDataMessageSerializer.cs index fc6e82870cf..0d59ac30250 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetNodeDataMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetNodeDataMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class GetNodeDataMessageSerializer : Eth66MessageSerializer - { - public GetNodeDataMessageSerializer() : base(new V63.Messages.GetNodeDataMessageSerializer()) - { - } - } + public class GetNodeDataMessageSerializer() + : Eth66MessageSerializer(new V63.Messages.GetNodeDataMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetPooledTransactionsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetPooledTransactionsMessageSerializer.cs index 86d8b7d1d0e..c81c4779926 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetPooledTransactionsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetPooledTransactionsMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class GetPooledTransactionsMessageSerializer : Eth66MessageSerializer - { - public GetPooledTransactionsMessageSerializer() : base(new V65.Messages.GetPooledTransactionsMessageSerializer()) - { - } - } + public class GetPooledTransactionsMessageSerializer() + : Eth66MessageSerializer(new V65.Messages.GetPooledTransactionsMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetReceiptsMessageSerializer.cs index ec89eb16a2f..9c3a6ff8c01 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetReceiptsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/GetReceiptsMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class GetReceiptsMessageSerializer : Eth66MessageSerializer - { - public GetReceiptsMessageSerializer() : base(new V63.Messages.GetReceiptsMessageSerializer()) - { - } - } + public class GetReceiptsMessageSerializer() + : Eth66MessageSerializer(new V63.Messages.GetReceiptsMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/NodeDataMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/NodeDataMessageSerializer.cs index bc52cce1434..5887191a14b 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/NodeDataMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/NodeDataMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class NodeDataMessageSerializer : Eth66MessageSerializer - { - public NodeDataMessageSerializer() : base(new V63.Messages.NodeDataMessageSerializer()) - { - } - } + public class NodeDataMessageSerializer() + : Eth66MessageSerializer(new V63.Messages.NodeDataMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/PooledTransactionsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/PooledTransactionsMessageSerializer.cs index 1b6832671cf..6f68ebf2209 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/PooledTransactionsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/PooledTransactionsMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class PooledTransactionsMessageSerializer : Eth66MessageSerializer - { - public PooledTransactionsMessageSerializer() : base(new V65.Messages.PooledTransactionsMessageSerializer()) - { - } - } + public class PooledTransactionsMessageSerializer() + : Eth66MessageSerializer(new V65.Messages.PooledTransactionsMessageSerializer()); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/ReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/ReceiptsMessageSerializer.cs index b853c6c5579..25a51531cc8 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/ReceiptsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Messages/ReceiptsMessageSerializer.cs @@ -3,10 +3,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages { - public class ReceiptsMessageSerializer : Eth66MessageSerializer - { - public ReceiptsMessageSerializer(IZeroInnerMessageSerializer ethMessageSerializer) : base(ethMessageSerializer) - { - } - } + public class ReceiptsMessageSerializer(IZeroInnerMessageSerializer ethMessageSerializer) + : Eth66MessageSerializer(ethMessageSerializer); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessage68.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessage68.cs index bfeb04e34b1..8526e32c469 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessage68.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessage68.cs @@ -16,8 +16,8 @@ public class NewPooledTransactionHashesMessage68( // of 102400 bytes which is used by Geth and us as max message size. (2925 items message has 102385 bytes) public const int MaxCount = 2048; - public override int PacketType { get; } = Eth68MessageCode.NewPooledTransactionHashes; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth68MessageCode.NewPooledTransactionHashes; + public override string Protocol => "eth"; public readonly IOwnedReadOnlyList Types = types; public readonly IOwnedReadOnlyList Sizes = sizes; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessageSerializer68.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessageSerializer68.cs index 64e80a7bd69..5e36ab48e84 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessageSerializer68.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Messages/NewPooledTransactionHashesMessageSerializer68.cs @@ -5,19 +5,24 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.Eth.V68.Messages { public class NewPooledTransactionHashesMessageSerializer : IZeroMessageSerializer { + private static readonly RlpLimit TypesRlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NewPooledTransactionHashesMessage68.Types)); + private static readonly RlpLimit SizesRlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NewPooledTransactionHashesMessage68.Sizes)); + private static readonly RlpLimit HashesRlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NewPooledTransactionHashesMessage68.Hashes)); + public NewPooledTransactionHashesMessage68 Deserialize(IByteBuffer byteBuffer) { NettyRlpStream rlpStream = new(byteBuffer); rlpStream.ReadSequenceLength(); - ArrayPoolList types = rlpStream.DecodeByteArrayPoolList(); - ArrayPoolList sizes = rlpStream.DecodeArrayPoolList(static item => item.DecodeInt()); - ArrayPoolList hashes = rlpStream.DecodeArrayPoolList(static item => item.DecodeKeccak()); + ArrayPoolList types = rlpStream.DecodeByteArrayPoolList(TypesRlpLimit); + ArrayPoolList sizes = rlpStream.DecodeArrayPoolList(static item => item.DecodeInt(), limit: SizesRlpLimit); + ArrayPoolList hashes = rlpStream.DecodeArrayPoolList(static item => item.DecodeKeccak(), limit: HashesRlpLimit); return new NewPooledTransactionHashesMessage68(types, sizes, hashes); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandler.cs index d11f7db844c..2691f187f59 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Eth69ProtocolHandler.cs @@ -4,12 +4,10 @@ using System; using System.Threading; using System.Threading.Tasks; -using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Synchronization; using Nethermind.Consensus; using Nethermind.Consensus.Scheduler; using Nethermind.Core; - using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Network.Contract.P2P; @@ -38,7 +36,6 @@ public class Eth69ProtocolHandler( IPooledTxsRequestor pooledTxsRequestor, IGossipPolicy gossipPolicy, IForkInfo forkInfo, - IBlockFinder blockFinder, ILogManager logManager, ITxGossipPolicy? transactionsGossipPolicy = null) : Eth68ProtocolHandler(session, serializer, nodeStatsManager, syncServer, backgroundTaskScheduler, txPool, @@ -66,23 +63,21 @@ public override void HandleMessage(ZeroPacket message) { case Eth69MessageCode.Status: StatusMessage69 statusMsg = Deserialize(message.Content); - base.ReportIn(statusMsg, size); - this.Handle(statusMsg); + ReportIn(statusMsg, size); + Handle(statusMsg); break; case Eth69MessageCode.Receipts: ReceiptsMessage69 receiptsMessage = Deserialize(message.Content); - base.ReportIn(receiptsMessage, size); + ReportIn(receiptsMessage, size); base.Handle(receiptsMessage, size); break; case Eth69MessageCode.GetReceipts: - GetReceiptsMessage getReceiptsMessage = Deserialize(message.Content); - ReportIn(getReceiptsMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getReceiptsMessage, this.Handle); + HandleInBackground(message, Handle); break; case Eth69MessageCode.BlockRangeUpdate: BlockRangeUpdateMessage blockRangeUpdateMsg = Deserialize(message.Content); ReportIn(blockRangeUpdateMsg, size); - this.Handle(blockRangeUpdateMsg); + Handle(blockRangeUpdateMsg); break; default: base.HandleMessage(message); @@ -155,7 +150,7 @@ protected override void NotifyOfStatus(BlockHeader head) NetworkId = SyncServer.NetworkId, GenesisHash = SyncServer.Genesis.Hash!, ForkId = _forkInfo.GetForkId(head.Number, head.Timestamp), - EarliestBlock = blockFinder.GetLowestBlock(), + EarliestBlock = SyncServer.LowestBlock, LatestBlock = head.Number, LatestBlockHash = head.Hash! }; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/BlockRangeUpdateMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/BlockRangeUpdateMessage.cs index 5805b773286..b36be88abcd 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/BlockRangeUpdateMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/BlockRangeUpdateMessage.cs @@ -12,8 +12,8 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V69.Messages; /// public class BlockRangeUpdateMessage : P2PMessage { - public override int PacketType { get; } = Eth69MessageCode.BlockRangeUpdate; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth69MessageCode.BlockRangeUpdate; + public override string Protocol => "eth"; /// /// Number of the earliest available full block. diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptMessageDecoder69.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptMessageDecoder69.cs index 2b594d7e7f2..578cbd42dc9 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptMessageDecoder69.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptMessageDecoder69.cs @@ -10,18 +10,12 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V69.Messages; [Rlp.SkipGlobalRegistration] // Created explicitly -public class ReceiptMessageDecoder69 : IRlpStreamDecoder, IRlpValueDecoder +public class ReceiptMessageDecoder69(bool skipStateAndStatus = false) : IRlpStreamDecoder, IRlpValueDecoder { - private readonly bool _skipStateAndStatus; - - public ReceiptMessageDecoder69(bool skipStateAndStatus = false) - { - _skipStateAndStatus = skipStateAndStatus; - } public TxReceipt? Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { Span span = rlpStream.PeekNextItem(); - Rlp.ValueDecoderContext ctx = new Rlp.ValueDecoderContext(span); + Rlp.ValueDecoderContext ctx = new(span); TxReceipt response = Decode(ref ctx, rlpBehaviors); rlpStream.SkipItem(); @@ -61,6 +55,7 @@ public ReceiptMessageDecoder69(bool skipStateAndStatus = false) int lastCheck = ctx.ReadSequenceLength() + ctx.Position; int numberOfReceipts = ctx.PeekNumberOfItemsRemaining(lastCheck); + ctx.GuardLimit(numberOfReceipts); LogEntry[] entries = new LogEntry[numberOfReceipts]; for (int i = 0; i < numberOfReceipts; i++) { @@ -88,7 +83,7 @@ public ReceiptMessageDecoder69(bool skipStateAndStatus = false) bool isEip658Receipts = (rlpBehaviors & RlpBehaviors.Eip658Receipts) == RlpBehaviors.Eip658Receipts; - if (!_skipStateAndStatus) + if (!skipStateAndStatus) { contentLength += isEip658Receipts ? Rlp.LengthOf(item.StatusCode) @@ -129,7 +124,7 @@ public void Encode(RlpStream rlpStream, TxReceipt? item, RlpBehaviors rlpBehavio rlpStream.Encode((byte)item.TxType); - if (!_skipStateAndStatus) + if (!skipStateAndStatus) { if ((rlpBehaviors & RlpBehaviors.Eip658Receipts) == RlpBehaviors.Eip658Receipts) { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsInnerMessage69.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsInnerMessage69.cs index 7d05b83f4e6..19a43e51759 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsInnerMessage69.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsInnerMessage69.cs @@ -6,7 +6,4 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V69.Messages; -public class ReceiptsInnerMessage69 : V63.Messages.ReceiptsMessage -{ - public ReceiptsInnerMessage69(IOwnedReadOnlyList txReceipts) : base(txReceipts) { } -} +public class ReceiptsInnerMessage69(IOwnedReadOnlyList txReceipts) : V63.Messages.ReceiptsMessage(txReceipts); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageInnerSerializer69.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageInnerSerializer69.cs index 82c5c0ca72f..fe4c5941d94 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageInnerSerializer69.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageInnerSerializer69.cs @@ -11,12 +11,10 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V69.Messages; /// "Inner" serializer here inherits and overrides parts of eth/63 implementation, /// while "wraps" it after, similar to eth/66 version. /// -public class ReceiptsMessageInnerSerializer69 : - ReceiptsMessageSerializer, +public class ReceiptsMessageInnerSerializer69(ISpecProvider specProvider) : + ReceiptsMessageSerializer(specProvider, new ReceiptMessageDecoder69()), IZeroInnerMessageSerializer { - public ReceiptsMessageInnerSerializer69(ISpecProvider specProvider) : base(specProvider, new ReceiptMessageDecoder69()) { } - int IZeroInnerMessageSerializer.GetLength(ReceiptsInnerMessage69 message, out int contentLength) => GetLength(message, out contentLength); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageSerializer69.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageSerializer69.cs index 442b274210c..9860a409286 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageSerializer69.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/ReceiptsMessageSerializer69.cs @@ -7,12 +7,10 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V69.Messages { - public class ReceiptsMessageSerializer69 : - ReceiptsMessageSerializer, + public class ReceiptsMessageSerializer69(ISpecProvider specProvider) : + ReceiptsMessageSerializer(new ReceiptsMessageInnerSerializer69(specProvider)), IZeroInnerMessageSerializer { - public ReceiptsMessageSerializer69(ISpecProvider specProvider) : base(new ReceiptsMessageInnerSerializer69(specProvider)) { } - int IZeroInnerMessageSerializer.GetLength(ReceiptsMessage69 message, out int contentLength) => base.GetLength(message, out contentLength); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/StatusMessage69.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/StatusMessage69.cs index fba19fc24a4..71e0e143af4 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/StatusMessage69.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V69/Messages/StatusMessage69.cs @@ -18,8 +18,8 @@ public class StatusMessage69 : P2PMessage public long LatestBlock { get; set; } public required Hash256 LatestBlockHash { get; set; } - public override int PacketType { get; } = Eth69MessageCode.Status; - public override string Protocol { get; } = "eth"; + public override int PacketType => Eth69MessageCode.Status; + public override string Protocol => "eth"; public override string ToString() { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessage.cs index b45e331b7ae..e714ddd6751 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessage.cs @@ -6,13 +6,8 @@ namespace Nethermind.Network.P2P.Subprotocols.NodeData.Messages; -public class GetNodeDataMessage : Eth.V63.Messages.GetNodeDataMessage +public class GetNodeDataMessage(IOwnedReadOnlyList keys) : Eth.V63.Messages.GetNodeDataMessage(keys) { - public override int PacketType { get; } = NodeDataMessageCode.GetNodeData; - public override string Protocol { get; } = "nodedata"; - - public GetNodeDataMessage(IOwnedReadOnlyList keys) - : base(keys) - { - } + public override int PacketType => NodeDataMessageCode.GetNodeData; + public override string Protocol => "nodedata"; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessageSerializer.cs index 531e35015fe..1ae3d7b0fa0 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/GetNodeDataMessageSerializer.cs @@ -5,14 +5,18 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Network.P2P.Subprotocols.Eth; +using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.NodeData.Messages; public class GetNodeDataMessageSerializer : HashesMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(GetNodeDataMessage.Hashes)); + public override GetNodeDataMessage Deserialize(IByteBuffer byteBuffer) { - IOwnedReadOnlyList keys = DeserializeHashesArrayPool(byteBuffer); + IOwnedReadOnlyList keys = DeserializeHashesArrayPool(byteBuffer, RlpLimit); return new GetNodeDataMessage(keys); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessage.cs index 3edbae80c91..693e1b5c21e 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessage.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessage.cs @@ -5,13 +5,8 @@ namespace Nethermind.Network.P2P.Subprotocols.NodeData.Messages; -public class NodeDataMessage : Eth.V63.Messages.NodeDataMessage +public class NodeDataMessage(IOwnedReadOnlyList? data) : Eth.V63.Messages.NodeDataMessage(data) { - public override int PacketType { get; } = NodeDataMessageCode.NodeData; - public override string Protocol { get; } = "nodedata"; - - public NodeDataMessage(IOwnedReadOnlyList? data) - : base(data) - { - } + public override int PacketType => NodeDataMessageCode.NodeData; + public override string Protocol => "nodedata"; } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessageSerializer.cs index 1ef6a7fb86d..2f1c4c72e60 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/Messages/NodeDataMessageSerializer.cs @@ -4,11 +4,14 @@ using DotNetty.Buffers; using Nethermind.Core.Collections; using Nethermind.Serialization.Rlp; +using Nethermind.Stats.SyncLimits; namespace Nethermind.Network.P2P.Subprotocols.NodeData.Messages; public class NodeDataMessageSerializer : IZeroInnerMessageSerializer { + private static readonly RlpLimit RlpLimit = RlpLimit.For(NethermindSyncLimits.MaxHashesFetch, nameof(NodeDataMessage.Data)); + public void Serialize(IByteBuffer byteBuffer, NodeDataMessage message) { int length = GetLength(message, out int contentLength); @@ -25,7 +28,7 @@ public void Serialize(IByteBuffer byteBuffer, NodeDataMessage message) public NodeDataMessage Deserialize(IByteBuffer byteBuffer) { RlpStream rlpStream = new NettyRlpStream(byteBuffer); - ArrayPoolList? result = rlpStream.DecodeArrayPoolList(static stream => stream.DecodeByteArray()); + ArrayPoolList? result = rlpStream.DecodeArrayPoolList(static stream => stream.DecodeByteArray(RlpLimit), limit: RlpLimit); return new NodeDataMessage(result); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/NodeDataProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/NodeDataProtocolHandler.cs index 9db1209ee49..e676b5279ac 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/NodeDataProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/NodeData/NodeDataProtocolHandler.cs @@ -27,7 +27,6 @@ public class NodeDataProtocolHandler : ZeroProtocolHandlerBase, INodeDataPeer { private readonly ISyncServer _syncServer; private readonly MessageQueue> _nodeDataRequests; - private readonly BackgroundTaskSchedulerWrapper _backgroundTaskScheduler; public override string Name => "nodedata1"; protected override TimeSpan InitTimeout => Timeouts.Eth; @@ -41,10 +40,9 @@ public NodeDataProtocolHandler(ISession session, ISyncServer syncServer, IBackgroundTaskScheduler backgroundTaskScheduler, ILogManager logManager) - : base(session, statsManager, serializer, logManager) + : base(session, statsManager, serializer, backgroundTaskScheduler, logManager) { _syncServer = syncServer ?? throw new ArgumentNullException(nameof(syncServer)); - _backgroundTaskScheduler = new BackgroundTaskSchedulerWrapper(this, backgroundTaskScheduler ?? throw new ArgumentNullException(nameof(backgroundTaskScheduler))); ; _nodeDataRequests = new MessageQueue>(Send); } public override void Init() @@ -77,9 +75,7 @@ public override void HandleMessage(ZeroPacket message) { case NodeDataMessageCode.GetNodeData: { - GetNodeDataMessage getNodeDataMessage = Deserialize(message.Content); - ReportIn(getNodeDataMessage, size); - _backgroundTaskScheduler.ScheduleSyncServe(getNodeDataMessage, Handle); + HandleInBackground(message, Handle); break; } case NodeDataMessageCode.NodeData: @@ -106,13 +102,7 @@ private Task Handle(GetNodeDataMessage getNodeDataMessage, Canc private NodeDataMessage FulfillNodeDataRequest(GetNodeDataMessage msg, CancellationToken cancellationToken) { - if (msg.Hashes.Count > 4096) - { - throw new EthSyncException("NODEDATA protocol: Incoming node data request for more than 4096 nodes"); - } - IOwnedReadOnlyList? nodeData = _syncServer.GetNodeData(msg.Hashes, cancellationToken); - return new NodeDataMessage(nodeData); } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/Messages/GetStorageRangesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/Messages/GetStorageRangesMessageSerializer.cs index 238adab58c3..5275e5f95c1 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/Messages/GetStorageRangesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/Messages/GetStorageRangesMessageSerializer.cs @@ -10,7 +10,6 @@ namespace Nethermind.Network.P2P.Subprotocols.Snap.Messages { public class GetStorageRangesMessageSerializer : SnapSerializerBase { - public override void Serialize(IByteBuffer byteBuffer, GetStorageRangeMessage message) { NettyRlpStream rlpStream = GetRlpStreamAndStartSequence(byteBuffer, message); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/SnapProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/SnapProtocolHandler.cs index 48c517ccf83..ff74fb076b1 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/SnapProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/SnapProtocolHandler.cs @@ -28,10 +28,9 @@ namespace Nethermind.Network.P2P.Subprotocols.Snap { public class SnapProtocolHandler : ZeroProtocolHandlerBase, ISnapSyncPeer { - private static readonly TrieNodesMessage EmptyTrieNodesMessage = new TrieNodesMessage(ArrayPoolList.Empty()); + private static readonly TrieNodesMessage EmptyTrieNodesMessage = new(ArrayPoolList.Empty()); private ISnapServer? SyncServer { get; } - private BackgroundTaskSchedulerWrapper BackgroundTaskScheduler { get; } private bool ServingEnabled { get; } public override string Name => "snap1"; @@ -55,14 +54,13 @@ public SnapProtocolHandler(ISession session, IBackgroundTaskScheduler backgroundTaskScheduler, ILogManager logManager, ISnapServer? snapServer = null) - : base(session, nodeStats, serializer, logManager) + : base(session, nodeStats, serializer, backgroundTaskScheduler, logManager) { _getAccountRangeRequests = new(Send); _getStorageRangeRequests = new(Send); _getByteCodesRequests = new(Send); _getTrieNodesRequests = new(Send); SyncServer = snapServer; - BackgroundTaskScheduler = new BackgroundTaskSchedulerWrapper(this, backgroundTaskScheduler); ServingEnabled = SyncServer is not null; } @@ -93,9 +91,7 @@ public override void HandleMessage(ZeroPacket message) case SnapMessageCode.GetAccountRange: if (ShouldServeSnap(nameof(GetAccountRangeMessage))) { - GetAccountRangeMessage getAccountRangeMessage = Deserialize(message.Content); - ReportIn(getAccountRangeMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getAccountRangeMessage, Handle); + HandleInBackground(message, Handle); } break; @@ -107,9 +103,7 @@ public override void HandleMessage(ZeroPacket message) case SnapMessageCode.GetStorageRanges: if (ShouldServeSnap(nameof(GetStorageRangeMessage))) { - GetStorageRangeMessage getStorageRangesMessage = Deserialize(message.Content); - ReportIn(getStorageRangesMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getStorageRangesMessage, Handle); + HandleInBackground(message, Handle); } break; @@ -121,9 +115,7 @@ public override void HandleMessage(ZeroPacket message) case SnapMessageCode.GetByteCodes: if (ShouldServeSnap(nameof(GetByteCodesMessage))) { - GetByteCodesMessage getByteCodesMessage = Deserialize(message.Content); - ReportIn(getByteCodesMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getByteCodesMessage, Handle); + HandleInBackground(message, Handle); } break; @@ -135,9 +127,7 @@ public override void HandleMessage(ZeroPacket message) case SnapMessageCode.GetTrieNodes: if (ShouldServeSnap(nameof(GetTrieNodes))) { - GetTrieNodesMessage getTrieNodesMessage = Deserialize(message.Content); - ReportIn(getTrieNodesMessage, size); - BackgroundTaskScheduler.ScheduleSyncServe(getTrieNodesMessage, Handle); + HandleInBackground(message, Handle); } break; diff --git a/src/Nethermind/Nethermind.Network/P2P/Utils/BackgroundTaskSchedulerWrapper.cs b/src/Nethermind/Nethermind.Network/P2P/Utils/BackgroundTaskSchedulerWrapper.cs index 733c2155882..db849810a5d 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Utils/BackgroundTaskSchedulerWrapper.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Utils/BackgroundTaskSchedulerWrapper.cs @@ -7,6 +7,7 @@ using Nethermind.Consensus.Scheduler; using Nethermind.Network.P2P.Messages; using Nethermind.Network.P2P.ProtocolHandlers; +using Nethermind.Serialization.Rlp; using Nethermind.Stats.Model; using Nethermind.Synchronization; @@ -19,33 +20,27 @@ namespace Nethermind.Network.P2P.Utils; /// public class BackgroundTaskSchedulerWrapper(ProtocolHandlerBase handler, IBackgroundTaskScheduler backgroundTaskScheduler) { - internal void ScheduleSyncServe(TReq request, Func> fulfillFunc) where TRes : P2PMessage - { + internal void ScheduleSyncServe(TReq request, Func> fulfillFunc) where TRes : P2PMessage => ScheduleBackgroundTask((request, fulfillFunc), BackgroundSyncSender); - } - internal void ScheduleSyncServe(TReq request, Func> fulfillFunc) where TRes : P2PMessage - { + internal void ScheduleSyncServe(TReq request, Func> fulfillFunc) where TRes : P2PMessage => ScheduleBackgroundTask((request, fulfillFunc), BackgroundSyncSenderValueTask); - } - internal void ScheduleBackgroundTask(TReq request, Func fulfillFunc) - { + internal void ScheduleBackgroundTask(TReq request, Func fulfillFunc) => backgroundTaskScheduler.ScheduleTask((request, fulfillFunc), BackgroundTaskFailureHandlerValueTask); - } - // I just don't want to create a closure.. so this happens. + // I just don't want to create a closure... so this happens. private async ValueTask BackgroundSyncSender( (TReq Request, Func> FullfillFunc) input, CancellationToken cancellationToken) where TRes : P2PMessage { - TRes response = await input.FullfillFunc.Invoke(input.Request, cancellationToken); + TRes response = await input.FullfillFunc(input.Request, cancellationToken); handler.Send(response); } private async ValueTask BackgroundSyncSenderValueTask( (TReq Request, Func> FullfillFunc) input, CancellationToken cancellationToken) where TRes : P2PMessage { - TRes response = await input.FullfillFunc.Invoke(input.Request, cancellationToken); + TRes response = await input.FullfillFunc(input.Request, cancellationToken); handler.Send(response); } @@ -53,11 +48,19 @@ private async Task BackgroundTaskFailureHandlerValueTask((TReq Request, Fu { try { - await input.BackgroundTask.Invoke(input.Request, cancellationToken); + await input.BackgroundTask(input.Request, cancellationToken); } catch (Exception e) { - handler.Session.InitiateDisconnect(e is EthSyncException ? DisconnectReason.EthSyncException : DisconnectReason.BackgroundTaskFailure, e.Message); + DisconnectReason disconnectReason = e switch + { + EthSyncException => DisconnectReason.EthSyncException, + RlpLimitException => DisconnectReason.MessageLimitsBreached, + RlpException => DisconnectReason.BreachOfProtocol, + _ => DisconnectReason.BackgroundTaskFailure + }; + + handler.Session.InitiateDisconnect(disconnectReason, e.Message); if (handler.Logger.IsDebug) handler.Logger.Debug($"Failure running background task on session {handler.Session}, {e}"); } } diff --git a/src/Nethermind/Nethermind.Network/ProtocolsManager.cs b/src/Nethermind/Nethermind.Network/ProtocolsManager.cs index 51b7d9cea08..ff6e9dbc7e5 100644 --- a/src/Nethermind/Nethermind.Network/ProtocolsManager.cs +++ b/src/Nethermind/Nethermind.Network/ProtocolsManager.cs @@ -7,10 +7,10 @@ using System.Linq; using System.Numerics; using Autofac.Features.AttributeFilters; -using Nethermind.Blockchain.Find; using Nethermind.Config; using Nethermind.Consensus; using Nethermind.Consensus.Scheduler; +using Nethermind.Core.Crypto; using Nethermind.Db; using Nethermind.Logging; using Nethermind.Network.Contract.P2P; @@ -72,7 +72,6 @@ public class ProtocolsManager : IProtocolsManager private readonly HashSet _capabilities = DefaultCapabilities.ToHashSet(); private readonly IBackgroundTaskScheduler _backgroundTaskScheduler; private readonly ISnapServer? _snapServer; - private readonly IBlockFinder _blockFinder; public ProtocolsManager( ISyncPeerPool syncPeerPool, @@ -89,7 +88,6 @@ public ProtocolsManager( IForkInfo forkInfo, IGossipPolicy gossipPolicy, IWorldStateManager worldStateManager, - IBlockFinder blockFinder, ILogManager logManager, ITxGossipPolicy? transactionsGossipPolicy = null) { @@ -109,7 +107,6 @@ public ProtocolsManager( _txGossipPolicy = transactionsGossipPolicy ?? ShouldGossip.Instance; _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); _snapServer = worldStateManager.SnapServer; - _blockFinder = blockFinder ?? throw new ArgumentNullException(nameof(blockFinder)); _logger = _logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _protocolFactories = GetProtocolFactories(); @@ -128,23 +125,36 @@ private void SessionDisconnected(object sender, DisconnectEventArgs e) ISession session = (ISession)sender; session.Initialized -= SessionInitialized; session.Disconnected -= SessionDisconnected; + _sessions.TryRemove(session.SessionId, out _); - if (_syncPeers.TryRemove(session.SessionId, out var removed)) + if (_logger.IsDebug && session.BestStateReached == SessionState.Initialized) + { + _logger.Debug($"{session.Direction} {session.Node:s} disconnected {e.DisconnectType} {e.DisconnectReason} {e.Details}"); + } + + if (session.Node is not null + && _hangingSatelliteProtocols.TryGetValue(session.Node, out ConcurrentDictionary? registrations) + && registrations is not null) + { + registrations.TryRemove(session.SessionId, out _); + } + + PublicKey? handlerKey = null; + if (_syncPeers.TryRemove(session.SessionId, out SyncPeerProtocolHandlerBase? removed) && removed is not null) { _syncPool.RemovePeer(removed); - _txPool.RemovePeer(removed.Node.Id); - if (session.BestStateReached == SessionState.Initialized) + if (removed.Node?.Id is not null) { - if (_logger.IsDebug) _logger.Debug($"{session.Direction} {session.Node:s} disconnected {e.DisconnectType} {e.DisconnectReason} {e.Details}"); + handlerKey = removed.Node.Id; + _txPool.RemovePeer(handlerKey); } } - if (_hangingSatelliteProtocols.TryGetValue(session.Node, out var registrations)) + PublicKey sessionKey = session.Node?.Id; + if (sessionKey is not null && sessionKey != handlerKey) { - registrations.TryRemove(session.SessionId, out _); + _txPool.RemovePeer(session.Node.Id); } - - _sessions.TryRemove(session.SessionId, out session); } private void SessionInitialized(object sender, EventArgs e) @@ -200,7 +210,7 @@ private IDictionary> GetProtocolFa { [Protocol.P2P] = (session, _) => { - P2PProtocolHandler handler = new(session, _rlpxHost.LocalNodeId, _stats, _serializer, _logManager); + P2PProtocolHandler handler = new(session, _rlpxHost.LocalNodeId, _stats, _serializer, _backgroundTaskScheduler, _logManager); session.PingSender = handler; InitP2PProtocol(session, handler); @@ -213,7 +223,7 @@ private IDictionary> GetProtocolFa 66 => new Eth66ProtocolHandler(session, _serializer, _stats, _syncServer, _backgroundTaskScheduler, _txPool, _pooledTxsRequestor, _gossipPolicy, _forkInfo, _logManager, _txGossipPolicy), 67 => new Eth67ProtocolHandler(session, _serializer, _stats, _syncServer, _backgroundTaskScheduler, _txPool, _pooledTxsRequestor, _gossipPolicy, _forkInfo, _logManager, _txGossipPolicy), 68 => new Eth68ProtocolHandler(session, _serializer, _stats, _syncServer, _backgroundTaskScheduler, _txPool, _pooledTxsRequestor, _gossipPolicy, _forkInfo, _logManager, _txGossipPolicy), - 69 => new Eth69ProtocolHandler(session, _serializer, _stats, _syncServer, _backgroundTaskScheduler, _txPool, _pooledTxsRequestor, _gossipPolicy, _forkInfo, _blockFinder, _logManager, _txGossipPolicy), + 69 => new Eth69ProtocolHandler(session, _serializer, _stats, _syncServer, _backgroundTaskScheduler, _txPool, _pooledTxsRequestor, _gossipPolicy, _forkInfo, _logManager, _txGossipPolicy), _ => throw new NotSupportedException($"Eth protocol version {version} is not supported.") }; diff --git a/src/Nethermind/Nethermind.Network/Rlpx/FrameHeaderReader.cs b/src/Nethermind/Nethermind.Network/Rlpx/FrameHeaderReader.cs index f4087825ef8..b029327ae99 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/FrameHeaderReader.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/FrameHeaderReader.cs @@ -33,20 +33,12 @@ public FrameInfo ReadFrameHeader(IByteBuffer input) return new FrameInfo(isChunked, isFirst, frameSize, totalPacketSize ?? frameSize); } - internal readonly struct FrameInfo + internal readonly struct FrameInfo(bool isChunked, bool isFirst, int size, int totalPacketSize) { - public FrameInfo(bool isChunked, bool isFirst, int size, int totalPacketSize) - { - IsChunked = isChunked; - IsFirst = isFirst; - Size = size; - TotalPacketSize = totalPacketSize; - } - - public bool IsChunked { get; } - public bool IsFirst { get; } - public int Size { get; } - public int TotalPacketSize { get; } + public bool IsChunked { get; } = isChunked; + public bool IsFirst { get; } = isFirst; + public int Size { get; } = size; + public int TotalPacketSize { get; } = totalPacketSize; public int Padding => Frame.CalculatePadding(Size); public int PayloadSize => Size + Padding; diff --git a/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AckEip8MessageSerializer.cs b/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AckEip8MessageSerializer.cs index 4bc7552cf93..f8424995d43 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AckEip8MessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AckEip8MessageSerializer.cs @@ -41,7 +41,7 @@ public AckEip8Message Deserialize(IByteBuffer msgBytes) NettyRlpStream rlpStream = new(msgBytes); AckEip8Message authEip8Message = new(); rlpStream.ReadSequenceLength(); - authEip8Message.EphemeralPublicKey = new PublicKey(rlpStream.DecodeByteArraySpan()); + authEip8Message.EphemeralPublicKey = new PublicKey(rlpStream.DecodeByteArraySpan(RlpLimit.L64)); authEip8Message.Nonce = rlpStream.DecodeByteArray(); return authEip8Message; } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AuthEip8MessageSerializer.cs b/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AuthEip8MessageSerializer.cs index d310ecec7cb..627a24ac726 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AuthEip8MessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/Handshake/AuthEip8MessageSerializer.cs @@ -46,10 +46,10 @@ public AuthEip8Message Deserialize(IByteBuffer msgBytes) NettyRlpStream rlpStream = new(msgBytes); AuthEip8Message authMessage = new(); rlpStream.ReadSequenceLength(); - ReadOnlySpan sigAllBytes = rlpStream.DecodeByteArraySpan(); + ReadOnlySpan sigAllBytes = rlpStream.DecodeByteArraySpan(RlpLimit.L65); Signature signature = new(sigAllBytes[..64], sigAllBytes[64]); // since Signature class is Ethereum style it expects V as the 65th byte, hence we use RecoveryID constructor authMessage.Signature = signature; - authMessage.PublicKey = new PublicKey(rlpStream.DecodeByteArraySpan()); + authMessage.PublicKey = new PublicKey(rlpStream.DecodeByteArraySpan(RlpLimit.L64)); authMessage.Nonce = rlpStream.DecodeByteArray(); return authMessage; } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/NettyHandshakeHandler.cs b/src/Nethermind/Nethermind.Network/Rlpx/NettyHandshakeHandler.cs index ca8d2f683ab..aaa6b2c156d 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/NettyHandshakeHandler.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/NettyHandshakeHandler.cs @@ -166,15 +166,15 @@ protected override void ChannelRead0(IChannelHandlerContext context, IByteBuffer using (FrameMacProcessor macProcessor = new(_session.RemoteNodeId, _handshake.Secrets)) { FrameCipher frameCipher = new(_handshake.Secrets.AesSecret); - context.Channel.Pipeline.AddLast(new ZeroFrameDecoder(frameCipher, macProcessor, _logManager)); + context.Channel.Pipeline.AddLast(new ZeroFrameDecoder(frameCipher, macProcessor)); if (_logger.IsTrace) _logger.Trace($"Registering {nameof(ZeroFrameEncoder)} for {RemoteId} @ {context.Channel.RemoteAddress}"); - context.Channel.Pipeline.AddLast(new ZeroFrameEncoder(frameCipher, macProcessor, _logManager)); + context.Channel.Pipeline.AddLast(new ZeroFrameEncoder(frameCipher, macProcessor)); } if (_logger.IsTrace) _logger.Trace($"Registering {nameof(ZeroFrameMerger)} for {RemoteId} @ {context.Channel.RemoteAddress}"); context.Channel.Pipeline.AddLast(new ZeroFrameMerger(_logManager)); if (_logger.IsTrace) _logger.Trace($"Registering {nameof(ZeroPacketSplitter)} for {RemoteId} @ {context.Channel.RemoteAddress}"); - context.Channel.Pipeline.AddLast(new ZeroPacketSplitter(_logManager)); + context.Channel.Pipeline.AddLast(new ZeroPacketSplitter()); PacketSender packetSender = new(_serializationService, _logManager, _sendLatency); if (_logger.IsTrace) _logger.Trace($"Registering {nameof(PacketSender)} for {_session.RemoteNodeId} @ {context.Channel.RemoteAddress}"); diff --git a/src/Nethermind/Nethermind.Network/Rlpx/Packet.cs b/src/Nethermind/Nethermind.Network/Rlpx/Packet.cs index 439aead8a3d..0ff0255dcbc 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/Packet.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/Packet.cs @@ -7,36 +7,24 @@ namespace Nethermind.Network.Rlpx { [DebuggerDisplay("{Protocol}.{PacketType}")] - public class Packet + public class Packet(string? protocol, int packetType, byte[] data) { - public byte[] Data; + public byte[] Data = data; - public Packet(ZeroPacket zeroPacket) + public Packet(ZeroPacket zeroPacket) : this(zeroPacket.Protocol, zeroPacket.PacketType, zeroPacket.Content.ReadAllBytesAsArray()) { - Data = zeroPacket.Content.ReadAllBytesAsArray(); - PacketType = zeroPacket.PacketType; - Protocol = zeroPacket.Protocol; } - public Packet(string protocol, int packetType, byte[] data) + public Packet(byte[] data) : this(null, 0, data) { - Data = data; - Protocol = protocol; - PacketType = packetType; } - public Packet(byte[] data) - { - Data = data; - } + public int PacketType { get; set; } = packetType; - public int PacketType { get; set; } + public string? Protocol { get; set; } = protocol; - public string Protocol { get; set; } + public override string ToString() => $"{Protocol ?? "???"}.{PacketType}"; - public override string ToString() - { - return $"{Protocol ?? "???"}.{PacketType}"; - } + public static implicit operator ZeroPacket(Packet packet) => new(packet); } } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/Padding.cs b/src/Nethermind/Nethermind.Network/Rlpx/Padding.cs index 0db9fc47c76..6f6f746d9ae 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/Padding.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/Padding.cs @@ -16,9 +16,6 @@ public static class Frame public const int DefaultMaxFrameSize = BlockSize * 64; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int CalculatePadding(int size) - { - return size % BlockSize == 0 ? 0 : BlockSize - size % BlockSize; - } + public static int CalculatePadding(int size) => size % BlockSize == 0 ? 0 : BlockSize - size % BlockSize; } } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/SnappyDecoder.cs b/src/Nethermind/Nethermind.Network/Rlpx/SnappyDecoder.cs index a41be3a915a..f21f4b6fa81 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/SnappyDecoder.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/SnappyDecoder.cs @@ -11,15 +11,8 @@ namespace Nethermind.Network.Rlpx; -public class SnappyDecoder : MessageToMessageDecoder +public class SnappyDecoder(ILogger logger) : MessageToMessageDecoder { - private readonly ILogger _logger; - - public SnappyDecoder(ILogger logger) - { - _logger = logger; - } - protected override void Decode(IChannelHandlerContext context, Packet message, List output) { if (Snappy.GetUncompressedLength(message.Data) > SnappyParameters.MaxSnappyLength) @@ -29,11 +22,11 @@ protected override void Decode(IChannelHandlerContext context, Packet message, L if (message.Data.Length > SnappyParameters.MaxSnappyLength / 4) { - if (_logger.IsWarn) _logger.Warn($"Big Snappy message of length {message.Data.Length}"); + if (logger.IsWarn) logger.Warn($"Big Snappy message of length {message.Data.Length}"); } else { - if (_logger.IsTrace) _logger.Trace($"Decompressing with Snappy a message of length {message.Data.Length}"); + if (logger.IsTrace) logger.Trace($"Decompressing with Snappy a message of length {message.Data.Length}"); } try @@ -42,7 +35,7 @@ protected override void Decode(IChannelHandlerContext context, Packet message, L } catch { - _logger.Error($"{message.Data.ToHexString()}"); + logger.Error($"{message.Data.ToHexString()}"); throw; } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameDecoder.cs b/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameDecoder.cs index ffc28431207..6d78a09b414 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameDecoder.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameDecoder.cs @@ -7,15 +7,14 @@ using DotNetty.Buffers; using DotNetty.Codecs; using DotNetty.Transport.Channels; -using Nethermind.Logging; namespace Nethermind.Network.Rlpx { - public class ZeroFrameDecoder : ByteToMessageDecoder + public class ZeroFrameDecoder(IFrameCipher frameCipher, FrameMacProcessor frameMacProcessor) + : ByteToMessageDecoder { - private readonly ILogger _logger; - private readonly IFrameCipher _cipher; - private readonly FrameMacProcessor _authenticator; + private readonly IFrameCipher _cipher = frameCipher ?? throw new ArgumentNullException(nameof(frameCipher)); + private readonly FrameMacProcessor _authenticator = frameMacProcessor ?? throw new ArgumentNullException(nameof(frameMacProcessor)); private readonly byte[] _headerBytes = new byte[Frame.HeaderSize]; private readonly byte[] _macBytes = new byte[Frame.MacSize]; @@ -27,13 +26,6 @@ public class ZeroFrameDecoder : ByteToMessageDecoder private int _frameSize; private int _remainingPayloadBlocks; - public ZeroFrameDecoder(IFrameCipher frameCipher, FrameMacProcessor frameMacProcessor, ILogManager logManager) - { - _cipher = frameCipher ?? throw new ArgumentNullException(nameof(frameCipher)); - _authenticator = frameMacProcessor ?? throw new ArgumentNullException(nameof(frameMacProcessor)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - public override void HandlerRemoved(IChannelHandlerContext context) { base.HandlerRemoved(context); diff --git a/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameEncoder.cs b/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameEncoder.cs index b5307cb624b..c322c8eb491 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameEncoder.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameEncoder.cs @@ -6,27 +6,21 @@ using DotNetty.Buffers; using DotNetty.Codecs; using DotNetty.Transport.Channels; -using Nethermind.Logging; namespace Nethermind.Network.Rlpx { - public class ZeroFrameEncoder : MessageToByteEncoder + public class ZeroFrameEncoder( + IFrameCipher frameCipher, + IFrameMacProcessor frameMacProcessor) + : MessageToByteEncoder { - private readonly ILogger _logger; - private readonly IFrameCipher _frameCipher; - private readonly IFrameMacProcessor _frameMacProcessor; + private readonly IFrameCipher _frameCipher = frameCipher ?? throw new ArgumentNullException(nameof(frameCipher)); + private readonly IFrameMacProcessor _frameMacProcessor = frameMacProcessor ?? throw new ArgumentNullException(nameof(frameMacProcessor)); private readonly FrameHeaderReader _headerReader = new(); private readonly byte[] _encryptBuffer = new byte[Frame.BlockSize]; private readonly byte[] _macBuffer = new byte[16]; - public ZeroFrameEncoder(IFrameCipher frameCipher, IFrameMacProcessor frameMacProcessor, ILogManager logManager) - { - _frameCipher = frameCipher ?? throw new ArgumentNullException(nameof(frameCipher)); - _frameMacProcessor = frameMacProcessor ?? throw new ArgumentNullException(nameof(frameMacProcessor)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - protected override void Encode(IChannelHandlerContext context, IByteBuffer input, IByteBuffer output) { while (input.IsReadable()) diff --git a/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameMerger.cs b/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameMerger.cs index 4e2b2b9f8cf..6efe15175f9 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameMerger.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/ZeroFrameMerger.cs @@ -12,18 +12,13 @@ namespace Nethermind.Network.Rlpx { - public class ZeroFrameMerger : ByteToMessageDecoder + public class ZeroFrameMerger(ILogManager logManager) : ByteToMessageDecoder { - private readonly ILogger _logger; + private readonly ILogger _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); private ZeroPacket? _zeroPacket; private readonly FrameHeaderReader _headerReader = new(); - public ZeroFrameMerger(ILogManager logManager) - { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - public override void HandlerRemoved(IChannelHandlerContext context) { base.HandlerRemoved(context); @@ -88,10 +83,12 @@ private void ReadFirstChunk(IChannelHandlerContext context, IByteBuffer input, i content = input.ReadRetainedSlice(frame.Size - 1); } - _zeroPacket = new ZeroPacket(content); - _zeroPacket.PacketType = GetPacketType(packetTypeRlp); + _zeroPacket = new ZeroPacket(content) + { + PacketType = GetPacketType(packetTypeRlp) + }; - // If not chunked then we already used a slice of the input, + // If not chunked, then we already used a slice of the input, // otherwise we need to read into the freshly allocated buffer. if (frame.IsChunked) { @@ -101,9 +98,6 @@ private void ReadFirstChunk(IChannelHandlerContext context, IByteBuffer input, i } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static byte GetPacketType(byte packetTypeRlp) - { - return packetTypeRlp == 128 ? (byte)0 : packetTypeRlp; - } + private static byte GetPacketType(byte packetTypeRlp) => packetTypeRlp == 128 ? (byte)0 : packetTypeRlp; } } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/ZeroPacketSplitter.cs b/src/Nethermind/Nethermind.Network/Rlpx/ZeroPacketSplitter.cs index b3fcc5ad2e3..96b9bbcb1e4 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/ZeroPacketSplitter.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/ZeroPacketSplitter.cs @@ -7,20 +7,12 @@ using DotNetty.Codecs; using DotNetty.Transport.Channels; using Nethermind.Core.Attributes; -using Nethermind.Logging; using Nethermind.Serialization.Rlp; namespace Nethermind.Network.Rlpx { - public class ZeroPacketSplitter : MessageToByteEncoder, IFramingAware + public class ZeroPacketSplitter() : MessageToByteEncoder, IFramingAware { - private readonly ILogger _logger; - - public ZeroPacketSplitter(ILogManager logManager) - { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - public void DisableFraming() { MaxFrameSize = int.MaxValue; diff --git a/src/Nethermind/Nethermind.Network/Rlpx/ZeroSnappyEncoder.cs b/src/Nethermind/Nethermind.Network/Rlpx/ZeroSnappyEncoder.cs index 6de5a151c2b..88b2f36d0f4 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/ZeroSnappyEncoder.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/ZeroSnappyEncoder.cs @@ -10,14 +10,9 @@ namespace Nethermind.Network.Rlpx; -public class ZeroSnappyEncoder : MessageToByteEncoder +public class ZeroSnappyEncoder(ILogManager logManager) : MessageToByteEncoder { - private readonly ILogger _logger; - - public ZeroSnappyEncoder(ILogManager logManager) - { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } + private readonly ILogger _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); protected override void Encode(IChannelHandlerContext context, IByteBuffer input, IByteBuffer output) { diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs b/src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs index 75c9265a6a8..89d983442ec 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs @@ -49,6 +49,7 @@ public OptimismTxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = int logEntriesCheck = rlpStream.ReadSequenceLength() + rlpStream.Position; int numberOfReceipts = rlpStream.PeekNumberOfItemsRemaining(logEntriesCheck); + rlpStream.GuardLimit(numberOfReceipts); LogEntry[] entries = new LogEntry[numberOfReceipts]; for (int i = 0; i < numberOfReceipts; i++) { @@ -127,8 +128,8 @@ public static int GetLogsLength(TxReceipt item) /// public int GetLength(TxReceipt item, RlpBehaviors rlpBehaviors) { - (int Total, _) = GetContentLength(item, rlpBehaviors); - int receiptPayloadLength = Rlp.LengthOfSequence(Total); + (int total, _) = GetContentLength(item, rlpBehaviors); + int receiptPayloadLength = Rlp.LengthOfSequence(total); bool isForTxRoot = (rlpBehaviors & RlpBehaviors.SkipTypedWrapping) == RlpBehaviors.SkipTypedWrapping; int result = item.TxType != TxType.Legacy diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs b/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs index 034ec815058..6434742d606 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs @@ -165,9 +165,9 @@ public void DecodeStructRef(scoped ref ValueDecoderContext decoderContext, RlpBe decoderContext.DecodeAddressStructRef(out item.Sender); item.GasUsedTotal = (long)decoderContext.DecodeUBigInt(); - (int PrefixLength, int ContentLength) = + (int prefixLength, int contentLength) = decoderContext.PeekPrefixAndContentLength(); - int logsBytes = ContentLength + PrefixLength; + int logsBytes = contentLength + prefixLength; item.LogsRlp = decoderContext.Data.Slice(decoderContext.Position, logsBytes); if (lastCheck > decoderContext.Position) diff --git a/src/Nethermind/Nethermind.Runner/NLog.config b/src/Nethermind/Nethermind.Runner/NLog.config index 308420fbc7c..1dd703dcb03 100644 --- a/src/Nethermind/Nethermind.Runner/NLog.config +++ b/src/Nethermind/Nethermind.Runner/NLog.config @@ -2,7 +2,9 @@ + autoReload="true" + throwExceptions="false" + flushAllBeforeShutdown="true"> diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs index f7fc6028159..642f55829cb 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs @@ -93,9 +93,6 @@ private int GetWithdrawalsLength(Withdrawal[] withdrawals) public BlockBody? DecodeUnwrapped(ref Rlp.ValueDecoderContext ctx, int lastPosition) { - - // quite significant allocations (>0.5%) here based on a sample 3M blocks sync - // (just on these delegates) Transaction[] transactions = ctx.DecodeArray(_txDecoder); BlockHeader[] uncles = ctx.DecodeArray(_headerDecoder); Withdrawal[]? withdrawals = null; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs index 80ef6e8681e..9833f5b881d 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs @@ -129,7 +129,7 @@ public void DecodeStructRef(scoped ref Rlp.ValueDecoderContext decoderContext, R decoderContext.SkipLength(); - ReadOnlySpan firstItem = decoderContext.DecodeByteArraySpan(); + ReadOnlySpan firstItem = decoderContext.DecodeByteArraySpan(RlpLimit.L32); if (firstItem.Length == 1) { item.StatusCode = firstItem[0]; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/KeyValueStoreRlpExtensions.cs b/src/Nethermind/Nethermind.Serialization.Rlp/KeyValueStoreRlpExtensions.cs index b78f1a223fb..a078240c27a 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/KeyValueStoreRlpExtensions.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/KeyValueStoreRlpExtensions.cs @@ -14,23 +14,24 @@ namespace Nethermind.Serialization.Rlp; public static class KeyValueStoreRlpExtensions { [SkipLocalsInit] - public static TItem? Get(this IReadOnlyKeyValueStore db, long blockNumber, ValueHash256 hash, IRlpStreamDecoder decoder, - ClockCache cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) where TItem : class + public static TItem? Get(this IReadOnlyKeyValueStore db, long blockNumber, Hash256AsKey hash, IRlpStreamDecoder decoder, out bool fromCache, + ClockCache cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) where TItem : class { Span dbKey = stackalloc byte[40]; - KeyValueStoreExtensions.GetBlockNumPrefixedKey(blockNumber, hash, dbKey); - return Get(db, hash, dbKey, decoder, cache, rlpBehaviors, shouldCache); + KeyValueStoreExtensions.GetBlockNumPrefixedKey(blockNumber, in hash.Value.ValueHash256, dbKey); + return Get(db, hash, dbKey, decoder, out fromCache, cache, rlpBehaviors, shouldCache); } - public static TItem? Get(this IReadOnlyKeyValueStore db, ValueHash256 key, IRlpStreamDecoder decoder, ClockCache cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) where TItem : class - { - return Get(db, key, key.Bytes, decoder, cache, rlpBehaviors, shouldCache); - } + public static TItem? Get(this IReadOnlyKeyValueStore db, Hash256AsKey key, IRlpStreamDecoder decoder, ClockCache cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) where TItem : class + => Get(db, key, key.Value.Bytes, decoder, out _, cache, rlpBehaviors, shouldCache); + + public static TItem? Get(this IReadOnlyKeyValueStore db, Hash256AsKey key, IRlpStreamDecoder decoder, out bool fromCache, ClockCache cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) where TItem : class + => Get(db, key, key.Value.Bytes, decoder, out fromCache, cache, rlpBehaviors, shouldCache); public static TItem? Get(this IReadOnlyKeyValueStore db, long key, IRlpStreamDecoder? decoder, ClockCache? cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) where TItem : class { ReadOnlySpan keyDb = key.ToBigEndianSpanWithoutLeadingZeros(out _); - return Get(db, key, keyDb, decoder, cache, rlpBehaviors, shouldCache); + return Get(db, key, keyDb, decoder, out _, cache, rlpBehaviors, shouldCache); } public static TItem? Get( @@ -38,6 +39,7 @@ public static class KeyValueStoreRlpExtensions TCacheKey cacheKey, ReadOnlySpan key, IRlpStreamDecoder decoder, + out bool fromCache, ClockCache cache = null, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true @@ -45,18 +47,22 @@ public static class KeyValueStoreRlpExtensions where TCacheKey : struct, IEquatable { TItem item = cache?.Get(cacheKey); - if (item is null) + if (item is not null) { - if (decoder is IRlpValueDecoder valueDecoder) - { - item = db is IReadOnlyNativeKeyValueStore native - ? Get(native, key, valueDecoder, rlpBehaviors) - : Get(db, key, valueDecoder, rlpBehaviors); - } - else - { - item = Get(db, key, decoder, rlpBehaviors); - } + fromCache = true; + return item; + } + + fromCache = false; + if (decoder is IRlpValueDecoder valueDecoder) + { + item = db is IReadOnlyNativeKeyValueStore native + ? Get(native, key, valueDecoder, rlpBehaviors) + : Get(db, key, valueDecoder, rlpBehaviors); + } + else + { + item = Get(db, key, decoder, rlpBehaviors); } if (shouldCache && cache is not null && item is not null) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/LogEntryDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/LogEntryDecoder.cs index c2bfe06fcd9..7fc7be2d202 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/LogEntryDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/LogEntryDecoder.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -142,11 +143,11 @@ public static void DecodeStructRef(scoped ref Rlp.ValueDecoderContext decoderCon decoderContext.ReadSequenceLength(); decoderContext.DecodeAddressStructRef(out var address); - var (PrefixLength, ContentLength) = decoderContext.PeekPrefixAndContentLength(); - var sequenceLength = PrefixLength + ContentLength; - var topics = decoderContext.Data.Slice(decoderContext.Position, sequenceLength); + var (prefixLength, contentLength) = decoderContext.PeekPrefixAndContentLength(); + int sequenceLength = prefixLength + contentLength; + ReadOnlySpan topics = decoderContext.Data.Slice(decoderContext.Position, sequenceLength); decoderContext.SkipItem(); - var data = decoderContext.DecodeByteArraySpan(); + ReadOnlySpan data = decoderContext.DecodeByteArraySpan(); item = new LogEntryStructRef(address, data, topics); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 9344ce4f608..5f796e5db26 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -18,6 +18,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Int256; +using Nethermind.Logging; namespace Nethermind.Serialization.Rlp { @@ -200,10 +201,12 @@ public static T Decode(Span bytes, RlpBehaviors rlpBehaviors = RlpBehav return Decode(ref valueContext, rlpBehaviors); } - public static T[] DecodeArray(RlpStream rlpStream, IRlpStreamDecoder? rlpDecoder, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + public static T[] DecodeArray(RlpStream rlpStream, IRlpStreamDecoder? rlpDecoder, RlpBehaviors rlpBehaviors = RlpBehaviors.None, RlpLimit? limit = null) { int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; - T[] result = new T[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + int length = rlpStream.PeekNumberOfItemsRemaining(checkPosition); + rlpStream.GuardLimit(length, limit); + T[] result = new T[length]; for (int i = 0; i < result.Length; i++) { result[i] = rlpDecoder.Decode(rlpStream, rlpBehaviors); @@ -212,22 +215,20 @@ public static T[] DecodeArray(RlpStream rlpStream, IRlpStreamDecoder? rlpD return result; } - public static ArrayPoolList DecodeArrayPool(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + public static ArrayPoolList DecodeArrayPool(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None, RlpLimit? limit = null) { IRlpStreamDecoder? rlpDecoder = GetStreamDecoder(); - if (rlpDecoder is not null) - { - return DecodeArrayPool(rlpStream, rlpDecoder, rlpBehaviors); - } - - throw new RlpException($"{nameof(Rlp)} does not support decoding {typeof(T).Name}"); + return rlpDecoder is not null + ? DecodeArrayPool(rlpStream, rlpDecoder, rlpBehaviors, limit) + : throw new RlpException($"{nameof(Rlp)} does not support decoding {typeof(T).Name}"); } - public static ArrayPoolList DecodeArrayPool(RlpStream rlpStream, IRlpStreamDecoder? rlpDecoder, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + public static ArrayPoolList DecodeArrayPool(RlpStream rlpStream, IRlpStreamDecoder? rlpDecoder, RlpBehaviors rlpBehaviors = RlpBehaviors.None, RlpLimit? limit = null) { int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; int length = rlpStream.PeekNumberOfItemsRemaining(checkPosition); - ArrayPoolList result = new ArrayPoolList(length); + rlpStream.GuardLimit(length, limit); + ArrayPoolList result = new(length); for (int i = 0; i < length; i++) { result.Add(rlpDecoder.Decode(rlpStream, rlpBehaviors)); @@ -1018,7 +1019,7 @@ public DecodeKeccakRlpException(in int prefix, in int position, in int dataLengt return null; } - ReadOnlySpan theSpan = DecodeByteArraySpan(); + ReadOnlySpan theSpan = DecodeByteArraySpan(RlpLimit.L32); byte[] keccakByte = new byte[32]; theSpan.CopyTo(keccakByte.AsSpan(32 - theSpan.Length)); return new Hash256(keccakByte); @@ -1087,7 +1088,7 @@ public void DecodeZeroPrefixedKeccakStructRef(out Hash256StructRef keccak, Span< } else { - ReadOnlySpan theSpan = DecodeByteArraySpan(); + ReadOnlySpan theSpan = DecodeByteArraySpan(RlpLimit.L32); if (theSpan.Length < 32) { buffer[..(32 - theSpan.Length)].Clear(); @@ -1132,7 +1133,7 @@ public void DecodeAddressStructRef(out AddressStructRef address) public UInt256 DecodeUInt256(int length = -1) { - ReadOnlySpan byteSpan = DecodeByteArraySpan(); + ReadOnlySpan byteSpan = DecodeByteArraySpan(RlpLimit.L32); if (byteSpan.Length > 32) { RlpHelpers.ThrowUnexpectedIntegerLength(byteSpan.Length); @@ -1156,7 +1157,7 @@ public UInt256 DecodeUInt256(int length = -1) public BigInteger DecodeUBigInt() { - ReadOnlySpan bytes = DecodeByteArraySpan(); + ReadOnlySpan bytes = DecodeByteArraySpan(RlpLimit.L32); if (bytes.Length > 1 && bytes[0] == 0) { RlpHelpers.ThrowNonCanonicalInteger(Position); @@ -1173,18 +1174,18 @@ public BigInteger DecodeUBigInt() if (Data[Position] == 249) { Position += 5; // tks: skip 249 1 2 129 127 and read 256 bytes - bloomBytes = Read(256); + bloomBytes = Read(Bloom.ByteLength); } else { - bloomBytes = DecodeByteArraySpan(); + bloomBytes = DecodeByteArraySpan(RlpLimit.Bloom); if (bloomBytes.Length == 0) { return null; } } - if (bloomBytes.Length != 256) + if (bloomBytes.Length != Bloom.ByteLength) { throw new InvalidOperationException("Incorrect bloom RLP"); } @@ -1201,11 +1202,11 @@ public void DecodeBloomStructRef(out BloomStructRef bloom) if (Data[Position] == 249) { Position += 5; // tks: skip 249 1 2 129 127 and read 256 bytes - bloomBytes = Read(256); + bloomBytes = Read(Bloom.ByteLength); } else { - bloomBytes = DecodeByteArraySpan(); + bloomBytes = DecodeByteArraySpan(RlpLimit.Bloom); if (bloomBytes.Length == 0) { bloom = new BloomStructRef(Bloom.Empty.Bytes); @@ -1213,7 +1214,7 @@ public void DecodeBloomStructRef(out BloomStructRef bloom) } } - if (bloomBytes.Length != 256) + if (bloomBytes.Length != Bloom.ByteLength) { throw new InvalidOperationException("Incorrect bloom RLP"); } @@ -1268,9 +1269,9 @@ public int DecodeInt() return result; } - public byte[] DecodeByteArray() => ByteSpanToArray(DecodeByteArraySpan()); + public byte[] DecodeByteArray(RlpLimit? limit = null) => ByteSpanToArray(DecodeByteArraySpan(limit)); - public ReadOnlySpan DecodeByteArraySpan() + public ReadOnlySpan DecodeByteArraySpan(RlpLimit? limit = null) { int prefix = ReadByte(); ReadOnlySpan span = RlpStream.SingleBytes; @@ -1287,6 +1288,7 @@ public ReadOnlySpan DecodeByteArraySpan() if (prefix <= 183) { int length = prefix - 128; + GuardLimit(length, limit); ReadOnlySpan buffer = Read(length); if (length == 1 && buffer[0] < 128) { @@ -1296,11 +1298,11 @@ public ReadOnlySpan DecodeByteArraySpan() return buffer; } - return DecodeLargerByteArraySpan(prefix); + return DecodeLargerByteArraySpan(prefix, limit); } [MethodImpl(MethodImplOptions.NoInlining)] - private ReadOnlySpan DecodeLargerByteArraySpan(int prefix) + private ReadOnlySpan DecodeLargerByteArraySpan(int prefix, RlpLimit? limit = null) { if (prefix < 192) { @@ -1315,6 +1317,7 @@ private ReadOnlySpan DecodeLargerByteArraySpan(int prefix) { RlpHelpers.ThrowUnexpectedLength(length); } + GuardLimit(length, limit); return Read(length); } @@ -1323,11 +1326,11 @@ private ReadOnlySpan DecodeLargerByteArraySpan(int prefix) return default; } - public Memory DecodeByteArrayMemory() + public Memory DecodeByteArrayMemory(RlpLimit? limit = null) { if (!_sliceMemory) { - return DecodeByteArraySpan().ToArray(); + return DecodeByteArraySpan(limit).ToArray(); } if (Memory is null) @@ -1368,6 +1371,7 @@ public Memory DecodeByteArrayMemory() { RlpHelpers.ThrowUnexpectedLength(length); } + GuardLimit(length, limit); return ReadSlicedMemory(length); } @@ -1381,6 +1385,7 @@ static void ThrowNotMemoryBacked() { throw new RlpException("Rlp not backed by a Memory"); } + } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -1439,9 +1444,9 @@ public bool DecodeBool() private void SkipBytes(int length) => Position += length; - public string DecodeString() + public string DecodeString(RlpLimit? limit = null) { - ReadOnlySpan bytes = DecodeByteArraySpan(); + ReadOnlySpan bytes = DecodeByteArraySpan(limit); return Encoding.UTF8.GetString(bytes); } @@ -1567,8 +1572,7 @@ public byte DecodeByte() return default; } - public T[] DecodeArray(IRlpValueDecoder? decoder = null, bool checkPositions = true, - T defaultElement = default) + public T[] DecodeArray(IRlpValueDecoder? decoder = null, bool checkPositions = true, T defaultElement = default, RlpLimit? limit = null) { if (decoder is null) { @@ -1580,10 +1584,11 @@ public T[] DecodeArray(IRlpValueDecoder? decoder = null, bool checkPositio } int positionCheck = ReadSequenceLength() + Position; int count = PeekNumberOfItemsRemaining(checkPositions ? positionCheck : null); + GuardLimit(count, limit); T[] result = new T[count]; for (int i = 0; i < result.Length; i++) { - if (PeekByte() == Rlp.OfEmptySequence[0]) + if (PeekByte() == OfEmptySequence[0]) { result[i] = defaultElement; Position++; @@ -1602,6 +1607,10 @@ public T[] DecodeArray(IRlpValueDecoder? decoder = null, bool checkPositio [DoesNotReturn, StackTraceHidden] private readonly void ThrowKeccakDecodeException(int prefix) => throw new DecodeKeccakRlpException(prefix, Position, Data.Length); + + [StackTraceHidden] + public void GuardLimit(int count, RlpLimit? limit = null) => + Rlp.GuardLimit(count, Length - Position, limit); } public override bool Equals(object? other) => Equals(other as Rlp); @@ -1762,15 +1771,7 @@ public static int LengthOf(IReadOnlyList array) return LengthOfByteString(array.Count, array[0]); } - public static int LengthOf(ReadOnlySpan array) - { - if (array.Length == 0) - { - return 1; - } - - return LengthOfByteString(array.Length, array[0]); - } + public static int LengthOf(ReadOnlySpan array) => array.Length == 0 ? 1 : LengthOfByteString(array.Length, array[0]); // Assumes that length is greater then 0 public static int LengthOfByteString(int length, byte firstByte) @@ -1824,9 +1825,7 @@ public static int LengthOf(string value) public static int LengthOf(BlockInfo item) => BlockInfoDecoder.Instance.GetLength(item, RlpBehaviors.None); [AttributeUsage(AttributeTargets.Class)] - public class SkipGlobalRegistration : Attribute - { - } + public class SkipGlobalRegistration : Attribute; /// /// Optional attribute for RLP decoders. @@ -1837,6 +1836,29 @@ public sealed class DecoderAttribute(string key = RlpDecoderKey.Default) : Attri { public string Key { get; } = key; } + + private static ILogger _logger = Static.LogManager.GetClassLogger(); + + [StackTraceHidden] + public static void GuardLimit(int count, int bytesLeft, RlpLimit? limit = null) + { + RlpLimit l = limit ?? RlpLimit.DefaultLimit; + if (count > bytesLeft || count > l.Limit) + { + ThrowCountOverLimit(count, bytesLeft, l); + } + } + + [DoesNotReturn] + [StackTraceHidden] + private static void ThrowCountOverLimit(int count, int bytesLeft, RlpLimit limit) + { + string message = string.IsNullOrEmpty(limit.CollectionExpression) + ? $"Collection count of {count} is over limit {limit.Limit} or {bytesLeft} bytes left" + : $"Collection count {limit.CollectionExpression} of {count} is over limit {limit.Limit} or {bytesLeft} bytes left"; + if (_logger.IsDebug) _logger.Error($"DEBUG/ERROR: {message}; {new StackTrace()}"); + throw new RlpLimitException(message); + } } public readonly struct RlpDecoderKey(Type type, string key = RlpDecoderKey.Default) : IEquatable diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs index 8c3183b911e..0f0c17e366c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs @@ -11,12 +11,14 @@ namespace Nethermind.Serialization.Rlp { public static class RlpDecoderExtensions { - private readonly static SpanSource[] s_intPreEncodes = CreatePreEncodes(); + private static readonly SpanSource[] s_intPreEncodes = CreatePreEncodes(); - public static T[] DecodeArray(this IRlpStreamDecoder decoder, RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + public static T[] DecodeArray(this IRlpStreamDecoder decoder, RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None, RlpLimit? limit = null) { int checkPosition = rlpStream.ReadSequenceLength() + rlpStream.Position; - T[] result = new T[rlpStream.PeekNumberOfItemsRemaining(checkPosition)]; + int length = rlpStream.PeekNumberOfItemsRemaining(checkPosition); + rlpStream.GuardLimit(length, limit); + T[] result = new T[length]; for (int i = 0; i < result.Length; i++) { result[i] = decoder.Decode(rlpStream, rlpBehaviors); @@ -25,10 +27,12 @@ public static T[] DecodeArray(this IRlpStreamDecoder decoder, RlpStream rl return result; } - public static T[] DecodeArray(this IRlpValueDecoder decoder, ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + public static T[] DecodeArray(this IRlpValueDecoder decoder, ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors = RlpBehaviors.None, RlpLimit? limit = null) { int checkPosition = decoderContext.ReadSequenceLength() + decoderContext.Position; - T[] result = new T[decoderContext.PeekNumberOfItemsRemaining(checkPosition)]; + int length = decoderContext.PeekNumberOfItemsRemaining(checkPosition); + decoderContext.GuardLimit(length, limit); + T[] result = new T[length]; for (int i = 0; i < result.Length; i++) { result[i] = decoder.Decode(ref decoderContext, rlpBehaviors); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpLimit.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpLimit.cs new file mode 100644 index 00000000000..f1f294dbf46 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpLimit.cs @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Runtime.CompilerServices; +using Nethermind.Core; +using Nethermind.Core.Extensions; + +namespace Nethermind.Serialization.Rlp; + +public record struct RlpLimit(int Limit, string TypeName = "", ReadOnlyMemory PropertyName = default) +{ + private const int Default = 256 * 1024 * 1024; + + // We shouldn't allocate any single array bigger than 1M + public static readonly RlpLimit DefaultLimit = new(Default); + public static readonly RlpLimit Bloom = For(Core.Bloom.ByteLength); + public static readonly RlpLimit L4 = new(4); + public static readonly RlpLimit L8 = new(8); + public static readonly RlpLimit L32 = new(32); + public static readonly RlpLimit L64 = new(64); + public static readonly RlpLimit L65 = new(65); + private string _collectionExpression; + + public RlpLimit() : this(Default) { } + + public string CollectionExpression + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => _collectionExpression ??= GenerateCollectionExpression(); + } + + private string GenerateCollectionExpression() => + PropertyName.IsEmpty + ? TypeName + : $"{TypeName}.{PropertyName}"; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RlpLimit For(int limit, string propertyName = "") => new(limit, typeof(T).Name, propertyName.AsMemory()); + + public override string ToString() => CollectionExpression; +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpLimitException.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpLimitException.cs new file mode 100644 index 00000000000..f881d4ccf49 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpLimitException.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Serialization.Rlp; + +public class RlpLimitException : RlpException +{ + public RlpLimitException(string message, Exception inner) + : base(message, inner) + { + } + + public RlpLimitException(string message) + : base(message) + { + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs index d776121f1dc..969ba0f86af 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs @@ -4,7 +4,11 @@ using System; using System.Buffers.Binary; using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; using System.Numerics; +using System.Reflection.Metadata; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -28,6 +32,9 @@ public class RlpStream private static readonly WithdrawalDecoder _withdrawalDecoder = new(); private static readonly LogEntryDecoder _logEntryDecoder = LogEntryDecoder.Instance; + internal static ReadOnlySpan SingleBytes => [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127]; + internal static readonly byte[][] SingleByteArrays = [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27], [28], [29], [30], [31], [32], [33], [34], [35], [36], [37], [38], [39], [40], [41], [42], [43], [44], [45], [46], [47], [48], [49], [50], [51], [52], [53], [54], [55], [56], [57], [58], [59], [60], [61], [62], [63], [64], [65], [66], [67], [68], [69], [70], [71], [72], [73], [74], [75], [76], [77], [78], [79], [80], [81], [82], [83], [84], [85], [86], [87], [88], [89], [90], [91], [92], [93], [94], [95], [96], [97], [98], [99], [100], [101], [102], [103], [104], [105], [106], [107], [108], [109], [110], [111], [112], [113], [114], [115], [116], [117], [118], [119], [120], [121], [122], [123], [124], [125], [126], [127]]; + private readonly CappedArray _data; private int _position = 0; @@ -766,7 +773,7 @@ public bool DecodeValueKeccak(out ValueHash256 keccak) return null; } - ReadOnlySpan theSpan = DecodeByteArraySpan(); + ReadOnlySpan theSpan = DecodeByteArraySpan(RlpLimit.L32); byte[] keccakByte = new byte[32]; theSpan.CopyTo(keccakByte.AsSpan(32 - theSpan.Length)); return new Hash256(keccakByte); @@ -804,7 +811,7 @@ public UInt256 DecodeUInt256(int length = -1) return byteValue; } - ReadOnlySpan byteSpan = DecodeByteArraySpan(); + ReadOnlySpan byteSpan = DecodeByteArraySpan(RlpLimit.L32); if (byteSpan.Length > 32) { @@ -828,7 +835,7 @@ public UInt256 DecodeUInt256(int length = -1) public BigInteger DecodeUBigInt() { - ReadOnlySpan bytes = DecodeByteArraySpan(); + ReadOnlySpan bytes = DecodeByteArraySpan(RlpLimit.L32); if (bytes.Length > 1 && bytes[0] == 0) { RlpHelpers.ThrowNonCanonicalInteger(Position); @@ -845,18 +852,18 @@ public BigInteger DecodeUBigInt() if (PeekByte() == 249) { SkipBytes(5); // tks: skip 249 1 2 129 127 and read 256 bytes - bloomBytes = Read(256); + bloomBytes = Read(Bloom.ByteLength); } else { - bloomBytes = DecodeByteArraySpan(); + bloomBytes = DecodeByteArraySpan(RlpLimit.Bloom); if (bloomBytes.Length == 0) { return null; } } - if (bloomBytes.Length != 256) + if (bloomBytes.Length != Bloom.ByteLength) { throw new RlpException("Incorrect bloom RLP"); } @@ -923,11 +930,11 @@ public bool DecodeBool() return default; } - public T[] DecodeArray(Func decodeItem, bool checkPositions = true, - T defaultElement = default) + public T[] DecodeArray(Func decodeItem, bool checkPositions = true, T defaultElement = default, RlpLimit? limit = null) { int positionCheck = ReadSequenceLength() + Position; - int count = PeekNumberOfItemsRemaining(checkPositions ? positionCheck : (int?)null); + int count = PeekNumberOfItemsRemaining(checkPositions ? positionCheck : null); + GuardLimit(count, limit); T[] result = new T[count]; for (int i = 0; i < result.Length; i++) { @@ -945,11 +952,11 @@ public T[] DecodeArray(Func decodeItem, bool checkPositions = t return result; } - public ArrayPoolList DecodeArrayPoolList(Func decodeItem, bool checkPositions = true, - T defaultElement = default) + public ArrayPoolList DecodeArrayPoolList(Func decodeItem, bool checkPositions = true, T defaultElement = default, RlpLimit? limit = null) { int positionCheck = ReadSequenceLength() + Position; - int count = PeekNumberOfItemsRemaining(checkPositions ? positionCheck : (int?)null); + int count = PeekNumberOfItemsRemaining(checkPositions ? positionCheck : null); + GuardLimit(count, limit); var result = new ArrayPoolList(count, count); for (int i = 0; i < result.Count; i++) { @@ -967,9 +974,9 @@ public ArrayPoolList DecodeArrayPoolList(Func decodeItem, bo return result; } - public string DecodeString() + public string DecodeString(RlpLimit? limit = null) { - ReadOnlySpan bytes = DecodeByteArraySpan(); + ReadOnlySpan bytes = DecodeByteArraySpan(limit); return Encoding.UTF8.GetString(bytes); } @@ -1039,7 +1046,7 @@ public int DecodeInt() public uint DecodeUInt() { - ReadOnlySpan bytes = DecodeByteArraySpan(); + ReadOnlySpan bytes = DecodeByteArraySpan(RlpLimit.L8); if (bytes.Length > 1 && bytes[0] == 0) { RlpHelpers.ThrowNonCanonicalInteger(Position); @@ -1127,7 +1134,7 @@ public ulong DecodeULong() public ulong DecodeUlong() { - ReadOnlySpan bytes = DecodeByteArraySpan(); + ReadOnlySpan bytes = DecodeByteArraySpan(RlpLimit.L8); if (bytes.Length > 1 && bytes[0] == 0) { RlpHelpers.ThrowNonCanonicalInteger(Position); @@ -1135,14 +1142,14 @@ public ulong DecodeUlong() return bytes.Length == 0 ? 0L : bytes.ReadEthUInt64(); } - public byte[] DecodeByteArray() => Rlp.ByteSpanToArray(DecodeByteArraySpan()); + public byte[] DecodeByteArray(RlpLimit? limit = null) => Rlp.ByteSpanToArray(DecodeByteArraySpan(limit)); - public ArrayPoolList DecodeByteArrayPoolList() => Rlp.ByteSpanToArrayPool(DecodeByteArraySpan()); + public ArrayPoolList DecodeByteArrayPoolList(RlpLimit? limit = null) => Rlp.ByteSpanToArrayPool(DecodeByteArraySpan(limit)); - public ReadOnlySpan DecodeByteArraySpan() + public ReadOnlySpan DecodeByteArraySpan(RlpLimit? limit = null) { int prefix = ReadByte(); - ReadOnlySpan span = RlpStream.SingleBytes; + ReadOnlySpan span = SingleBytes; if ((uint)prefix < (uint)span.Length) { return span.Slice(prefix, 1); @@ -1156,6 +1163,7 @@ public ReadOnlySpan DecodeByteArraySpan() if (prefix <= 183) { int length = prefix - 128; + GuardLimit(length, limit); ReadOnlySpan buffer = Read(length); if (buffer.Length == 1 && buffer[0] < 128) { @@ -1165,11 +1173,11 @@ public ReadOnlySpan DecodeByteArraySpan() return buffer; } - return DecodeLargerByteArraySpan(prefix); + return DecodeLargerByteArraySpan(prefix, limit); } [MethodImpl(MethodImplOptions.NoInlining)] - private ReadOnlySpan DecodeLargerByteArraySpan(int prefix) + private ReadOnlySpan DecodeLargerByteArraySpan(int prefix, RlpLimit? limit = null) { if (prefix < 192) { @@ -1184,6 +1192,7 @@ private ReadOnlySpan DecodeLargerByteArraySpan(int prefix) { RlpHelpers.ThrowUnexpectedLength(length); } + GuardLimit(length, limit); return Read(length); } @@ -1206,7 +1215,7 @@ private ReadOnlySpan DecodeLargerByteArraySpan(int prefix) public override string ToString() => $"[{nameof(RlpStream)}|{Position}/{Length}]"; - public byte[][] DecodeByteArrays() + public byte[][] DecodeByteArrays(RlpLimit? limit = null) { int length = ReadSequenceLength(); if (length is 0) @@ -1215,6 +1224,7 @@ public byte[][] DecodeByteArrays() } int itemsCount = PeekNumberOfItemsRemaining(Position + length); + GuardLimit(itemsCount, limit); byte[][] result = new byte[itemsCount][]; for (int i = 0; i < itemsCount; i++) @@ -1225,7 +1235,8 @@ public byte[][] DecodeByteArrays() return result; } - internal static ReadOnlySpan SingleBytes => new byte[128] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127 }; - internal static readonly byte[][] SingleByteArrays = new byte[128][] { [0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27], [28], [29], [30], [31], [32], [33], [34], [35], [36], [37], [38], [39], [40], [41], [42], [43], [44], [45], [46], [47], [48], [49], [50], [51], [52], [53], [54], [55], [56], [57], [58], [59], [60], [61], [62], [63], [64], [65], [66], [67], [68], [69], [70], [71], [72], [73], [74], [75], [76], [77], [78], [79], [80], [81], [82], [83], [84], [85], [86], [87], [88], [89], [90], [91], [92], [93], [94], [95], [96], [97], [98], [99], [100], [101], [102], [103], [104], [105], [106], [107], [108], [109], [110], [111], [112], [113], [114], [115], [116], [117], [118], [119], [120], [121], [122], [123], [124], [125], [126], [127] }; + [StackTraceHidden] + public void GuardLimit(int count, RlpLimit? limit = null) => + Rlp.GuardLimit(count, Length - Position, limit); } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/BaseTxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/BaseTxDecoder.cs index c6522607a26..d7115f9d72c 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/BaseTxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/BaseTxDecoder.cs @@ -14,6 +14,9 @@ public abstract class BaseTxDecoder(TxType txType, Func? transactionFactor private const int MaxDelayedHashTxnSize = 32768; private readonly Func _createTransaction = transactionFactory ?? (static () => new T()); + // 30MB should be good enough for 300MGas block just filled with call data + private static readonly RlpLimit _dataRlpLimit = RlpLimit.For((int)30.MiB(), nameof(Transaction.Data)); + public TxType Type => txType; public virtual Transaction? Decode(Span transactionSequence, RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) @@ -132,7 +135,7 @@ protected virtual void DecodePayload(Transaction transaction, RlpStream rlpStrea transaction.GasLimit = rlpStream.DecodeLong(); transaction.To = rlpStream.DecodeAddress(); transaction.Value = rlpStream.DecodeUInt256(); - transaction.Data = rlpStream.DecodeByteArray(); + transaction.Data = rlpStream.DecodeByteArray(_dataRlpLimit); } protected virtual void DecodeGasPrice(Transaction transaction, RlpStream rlpStream) @@ -147,7 +150,7 @@ protected virtual void DecodePayload(Transaction transaction, ref Rlp.ValueDecod transaction.GasLimit = decoderContext.DecodeLong(); transaction.To = decoderContext.DecodeAddress(); transaction.Value = decoderContext.DecodeUInt256(); - transaction.Data = decoderContext.DecodeByteArrayMemory(); + transaction.Data = decoderContext.DecodeByteArrayMemory(_dataRlpLimit); } protected virtual void DecodeGasPrice(Transaction transaction, ref Rlp.ValueDecoderContext decoderContext) @@ -158,16 +161,16 @@ protected virtual void DecodeGasPrice(Transaction transaction, ref Rlp.ValueDeco protected Signature? DecodeSignature(Transaction transaction, RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { ulong v = rlpStream.DecodeULong(); - ReadOnlySpan rBytes = rlpStream.DecodeByteArraySpan(); - ReadOnlySpan sBytes = rlpStream.DecodeByteArraySpan(); + ReadOnlySpan rBytes = rlpStream.DecodeByteArraySpan(RlpLimit.L32); + ReadOnlySpan sBytes = rlpStream.DecodeByteArraySpan(RlpLimit.L32); return DecodeSignature(v, rBytes, sBytes, transaction.Signature, rlpBehaviors); } protected Signature? DecodeSignature(Transaction transaction, ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { ulong v = decoderContext.DecodeULong(); - ReadOnlySpan rBytes = decoderContext.DecodeByteArraySpan(); - ReadOnlySpan sBytes = decoderContext.DecodeByteArraySpan(); + ReadOnlySpan rBytes = decoderContext.DecodeByteArraySpan(RlpLimit.L32); + ReadOnlySpan sBytes = decoderContext.DecodeByteArraySpan(RlpLimit.L32); return DecodeSignature(v, rBytes, sBytes, transaction.Signature, rlpBehaviors); } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/SetCodeTxDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/SetCodeTxDecoder.cs index 3b121bffcbe..0b6dff7fb32 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/SetCodeTxDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/TxDecoders/SetCodeTxDecoder.cs @@ -14,7 +14,7 @@ public sealed class SetCodeTxDecoder(Func? transactionFactory = null) protected override void DecodePayload(Transaction transaction, RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { base.DecodePayload(transaction, rlpStream, rlpBehaviors); - transaction.AuthorizationList = rlpStream.DecodeArray((s) => _authTupleDecoder.Decode(s, rlpBehaviors)); + transaction.AuthorizationList = rlpStream.DecodeArray((s) => _authTupleDecoder.Decode(s, rlpBehaviors)); } protected override void DecodePayload(Transaction transaction, ref Rlp.ValueDecoderContext decoderContext, diff --git a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs index 2f129dc1853..b94c757e384 100644 --- a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs @@ -603,7 +603,13 @@ public static IEnumerable MainnetActivations yield return new TestCaseData(MainnetSpecProvider.CancunActivation) { TestName = "Cancun" }; yield return new TestCaseData(new ForkActivation(MainnetSpecProvider.ParisBlockNumber, MainnetSpecProvider.PragueBlockTimestamp - 1)) { TestName = "Before Prague" }; yield return new TestCaseData(MainnetSpecProvider.PragueActivation) { TestName = "Prague" }; - yield return new TestCaseData(new ForkActivation(MainnetSpecProvider.ParisBlockNumber, MainnetSpecProvider.PragueBlockTimestamp + 100000000)) { TestName = "Future" }; + yield return new TestCaseData(new ForkActivation(MainnetSpecProvider.ParisBlockNumber, MainnetSpecProvider.OsakaBlockTimestamp - 1)) { TestName = "Before Osaka" }; + yield return new TestCaseData(MainnetSpecProvider.OsakaActivation) { TestName = "Osaka" }; + yield return new TestCaseData(new ForkActivation(MainnetSpecProvider.ParisBlockNumber, MainnetSpecProvider.BPO1BlockTimestamp - 1)) { TestName = "Before BPO1" }; + yield return new TestCaseData(MainnetSpecProvider.BPO1Activation) { TestName = "BPO1" }; + yield return new TestCaseData(new ForkActivation(MainnetSpecProvider.ParisBlockNumber, MainnetSpecProvider.BPO2BlockTimestamp - 1)) { TestName = "Before BPO2" }; + yield return new TestCaseData(MainnetSpecProvider.BPO2Activation) { TestName = "BPO2" }; + yield return new TestCaseData(new ForkActivation(MainnetSpecProvider.ParisBlockNumber, MainnetSpecProvider.BPO2BlockTimestamp + 100000000)) { TestName = "Future" }; } } @@ -650,9 +656,13 @@ public void Mainnet_loads_properly(ForkActivation forkActivation) IReleaseSpec postCancunSpec = provider.GetSpec(MainnetSpecProvider.CancunActivation); IReleaseSpec postPragueSpec = provider.GetSpec(MainnetSpecProvider.PragueActivation); + IReleaseSpec postOsakaSpec = provider.GetSpec(MainnetSpecProvider.OsakaActivation); + IReleaseSpec postBPO1Spec = provider.GetSpec(MainnetSpecProvider.BPO1Activation); + IReleaseSpec postBPO2Spec = provider.GetSpec(MainnetSpecProvider.BPO2Activation); - VerifyCancunSpecificsForMainnetAndHoleskyAndSepolia(postCancunSpec); - VerifyPragueSpecificsForMainnetHoleskyHoodiAndSepolia(provider.ChainId, postPragueSpec); + VerifyCancunSpecificsForMainnetAndSepolia(postCancunSpec); + VerifyPragueSpecificsForMainnetHoodiAndSepolia(provider.ChainId, postPragueSpec); + VerifyOsakaSpecificsForMainnetHoleskyHoodiAndSepolia(provider.ChainId, postOsakaSpec, postBPO1Spec, postBPO2Spec); } [Flags] diff --git a/src/Nethermind/Nethermind.Specs/MainnetSpecProvider.cs b/src/Nethermind/Nethermind.Specs/MainnetSpecProvider.cs index 7352ae53fc0..1c3425fe3ba 100644 --- a/src/Nethermind/Nethermind.Specs/MainnetSpecProvider.cs +++ b/src/Nethermind/Nethermind.Specs/MainnetSpecProvider.cs @@ -27,7 +27,9 @@ public class MainnetSpecProvider : ISpecProvider public const ulong ShanghaiBlockTimestamp = 0x64373057; public const ulong CancunBlockTimestamp = 0x65F1B057; public const ulong PragueBlockTimestamp = 0x681b3057; - public const ulong OsakaBlockTimestamp = ulong.MaxValue - 1; + public const ulong OsakaBlockTimestamp = 0x6930b057; + public const ulong BPO1BlockTimestamp = 0x69383057; + public const ulong BPO2BlockTimestamp = 0x695db057; IReleaseSpec ISpecProvider.GetSpecInternal(ForkActivation forkActivation) => forkActivation switch @@ -49,7 +51,9 @@ IReleaseSpec ISpecProvider.GetSpecInternal(ForkActivation forkActivation) => { Timestamp: < CancunBlockTimestamp } => Shanghai.Instance, { Timestamp: < PragueBlockTimestamp } => Cancun.Instance, { Timestamp: < OsakaBlockTimestamp } => Prague.Instance, - _ => Osaka.Instance + { Timestamp: < BPO1BlockTimestamp } => Osaka.Instance, + { Timestamp: < BPO2BlockTimestamp } => BPO1.Instance, + _ => BPO2.Instance }; public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalDifficulty = null) @@ -73,6 +77,8 @@ public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalD public static ForkActivation CancunActivation { get; } = (ParisBlockNumber + 2, CancunBlockTimestamp); public static ForkActivation PragueActivation { get; } = (ParisBlockNumber + 3, PragueBlockTimestamp); public static ForkActivation OsakaActivation { get; } = (ParisBlockNumber + 4, OsakaBlockTimestamp); + public static ForkActivation BPO1Activation { get; } = (ParisBlockNumber + 5, BPO1BlockTimestamp); + public static ForkActivation BPO2Activation { get; } = (ParisBlockNumber + 6, BPO2BlockTimestamp); public ForkActivation[] TransitionActivations { get; } = { (ForkActivation)HomesteadBlockNumber, @@ -91,6 +97,8 @@ public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalD CancunActivation, PragueActivation, OsakaActivation, + BPO1Activation, + BPO2Activation, }; public static MainnetSpecProvider Instance { get; } = new(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/FullStateFinderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/FullStateFinderTests.cs new file mode 100644 index 00000000000..412bb7384e4 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/FullStateFinderTests.cs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test.Builders; +using Nethermind.State; +using Nethermind.Synchronization.ParallelSync; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.ParallelSync; + +public class FullStateFinderTests +{ + private readonly Hash256 _goodRoot = Keccak.Compute("test"); + private readonly Hash256 _badRoot = Keccak.Compute("test2"); + + [Test] + public void TestWillCheckForState() + { + IBlockTree blockTree = Build.A.BlockTree() + .WithStateRoot((b) => b.Number == 950 ? _goodRoot : _badRoot) + .OfChainLength(1000) + .TestObject; + + IStateReader stateReader = Substitute.For(); + stateReader.HasStateForBlock(Arg.Is((header) => header.StateRoot == _goodRoot)).Returns(true); + + FullStateFinder finder = new FullStateFinder(blockTree, stateReader); + finder.FindBestFullState().Should().Be(950); + } + + [Test] + public void TestWillCheckForStateWhenItWasPreviouslyFound() + { + IBlockTree blockTree = Build.A.BlockTree() + .WithStateRoot((b) => b.Number == 50 ? _goodRoot : _badRoot) + .OfChainLength(100) + .TestObject; + + IStateReader stateReader = Substitute.For(); + stateReader.HasStateForBlock(Arg.Is((header) => header.StateRoot == _goodRoot)).Returns(true); + + FullStateFinder finder = new FullStateFinder(blockTree, stateReader); + finder.FindBestFullState().Should().Be(50); + + BlockHeader parent = blockTree.FindHeader(50, BlockTreeLookupOptions.None)!; + + for (int i = 0; i < 500; i++) + { + Block block = Build.A.Block + .WithParent(parent) + .WithStateRoot(_badRoot) + .TestObject; + + blockTree.SuggestBlock(block); + + parent = block.Header; + } + + finder.FindBestFullState().Should().Be(50); + } + +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs index 031426920ff..d0b2db9bd46 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs @@ -48,7 +48,7 @@ public class SyncServerTests public void When_finding_hash_it_does_not_load_headers() { Context ctx = new(); - ctx.BlockTree.FindHash(123).Returns(TestItem.KeccakA); + ctx.BlockTree.FindBlockHash(123).Returns(TestItem.KeccakA); Hash256 result = ctx.SyncServer.FindHash(123)!; ctx.BlockTree.DidNotReceive().FindHeader(Arg.Any(), Arg.Any()); @@ -775,6 +775,14 @@ public void GetNodeData_returns_cached_trie_nodes() ctx.SyncServer.GetNodeData(new[] { nodeKey }, CancellationToken.None, NodeDataType.All).Should().BeEquivalentTo(new[] { TestItem.KeccakB.BytesToArray() }); } + [Test] + public void Correctly_clips_lowestBlock() + { + Context ctx = new(); + ctx.BlockTree.GetLowestBlock().Returns(5); + ctx.SyncServer.LowestBlock.Should().Be(0); + } + private class Context { public Context() diff --git a/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs b/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs index 3ee241c79ea..72f5d6295fe 100644 --- a/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs +++ b/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs @@ -26,5 +26,6 @@ public interface ISyncServer : IDisposable ulong NetworkId { get; } BlockHeader Genesis { get; } BlockHeader? Head { get; } + long LowestBlock { get; } } } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/FullStateFinder.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/FullStateFinder.cs index 94edf092289..4e0e3be80cf 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/FullStateFinder.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/FullStateFinder.cs @@ -3,6 +3,7 @@ using System; using Nethermind.Blockchain; +using Nethermind.Blockchain.Find; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.State; @@ -17,6 +18,7 @@ public class FullStateFinder : IFullStateFinder private const int MaxLookupBack = 128; private readonly IStateReader _stateReader; private readonly IBlockTree _blockTree; + private long _lastKnownState = 0; public FullStateFinder( IBlockTree blockTree, @@ -65,13 +67,24 @@ public long FindBestFullState() } } + if (bestFullState != 0) + { + _lastKnownState = bestFullState; + } + return bestFullState; } private long SearchForFullState(BlockHeader startHeader) { long bestFullState = 0; - for (int i = 0; i < MaxLookupBack; i++) + long maxLookupBack = MaxLookupBack; + if (_lastKnownState != 0) + { + maxLookupBack = long.Max(maxLookupBack, startHeader.Number - _lastKnownState + 1); + } + + for (int i = 0; i < maxLookupBack; i++) { if (startHeader is null) { @@ -84,7 +97,7 @@ private long SearchForFullState(BlockHeader startHeader) break; } - startHeader = _blockTree.FindHeader(startHeader.ParentHash!, BlockTreeLookupOptions.TotalDifficultyNotNeeded); + startHeader = _blockTree.FindParentHeader(startHeader!, BlockTreeLookupOptions.TotalDifficultyNotNeeded); } return bestFullState; diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index e3f4715ced9..5e681453fd2 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -407,6 +407,7 @@ public void EnqueueNextSlot(StorageRange parentRequest, int accountIndex, ValueH public void RetryStorageRange(StorageRange storageRange) { + bool dispose = false; if (storageRange.Accounts.Count == 1) { EnqueueNextSlot(storageRange); @@ -417,9 +418,12 @@ public void RetryStorageRange(StorageRange storageRange) { EnqueueAccountStorage(account); } + + dispose = true; } Interlocked.Add(ref _activeStorageRequests, -(storageRange?.Accounts.Count ?? 0)); + if (dispose) storageRange.Dispose(); } public void ReportAccountRangePartitionFinished(in ValueHash256 hashLimit) diff --git a/src/Nethermind/Nethermind.Synchronization/SyncServer.cs b/src/Nethermind/Nethermind.Synchronization/SyncServer.cs index 689a09b53e7..b06a36cef27 100644 --- a/src/Nethermind/Nethermind.Synchronization/SyncServer.cs +++ b/src/Nethermind/Nethermind.Synchronization/SyncServer.cs @@ -124,6 +124,8 @@ public BlockHeader? Head } } + public long LowestBlock => Math.Min(Head?.Number ?? 0, _blockTree.GetLowestBlock()); + public int GetPeerCount() => _pool.PeerCount; private readonly Guid _sealValidatorUserGuid = Guid.NewGuid(); @@ -414,7 +416,7 @@ public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlo { try { - Hash256? hash = _blockTree.FindHash(number); + Hash256? hash = _blockTree.FindBlockHash(number); return hash; } catch (Exception) diff --git a/src/Nethermind/Nethermind.Taiko/L1OriginStore.cs b/src/Nethermind/Nethermind.Taiko/L1OriginStore.cs index 2ab8778f43f..8d0c7cada91 100644 --- a/src/Nethermind/Nethermind.Taiko/L1OriginStore.cs +++ b/src/Nethermind/Nethermind.Taiko/L1OriginStore.cs @@ -23,7 +23,7 @@ public class L1OriginStore([KeyFilter(L1OriginStore.L1OriginDbName)] IDb db, IRl Span keyBytes = stackalloc byte[UInt256BytesLength]; blockId.ToBigEndian(keyBytes); - return db.Get(new ValueHash256(keyBytes), decoder); + return db.Get(new Hash256(keyBytes), decoder); } public void WriteL1Origin(UInt256 blockId, L1Origin l1Origin) diff --git a/src/Nethermind/Nethermind.Taiko/TaikoPayloadPreparationService.cs b/src/Nethermind/Nethermind.Taiko/TaikoPayloadPreparationService.cs index 90a48489e41..583156ab26e 100644 --- a/src/Nethermind/Nethermind.Taiko/TaikoPayloadPreparationService.cs +++ b/src/Nethermind/Nethermind.Taiko/TaikoPayloadPreparationService.cs @@ -124,6 +124,7 @@ private Transaction[] BuildTransactions(TaikoPayloadAttributes payloadAttributes int transactionsCheck = rlpStream.Position + transactionsSequenceLength; int txCount = rlpStream.PeekNumberOfItemsRemaining(transactionsCheck); + rlpStream.GuardLimit(txCount); Transaction[] transactions = new Transaction[txCount]; int txIndex = 0; diff --git a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs index f5267e2b0d2..e01f103d174 100644 --- a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; @@ -559,52 +560,6 @@ public void Will_get_dropped_on_snapshot_if_it_was_a_transient_node() fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); } - private class BadDb : IKeyValueStoreWithBatching - { - private readonly Dictionary _db = new(); - - public byte[]? this[ReadOnlySpan key] - { - get => Get(key); - set => Set(key, value); - } - - public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) - { - _db[key.ToArray()] = value; - } - - public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) - { - return _db[key.ToArray()]; - } - - public IWriteBatch StartWriteBatch() - { - return new BadWriteBatch(); - } - - private class BadWriteBatch : IWriteBatch - { - private readonly Dictionary _inBatched = new(); - - public void Dispose() - { - } - - public byte[]? this[ReadOnlySpan key] - { - set => Set(key, value); - } - - public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) - { - _inBatched[key.ToArray()] = value; - } - } - } - - [Test] public void Trie_store_multi_threaded_scenario() { @@ -1051,50 +1006,6 @@ public async Task Will_NotRemove_ReCommittedNode() memDb.Count.Should().Be(4); } - [Test] - public Task Will_RePersist_PersistedReCommittedNode() - { - MemDb memDb = new(); - - using TrieStore fullTrieStore = CreateTrieStore( - kvStore: memDb, - pruningStrategy: new TestPruningStrategy(true, true), - persistenceStrategy: No.Persistence, - pruningConfig: new PruningConfig() - { - PruningBoundary = 3, - TrackPastKeys = true - }); - - PatriciaTree topTree = new PatriciaTree(fullTrieStore.GetTrieStore(null), LimboLogs.Instance); - - byte[] key1 = Bytes.FromHexString("0000000000000000000000000000000000000000000000000000000000000000"); - byte[] key2 = Bytes.FromHexString("0011000000000000000000000000000000000000000000000000000000000000"); - - BlockHeader? baseBlock = null; - for (int i = 0; i < 64; i++) - { - using (fullTrieStore.BeginScope(baseBlock)) - { - topTree.Set(key1, [1, 2]); - topTree.Set(key2, [4, (byte)(i % 4)]); - - using (fullTrieStore.BeginStateBlockCommit(i, topTree.Root)) - { - topTree.Commit(); - } - - baseBlock = Build.A.BlockHeader.WithParentOptional(baseBlock).WithStateRoot(topTree.RootHash).TestObject; - } - - fullTrieStore.WaitForPruning(); - } - - memDb.Count.Should().Be(13); - memDb.WritesCount.Should().Be(184); - return Task.CompletedTask; - } - [Test] public void When_SomeKindOfNonResolvedNotInMainWorldState_OnPrune_DoNotDeleteNode() { @@ -1222,5 +1133,298 @@ public void OnDispose_PersistAtLeastOneCommitSet() fullTrieStore.Dispose(); memDb.Count.Should().Be(1); } + + [Test] + public void Will_NotPruneTopLevelNode() + { + if (_scheme == INodeStorage.KeyScheme.Hash) Assert.Ignore("Not applicable for hash"); + + MemDb memDb = new(); + TestPruningStrategy testPruningStrategy = new TestPruningStrategy( + shouldPrune: false, + deleteObsoleteKeys: true + ); + + TrieStore fullTrieStore = CreateTrieStore( + kvStore: memDb, + pruningStrategy: testPruningStrategy, + persistenceStrategy: No.Persistence, + pruningConfig: new PruningConfig() + { + PruningBoundary = 4, + PrunePersistedNodePortion = 1.0, + DirtyNodeShardBit = 4, + MaxBufferedCommitCount = 0, + TrackPastKeys = true + }); + + PatriciaTree ptree = new PatriciaTree(fullTrieStore.GetTrieStore(null), LimboLogs.Instance); + + void WriteRandomData(int seed) + { + ptree.Set(Keccak.Compute(seed.ToBigEndianByteArray()).Bytes, Keccak.Compute(seed.ToBigEndianByteArray()).BytesToArray()); + ptree.Commit(); + } + + for (int i = 0; i < 10; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + if (i == 0) + { + ptree.Set(Keccak.Compute(10000.ToBigEndianByteArray()).Bytes, Keccak.Compute(i.ToBigEndianByteArray()).BytesToArray()); + } + WriteRandomData(i); + } + } + fullTrieStore.PersistAndPruneDirtyCache(); + + for (int i = 10; i < 15; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + WriteRandomData(i); + } + } + // Do a branch + for (int i = 10; i < 15; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + WriteRandomData(i * 10); + } + } + fullTrieStore.PersistAndPruneDirtyCache(); + + for (int i = 15; i < 20; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + WriteRandomData(i); + } + } + fullTrieStore.WaitForPruning(); + + fullTrieStore.PrunePersistedNodes(); + fullTrieStore.CachedNodesCount.Should().Be(52); + + fullTrieStore.PersistAndPruneDirtyCache(); + fullTrieStore.PrunePersistedNodes(); + fullTrieStore.CachedNodesCount.Should().Be(20); + } + + [TestCase(27, 1000, 31, 7)] + [TestCase(27, 1000, 2, 2)] + public void Will_HaveConsistentState_AfterPrune(int possibleSeed, int totalBlock, int snapshotInterval, int prunePersistedInterval) + { + MemDb memDb = new MemDb(writeDelay: 5, readDelay: 0); + TestPruningStrategy testPruningStrategy = new TestPruningStrategy( + shouldPrune: false, + deleteObsoleteKeys: true + ); + + TrieStore fullTrieStore = CreateTrieStore( + kvStore: memDb, + pruningStrategy: testPruningStrategy, + persistenceStrategy: No.Persistence, + pruningConfig: new PruningConfig() + { + PruningBoundary = 4, + PrunePersistedNodePortion = 0.1, + DirtyNodeShardBit = 8, // More shard this time + MaxBufferedCommitCount = 20, + PrunePersistedNodeMinimumTarget = 0, + TrackPastKeys = true + }); + + PatriciaTree ptree = new PatriciaTree(fullTrieStore.GetTrieStore(null), LimboLogs.Instance); + + void WriteRandomData(int seed) + { + ptree.Set(Keccak.Compute(seed.ToBigEndianByteArray()).Bytes, Keccak.Compute(seed.ToBigEndianByteArray()).BytesToArray()); + ptree.Set(Keccak.Compute((seed * 10000).ToBigEndianByteArray()).Bytes, Keccak.Compute(seed.ToBigEndianByteArray()).BytesToArray()); + ptree.Set(TestItem.KeccakA.Bytes, Keccak.Compute(seed.ToBigEndianByteArray()).BytesToArray()); + ptree.Commit(); + } + + HashSet rootsToTests = new HashSet(); + void VerifyAllTrie() + { + PatriciaTree readOnlyPTree = new PatriciaTree(fullTrieStore.AsReadOnly().GetTrieStore(null), LimboLogs.Instance); + MemDb stubCodeDb = new MemDb(); + foreach (Hash256 rootsToTest in rootsToTests) + { + if (!fullTrieStore.HasRoot(rootsToTest)) continue; + TrieStatsCollector collector = new TrieStatsCollector(stubCodeDb, LimboLogs.Instance, expectAccounts: false); + ptree.Accept(collector, rootHash: rootsToTest); + collector.Stats.MissingNodes.Should().Be(0); + + collector = new TrieStatsCollector(stubCodeDb, LimboLogs.Instance, expectAccounts: false); + readOnlyPTree.Accept(collector, rootHash: rootsToTest); + collector.Stats.MissingNodes.Should().Be(0); + } + } + + BlockHeader baseBlock = Build.A.BlockHeader.WithStateRoot(Keccak.EmptyTreeHash).TestObject; + for (int i = 0; i < totalBlock; i++) + { + int seed = i % possibleSeed; + Hash256 parentRoot = ptree.RootHash ?? Keccak.EmptyTreeHash; + using (fullTrieStore.BeginScope(baseBlock)) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + ptree.RootHash = parentRoot; + WriteRandomData(seed); + rootsToTests.Add(ptree.RootHash); + } + } + + // Branches sometimes + if ((i / 20) % 2 == 0) + { + using (fullTrieStore.BeginScope(baseBlock)) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + ptree.RootHash = parentRoot; + WriteRandomData(seed * 10); + rootsToTests.Add(ptree.RootHash); + } + } + } + + // Persist sometimes + testPruningStrategy.ShouldPruneEnabled = i % snapshotInterval == 0; + testPruningStrategy.ShouldPrunePersistedEnabled = i % prunePersistedInterval == 0; + fullTrieStore.SyncPruneCheck(); + testPruningStrategy.ShouldPruneEnabled = false; + testPruningStrategy.ShouldPrunePersistedEnabled = false; + + VerifyAllTrie(); + baseBlock = Build.A.BlockHeader.WithParent(baseBlock).WithStateRoot(ptree.RootHash).TestObject; + } + } + + [Test] + public async Task Will_Persist_ReCommittedPersistedNode_FromCommitBuffer() + { + int pruningBoundary = 4; + + ManualResetEvent writeBlocker = new ManualResetEvent(true); + TestMemDb memDb = new(); + memDb.WriteFunc = (k, v) => + { + writeBlocker.WaitOne(); + return true; + }; + TestPruningStrategy testPruningStrategy = new TestPruningStrategy( + shouldPrune: false, + deleteObsoleteKeys: true + ); + + TrieStore fullTrieStore = CreateTrieStore( + kvStore: memDb, + pruningStrategy: testPruningStrategy, + persistenceStrategy: No.Persistence, + pruningConfig: new PruningConfig() + { + PruningBoundary = pruningBoundary, + PrunePersistedNodePortion = 1.0, + DirtyNodeShardBit = 4, + MaxBufferedCommitCount = 1, + TrackPastKeys = true + }); + + PatriciaTree ptree = new PatriciaTree(fullTrieStore.GetTrieStore(null), LimboLogs.Instance); + + void WriteRandomData(int seed) + { + ptree.Set(TestItem.KeccakA.Bytes, Keccak.Compute(seed.ToBigEndianByteArray()).BytesToArray()); + ptree.Commit(); + } + + Hash256 persistedRootHash = null; + int persistedBlockNumber = 5; + + for (int i = 0; i < 10; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + if (i == 0) + { + ptree.Set(Keccak.Compute(10000.ToBigEndianByteArray()).Bytes, Keccak.Compute(i.ToBigEndianByteArray()).BytesToArray()); + } + WriteRandomData(i); + + if (i == persistedBlockNumber) + { + persistedRootHash = ptree.RootHash; + } + } + } + + // Persisted nodes should be from block 5 + fullTrieStore.PersistAndPruneDirtyCache(); + fullTrieStore.LastPersistedBlockNumber.Should().Be(persistedBlockNumber); + + // Write a bit more + for (int i = 10; i < 12; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + WriteRandomData(i); + } + } + + // Block writes + writeBlocker.Reset(); + + // Background pruning + Task persistTask = Task.Run(() => + { + testPruningStrategy.ShouldPruneEnabled = true; + fullTrieStore.SyncPruneCheck(); + testPruningStrategy.ShouldPruneEnabled = false; + }); + Thread.Sleep(100); + + // Bring block 5's node to block 12 + // This is done in commit buffer. + using (fullTrieStore.BeginScope(Build.A.BlockHeader.WithStateRoot(ptree.RootHash).TestObject)) + { + fullTrieStore.IsInCommitBufferMode.Should().BeTrue(); + using (fullTrieStore.BeginBlockCommit(12)) + { + WriteRandomData(5); + ptree.RootHash.Should().Be(persistedRootHash); + } + } + + writeBlocker.Set(); + + await persistTask; + + // Write a bit more + for (int i = 13; i < 13 + pruningBoundary; i++) + { + using (fullTrieStore.BeginBlockCommit(i)) + { + WriteRandomData(i); + } + } + + // Persisted nodes should be from block 12 + testPruningStrategy.ShouldPruneEnabled = true; + fullTrieStore.SyncPruneCheck(); + testPruningStrategy.ShouldPruneEnabled = false; + fullTrieStore.LastPersistedBlockNumber.Should().Be(12); + + fullTrieStore.PrunePersistedNodes(); + + TrieStatsCollector collector = new TrieStatsCollector(new MemDb(), SimpleConsoleLogManager.Instance, expectAccounts: false); + ptree.Accept(collector, rootHash: persistedRootHash); + collector.Stats.MissingNodes.Should().Be(0); + } } } diff --git a/src/Nethermind/Nethermind.Trie.Test/PruningScenariosTests.cs b/src/Nethermind/Nethermind.Trie.Test/PruningScenariosTests.cs index 06a9ffcba96..3996fa8a872 100644 --- a/src/Nethermind/Nethermind.Trie.Test/PruningScenariosTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/PruningScenariosTests.cs @@ -312,6 +312,12 @@ public PruningContext AssertThatCachedNodeCountIs(long cachedNodeCount) return this; } + public PruningContext AssertThatCachedPersistedNodeCountIs(long cachedNodeCount) + { + (_trieStore.CachedNodesCount - _trieStore.DirtyCachedNodesCount).Should().Be(cachedNodeCount); + return this; + } + public PruningContext AssertThatCachedNodeCountMoreThan(long cachedNodeCount) { _trieStore.CachedNodesCount.Should().BeGreaterThan(cachedNodeCount); @@ -345,7 +351,7 @@ public PruningContext RestoreBranchingPoint(string name) return this; } - public PruningContext WithPersistedMemoryLimit(long persistedMemoryLimit) + public PruningContext WithPersistedMemoryLimit(long? persistedMemoryLimit) { _pruningStrategy.WithPersistedMemoryLimit = persistedMemoryLimit; return this; @@ -977,6 +983,31 @@ public void Retain_Some_PersistedNodes() .AssertThatCachedNodeCountMoreThan(280); } + [Test] + public void Can_Prune_AllPersistedNodeInOnePrune() + { + PruningContext ctx = PruningContext.InMemoryWithPastKeyTracking + .WithMaxDepth(2) + .WithPersistedMemoryLimit(null) + .WithPrunePersistedNodeParameter(0, 0.1) + .TurnOffAlwaysPrunePersistedNode(); + + for (int i = 0; i < 256; i++) + { + if (i == 256 - 1) + { + ctx.TurnOnPrune(); + } + + ctx + .SetAccountBalance(i, (UInt256)i) + .CommitAndWaitForPruning(); + } + + ctx + .AssertThatCachedPersistedNodeCountIs(3); + } + [TestCase(10)] [TestCase(64)] [TestCase(100)] @@ -1001,6 +1032,30 @@ public void Can_ContinueCommittingEvenWhenPruning(int maxDepth) } } + [TestCase(10)] + [TestCase(64)] + [TestCase(100)] + public void Can_ContinueCommittingEvenWhenPruning_WithKeyTracking(int maxDepth) + { + PruningContext ctx = PruningContext.InMemoryWithPastKeyTracking + .WithMaxDepth(maxDepth) + .TurnOnPrune(); + + using ArrayPoolList stateRoots = new ArrayPoolList(256); + for (int i = 0; i < 256; i++) + { + ctx + .SetAccountBalance(0, (UInt256)i) + .CommitAndWaitForPruning(); + stateRoots.Add(ctx.CurrentStateRoot); + } + + for (int i = 0; i < 256; i++) + { + ctx.VerifyNodeInCache(stateRoots[i], i >= 255 - maxDepth); + } + } + [TestCase(100, 50, false)] [TestCase(100, 150, true)] public async Task Can_ContinueEvenWhenPruningIsBlocked(int maxBufferedCommit, int blockCount, bool isBlocked) diff --git a/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs b/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs index 69e9ae81987..01b56210412 100644 --- a/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs @@ -23,11 +23,11 @@ namespace Nethermind.Trie.Test; public class VisitingTests { [TestCaseSource(nameof(GetOptions))] - public void Visitors_state(VisitingOptions options) + public void Visitors_state(VisitingOptions options, INodeStorage.KeyScheme scheme) { MemDb memDb = new(); - using TrieStore trieStore = TestTrieStoreFactory.Build(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, LimboLogs.Instance); + using ITrieStore trieStore = TestTrieStoreFactory.Build(new NodeStorage(memDb, scheme), LimboLogs.Instance); PatriciaTree patriciaTree = new(trieStore, LimboLogs.Instance); Span raw = stackalloc byte[32]; @@ -62,11 +62,11 @@ public void Visitors_state(VisitingOptions options) } [TestCaseSource(nameof(GetOptions))] - public void Visitors_storage(VisitingOptions options) + public void Visitors_storage(VisitingOptions options, INodeStorage.KeyScheme scheme) { MemDb memDb = new(); - using TrieStore trieStore = TestTrieStoreFactory.Build(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, LimboLogs.Instance); + using ITrieStore trieStore = TestTrieStoreFactory.Build(new NodeStorage(memDb, scheme), LimboLogs.Instance); byte[] value = Enumerable.Range(1, 32).Select(static i => (byte)i).ToArray(); Hash256 stateRootHash = Keccak.Zero; @@ -109,8 +109,11 @@ public void Visitors_storage(VisitingOptions options) stateTree.Accept(visitor, stateTree.RootHash, options); + int totalPath = 0; + foreach (var path in visitor.LeafPaths) { + totalPath++; if (path.Length == 64) { AssertPath(path); @@ -127,6 +130,8 @@ public void Visitors_storage(VisitingOptions options) } } + totalPath.Should().Be(4160); + return; static void AssertPath(ReadOnlySpan path) @@ -142,13 +147,23 @@ private static IEnumerable GetOptions() { yield return new TestCaseData(new VisitingOptions { - }).SetName("Default"); + }, INodeStorage.KeyScheme.HalfPath).SetName("Default"); + + yield return new TestCaseData(new VisitingOptions + { + }, INodeStorage.KeyScheme.Hash).SetName("Default Hash"); + + yield return new TestCaseData(new VisitingOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + FullScanMemoryBudget = 1.MiB(), + }, INodeStorage.KeyScheme.HalfPath).SetName("Parallel"); yield return new TestCaseData(new VisitingOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, FullScanMemoryBudget = 1.MiB(), - }).SetName("Parallel"); + }, INodeStorage.KeyScheme.Hash).SetName("Parallel Hash"); } public class AppendingVisitor(bool expectAccount) : ITreeVisitor diff --git a/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs b/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs index 45e9c9607d7..65307df47fc 100644 --- a/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs +++ b/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs @@ -367,10 +367,10 @@ private void BatchedThread() return; - static void ThrowUnableToResolve(in SmallTrieVisitContext ctx) + void ThrowUnableToResolve(in SmallTrieVisitContext ctx) { throw new TrieException( - $"Unable to resolve node without Keccak. ctx: {ctx.Level}, {ctx.ExpectAccounts}, {ctx.IsStorage}"); + $"Unable to resolve node without Keccak. ctx: {ctx.Level}, {_visitor.ExpectAccounts}, {ctx.IsStorage}"); } } @@ -434,7 +434,7 @@ internal void AcceptResolvedNode(TrieNode node, in TNodeContext nodeContext, ITr { _visitor.VisitLeaf(nodeContext, node); - if (!trieVisitContext.IsStorage && trieVisitContext.ExpectAccounts) // can combine these conditions + if (!trieVisitContext.IsStorage && _visitor.ExpectAccounts) // can combine these conditions { TNodeContext childContext = nodeContext.Add(node.Key!); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 1e14a0875b5..bb3ce42ffb2 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -49,7 +49,7 @@ public sealed class TrieStore : ITrieStore, IPruningTrieStore // Small optimization to not re-create CommitBuffer private CommitBuffer? _commitBufferUnused = null; - private bool IsInCommitBufferMode => _commitBuffer is not null; + internal bool IsInCommitBufferMode => _commitBuffer is not null; // Only one scope can be active at the same time. Any mutation to trieStore as part of block processing need to // acquire _scopeLock. @@ -62,6 +62,7 @@ public sealed class TrieStore : ITrieStore, IPruningTrieStore private readonly bool _pastKeyTrackingEnabled = false; private bool _lastPersistedReachedReorgBoundary; + private long _toBePersistedBlockNumber = -1; private Task _pruningTask = Task.CompletedTask; private readonly CancellationTokenSource _pruningTaskCancellationTokenSource = new(); @@ -84,6 +85,7 @@ public TrieStore( _prunePersistedNodeMinimumTarget = pruningConfig.PrunePersistedNodeMinimumTarget; _maxBufferedCommitCount = pruningConfig.MaxBufferedCommitCount; + _deleteOldNodes = _pruningStrategy.DeleteObsoleteKeys; _shardBit = pruningConfig.DirtyNodeShardBit; _shardedDirtyNodeCount = 1 << _shardBit; _dirtyNodes = new TrieStoreDirtyNodesCache[_shardedDirtyNodeCount]; @@ -91,11 +93,10 @@ public TrieStore( _persistedHashes = new ConcurrentDictionary[_shardedDirtyNodeCount]; for (int i = 0; i < _shardedDirtyNodeCount; i++) { - _dirtyNodes[i] = new TrieStoreDirtyNodesCache(this, !_nodeStorage.RequirePath, _logger); + _dirtyNodes[i] = new TrieStoreDirtyNodesCache(this, !_nodeStorage.RequirePath, keepRoot: _deleteOldNodes, _logger); _persistedHashes[i] = new ConcurrentDictionary(); } - _deleteOldNodes = _pruningStrategy.DeleteObsoleteKeys; _pastKeyTrackingEnabled = pruningConfig.TrackPastKeys && nodeStorage.RequirePath; } @@ -321,22 +322,52 @@ long blockNumber public IDisposable BeginScope(BlockHeader? baseBlock) { _scopeLock.Enter(); - if (_pruningLock.TryEnter()) + + SpinWait spinWait = new SpinWait(); + while (true) { - // When in non commit buffer mode, FindCachedOrUnknown can also modify the dirty cache which has - // a notable performance benefit. So we try to clear the buffer before. - FlushCommitBufferNoLock(); + if (_pruningLock.TryEnter()) + { + // When in non commit buffer mode, FindCachedOrUnknown can also modify the dirty cache which has + // a notable performance benefit. So we try to clear the buffer before. + FlushCommitBufferNoLock(); - return new Reactive.AnonymousDisposable(() => + return new Reactive.AnonymousDisposable(() => + { + _pruningLock.Exit(); + _scopeLock.Exit(); + }); + } + + if (_commitBuffer is null) { - _pruningLock.Exit(); - _scopeLock.Exit(); - }); + long persistedBoundary = Interlocked.Read(ref _toBePersistedBlockNumber); + if (persistedBoundary == -1) + { + // This can happen in the tiny time in between pruningLock was acquired but the exact block to + // persist was not determined yet. + spinWait.SpinOnce(); + continue; + } + + // _dirtyNodesLock was not acquired, likely due to memory pruning. + // Will continue with commit buffer. + CommitBuffer? commitBuffer = _commitBufferUnused; + if (commitBuffer is null) + { + commitBuffer = new CommitBuffer(this, persistedBoundary); + } + else + { + commitBuffer.Reset(persistedBoundary); + } + + _commitBuffer = commitBuffer; + } + + break; } - // _dirtyNodesLock was not acquired, likely due to memory pruning. - // Will continue with commit buffer. - if (_commitBuffer is null) _commitBuffer = _commitBufferUnused ?? new CommitBuffer(this); if (_commitBuffer.CommitCount >= _maxBufferedCommitCount) { // Prevent commit buffer from becoming too large. @@ -538,32 +569,45 @@ public void Prune() var state = CaptureCurrentState(); if ((_pruningStrategy.ShouldPruneDirtyNode(state) || _pruningStrategy.ShouldPrunePersistedNode(state)) && _pruningTask.IsCompleted) { - _pruningTask = Task.Run(() => + _pruningTask = Task.Run(SyncPruneCheck); + } + } + + internal void SyncPruneCheck() + { + using (var _ = _pruningLock.EnterScope()) + { + if (_pruningStrategy.ShouldPruneDirtyNode(CaptureCurrentState())) { - using (var _ = _pruningLock.EnterScope()) - { - if (_pruningStrategy.ShouldPruneDirtyNode(CaptureCurrentState())) - { - PersistAndPruneDirtyCache(); - } + PersistAndPruneDirtyCache(); + } - if (_pruningStrategy.ShouldPrunePersistedNode(CaptureCurrentState())) - { - PrunePersistedNodes(); - } + if (_prunePersistedNodePortion > 0) + { + // `PrunePersistedNodes` only work on part of the partition at any one time. With commit buffer, + // it is possible that the commit buffer once flushed will immediately trigger another prune, which + // mean `PrunePersistedNodes` was not able to re-trigger multiple time, which make the persisted node + // cache even bigger which causes longer prune which causes bigger commit buffer, etc. + // So we loop it here until `ShouldPrunePersistedNode` return false. + int maxTry = _shardedDirtyNodeCount; + int i = 0; + while (i < maxTry && _pruningStrategy.ShouldPrunePersistedNode(CaptureCurrentState())) + { + PrunePersistedNodes(); + i++; } - - TryExitCommitBufferMode(); - }); + } } + + TryExitCommitBufferMode(); } - private void PersistAndPruneDirtyCache() + internal void PersistAndPruneDirtyCache() { try { long start = Stopwatch.GetTimestamp(); - if (_logger.IsDebug) _logger.Debug($"Locked {nameof(TrieStore)} for pruning."); + if (_logger.IsInfo) _logger.Info($"Starting memory pruning. Dirty memory {DirtyMemoryUsedByDirtyCache / 1.MiB()}MB, Persisted node memory {(PersistedMemoryUsedByDirtyCache / 1.MiB())}MB"); long memoryUsedByDirtyCache = DirtyMemoryUsedByDirtyCache; SaveSnapshot(); @@ -582,6 +626,10 @@ private void PersistAndPruneDirtyCache() { if (_logger.IsError) _logger.Error("Pruning failed with exception.", e); } + finally + { + _toBePersistedBlockNumber = -1; + } } private void SaveSnapshot() @@ -614,6 +662,16 @@ private void SaveSnapshot() } } + if (candidateSets.Count > 0) + { + long minToBePersistedBlock = long.MaxValue; + foreach (BlockCommitSet blockCommitSet in candidateSets) + { + minToBePersistedBlock = Math.Min(minToBePersistedBlock, blockCommitSet.BlockNumber); + } + _toBePersistedBlockNumber = minToBePersistedBlock; + } + Action persistedNodeRecorder = shouldTrackPastKey ? _persistedNodeRecorder : _persistedNodeRecorderNoop; for (int index = 0; index < candidateSets.Count; index++) @@ -760,7 +818,7 @@ private void PruneCache(bool prunePersisted = false, bool dontRemoveNodes = fals /// /// Only prune persisted nodes. This method attempt to pick only some shard for pruning. /// - private void PrunePersistedNodes() + internal void PrunePersistedNodes() { try { @@ -912,7 +970,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) if (path.Length < parallelBoundaryPathLength) { persistedNodeRecorder.Invoke(path, address2, tn); - PersistNode(address2, path, tn, topLevelWriteBatch, writeFlags); + PersistNode(address2, path, tn, commitSet.BlockNumber, topLevelWriteBatch, writeFlags); } else { @@ -950,7 +1008,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) } using ArrayPoolList persistNodeStartingFromTasks = parallelStartNodes.Select( - entry => Task.Run(() => PersistNodeStartingFrom(entry.trieNode, entry.address2, entry.path, persistedNodeRecorder, writeFlags, disposeQueue))) + entry => Task.Run(() => PersistNodeStartingFrom(entry.trieNode, entry.address2, entry.path, commitSet.BlockNumber, persistedNodeRecorder, writeFlags, disposeQueue))) .ToPooledList(parallelStartNodes.Count); Task.WaitAll(persistNodeStartingFromTasks.AsSpan()); @@ -975,6 +1033,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) } private async Task PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath path, + long blockNumber, Action persistedNodeRecorder, WriteFlags writeFlags, Channel disposeQueue) { @@ -984,7 +1043,7 @@ private async Task PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePa async ValueTask DoPersist(TrieNode node, Hash256? address3, TreePath path2) { persistedNodeRecorder.Invoke(path2, address3, node); - PersistNode(address3, path2, node, writeBatch, writeFlags); + PersistNode(address3, path2, node, blockNumber, writeBatch, writeFlags); persistedNodeCount++; if (persistedNodeCount % 512 == 0) @@ -998,12 +1057,17 @@ async ValueTask DoPersist(TrieNode node, Hash256? address3, TreePath path2) await disposeQueue.Writer.WriteAsync(writeBatch); } - private void PersistNode(Hash256? address, in TreePath path, TrieNode currentNode, INodeStorage.IWriteBatch writeBatch, WriteFlags writeFlags = WriteFlags.None) + private void PersistNode(Hash256? address, in TreePath path, TrieNode currentNode, long blockNumber, INodeStorage.IWriteBatch writeBatch, WriteFlags writeFlags = WriteFlags.None) { ArgumentNullException.ThrowIfNull(currentNode); if (currentNode.Keccak is not null) { + TrieStoreDirtyNodesCache.Key key = new TrieStoreDirtyNodesCache.Key(address, path, currentNode.Keccak); + // Unpersisted note may have lower commit number than its parent. This can when its child is created + // on a different block than its parent. + GetDirtyNodeShard(key).GetOrAdd(key, new TrieStoreDirtyNodesCache.NodeRecord(currentNode, blockNumber)); + if (_logger.IsTrace) _logger.Trace($"Persisting {nameof(TrieNode)} {currentNode}."); writeBatch.Set(address, path, currentNode.Keccak, currentNode.FullRlp.Span, writeFlags); currentNode.IsPersisted = true; @@ -1431,18 +1495,25 @@ private class CommitBuffer private readonly ILogger _logger; public int CommitCount => _commitSetQueueBuffer.Count; + private long _minCommitBlockNumber; - public CommitBuffer(TrieStore trieStore) + public CommitBuffer(TrieStore trieStore, long minCommitBlockNumber) { + _minCommitBlockNumber = minCommitBlockNumber; _trieStore = trieStore; _logger = trieStore._logger; _dirtyNodesBuffer = new TrieStoreDirtyNodesCache[trieStore._dirtyNodes.Length]; for (int i = 0; i < trieStore._shardedDirtyNodeCount; i++) { - _dirtyNodesBuffer[i] = new TrieStoreDirtyNodesCache(trieStore, !trieStore._nodeStorage.RequirePath, trieStore._logger); + _dirtyNodesBuffer[i] = new TrieStoreDirtyNodesCache(trieStore, !trieStore._nodeStorage.RequirePath, _trieStore._deleteOldNodes, trieStore._logger); } } + public void Reset(long minCommitBlockNumber) + { + _minCommitBlockNumber = minCommitBlockNumber; + } + public void EnqueueCommitSet(BlockCommitSet set) { _commitSetQueueBuffer.Enqueue(set); @@ -1485,47 +1556,50 @@ public TrieNode FindCachedOrUnknown(TrieStoreDirtyNodesCache.Key key, bool isRea TrieStoreDirtyNodesCache mainShard = _trieStore._dirtyNodes[shardIdx]; var hasInBuffer = bufferShard.TryGetValue(key, out TrieNode bufferNode); - if (!hasInBuffer && mainShard.TryGetRecord(key, out TrieStoreDirtyNodesCache.NodeRecord nodeRecord)) + if (isReadOnly) { - if (_trieStore.IsStillNeeded(nodeRecord.LastCommit)) + if (hasInBuffer) { - var rlp = nodeRecord.Node.FullRlp; - if (rlp.IsNull) - { - bufferShard.GetOrAdd(key, nodeRecord); - if (!isReadOnly) return nodeRecord.Node; - } - else - { - // clone is as if it read only - TrieNode node = nodeRecord.Node.Clone(); - if (nodeRecord.Node.IsSealed) node.Seal(); - if (nodeRecord.Node.IsPersisted) node.IsPersisted = true; - node.Keccak = nodeRecord.Node.Keccak; - bufferShard.GetOrAdd(key, nodeRecord); - if (!isReadOnly) return nodeRecord.Node; - } + return _trieStore.CloneForReadOnly(key, bufferNode); } + + return mainShard.FromCachedRlpOrUnknown(key); } - if (hasInBuffer) + if (!hasInBuffer && mainShard.TryGetRecord(key, out TrieStoreDirtyNodesCache.NodeRecord nodeRecord)) { - if (!isReadOnly) + if (nodeRecord.Node.IsPersisted) { - return bufferNode; + // If a node is persisted, then it is either a node that was previously not persisted and not yet + // in disk, or a node that will be deleted. We must never get a node that will be deleted. + if (nodeRecord.LastCommit >= _minCommitBlockNumber) + { + bufferShard.GetOrAdd(key, new TrieStoreDirtyNodesCache.NodeRecord(nodeRecord.Node, -1)); + return nodeRecord.Node; + } } else { - return _trieStore.CloneForReadOnly(key, bufferNode); + // If it is not persisted, then its child is still referred directly. + // The child will not get unreferred until after later it and all its children was persisted. + bufferShard.GetOrAdd(key, new TrieStoreDirtyNodesCache.NodeRecord(nodeRecord.Node, -1)); + return nodeRecord.Node; } } - return isReadOnly ? bufferShard.FromCachedRlpOrUnknown(key) : bufferShard.FindCachedOrUnknown(key); + return hasInBuffer ? bufferNode : bufferShard.FindCachedOrUnknown(key); } } internal TrieNode CloneForReadOnly(in TrieStoreDirtyNodesCache.Key key, TrieNode node) { + if (node!.FullRlp.IsNull) + { + // // this happens in SyncProgressResolver + // throw new InvalidAsynchronousStateException("Read only trie store is trying to read a transient node."); + return new TrieNode(NodeType.Unknown, key.Keccak); + } + // we returning a copy to avoid multithreaded access var trieNode = new TrieNode(NodeType.Unknown, key.Keccak, node.FullRlp); trieNode.ResolveNode(GetTrieStore(key.Address), key.Path); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStoreDirtyNodesCache.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStoreDirtyNodesCache.cs index c74fa571162..98009ab93f1 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStoreDirtyNodesCache.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStoreDirtyNodesCache.cs @@ -35,14 +35,21 @@ internal class TrieStoreDirtyNodesCache public long TotalDirtyMemory => _totalDirtyMemory; public readonly long KeyMemoryUsage; + private readonly bool _keepRoot; - public TrieStoreDirtyNodesCache(TrieStore trieStore, bool storeByHash, ILogger logger) + public TrieStoreDirtyNodesCache(TrieStore trieStore, bool storeByHash, bool keepRoot, ILogger logger) { _trieStore = trieStore; _logger = logger; // If the nodestore indicated that path is not required, // we will use a map with hash as its key instead of the full Key to reduce memory usage. _storeByHash = storeByHash; + + // Keep root causes persisted root nodes to not get pruned out of the cache. This ensure that it will + // be deleted when another canonical state is persisted which prevent having incomplete state which can happen + // when inner nodes get deleted but the root does not. + _keepRoot = keepRoot; + // NOTE: DirtyNodesCache is already sharded. int concurrencyLevel = Math.Min(Environment.ProcessorCount * 4, 32); int initialBuckets = TrieStore.HashHelpers.GetPrime(Math.Max(31, concurrencyLevel)); @@ -87,13 +94,6 @@ public TrieNode FromCachedRlpOrUnknown(in Key key) // ReSharper disable once ConditionIsAlwaysTrueOrFalse if (TryGetValue(key, out TrieNode trieNode)) { - if (trieNode!.FullRlp.IsNull) - { - // // this happens in SyncProgressResolver - // throw new InvalidAsynchronousStateException("Read only trie store is trying to read a transient node."); - return new TrieNode(NodeType.Unknown, key.Keccak); - } - trieNode = _trieStore.CloneForReadOnly(key, trieNode); Metrics.LoadedFromCacheNodesCount++; @@ -325,7 +325,7 @@ public void PruneCache( continue; } - if (_trieStore.IsNoLongerNeeded(lastCommit)) + if (_trieStore.IsNoLongerNeeded(lastCommit) && !(_keepRoot && key.IsRoot())) { RemoveNodeFromCache(key, node, ref Metrics.PrunedPersistedNodesCount); continue; @@ -499,6 +499,11 @@ public override string ToString() { return $"A:{Address} P:{Path} K:{Keccak}"; } + + public bool IsRoot() + { + return Path.Length == 0; + } } public void CopyTo(TrieStoreDirtyNodesCache otherCache) diff --git a/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs b/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs index f799f5eb20b..a49faf1175b 100644 --- a/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs +++ b/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs @@ -58,10 +58,13 @@ public readonly Context AddStorage(in ValueHash256 storage) } } - public TrieStatsCollector(IKeyValueStore codeKeyValueStore, ILogManager logManager, CancellationToken cancellationToken = default) + public bool ExpectAccounts { get; } + + public TrieStatsCollector(IKeyValueStore codeKeyValueStore, ILogManager logManager, CancellationToken cancellationToken = default, bool expectAccounts = true) { _codeKeyValueStore = codeKeyValueStore ?? throw new ArgumentNullException(nameof(codeKeyValueStore)); _logger = logManager.GetClassLogger(); + ExpectAccounts = expectAccounts; _cancellationToken = cancellationToken; } diff --git a/src/Nethermind/Nethermind.Trie/VisitContext.cs b/src/Nethermind/Nethermind.Trie/VisitContext.cs index 72c8e0d46cf..5d8f3f0c0c7 100644 --- a/src/Nethermind/Nethermind.Trie/VisitContext.cs +++ b/src/Nethermind/Nethermind.Trie/VisitContext.cs @@ -62,7 +62,6 @@ public SmallTrieVisitContext(TrieVisitContext trieVisitContext) private byte _flags = 0; private const byte StorageFlag = 1; - private const byte ExpectAccountsFlag = 2; public bool IsStorage { @@ -79,21 +78,5 @@ internal set } } } - - public bool ExpectAccounts - { - readonly get => (_flags & ExpectAccountsFlag) == ExpectAccountsFlag; - internal set - { - if (value) - { - _flags = (byte)(_flags | ExpectAccountsFlag); - } - else - { - _flags = (byte)(_flags & ~ExpectAccountsFlag); - } - } - } } } diff --git a/src/Nethermind/Nethermind.sln.DotSettings b/src/Nethermind/Nethermind.sln.DotSettings new file mode 100644 index 00000000000..ac13d17e457 --- /dev/null +++ b/src/Nethermind/Nethermind.sln.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file