From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aere Network Date: Tue, 22 Sep 2026 22:21:16 +0000 Subject: [PATCH] Aere Network: AIP-21, verifiable post-quantum finality of every block AERE patch 0007: AIP-21, verifiable post-quantum finality of every block Why this patch exists: since 2026-09-22 every validator of chain 2800 keeps the Falcon-512 seals it hears on QBFT Commit messages in a window on disk and answers the JSON-RPC method aere_getPqFinality for any block in that window: which distinct validators sealed that block hash with a post-quantum signature, re-verified now against the registry epoch of that height, and whether they reach the commit quorum. A node built without this patch follows the chain exactly as before (nothing here changes block validity, so there is no activation height), but it cannot answer the method and it does not keep the window. The window survives a restart (segments of 64 heights under /aere-pq-seal-window, each frame bound to the canonical block hash and re-verified on restore); aere.pq.finality.persist=false disables the journal. What it changes, as one delta over the tree produced by patches 0001 to 0006: the seal cache of the BFT layer (PqSealCache: keyed by the on-chain block hash, with a retention window instead of a fixed depth), the QBFT controller builder (wiring of the window store), the QBFT JSON-RPC method set and RpcMethod (the new method), plus the new files PqSealWindowStore, AereGetPqFinality and their tests. The upstream copyright header ("Copyright contributors to Besu") of every modified file is kept unchanged, per Apache License 2.0 section 4(c); the two upstream files this patch modifies that had no notice yet (RpcMethod, QbftJsonRpcMethods) receive, in this diff, the in-file modification notice that section 4(b) requires, QbftBesuControllerBuilder already carries it from patch 0003, and PqSealCache is an Aere Network file introduced by patch 0003 (section 4(b) does not apply to it); the changed files are also listed here and the holder is named in NOTICE, per section 4(d). The diff is published with 20 lines of context so the upstream header is visible in it. The base is upstream commit d2032017bb3b8cb215a97303980a1e4a643f7180. How it was measured, 2026-09-22: see patch 0006; the same build reproduces the fleet distribution wt3f12ef0d class by class on consensus and on these files. Specification: aips/AIP-21.md; the two-client conformance record is in aips/aip21-conformitate/. Apache License 2.0, section 4(b): the files this patch modifies that existed before it (upstream files of Hyperledger Besu, or Aere Network files introduced by patch 0003, all with their copyright notices kept unchanged) are, and this notice is the statement that contributors to the Aere Network changed them: app/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java The in-file "Modifications Copyright" blocks for the upstream files among them are not in this delta on purpose: the delta is exact to the fleet distribution wt3f12ef0d, and the notices will enter the production sources with the next fleet distribution. --- diff --git a/app/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java b/app/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java index 281add4c9..252f65583 100644 --- a/app/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java +++ b/app/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java @@ -470,40 +470,64 @@ public class QbftBesuControllerBuilder extends BesuControllerBuilder { PqSealCache.instance() .restoreFromDisk( aereCapSigilii.getNumber(), aereCapSigilii.getHash(), PqSignerRegistry.falconSealSupport()); LOG.info( "AERE PERSISTENTA-SIGILII: {} VERIFIED seal(s) restored for chain head {} ({}). The " + "threshold demanded at the next height is K={}. Zero means this node cannot " + "propose until it hears a Commit, which after a fleet restart with K>0 never " + "happens again on its own.", aereSigiliiRestaurate, aereCapSigilii.getNumber(), aereCapSigilii.getHash(), aereAnchorConfig.minSealsAt(aereCapSigilii.getNumber() + 1L)); } catch (final Exception e) { LOG.warn( "AERE PERSISTENTA-SIGILII: restoring the chain head seals could not be done ({}); the " + "node starts with an empty cache, exactly as before this repair.", e.toString()); } + // AIP-21, SECOND HALF (2026-09-18): the finality WINDOW survives the restart too. Same + // security argument as above, one step further: a journal frame counts only if its hash is + // the canonical hash THIS chain holds at that height, and each seal only if it verifies now + // against the registry epoch bound there. A forged journal buys an empty window. + try { + final BlockHeader aereCapFereastra = blockchain.getChainHeadHeader(); + final int aereFereastraRestaurata = + PqSealCache.instance() + .restoreWindowFromDisk( + aereCapFereastra.getNumber(), + aereInaltime -> + blockchain.getBlockHeader(aereInaltime).map(BlockHeader::getHash), + PqSignerRegistry.falconSealSupport()); + LOG.info( + "AERE FEREASTRA-SIGILII (AIP-21): {} VERIFIED seal(s) restored into the finality window " + + "below chain head {}; aere_getPqFinality answers for those blocks again.", + aereFereastraRestaurata, + aereCapFereastra.getNumber()); + } catch (final Exception e) { + LOG.warn( + "AERE FEREASTRA-SIGILII (AIP-21): the finality window could not be restored ({}); the " + + "node answers 'unavailable' for blocks heard before this restart, as before.", + e.toString()); + } } final BftExecutors bftExecutors = BftExecutors.create(metricsSystem, BftExecutors.ConsensusType.QBFT); final QbftBlockCodec blockEncoder = new QbftBlockCodecAdaptor(qbftExtraDataCodec); final Address localAddress = Util.publicKeyToAddress(nodeKey.getPublicKey()); final BftProtocolSchedule bftProtocolSchedule = (BftProtocolSchedule) protocolSchedule; QbftProtocolSchedule qbftProtocolSchedule = new QbftProtocolScheduleAdaptor(bftProtocolSchedule, protocolContext); final QbftBlockCreatorFactory blockCreatorFactory = new QbftBlockCreatorFactory( transactionPool, protocolContext, bftProtocolSchedule, qbftForksSchedule, miningConfiguration, localAddress, qbftExtraDataCodec, ethProtocolManager.ethContext().getScheduler()); diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java index dac0d63dd..716c67e0f 100755 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealCache.java @@ -52,104 +52,146 @@ import org.slf4j.LoggerFactory; * amount of waiting healed it. The seals over M(head) exist nowhere else: they travel only on the * Commit messages of the head block, and those are never replayed. * *

AND IT IS NOT DEFECT A8 IN ANOTHER COAT. A8 was a registry of public keys read from a file and * BELIEVED. Every seal read back here is re-verified, cryptographically, against the anchored * registry over M rebuilt from the header this process just loaded - see {@link PqSealStore}. A * forged file cannot inject a seal without forging a Falcon-512 signature; the worst it achieves is * the empty cache an absent file already gives. Persistence is OFF unless a caller enables it, and * the only caller that does is the QBFT controller builder, only when the anchor is actually armed. * *

WHAT IT DOES NOT DO. The in-memory path verifies nothing. Whether a seal is valid, whether its * index maps to an eligible validator, and whether there are enough of them, are decided at * SELECTION time by the producer, against the message M of the specific parent. A seal that was * signed under the pre-fork message simply fails that check and is dropped, which is what makes the * height switch safe without any coordination beyond the shared activation height. */ public final class PqSealCache { private static final Logger LOG = LoggerFactory.getLogger(PqSealCache.class); - /** How many distinct block hashes are retained. Several rounds at one height each add one. */ - private static final int MAX_ENTRIES = 1024; + /** + * AIP-21 (AERE, 2026-09-18): how far below the highest seen height entries are kept, in blocks. The + * consensus default stays 256 (what the producer needs is the parent); a node that serves + * aere_getPqFinality may keep more through the system property, at ~7 kB per block for ten seals. + * Read once, never below 256, never a consensus value: two nodes with different windows import + * the same chain. + */ + public static final String PROPERTY_FINALITY_WINDOW = "aere.pq.finality.window"; + + private static final long DEFAULT_HEIGHT_WINDOW = 256L; /** How far below the highest seen height entries are kept, in blocks. */ - private static final long HEIGHT_WINDOW = 256L; + private static final long HEIGHT_WINDOW = readWindow(); + + /** How many distinct block hashes are retained. Several rounds at one height each add one. */ + private static final int MAX_ENTRIES = (int) Math.max(1024L, Math.min(HEIGHT_WINDOW * 4L, 1 << 20)); + + private static long readWindow() { + final Long v = Long.getLong(PROPERTY_FINALITY_WINDOW); + if (v == null) { + return DEFAULT_HEIGHT_WINDOW; + } + return Math.max(DEFAULT_HEIGHT_WINDOW, Math.min(v, 1L << 18)); + } + + /** + * The retention window in blocks, as this node runs it. + * + * @return the window + */ + public long retentionBlocks() { + return HEIGHT_WINDOW; + } private static final PqSealCache INSTANCE = new PqSealCache(); private final LinkedHashMap byHash = new LinkedHashMap<>(); private long highestSeen = -1L; /** Guards the file, never the map, so a slow disk cannot hold up the consensus thread's map. */ private final Object writeLock = new Object(); private volatile Path persistenceFile; private volatile long persistenceChainId; private long persistedBlock = -1L; private Hash persistedHash; private int persistedCount = -1; + // AIP-21, SECOND HALF (2026-09-18): the finality window survives a restart. Heard heights are + // appended to a segmented journal (PqSealWindowStore) when the NEXT height begins, and read back, + // re-verified seal by seal against the canonical chain, at startup. Null = journal off. + private volatile Path windowDirectory; + private volatile long windowPrunedSegment = -1L; + private long windowJournaledThrough = -1L; + + /** Heights restored at startup at most, newest first, whatever the configured window. */ + private static final long MAX_RESTORED_HEIGHTS = 4096L; + private PqSealCache() {} /** * The process-wide cache. * * @return the singleton */ public static PqSealCache instance() { return INSTANCE; } /** * Turn on writing the chain head's seals to the node's data directory. * *

The path is DERIVED from the data directory Besu was started with, never configured * separately: a second configurable path is a second thing that can be pointed at the wrong node's * state, and this file is per-node by construction. * * @param dataDirectory the node's data directory, or null to leave persistence off * @param chainId the chain id used in the M pre-image, which must be the SAME value the producer * uses or every restored seal would fail verification */ public void enablePersistence(final Path dataDirectory, final long chainId) { if (dataDirectory == null) { LOG.warn( "AERE PERSISTENTA-SIGILII: no data directory was supplied, so heard seals stay in memory " + "only. A simultaneous restart of the whole fleet with K>0 stops the chain."); return; } this.persistenceChainId = chainId; this.persistenceFile = PqSealStore.fileIn(dataDirectory); + this.windowPrunedSegment = -1L; + this.windowDirectory = + PqSealWindowStore.enabled() ? PqSealWindowStore.directoryIn(dataDirectory) : null; LOG.info( "AERE PERSISTENTA-SIGILII: heard seals for the chain head are kept in {} (chainId={}). " + "Every seal read back is re-verified against the anchored registry before it is used.", this.persistenceFile, chainId); } /** * Turn persistence off again. For tests, and for an operator who wants the previous behaviour. */ public void disablePersistence() { this.persistenceFile = null; + this.windowDirectory = null; synchronized (writeLock) { persistedBlock = -1L; persistedHash = null; persistedCount = -1; } } /** * The file heard seals are written to, or null when persistence is off. * * @return the seal file, or null */ public Path persistenceFile() { return persistenceFile; } /** * Refill the cache from disk for a specific block, normally this node's chain head at startup. * *

Call this AFTER the Falcon registry has been activated, or every seal will be rejected for @@ -209,65 +251,71 @@ public final class PqSealCache { } if (blockNumber > highestSeen) { highestSeen = blockNumber; } prune(); } return verified.size(); } /** * Remember the seals heard for a block. Idempotent per validator index: the first seal seen for * an index wins, so a later duplicate cannot displace it. * * @param blockNumber the height of the block the seals attest * @param onchainBlockHash the ON-CHAIN hash of that block * @param seals the seals heard, possibly empty */ public void record( final long blockNumber, final Hash onchainBlockHash, final Collection seals) { final List toPersist; + List windowToJournal = List.of(); synchronized (this) { if (onchainBlockHash == null || seals == null || seals.isEmpty()) { return; } final Entry entry = byHash.computeIfAbsent(onchainBlockHash, unused -> new Entry(blockNumber)); for (final FalconSeal seal : seals) { if (seal != null && seal.getValidatorIndex() >= 0 && seal.getSignature() != null) { entry.seals.putIfAbsent(seal.getValidatorIndex(), seal); } } if (blockNumber > highestSeen) { + // AIP-21 second half: a NEW height begins, so what was heard for the heights below it is as + // complete as this node will ever hold it. It goes to the window journal, outside this lock. + windowToJournal = windowSnapshotBelow(blockNumber); highestSeen = blockNumber; } prune(); if (persistenceFile == null || blockNumber < highestSeen) { // Only the HIGHEST block is worth a file: the producer of the next block asks for the head's // seals and nothing else, and writing older heights would just churn the disk. return; } toPersist = PqAnchor.sortedByIndex(new ArrayList<>(entry.seals.values())); } // Deliberately OUTSIDE the map lock: a file write must never be able to stall a caller that only // wants to read the cache. persist(blockNumber, onchainBlockHash, toPersist); + // the head file first (liveness depends on it), the window journal after (best effort) + persistWindow(windowToJournal); } private void persist( final long blockNumber, final Hash onchainBlockHash, final List seals) { final Path file = persistenceFile; if (file == null) { return; } synchronized (writeLock) { if (blockNumber < persistedBlock) { return; } if (blockNumber == persistedBlock && onchainBlockHash.equals(persistedHash) && seals.size() <= persistedCount) { return; } try { // MAGAZIA TINE VARFUL (D-330, 2026-09-03): before the main file moves to a HIGHER height, what it // held becomes the previous slot. The main file may hold a proposal that never imports; the @@ -379,47 +427,199 @@ public final class PqSealCache { } /** * How many seals are retained for a block hash. * * @param onchainBlockHash the ON-CHAIN hash * @return the seal count, 0 when nothing is retained */ public synchronized int sealCount(final Hash onchainBlockHash) { final Entry entry = byHash.get(onchainBlockHash); return entry == null ? 0 : entry.seals.size(); } /** * Drop everything held in memory. For tests, and for a deliberate operator reset. The file, if * one exists, is deliberately left alone: it is rewritten by the next commit this node hears. */ public synchronized void clear() { byHash.clear(); highestSeen = -1L; + windowJournaledThrough = -1L; // what clear() forgets, the journal mark forgets too synchronized (writeLock) { persistedBlock = -1L; persistedHash = null; persistedCount = -1; } } + /** + * AIP-21, second half. Everything heard BELOW a height that has just begun and not yet handed to + * the window journal. Caller holds the map lock. + * + * @param newHeight the height that has just begun + * @return the heard blocks to journal, possibly empty + */ + private List windowSnapshotBelow(final long newHeight) { + if (windowDirectory == null) { + return List.of(); + } + final List out = new ArrayList<>(); + for (final Map.Entry e : byHash.entrySet()) { + final long n = e.getValue().blockNumber; + if (n > windowJournaledThrough && n < newHeight && !e.getValue().seals.isEmpty()) { + out.add(PqSealWindowStore.heard(n, e.getKey().getBytes(), e.getValue().seals.values())); + } + } + windowJournaledThrough = Math.max(windowJournaledThrough, newHeight - 1L); + return out; + } + + private void persistWindow(final List heard) { + final Path directory = windowDirectory; + if (directory == null || heard.isEmpty()) { + return; + } + synchronized (writeLock) { + try { + long highest = -1L; + for (final PqSealWindowStore.Heard h : heard) { + PqSealWindowStore.append(directory, persistenceChainId, h); + highest = Math.max(highest, h.blockNumber()); + } + final long segment = highest / PqSealWindowStore.SEGMENT_BLOCKS; + if (segment != windowPrunedSegment) { + PqSealWindowStore.pruneSegments(directory, highest, HEIGHT_WINDOW); + windowPrunedSegment = segment; + } + } catch (final Exception e) { + // A disk fault must not be able to stop this node from taking part in consensus. It costs the + // answer aere_getPqFinality gives for these blocks after a restart, and nothing else. + LOG.warn( + "AERE FEREASTRA-SIGILII: could not append {} heard block(s) to the window journal {} ({}). " + + "This node keeps working; after a restart it answers 'unavailable' for them.", + heard.size(), + directory, + e.toString()); + } + } + } + + /** + * AIP-21, second half. Bring the finality window back after a restart. Nothing in the journal is + * believed: a frame counts only if its hash is the CANONICAL hash this node's own chain holds at + * that height, and each seal only if it verifies, against the registry epoch bound at that height, + * over the commit message rebuilt from that canonical hash. Never throws. + * + * @param headNumber this node's chain head + * @param canonicalHashAt the canonical on-chain hash at a height, empty when the chain holds none + * @param registry the anchored registry seals are verified against + * @return how many verified seals entered the window + */ + public int restoreWindowFromDisk( + final long headNumber, + final java.util.function.LongFunction> canonicalHashAt, + final PqSignerRegistry registry) { + final Path directory = windowDirectory; + if (directory == null || canonicalHashAt == null || registry == null) { + return 0; + } + int restored = 0; + try { + final long from = Math.max(0L, headNumber - Math.min(HEIGHT_WINDOW, MAX_RESTORED_HEIGHTS)); + final List frames = + PqSealWindowStore.readAll(directory, persistenceChainId, from, headNumber); + long highestInJournal = -1L; + int notCanonical = 0; + int rejected = 0; + final LinkedHashMap verified = new LinkedHashMap<>(); + for (final PqSealWindowStore.Heard frame : frames) { + final long n = frame.blockNumber(); + highestInJournal = Math.max(highestInJournal, n); + final java.util.Optional canonical = canonicalHashAt.apply(n); + if (canonical == null + || canonical.isEmpty() + || !canonical.get().getBytes().equals(frame.blockHash())) { + notCanonical++; + continue; + } + final org.apache.tuweni.bytes.Bytes32 message = + PqAnchor.commitMessage(persistenceChainId, n, frame.blockHash()); + for (final FalconSeal seal : frame.seals()) { + if (seal == null + || seal.getValidatorIndex() < 0 + || seal.getSignature() == null + || !registry.verifyAtHistoric( + n, seal.getValidatorIndex(), message, seal.getSignature())) { + rejected++; + continue; + } + final Entry entry = verified.computeIfAbsent(canonical.get(), unused -> new Entry(n)); + if (entry.seals.putIfAbsent(seal.getValidatorIndex(), seal) == null) { + restored++; + } + } + } + synchronized (this) { + // Oldest first, so the size cap still evicts the oldest: the restored heights go in BEFORE + // what this node already holds (its head, restored a moment ago from the head file). + final LinkedHashMap held = new LinkedHashMap<>(byHash); + byHash.clear(); + byHash.putAll(verified); + for (final Map.Entry e : held.entrySet()) { + final Entry target = byHash.get(e.getKey()); + if (target == null) { + byHash.put(e.getKey(), e.getValue()); + } else { + e.getValue().seals.forEach(target.seals::putIfAbsent); + e.getValue().extras.forEach(target.extras::putIfAbsent); + } + } + for (final Entry entry : verified.values()) { + if (entry.blockNumber > highestSeen) { + highestSeen = entry.blockNumber; + } + } + windowJournaledThrough = Math.max(windowJournaledThrough, highestInJournal); + prune(); + } + LOG.info( + "AERE FEREASTRA-SIGILII: {} verified seal(s) over {} block(s) restored into the finality " + + "window from {} ({} frame(s) read, {} not canonical on this chain, {} seal(s) FAILED " + + "verification and were dropped).", + restored, + verified.size(), + directory, + frames.size(), + notCanonical, + rejected); + } catch (final RuntimeException e) { + LOG.warn( + "AERE FEREASTRA-SIGILII: restoring the finality window from {} failed unexpectedly ({}); " + + "the window starts empty, exactly as before this journal existed.", + directory, + e.toString()); + return 0; + } + return restored; + } + private void prune() { final Iterator> it = byHash.entrySet().iterator(); while (it.hasNext()) { final Map.Entry e = it.next(); if (highestSeen - e.getValue().blockNumber > HEIGHT_WINDOW) { it.remove(); } } while (byHash.size() > MAX_ENTRIES) { final Iterator oldest = byHash.keySet().iterator(); oldest.next(); oldest.remove(); } } private static final class Entry { private final long blockNumber; private final Map seals = new LinkedHashMap<>(); /** AERE ANCHOR V2: heard extra (non-Falcon) scheme seals, keyed (index << 8 | wire). */ private final Map extras = new LinkedHashMap<>(); diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java new file mode 100755 index 000000000..b224d8fb2 --- /dev/null +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowStore.java @@ -0,0 +1,387 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.common.bft; + +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.CRC32; + +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * AIP-21, SECOND HALF (2026-09-18): the finality window on disk. + * + *

The first half gave every validator a window of heard Falcon seals ({@link PqSealCache}) and a + * method that reads it back ({@code aere_getPqFinality}). The window lived in memory only, so a node + * that restarted answered {@code unavailable} for every block it had heard before the restart. This + * class is the journal that carries the window across a restart. + * + *

SHAPE. One directory under the node's data directory, {@value #DIRECTORY_NAME}, holding + * SEGMENT files of {@value #SEGMENT_BLOCKS} heights each ({@code w-.rlp}). A segment is + * a sequence of FRAMES: {@code length(4, big endian) || payload || crc32(payload)(4)}, the payload + * being the RLP list {@code [DOMAIN, chainId, blockNumber, blockHash, certificate]} with the + * certificate encoded exactly as {@link PqSealStore} and the header encode it. Frames are only ever + * APPENDED, one per heard block, when the next height begins; whole segments below the window are + * deleted. The footprint is therefore bounded by the window, whatever the age of the node. + * + *

WHY THIS IS NOT A8 IN ANOTHER COAT, the same argument as {@link PqSealStore}: a seal is + * SELF-VERIFYING. Nothing read from this directory is believed. {@link PqSealCache} re-verifies + * every seal against the registry epoch bound at its height, over the message rebuilt from the + * CANONICAL hash this process reads from its own chain at that height; a frame for a hash that is + * not canonical is dropped before a single signature is looked at. A forged journal can obtain the + * empty window an absent journal already gives, and nothing else. + * + *

WHAT A FAULT COSTS. Every method here is written not to throw on bad input: a torn tail (the + * node died mid-append), a flipped byte, a directory where a file should be, a journal from another + * chain. A bad frame costs that frame and what FOLLOWS it in the same segment (a frame boundary + * cannot be trusted after a bad length), never the node. Writes are best effort and unsynced: the + * head file of {@link PqSealStore} is the one liveness depends on, and it keeps its fsync. + */ +public final class PqSealWindowStore { + + private static final Logger LOG = LoggerFactory.getLogger(PqSealWindowStore.class); + + /** Directory name under the node's data directory. */ + public static final String DIRECTORY_NAME = "aere-pq-seal-window"; + + /** Domain separator of a frame payload. */ + public static final String DOMAIN = "AERE-PQ-SEALWINDOW-1"; + + private static final Bytes DOMAIN_BYTES = Bytes.wrap(DOMAIN.getBytes(StandardCharsets.US_ASCII)); + + /** Heights per segment file. */ + public static final long SEGMENT_BLOCKS = 64L; + + /** A segment larger than this is ignored whole. */ + public static final long MAX_SEGMENT_BYTES = 64L * 1024L * 1024L; + + /** A frame payload larger than this ends the reading of its segment. */ + public static final int MAX_FRAME_BYTES = 4 * 1024 * 1024; + + /** More seals than this in one frame is not a certificate. */ + public static final int MAX_SEALS = 1024; + + /** System property: {@code false} turns the journal off (the window is then memory only). */ + public static final String PROPERTY_PERSIST = "aere.pq.finality.persist"; + + private static final Pattern SEGMENT_NAME = Pattern.compile("^w-(\\d{1,18})\\.rlp$"); + + private PqSealWindowStore() {} + + /** + * One heard block as the journal holds it. Unverified: the caller verifies. + * + * @param blockNumber the height + * @param blockHash the round-independent on-chain hash the seals were heard for + * @param seals the heard seals, index-sorted + */ + public record Heard(long blockNumber, Bytes blockHash, List seals) {} + + /** + * Whether the journal is enabled for this process. + * + * @return false only when {@value #PROPERTY_PERSIST} is {@code false} + */ + public static boolean enabled() { + return !"false".equalsIgnoreCase(System.getProperty(PROPERTY_PERSIST)); + } + + /** + * The journal directory of a node. + * + * @param dataDirectory the node's data directory + * @return the directory the segments live in + */ + public static Path directoryIn(final Path dataDirectory) { + return dataDirectory.resolve(DIRECTORY_NAME); + } + + /** + * The segment file a height belongs to. + * + * @param directory the journal directory + * @param blockNumber the height + * @return the segment path + */ + public static Path segmentFor(final Path directory, final long blockNumber) { + return directory.resolve("w-" + (blockNumber / SEGMENT_BLOCKS) + ".rlp"); + } + + /** + * Encode one frame payload. + * + * @param chainId the chain id + * @param heard the heard block + * @return the RLP payload + */ + public static byte[] encode(final long chainId, final Heard heard) { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + out.startList(); + out.writeBytes(DOMAIN_BYTES); + out.writeLongScalar(chainId); + out.writeLongScalar(heard.blockNumber()); + out.writeBytes(heard.blockHash()); + PqAnchor.writeCertificate(out, heard.seals()); + out.endList(); + return out.encoded().toArrayUnsafe(); + } + + /** + * Append one heard block to its segment. Best effort, unsynced. + * + * @param directory the journal directory + * @param chainId the chain id + * @param heard the heard block + * @throws IOException when the file system refuses; the caller logs and carries on + */ + public static void append(final Path directory, final long chainId, final Heard heard) + throws IOException { + Files.createDirectories(directory); + final byte[] payload = encode(chainId, heard); + final CRC32 crc = new CRC32(); + crc.update(payload, 0, payload.length); + final ByteBuffer frame = ByteBuffer.allocate(4 + payload.length + 4); + frame.putInt(payload.length); + frame.put(payload); + frame.putInt((int) crc.getValue()); + frame.flip(); + try (FileChannel channel = + FileChannel.open( + segmentFor(directory, heard.blockNumber()), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.APPEND)) { + while (frame.hasRemaining()) { + channel.write(frame); + } + } + } + + /** + * Delete every segment that lies WHOLLY below the window. + * + * @param directory the journal directory + * @param highestBlock the highest height heard + * @param window the retention window in blocks + * @return how many segment files were deleted + */ + public static int pruneSegments(final Path directory, final long highestBlock, final long window) { + if (directory == null || !Files.isDirectory(directory)) { + return 0; + } + final long oldestKept = Math.max(0L, highestBlock - window); + int deleted = 0; + try (DirectoryStream stream = Files.newDirectoryStream(directory)) { + for (final Path file : stream) { + final Matcher m = SEGMENT_NAME.matcher(file.getFileName().toString()); + if (!m.matches()) { + continue; + } + final long lastHeightOfSegment = (Long.parseLong(m.group(1)) + 1L) * SEGMENT_BLOCKS - 1L; + if (lastHeightOfSegment < oldestKept) { + try { + Files.deleteIfExists(file); + deleted++; + } catch (final IOException e) { + LOG.debug("AERE FEREASTRA-SIGILII: could not delete {} ({}).", file, e.toString()); + } + } + } + } catch (final IOException | RuntimeException e) { + LOG.debug("AERE FEREASTRA-SIGILII: could not list {} ({}).", directory, e.toString()); + } + return deleted; + } + + /** + * Read every frame for a height range, oldest first. Never throws; never verifies. + * + * @param directory the journal directory + * @param chainId the chain id this node runs + * @param fromBlock the lowest height wanted, inclusive + * @param toBlock the highest height wanted, inclusive + * @return the frames that decoded, for this chain, within the range + */ + public static List readAll( + final Path directory, final long chainId, final long fromBlock, final long toBlock) { + final List out = new ArrayList<>(); + if (directory == null || toBlock < fromBlock || !Files.isDirectory(directory)) { + return out; + } + final List segments = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(directory)) { + for (final Path file : stream) { + final Matcher m = SEGMENT_NAME.matcher(file.getFileName().toString()); + if (!m.matches()) { + continue; + } + final long first = Long.parseLong(m.group(1)) * SEGMENT_BLOCKS; + final long last = first + SEGMENT_BLOCKS - 1L; + if (last >= fromBlock && first <= toBlock) { + segments.add(file); + } + } + } catch (final IOException | RuntimeException e) { + LOG.warn( + "AERE FEREASTRA-SIGILII: the window journal {} could not be listed ({}); IGNORED, the " + + "window starts empty exactly as it did before this journal existed.", + directory, + e.toString()); + return out; + } + for (final Path segment : segments) { + readSegment(segment, chainId, fromBlock, toBlock, out); + } + out.sort(Comparator.comparingLong(Heard::blockNumber)); + return out; + } + + private static void readSegment( + final Path segment, + final long chainId, + final long fromBlock, + final long toBlock, + final List out) { + final byte[] raw; + try { + if (!Files.isRegularFile(segment)) { + return; + } + final long size = Files.size(segment); + if (size <= 0 || size > MAX_SEGMENT_BYTES) { + LOG.warn( + "AERE FEREASTRA-SIGILII: segment {} has an implausible size ({} bytes); IGNORED.", + segment, + size); + return; + } + raw = Files.readAllBytes(segment); + } catch (final IOException | RuntimeException e) { + LOG.warn( + "AERE FEREASTRA-SIGILII: segment {} could not be read ({}); IGNORED.", + segment, + e.toString()); + return; + } + final ByteBuffer buffer = ByteBuffer.wrap(raw); + int frames = 0; + while (buffer.remaining() >= 8) { + final int length = buffer.getInt(); + if (length <= 0 || length > MAX_FRAME_BYTES || buffer.remaining() < length + 4) { + LOG.info( + "AERE FEREASTRA-SIGILII: segment {} ends in a torn or implausible frame after {} good " + + "frame(s); what follows it is dropped.", + segment, + frames); + return; + } + final byte[] payload = new byte[length]; + buffer.get(payload); + final int storedCrc = buffer.getInt(); + final CRC32 crc = new CRC32(); + crc.update(payload, 0, payload.length); + if ((int) crc.getValue() != storedCrc) { + LOG.warn( + "AERE FEREASTRA-SIGILII: segment {} holds a frame whose checksum does not match after " + + "{} good frame(s); it and what follows it are dropped.", + segment, + frames); + return; + } + frames++; + final Heard heard = decode(payload, chainId); + if (heard != null && heard.blockNumber() >= fromBlock && heard.blockNumber() <= toBlock) { + out.add(heard); + } + } + } + + private static Heard decode(final byte[] payload, final long chainId) { + try { + final RLPInput in = new BytesValueRLPInput(Bytes.wrap(payload), false); + in.enterList(); + if (!DOMAIN_BYTES.equals(in.readBytes())) { + return null; + } + if (in.readLongScalar() != chainId) { + return null; + } + final long blockNumber = in.readLongScalar(); + final Bytes blockHash = in.readBytes(); + if (blockNumber < 0 || blockHash.size() != 32) { + return null; + } + final List seals = new ArrayList<>(); + in.enterList(); + while (!in.isEndOfCurrentList()) { + if (seals.size() >= MAX_SEALS) { + return null; + } + in.enterList(); + final int index = in.readIntScalar(); + final Bytes signature = in.readBytes(); + in.leaveList(); + seals.add(new FalconSeal(index, signature)); + } + in.leaveList(); + in.leaveList(); + return new Heard(blockNumber, blockHash, seals); + } catch (final RuntimeException e) { + return null; + } + } + + /** + * Count the frames of a journal, for tests and for the operator's curiosity. Never throws. + * + * @param directory the journal directory + * @param chainId the chain id + * @return how many frames decode for this chain, at any height + */ + public static int frameCount(final Path directory, final long chainId) { + return readAll(directory, chainId, 0L, Long.MAX_VALUE).size(); + } + + /** + * The seals of one frame collection, for callers that hold a plain collection. + * + * @param blockNumber the height + * @param blockHash the on-chain hash + * @param seals the heard seals + * @return the frame value, seals index-sorted + */ + public static Heard heard( + final long blockNumber, final Bytes blockHash, final Collection seals) { + return new Heard(blockNumber, blockHash, PqAnchor.sortedByIndex(new ArrayList<>(seals))); + } +} diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java new file mode 100755 index 000000000..5c661d32d --- /dev/null +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/PqSealWindowPersistenceTest.java @@ -0,0 +1,478 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.common.bft; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import org.hyperledger.besu.consensus.common.bft.blockcreation.PqAnchorProducer; +import org.hyperledger.besu.crypto.SecureRandomProvider; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; + +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.LongFunction; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.crypto.digests.KeccakDigest; +import org.bouncycastle.pqc.crypto.falcon.FalconPrivateKeyParameters; +import org.bouncycastle.pqc.crypto.falcon.FalconSigner; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * AIP-21, SECOND HALF (2026-09-18): the finality window survives a restart. + * + *

The rule of the house: no proof without its negative pair. Real Falcon-512 keys, the real + * genesis-anchored registry, the real commit message, a real directory. What each test pins down: + * + *

    + *
  1. {@link #theWindowSurvivesARestartAndOnlyCanonicalBlocksComeBack()} - the point of the whole + * thing, with its negative: a journal frame for a hash that is NOT canonical at its height + * never comes back, and neither does a height this node's chain does not hold. + *
  2. {@link #aForgedSealInTheJournalNeverEntersTheWindow()} - THE load-bearing assertion, the one + * the build-time negative control plants against: three forgeries beside two genuine seals. + *
  3. {@link #aTornTailOrAFlippedByteCostsOnlyWhatFollowsIt()} - a node that died mid-append, a + * flipped byte, garbage, a directory where a file should be: never an exception, and the good + * frames BEFORE the damage still come back. + *
  4. {@link #aJournalFromAnotherChainOrOutsideTheWindowIsIgnored()}. + *
  5. {@link #oldSegmentsAreDeletedSoTheFootprintStaysBounded()}. + *
  6. {@link #theJournalIsOffWhenPersistenceIsOffOrThePropertySaysSo()}. + *
  7. {@link #aRestoredWindowIsNotAppendedASecondTime()} - the journal does not grow by what it + * already holds every time the node restarts. + *
  8. {@link #whatOneAppendCostsAgainstTheBlockInterval()} - the price on the consensus thread, + * as a number. + *
+ * + *

NOT MEASURED here: that a live validator answers {@code post-quantum} for a block heard before + * its restart. That is measured on testnet 28001 and recorded in the AIP. + */ +public class PqSealWindowPersistenceTest { + + private static final long H = 1_000L; + private static final long ATTACH = 900L; + private static final int N = 7; + private static final long CHAIN_ID = 2_800L; + private static final long BLOCK_INTERVAL_MS = 523L; + private static final long HEAD = 5_000L; + + @TempDir private Path tmp; + + private Path dataDirectory; + private final List privateKeys = new ArrayList<>(); + private final Map chain = new HashMap<>(); + + @BeforeEach + public void setUp() throws Exception { + dataDirectory = Files.createDirectories(tmp.resolve("besu-data")); + final KeccakDigest kd = new KeccakDigest(256); + final StringBuilder manifest = + new StringBuilder("{\"config\":{\"aereFalconRegistry\":{") + .append(PqV2Fixture.manifestHeader(N, CHAIN_ID, H)); + for (int i = 0; i < N; i++) { + privateKeys.add(PqV2Fixture.privateKey(i)); + final byte[] anchoredRow = PqV2Fixture.anchorPreimageRow(i); + kd.update(anchoredRow, 0, anchoredRow.length); + manifest.append(',').append(PqV2Fixture.manifestEntry(i, N, CHAIN_ID, H)); + } + final byte[] anchoredHash = new byte[32]; + kd.doFinal(anchoredHash, 0); + manifest + .append("}},\"alloc\":{\"0000000000000000000000000000000000000fa1\":{\"storage\":{\"0x") + .append("0".repeat(64)) + .append("\":\"0x") + .append(Bytes.wrap(anchoredHash).toUnprefixedHexString()) + .append("\"}}}}"); + final Path genesisPath = tmp.resolve("genesis-registry.json"); + Files.writeString(genesisPath, manifest.toString()); + System.setProperty("aere.falcon.genesis", genesisPath.toAbsolutePath().toString()); + System.setProperty("aere.falcon.attachBlock", Long.toString(ATTACH)); + System.clearProperty(PqSealWindowStore.PROPERTY_PERSIST); + + resetFalconSingleton(); + bindTheRegistryScheduleTheWayALiveNodeDoes(genesisPath); + PqSealCache.instance().disablePersistence(); + PqSealCache.instance().clear(); + PqAnchorProducer.useConfigForTesting( + new PqAnchorConfig(CHAIN_ID, H, Map.of(H, 0, H + 10L, 3), OptionalInt.empty(), false)); + chain.clear(); + } + + /** + * The HISTORY door ({@code verifyAtHistoric}) refuses to resolve a key above the arming height + * unless a registry schedule is loaded (D2, 2026-08-06): it will not fall back to the head + * registry. A live node holds that schedule from its genesis ({@code config.pqRegistryHash}); the + * window restore judges heights BELOW the head, so it goes through that door, and so this fixture + * binds the schedule exactly the way a node does. Measured 2026-09-18: without this every restore + * here returned zero, with the refusal in the log. + */ + private void bindTheRegistryScheduleTheWayALiveNodeDoes(final Path genesisPath) throws Exception { + final PqRegistryHash.Registry held = PqRegistryHash.loadAuto(genesisPath); + String hash = PqRegistryHash.hashFor(held, CHAIN_ID); + if (!hash.startsWith("0x")) { + hash = "0x" + hash; + } + final Path schedule = tmp.resolve("genesis-schedule.json"); + Files.writeString( + schedule, + "{\"config\":{\"pqRegistryHash\":[{\"block\":" + H + ",\"hash\":\"" + hash + "\"}]}}"); + FalconSealSupport.instance() + .verifyRegistryBindingOrAbort( + 0L, CHAIN_ID, PqRegistryHash.loadScheduleFromGenesis(schedule)); + } + + @AfterEach + public void tearDown() throws Exception { + System.clearProperty("aere.falcon.genesis"); + System.clearProperty("aere.falcon.attachBlock"); + System.clearProperty(PqSealWindowStore.PROPERTY_PERSIST); + PqSealCache.instance().disablePersistence(); + PqSealCache.instance().clear(); + PqAnchorProducer.useConfigForTesting(null); + resetFalconSingleton(); + } + + @Test + public void theWindowSurvivesARestartAndOnlyCanonicalBlocksComeBack() { + assertThat(FalconSealSupport.instance().genesisAnchored()).isTrue(); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + + // Ten heights heard live, three genuine seals each. A height reaches the journal when the NEXT + // one begins, so after hearing HEAD the journal holds HEAD-9 .. HEAD-1 and the head file HEAD. + for (long n = HEAD - 9L; n <= HEAD; n++) { + for (int i = 0; i < 3; i++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(i, header(n)))); + } + } + // and one losing proposal at HEAD-4: heard, journaled, NOT canonical + final BlockHeader loser = + new BlockHeaderTestFixture().number(HEAD - 4L).extraData(Bytes.of(7, 7, 7)).buildHeader(); + assertThat(loser.getHash()).isNotEqualTo(header(HEAD - 4L).getHash()); + + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + assertThat(journal).isDirectory(); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)).isEqualTo(9); + appendUnchecked(journal, loser, List.of(genuineSeal(0, loser))); + + // THE RESTART: the in-memory window is gone + PqSealCache.instance().clear(); + assertThat(PqSealCache.instance().sealCount(header(HEAD - 5L).getHash())).isZero(); + + // this node's chain holds every height except HEAD-7 (the negative: a height it cannot vouch for) + final LongFunction> canonical = + n -> n == HEAD - 7L ? Optional.empty() : Optional.of(header(n).getHash()); + final int restored = + PqSealCache.instance() + .restoreWindowFromDisk(HEAD, canonical, PqSignerRegistry.falconSealSupport()); + + assertThat(restored) + .describedAs("nine journaled heights, minus the one the chain does not hold, three seals each") + .isEqualTo(8 * 3); + for (long n = HEAD - 9L; n < HEAD; n++) { + assertThat(PqSealCache.instance().sealCount(header(n).getHash())) + .describedAs("height %d", n) + .isEqualTo(n == HEAD - 7L ? 0 : 3); + } + assertThat(PqSealCache.instance().sealCount(loser.getHash())) + .describedAs("a frame for a hash that is not canonical at its height never comes back") + .isZero(); + assertThat(PqSealCache.instance().sealsFor(HEAD - 2L, header(HEAD - 2L).getHash())) + .extracting(FalconSeal::getValidatorIndex) + .containsExactly(0, 1, 2); + } + + @Test + public void aForgedSealInTheJournalNeverEntersTheWindow() throws Exception { + final BlockHeader block = header(HEAD - 3L); + final FalconSeal genuine0 = genuineSeal(0, block); + final FalconSeal genuine1 = genuineSeal(1, block); + final Bytes32 anotherBlocksMessage = + PqAnchor.commitMessage(CHAIN_ID, block.getNumber() - 1L, Bytes32.leftPad(Bytes.of(9))); + final byte[] randomBytes = new byte[genuine0.getSignature().size()]; + SecureRandomProvider.createSecureRandom().nextBytes(randomBytes); + final List all = + List.of( + genuine0, + genuine1, + new FalconSeal(2, Bytes.wrap(falconSign(privateKeys.get(2), anotherBlocksMessage))), + new FalconSeal(3, Bytes.wrap(randomBytes)), + new FalconSeal(4, genuine0.getSignature())); + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + PqSealWindowStore.append( + journal, CHAIN_ID, PqSealWindowStore.heard(block.getNumber(), block.getHash().getBytes(), all)); + + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + final int restored = + PqSealCache.instance() + .restoreWindowFromDisk( + HEAD, n -> Optional.of(header(n).getHash()), PqSignerRegistry.falconSealSupport()); + + assertThat(restored) + .describedAs( + "THE LOAD-BEARING ASSERTION. Five seals were in the frame and only the two genuine ones " + + "may come out: a signature over another block's message, random bytes and a " + + "genuine seal re-labelled under another index all die at the verifier. Take the " + + "verifier out of restoreWindowFromDisk and this line goes red, which is what the " + + "build-time negative control proves.") + .isEqualTo(2); + assertThat(PqSealCache.instance().sealsFor(block.getNumber(), block.getHash())) + .extracting(FalconSeal::getValidatorIndex) + .containsExactly(0, 1); + } + + @Test + public void aTornTailOrAFlippedByteCostsOnlyWhatFollowsIt() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + // five heights in ONE segment (64 heights per segment; HEAD-5 .. HEAD-1 share 4992/64 = 78) + for (long n = HEAD - 5L; n < HEAD; n++) { + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard(n, header(n).getHash().getBytes(), List.of(genuineSeal(0, header(n))))); + } + final Path segment = PqSealWindowStore.segmentFor(journal, HEAD - 1L); + assertThat(PqSealWindowStore.segmentFor(journal, HEAD - 5L)).isEqualTo(segment); + final byte[] whole = Files.readAllBytes(segment); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + + // positive control first: the undamaged journal gives all five + assertThat(restoreAll()).isEqualTo(5); + + // (a) torn tail: the node died in the middle of the fifth append + Files.write(segment, java.util.Arrays.copyOf(whole, whole.length - 40)); + PqSealCache.instance().clear(); + assertThat(restoreAll()).describedAs("the four whole frames before the torn one").isEqualTo(4); + + // (b) one byte flipped inside the THIRD frame: the checksum catches it, frames one and two live + final int frame = whole.length / 5; + final byte[] flipped = whole.clone(); + flipped[2 * frame + frame / 2] ^= 0x20; + Files.write(segment, flipped); + PqSealCache.instance().clear(); + assertThat(restoreAll()).describedAs("the two frames before the damaged one").isEqualTo(2); + + // (c) garbage, empty, a directory where the segment should be: nothing, and no exception + Files.write(segment, new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9}); + PqSealCache.instance().clear(); + assertThat(restoreAll()).isZero(); + Files.write(segment, new byte[0]); + assertThat(restoreAll()).isZero(); + Files.delete(segment); + Files.createDirectory(segment); + assertThatCode(this::restoreAll).doesNotThrowAnyException(); + assertThat(restoreAll()).isZero(); + } + + @Test + public void aJournalFromAnotherChainOrOutsideTheWindowIsIgnored() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + final BlockHeader inWindow = header(HEAD - 10L); + final BlockHeader tooOld = header(HEAD - PqSealCache.instance().retentionBlocks() - 50L); + PqSealWindowStore.append( + journal, + CHAIN_ID + 1L, + PqSealWindowStore.heard( + inWindow.getNumber(), inWindow.getHash().getBytes(), List.of(genuineSeal(0, inWindow)))); + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard( + tooOld.getNumber(), tooOld.getHash().getBytes(), List.of(genuineSeal(0, tooOld)))); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + assertThat(restoreAll()) + .describedAs("another chain's frame and a height below the window: neither comes back") + .isZero(); + // positive pair: the same height, this chain + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard( + inWindow.getNumber(), inWindow.getHash().getBytes(), List.of(genuineSeal(0, inWindow)))); + assertThat(restoreAll()).isEqualTo(1); + } + + @Test + public void oldSegmentsAreDeletedSoTheFootprintStaysBounded() throws Exception { + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + final long window = PqSealCache.instance().retentionBlocks(); + final long heights = window + 5L * PqSealWindowStore.SEGMENT_BLOCKS; + final FalconSeal one = new FalconSeal(0, Bytes.wrap(new byte[650])); + for (long n = HEAD; n < HEAD + heights; n++) { + PqSealCache.instance().record(n, Hash.hash(Bytes.ofUnsignedLong(n)), List.of(one)); + } + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + final long highest = HEAD + heights - 1L; + try (Stream files = Files.list(journal)) { + final List segments = files.toList(); + assertThat(segments.size()) + .describedAs("segments kept: the window plus the one being written plus one of slack") + .isLessThanOrEqualTo((int) (window / PqSealWindowStore.SEGMENT_BLOCKS) + 2); + assertThat(segments) + .describedAs("no segment wholly below the window is left behind") + .doesNotContain(PqSealWindowStore.segmentFor(journal, highest - window - 2L * PqSealWindowStore.SEGMENT_BLOCKS)); + assertThat(segments).contains(PqSealWindowStore.segmentFor(journal, highest - 1L)); + } + } + + @Test + public void theJournalIsOffWhenPersistenceIsOffOrThePropertySaysSo() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + // persistence never enabled: memory only, exactly as before this class existed + for (long n = HEAD; n < HEAD + 3L; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + assertThat(journal).doesNotExist(); + + // the operator's switch + System.setProperty(PqSealWindowStore.PROPERTY_PERSIST, "false"); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + for (long n = HEAD + 3L; n < HEAD + 6L; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + assertThat(journal).doesNotExist(); + assertThat(PqSealStore.fileIn(dataDirectory)) + .describedAs("the head file, which liveness depends on, is NOT governed by this switch") + .exists(); + + // positive pair: the switch back on + System.clearProperty(PqSealWindowStore.PROPERTY_PERSIST); + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + for (long n = HEAD + 6L; n < HEAD + 9L; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + assertThat(journal).isDirectory(); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)).isGreaterThanOrEqualTo(2); + } + + @Test + public void aRestoredWindowIsNotAppendedASecondTime() { + PqSealCache.instance().enablePersistence(dataDirectory, CHAIN_ID); + for (long n = HEAD - 4L; n <= HEAD; n++) { + PqSealCache.instance().record(n, header(n).getHash(), List.of(genuineSeal(0, header(n)))); + } + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)).isEqualTo(4); + + PqSealCache.instance().clear(); + PqSealCache.instance() + .restoreFromDisk(HEAD, header(HEAD).getHash(), PqSignerRegistry.falconSealSupport()); + assertThat(restoreAll()).isEqualTo(4); + + // the chain moves on: exactly ONE new frame (the head, which was only in the head file) + PqSealCache.instance() + .record(HEAD + 1L, header(HEAD + 1L).getHash(), List.of(genuineSeal(0, header(HEAD + 1L)))); + assertThat(PqSealWindowStore.frameCount(journal, CHAIN_ID)) + .describedAs("four restored heights are not written again; the head is written once") + .isEqualTo(5); + } + + @Test + public void whatOneAppendCostsAgainstTheBlockInterval() throws Exception { + final Path journal = PqSealWindowStore.directoryIn(dataDirectory); + final List ten = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + ten.add(new FalconSeal(i, Bytes.wrap(new byte[660]))); + } + final int rounds = 200; + final long t0 = System.nanoTime(); + for (int r = 0; r < rounds; r++) { + PqSealWindowStore.append( + journal, CHAIN_ID, PqSealWindowStore.heard(HEAD + r, Bytes32.leftPad(Bytes.of(r)), ten)); + } + final double meanMs = (System.nanoTime() - t0) / 1_000_000.0 / rounds; + System.out.printf( + "AERE FEREASTRA-SIGILII: one append of 10 seals costs %.3f ms on average, %.2f%% of a %d ms block%n", + meanMs, 100.0 * meanMs / BLOCK_INTERVAL_MS, BLOCK_INTERVAL_MS); + assertThat(meanMs) + .describedAs("an append on the consensus thread must stay far below the block interval") + .isLessThan(BLOCK_INTERVAL_MS * 0.05); + } + + /** + * THE SHARED VECTOR, literal for literal in the Nethermind-derived client's + * AereAip21SealWindowJournalProofTests: the two clients write the SAME frame payload. Chain 28001, + * block 5, hash 32 x 0xaa, one seal (index 1, signature 0x0102). The frame around it is + * length(4, big endian) || payload || crc32(payload)(4, big endian), java.util.zip.CRC32. + */ + @Test + public void theBytesOnDiskAreTheOnesTheOtherClientWrites() { + final Bytes hash = Bytes.fromHexString("0x" + "aa".repeat(32)); + final byte[] payload = + PqSealWindowStore.encode( + 28_001L, PqSealWindowStore.heard(5L, hash, List.of(new FalconSeal(1, Bytes.of(1, 2))))); + assertThat(Bytes.wrap(payload).toUnprefixedHexString()) + .isEqualTo( + "f84094414552452d50512d5345414c57494e444f572d31826d6105a0" + + "aa".repeat(32) + + "c5c401820102"); + final java.util.zip.CRC32 crc = new java.util.zip.CRC32(); + crc.update("123456789".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + assertThat(crc.getValue()).isEqualTo(0xCBF43926L); + } + + private int restoreAll() { + return PqSealCache.instance() + .restoreWindowFromDisk( + HEAD, n -> Optional.of(header(n).getHash()), PqSignerRegistry.falconSealSupport()); + } + + private BlockHeader header(final long number) { + return chain.computeIfAbsent( + number, n -> new BlockHeaderTestFixture().number(n).buildHeader()); + } + + private FalconSeal genuineSeal(final int index, final BlockHeader block) { + final Bytes32 m = + PqAnchor.commitMessage(CHAIN_ID, block.getNumber(), block.getHash().getBytes()); + return new FalconSeal(index, Bytes.wrap(falconSign(privateKeys.get(index), m))); + } + + private static void appendUnchecked( + final Path journal, final BlockHeader block, final List seals) { + try { + PqSealWindowStore.append( + journal, + CHAIN_ID, + PqSealWindowStore.heard(block.getNumber(), block.getHash().getBytes(), seals)); + } catch (final java.io.IOException e) { + throw new java.io.UncheckedIOException(e); + } + } + + private static byte[] falconSign(final FalconPrivateKeyParameters key, final Bytes32 m) { + final FalconSigner signer = new FalconSigner(); + signer.init(true, key); + return signer.generateSignature(m.toArray()); + } + + private static void resetFalconSingleton() throws Exception { + final Field f = FalconSealSupport.class.getDeclaredField("instance"); + f.setAccessible(true); + f.set(null, null); + } +} diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java index a0aaf3a16..674b39e15 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/QbftJsonRpcMethods.java @@ -1,42 +1,49 @@ /* * Copyright ConsenSys AG. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0007-aere-pq-finality.patch. */ package org.hyperledger.besu.consensus.qbft.jsonrpc; import org.hyperledger.besu.config.BftConfigOptions; import org.hyperledger.besu.consensus.common.BlockInterface; import org.hyperledger.besu.consensus.common.bft.BftContext; import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftDiscardValidatorVote; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetPendingVotes; +import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.AereGetPqFinality; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetRequestTimeoutSeconds; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetSignerMetrics; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetValidatorsByBlockHash; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftGetValidatorsByBlockNumber; import org.hyperledger.besu.consensus.qbft.jsonrpc.methods.QbftProposeValidatorVote; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.ApiGroupJsonRpcMethods; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.MiningConfiguration; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import java.util.Map; /** The Qbft json rpc methods. */ public class QbftJsonRpcMethods extends ApiGroupJsonRpcMethods { private final ProtocolContext context; private final ValidatorProvider readOnlyValidatorProvider; @@ -74,25 +81,27 @@ public class QbftJsonRpcMethods extends ApiGroupJsonRpcMethods { @Override protected Map create() { final BlockchainQueries blockchainQueries = new BlockchainQueries( protocolSchedule, context.getBlockchain(), context.getWorldStateArchive(), miningConfiguration); final BftContext bftContext = context.getConsensusContext(BftContext.class); final BlockInterface blockInterface = bftContext.getBlockInterface(); final ValidatorProvider validatorProvider = bftContext.getValidatorProvider(); Map methods = mapOf( new QbftProposeValidatorVote(validatorProvider), new QbftGetValidatorsByBlockNumber(blockchainQueries, readOnlyValidatorProvider), new QbftDiscardValidatorVote(validatorProvider), new QbftGetValidatorsByBlockHash(context.getBlockchain(), readOnlyValidatorProvider), new QbftGetSignerMetrics(readOnlyValidatorProvider, blockInterface, blockchainQueries), new QbftGetPendingVotes(validatorProvider), - new QbftGetRequestTimeoutSeconds(bftConfig)); + new QbftGetRequestTimeoutSeconds(bftConfig), + // AIP-21 (AERE): per-block post-quantum finality, non-consensus, verified against the anchored registry + new AereGetPqFinality(blockchainQueries, readOnlyValidatorProvider)); return methods; } } diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java new file mode 100755 index 000000000..7fc5d24a5 --- /dev/null +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinality.java @@ -0,0 +1,254 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.qbft.jsonrpc.methods; + +import org.hyperledger.besu.consensus.common.bft.BftHelpers; +import org.hyperledger.besu.consensus.common.bft.FalconSeal; +import org.hyperledger.besu.consensus.common.bft.PqSealCache; +import org.hyperledger.besu.consensus.common.bft.PqSignerRegistry; +import org.hyperledger.besu.consensus.common.bft.SchemeSeal; +import org.hyperledger.besu.consensus.common.bft.SealSchemes; +import org.hyperledger.besu.consensus.common.bft.blockcreation.PqAnchorProducer; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.AbstractBlockParameterMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeSet; + +import org.apache.tuweni.bytes.Bytes; + +/** + * AIP-21 (AERE, 2026-09-18), first half: the post-quantum finality of ONE block, verifiable per block, + * without growing the header. + * + *

WHAT IT ANSWERS. Every block of chain 2800 is committed with a Falcon-512 seal on each Commit + * message (enforced since H1, 2026-09-05), and every validator keeps the seals it heard for recent + * blocks in {@link PqSealCache}. Those seals never reach the header (only every 128th block carries a + * certificate, over its parent), so until now the immediate post-quantum finality of a block was true + * and NOT observable. This method exposes it: for a block it returns the seals heard, re-verifies each + * one right now against the anchored registry over the exact message the validators signed, and gives + * the verdict "verified distinct signers >= the chain's commit quorum". + * + *

WHAT IT DOES NOT CLAIM. Seals travel only on Commit messages, which validators send to + * validators: a reader node hears none and answers {@code finality: "unavailable"} honestly. The + * retention window is the cache's (system property {@code aere.pq.finality.window}, blocks; default + * 256), so an older block answers "unavailable" too, and the permanent record for it is the anchor + * certificate. Hybrid extras (SLH-DSA seals on anchor parents) are listed as heard but NOT verified + * here ({@code verified: null}); their verification is the anchor certificate's job. Below the anchor + * form (never on a live chain today) the signed message is the round-specific commit digest, which + * cannot be rebuilt from the header, so the method says so instead of guessing. + * + *

NON-CONSENSUS. Nothing here changes what a block is; a node with or without this method imports + * the same chain. That is why it needs no activation height. + */ +public class AereGetPqFinality extends AbstractBlockParameterMethod { + + /** Verifies one Falcon seal over the message signed for a block; injectable for tests. */ + @FunctionalInterface + public interface SealVerifier { + /** + * Verify. + * + * @param blockNumber the height + * @param validatorIndex the signer index in the anchored registry + * @param message the message that was signed + * @param signature the Falcon signature + * @return true iff it verifies against the key bound at that height + */ + boolean verify(long blockNumber, int validatorIndex, Bytes message, Bytes signature); + } + + /** Rebuilds the message the seals of a block signed, when that is possible from the header. */ + @FunctionalInterface + public interface MessageBuilder { + /** + * The signed message. + * + * @param blockNumber the height + * @param onchainHash the round-independent on-chain hash + * @return the message, or empty when the form at that height cannot be rebuilt from the header + */ + Optional messageFor(long blockNumber, Hash onchainHash); + } + + private final ValidatorProvider validatorProvider; + private final PqSealCache cache; + private final SealVerifier verifier; + private final MessageBuilder messageBuilder; + + /** + * Production wiring: the process-wide seal cache, the anchored Falcon registry at the block's + * height, and the anchor-form commit message. + * + * @param blockchainQueries the blockchain queries + * @param validatorProvider the read-only validator provider + */ + public AereGetPqFinality( + final BlockchainQueries blockchainQueries, final ValidatorProvider validatorProvider) { + this( + blockchainQueries, + validatorProvider, + PqSealCache.instance(), + (n, i, m, s) -> PqSignerRegistry.falconSealSupport().verifyAtHistoric(n, i, m, s), + (n, h) -> + PqAnchorProducer.sealMessageIsAnchorForm(n) + ? Optional.of(PqAnchorProducer.commitSealMessage(n, h.getBytes(), Hash.ZERO)) + : Optional.empty()); + } + + /** + * Full wiring, for tests. + * + * @param blockchainQueries the blockchain queries + * @param validatorProvider the validator provider + * @param cache the seal cache to read + * @param verifier the seal verifier + * @param messageBuilder the signed-message builder + */ + public AereGetPqFinality( + final BlockchainQueries blockchainQueries, + final ValidatorProvider validatorProvider, + final PqSealCache cache, + final SealVerifier verifier, + final MessageBuilder messageBuilder) { + super(blockchainQueries); + this.validatorProvider = validatorProvider; + this.cache = cache; + this.verifier = verifier; + this.messageBuilder = messageBuilder; + } + + @Override + public String getName() { + return RpcMethod.AERE_GET_PQ_FINALITY.getMethodName(); + } + + @Override + protected BlockParameter blockParameter(final JsonRpcRequestContext request) { + try { + return request.getOptionalParameter(0, BlockParameter.class).orElse(BlockParameter.LATEST); + } catch (final JsonRpcParameterException e) { + throw new InvalidJsonRpcParameters( + "Invalid block parameter (index 0)", RpcErrorType.INVALID_BLOCK_PARAMS, e); + } + } + + @Override + protected Object pendingResult(final JsonRpcRequestContext request) { + return describe(getBlockchainQueries().headBlockHeader()); + } + + @Override + protected Object resultByBlockNumber(final JsonRpcRequestContext request, final long blockNumber) { + return getBlockchainQueries().getBlockHeaderByNumber(blockNumber).map(this::describe).orElse(null); + } + + /** + * The finality record of one block. + * + * @param header the block's header + * @return the JSON-shaped record + */ + public Map describe(final BlockHeader header) { + final long number = header.getNumber(); + final Hash hash = header.getHash(); + final int validators = validatorProvider.getValidatorsForBlock(header).size(); + final int quorum = validators == 0 ? 0 : BftHelpers.calculateRequiredValidatorQuorum(validators); + final Optional message = messageBuilder.messageFor(number, hash); + final List heard = cache.sealsFor(number, hash); + final List extras = cache.extrasFor(number, hash); + + final List> seals = new ArrayList<>(); + final TreeSet verifiedSigners = new TreeSet<>(); + for (final FalconSeal seal : heard) { + final boolean ok = + message.isPresent() + && verifier.verify(number, seal.getValidatorIndex(), message.get(), seal.getSignature()); + if (ok) { + verifiedSigners.add(seal.getValidatorIndex()); + } + final Map s = new LinkedHashMap<>(); + s.put("scheme", "falcon-512"); + s.put("index", seal.getValidatorIndex()); + s.put("verified", ok); + s.put("signature", seal.getSignature().toHexString()); + seals.add(s); + } + final List> hybrid = new ArrayList<>(); + for (final SchemeSeal seal : extras) { + final Map s = new LinkedHashMap<>(); + s.put( + "scheme", + SealSchemes.byWireId(seal.getSchemeWireId()).map(x -> x.id()).orElse("wire-" + (seal.getSchemeWireId() & 0xff))); + s.put("index", seal.getValidatorIndex()); + s.put("verified", null); + s.put("signature", seal.getSignature().toHexString()); + hybrid.add(s); + } + + final String finality; + final String note; + if (message.isEmpty()) { + finality = "unverifiable-form"; + note = + "below the anchor form the signed message is the round-specific commit digest, which this node cannot rebuild from the header; nothing is guessed"; + } else if (heard.isEmpty()) { + finality = "unavailable"; + note = + "no seal heard for this block: seals travel on Commit messages between validators, so a reader node hears none, and a validator keeps only the last " + + cache.retentionBlocks() + + " blocks; the permanent record for older blocks is the anchor certificate"; + } else if (verifiedSigners.size() >= quorum && quorum > 0) { + finality = "post-quantum"; + note = "at least the commit quorum of distinct validators sealed this block hash with Falcon-512 and every counted seal verifies now against the anchored registry"; + } else if (!verifiedSigners.isEmpty()) { + finality = "partial"; + note = "fewer verified distinct signers than the commit quorum: this node did not hear enough seals, or some it heard no longer verify"; + } else { + finality = "none"; + note = "seals were heard but none verifies against the anchored registry at this height"; + } + + final Map out = new LinkedHashMap<>(); + out.put("blockNumber", number); + out.put("blockHash", hash.getBytes().toHexString()); + out.put("validators", validators); + out.put("quorum", quorum); + out.put("sealForm", message.isPresent() ? "anchor" : "commit-digest"); + out.put("signedMessage", message.map(Bytes::toHexString).orElse(null)); + out.put("falconSealsHeard", heard.size()); + out.put("falconSealsVerified", verifiedSigners.size()); + out.put("verifiedIndexes", new ArrayList<>(verifiedSigners)); + out.put("falconSeals", seals); + out.put("hybridSeals", hybrid); + out.put("finality", finality); + out.put("retentionBlocks", cache.retentionBlocks()); + out.put("note", note); + return out; + } +} diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java new file mode 100755 index 000000000..b3b6d95ae --- /dev/null +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/jsonrpc/methods/AereGetPqFinalityTest.java @@ -0,0 +1,205 @@ +/* + * Copyright contributors to Besu / AERE Network. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.consensus.qbft.jsonrpc.methods; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.consensus.common.bft.FalconSeal; +import org.hyperledger.besu.consensus.common.bft.PqSealCache; +import org.hyperledger.besu.consensus.common.validator.ValidatorProvider; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * AIP-21: the finality record is a MEASUREMENT of the seals heard, re-verified, against the chain's + * commit quorum; every claim it can make has its negative pair here (too few signers, none verifying, + * nothing heard, a form that cannot be rebuilt), and the verifier is consulted with the exact message + * and signature, so a stub that verifies nothing turns the verdict from post-quantum to none. + */ +@ExtendWith(MockitoExtension.class) +public class AereGetPqFinalityTest { + + private static final long NUMBER = 19_018_096L; + private static final Hash HASH = Hash.fromHexString("0xf776f80e1710dc0e0f1927958dd589b98fc35ce8b9854e4fdb12320459fbb3f1"); + private static final Bytes MESSAGE = Bytes32.fromHexStringLenient("0x1122"); + + @Mock private BlockchainQueries blockchainQueries; + @Mock private BlockHeader header; + @Mock private ValidatorProvider validatorProvider; + + private final PqSealCache cache = PqSealCache.instance(); + private final List verifierCalls = new ArrayList<>(); + + @BeforeEach + public void setUp() { + cache.clear(); + // lenient: the name and unknown-block cases never touch the header, and strict stubs would call that a defect + lenient().when(header.getNumber()).thenReturn(NUMBER); + lenient().when(header.getHash()).thenReturn(HASH); + } + + @AfterEach + public void tearDown() { + cache.clear(); + } + + private static List

validators(final int n) { + final List
out = new ArrayList<>(); + for (int i = 0; i < n; i++) { + out.add(Address.fromHexString(String.format("0x%040x", 0xA000 + i))); + } + return out; + } + + private void heard(final int count) { + final List seals = new ArrayList<>(); + for (int i = 0; i < count; i++) { + seals.add(new FalconSeal(i, Bytes.of(0x29, i, 7, 7, 7))); + } + cache.record(NUMBER, HASH, seals); + } + + private AereGetPqFinality method(final int verifyBelowIndex, final Optional message) { + return new AereGetPqFinality( + blockchainQueries, + validatorProvider, + cache, + (n, i, m, s) -> { + verifierCalls.add(new Object[] {n, i, m, s}); + return i < verifyBelowIndex; + }, + (n, h) -> message); + } + + @SuppressWarnings("unchecked") + private Map call(final AereGetPqFinality method, final Object... params) { + final JsonRpcRequestContext request = + new JsonRpcRequestContext(new JsonRpcRequest("2.0", method.getName(), params)); + final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) method.response(request); + return (Map) response.getResult(); + } + + @Test + public void nameIsTheAipMethod() { + assertThat(method(7, Optional.of(MESSAGE)).getName()).isEqualTo("aere_getPqFinality"); + } + + @Test + public void quorumOfVerifiedDistinctSignersIsPostQuantumFinality() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(5, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("post-quantum"); + assertThat(r.get("quorum")).isEqualTo(5); + assertThat(r.get("falconSealsHeard")).isEqualTo(7); + assertThat(r.get("falconSealsVerified")).isEqualTo(5); + assertThat(r.get("sealForm")).isEqualTo("anchor"); + assertThat(r.get("signedMessage")).isEqualTo(MESSAGE.toHexString()); + // the verifier saw exactly the message and each seal's own signature + assertThat(verifierCalls).hasSize(7); + assertThat(verifierCalls.get(3)[1]).isEqualTo(3); + assertThat(verifierCalls.get(3)[2]).isEqualTo(MESSAGE); + assertThat(verifierCalls.get(3)[3]).isEqualTo(Bytes.of(0x29, 3, 7, 7, 7)); + } + + @Test + public void oneSignerShortOfQuorumIsPartial() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(4, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("partial"); + assertThat(r.get("falconSealsVerified")).isEqualTo(4); + } + + @Test + public void negativeControlNothingVerifiesIsNone() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(0, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("none"); + assertThat(r.get("falconSealsVerified")).isEqualTo(0); + assertThat(r.get("falconSealsHeard")).isEqualTo(7); + } + + @Test + public void nothingHeardIsUnavailableNotFalse() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + final Map r = call(method(7, Optional.of(MESSAGE)), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("unavailable"); + assertThat(r.get("falconSealsHeard")).isEqualTo(0); + assertThat(verifierCalls).isEmpty(); + } + + @Test + public void aFormThatCannotBeRebuiltIsSaidNotGuessed() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(7)); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(7); + final Map r = call(method(7, Optional.empty()), "0x" + Long.toHexString(NUMBER)); + assertThat(r.get("finality")).isEqualTo("unverifiable-form"); + assertThat(r.get("sealForm")).isEqualTo("commit-digest"); + assertThat(r.get("falconSealsVerified")).isEqualTo(0); + assertThat(verifierCalls).isEmpty(); + } + + @Test + public void latestIsTheDefaultBlock() { + when(validatorProvider.getValidatorsForBlock(any())).thenReturn(validators(10)); + // "latest" resolves through the head NUMBER, then the header by number (AbstractBlockParameterMethod) + when(blockchainQueries.headBlockNumber()).thenReturn(NUMBER); + when(blockchainQueries.getBlockHeaderByNumber(NUMBER)).thenReturn(Optional.of(header)); + heard(10); + final Map r = call(method(10, Optional.of(MESSAGE))); + assertThat(r.get("quorum")).isEqualTo(7); + assertThat(r.get("finality")).isEqualTo("post-quantum"); + assertThat(r.get("blockNumber")).isEqualTo(NUMBER); + } + + @Test + public void unknownBlockIsNull() { + when(blockchainQueries.getBlockHeaderByNumber(5L)).thenReturn(Optional.empty()); + final JsonRpcRequestContext request = + new JsonRpcRequestContext(new JsonRpcRequest("2.0", "aere_getPqFinality", new Object[] {"0x5"})); + final JsonRpcSuccessResponse response = + (JsonRpcSuccessResponse) method(7, Optional.of(MESSAGE)).response(request); + assertThat(response.getResult()).isNull(); + } +} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java index ecebef299..cdcaafcbe 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java @@ -1,33 +1,39 @@ /* * Copyright ConsenSys AG. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright contributors to the Aere Network. + * + * This file was modified by contributors to the Aere Network, as required by section 4(b) of the + * Apache License 2.0. The copyright header above is the upstream one and is left exactly as it was + * found, as section 4(c) requires. The change itself is in patches/0007-aere-pq-finality.patch. */ package org.hyperledger.besu.ethereum.api.jsonrpc; import java.util.Collection; import java.util.HashSet; public enum RpcMethod { ADMIN_ADD_PEER("admin_addPeer"), ADMIN_NODE_INFO("admin_nodeInfo"), ADMIN_PEERS("admin_peers"), ADMIN_REMOVE_PEER("admin_removePeer"), ADMIN_CHANGE_LOG_LEVEL("admin_changeLogLevel"), ADMIN_GENERATE_LOG_BLOOM_CACHE("admin_generateLogBloomCache"), ADMIN_LOGS_REPAIR_CACHE("admin_logsRepairCache"), ADMIN_LOGS_REMOVE_CACHE("admin_logsRemoveCache"), DEBUG_ACCOUNT_AT("debug_accountAt"), DEBUG_ACCOUNT_RANGE("debug_accountRange"), DEBUG_METRICS("debug_metrics"), DEBUG_RESYNC_WORLDSTATE("debug_resyncWorldState"), DEBUG_SET_HEAD("debug_setHead"), @@ -114,40 +120,42 @@ public enum RpcMethod { ETH_SEND_RAW_TRANSACTION("eth_sendRawTransaction"), ETH_SEND_TRANSACTION("eth_sendTransaction"), ETH_SIMULATE_V1("eth_simulateV1"), ETH_SUBSCRIBE("eth_subscribe"), ETH_SYNCING("eth_syncing"), ETH_UNINSTALL_FILTER("eth_uninstallFilter"), ETH_UNSUBSCRIBE("eth_unsubscribe"), IBFT_DISCARD_VALIDATOR_VOTE("ibft_discardValidatorVote"), IBFT_GET_PENDING_VOTES("ibft_getPendingVotes"), IBFT_GET_VALIDATORS_BY_BLOCK_HASH("ibft_getValidatorsByBlockHash"), IBFT_GET_VALIDATORS_BY_BLOCK_NUMBER("ibft_getValidatorsByBlockNumber"), IBFT_PROPOSE_VALIDATOR_VOTE("ibft_proposeValidatorVote"), IBFT_GET_SIGNER_METRICS("ibft_getSignerMetrics"), QBFT_DISCARD_VALIDATOR_VOTE("qbft_discardValidatorVote"), QBFT_GET_PENDING_VOTES("qbft_getPendingVotes"), QBFT_GET_VALIDATORS_BY_BLOCK_HASH("qbft_getValidatorsByBlockHash"), QBFT_GET_VALIDATORS_BY_BLOCK_NUMBER("qbft_getValidatorsByBlockNumber"), QBFT_PROPOSE_VALIDATOR_VOTE("qbft_proposeValidatorVote"), QBFT_GET_SIGNER_METRICS("qbft_getSignerMetrics"), QBFT_GET_REQUEST_TIMEOUT_SECONDS("qbft_getRequestTimeoutSeconds"), + // AIP-21 (AERE, 2026-09-18): the per-block post-quantum finality record, read from the seals heard on Commit messages + AERE_GET_PQ_FINALITY("aere_getPqFinality"), MINER_CHANGE_TARGET_GAS_LIMIT("miner_changeTargetGasLimit"), MINER_START("miner_start"), MINER_STOP("miner_stop"), MINER_GET_MIN_PRIORITY_FEE("miner_getMinPriorityFee"), MINER_SET_MIN_PRIORITY_FEE("miner_setMinPriorityFee"), MINER_GET_MIN_GAS_PRICE("miner_getMinGasPrice"), MINER_SET_MIN_GAS_PRICE("miner_setMinGasPrice"), MINER_GET_EXTRA_DATA("miner_getExtraData"), MINER_SET_EXTRA_DATA("miner_setExtraData"), NET_ENODE("net_enode"), NET_LISTENING("net_listening"), NET_PEER_COUNT("net_peerCount"), NET_SERVICES("net_services"), NET_VERSION("net_version"), PERM_ADD_ACCOUNTS_TO_WHITELIST("perm_addAccountsToWhitelist"), PERM_ADD_ACCOUNTS_TO_ALLOWLIST("perm_addAccountsToAllowlist"), PERM_ADD_NODES_TO_WHITELIST("perm_addNodesToWhitelist"), PERM_ADD_NODES_TO_ALLOWLIST("perm_addNodesToAllowlist"), PERM_GET_ACCOUNTS_WHITELIST("perm_getAccountsWhitelist"), PERM_GET_ACCOUNTS_ALLOWLIST("perm_getAccountsAllowlist"),