aere-research/pq-stark/mmcs_babybear_reference.py
Aere Network 4a0b48588c Initial public release
Aere Network public source. Everything here can be checked against the live
chain (chain id 2800, https://rpc.aere.network).

Scope note, stated up front rather than buried: consensus on chain 2800 is
classical secp256k1 ECDSA QBFT. The post-quantum work in this repository is at
the signature, precompile, account and transport layers. Nothing here makes the
consensus post-quantum, and no document in it should be read as claiming so.
2026-07-20 01:02:30 +03:00

481 lines
22 KiB
Python

#!/usr/bin/env python3
# Independent reference for the Mixed Matrix Commitment Scheme (MMCS) that Plonky3/SP1 use over
# BabyBear, component (c) of the PQ STARK-verify precompile 0x0AE8 (direct SP1/Plonky3 inner FRI/STARK
# verify). The MMCS is the Merkle-tree vector commitment FRI (component (d)) and the trace commitment
# are built on. See docs/AERE-STARK-VERIFIER-PORT-SPEC.md section 4.
#
# ============================ HONEST SCOPE (read first) ============================
# This module implements the MMCS root computation and opening-verification over the CONFIRMED
# Poseidon2-BabyBear permutation (component (b)), and its construction + outputs are now
# CONFIRMED-FROM-SOURCE against the pinned Plonky3 revision, with a real known-answer test that PASSES.
# - CONFIRMED construction (2026-07-19): the exact SP1 inner config, verbatim from p3-merkle-tree's
# own mmcs.rs tests:
# Perm = Poseidon2<BabyBear, Poseidon2ExternalMatrixGeneral, DiffusionMatrixBabyBear,16,7>
# MyHash = PaddingFreeSponge<Perm, WIDTH=16, RATE=8, OUT=8> (the leaf hasher)
# MyCompress = TruncatedPermutation<Perm, N=2, CHUNK=8, WIDTH=16> (the 2-to-1 node compressor)
# MyMmcs = FieldMerkleTreeMmcs<Packing, Packing, MyHash, MyCompress, DIGEST_ELEMS=8>
# So a digest is 8 BabyBear elements (32 bytes). Source: p3-merkle-tree / p3-symmetric / p3-commit
# 0.4.3-succinct (the exact crates pinned in the repo Cargo.lock; checksums matched, see
# spec-mmcs-babybear.md).
# - CONFIRMED outputs (real KAT PASSED): a small Rust extractor depending on the pinned crates was
# compiled and run; it commits to several known matrix batches and emits the Merkle root plus an
# open_batch opening (opened rows + sibling path), and asserts the library's own verify_batch
# accepts them. This reference reproduces every root, opening, and proof byte-for-byte (canonical
# u32), and its verify_batch accepts the emitted openings and rejects tampered ones. See
# conformance() / test_mmcs_babybear.py and spec-mmcs-babybear.md.
# - MONTGOMERY note: field elements are serialized as canonical u32 (PrimeField32::as_canonical_u32),
# the same canonical form the Poseidon2 confirmation used. The permutation carries the Montgomery
# R^{-1} internal-layer factor internally (component (b)); the MMCS layer above it is pure field
# data + hashing, no additional Montgomery subtlety. The extractor's permute([0;16]) is emitted as
# a perm-sanity KAT and equals the confirmed Poseidon2 "zeros" vector, proving the extractor uses
# the same confirmed permutation this reference does.
#
# What this module still does NOT do: it does not verify any STARK proof, and it is deliberately NOT
# wired to make the top-level 0x0AE8 accept anything. The MMCS being confirmed is necessary but not
# sufficient: the duplex-sponge challenger (component (f)), the FRI folding/consistency arithmetic
# (component (d)), and the SP1 recursion-AIR constraint evaluation (component (e)) are still un-ported,
# so the precompile stays fail-closed (returns EMPTY for every input). See the README and port spec.
#
# Source (pinned, checksum-matched to the repo Cargo.lock):
# p3-merkle-tree 0.4.3-succinct d5703d9229d52a8c09970e4d722c3a8b4d37e688c306c3a1c03b872efcd204e6
# p3-symmetric 0.4.3-succinct 9047ce85c086a9b3f118e10078f10636f7bfeed5da871a04da0b61400af8793a
# p3-commit 0.4.3-succinct 50acacc7219fce6c01db938f82c1b21b5e7133990b7fff861f91534aeb569419
# p3-matrix 0.4.3-succinct 75c3f150ceb90e09539413bf481e618d05ee19210b4e467d2902eb82d2e15281
# (leaf hash/compress permutation: p3-baby-bear / p3-poseidon2 0.4.3-succinct, component (b)).
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import poseidon2_babybear_reference as P2 # the CONFIRMED Poseidon2-BabyBear permutation (component (b))
P = P2.P # BabyBear prime
WIDTH = 16 # Poseidon2 state width
RATE = 8 # PaddingFreeSponge rate
OUT = 8 # digest size in field elements (DIGEST_ELEMS)
DIGEST_ELEMS = 8
DEFAULT_DIGEST = [0] * DIGEST_ELEMS # the zero (padding) digest
# ============================ primitives ============================
def permute(state16):
"""The CONFIRMED Poseidon2-BabyBear width-16 permutation (component (b))."""
return P2.permute(state16)
def hash_iter(elems):
"""PaddingFreeSponge<Perm, WIDTH=16, RATE=8, OUT=8> over a sequence of BabyBear elements.
Overwrite-mode, padding-free sponge (p3-symmetric sponge.rs): state starts all-zero; for each
chunk of RATE elements, OVERWRITE the first len(chunk) lanes (leaving the remaining lanes at their
prior value), then permute; the digest is the first OUT lanes. An empty input yields the all-zero
digest with no permutation (the Rust `for chunk in chunks(RATE)` loop runs zero times)."""
state = [0] * WIDTH
elems = [e % P for e in elems]
i = 0
n = len(elems)
while i < n:
chunk = elems[i:i + RATE]
for j in range(len(chunk)):
state[j] = chunk[j]
state = permute(state)
i += RATE
return state[:OUT]
def hash_slice(elems):
"""CryptographicHasher::hash_slice: hash a single row of field elements."""
return hash_iter(list(elems))
def hash_item(x):
"""CryptographicHasher::hash_item: hash a single field element."""
return hash_iter([x])
def compress2to1(left8, right8):
"""TruncatedPermutation<Perm, N=2, CHUNK=8, WIDTH=16>: permute(left||right) truncated to 8 lanes."""
if len(left8) != DIGEST_ELEMS or len(right8) != DIGEST_ELEMS:
raise ValueError("compress inputs must be length 8")
pre = [x % P for x in left8] + [x % P for x in right8]
return permute(pre)[:DIGEST_ELEMS]
# ============================ helpers ============================
def _log2_ceil(n):
"""log2_ceil_usize: smallest k with 2^k >= n (0 for n<=1). Matches p3-util."""
if n <= 1:
return 0
return (n - 1).bit_length()
def _next_pow2(n):
if n <= 1:
return 1
return 1 << ((n - 1).bit_length())
class Matrix:
"""A row-major matrix of BabyBear elements; rows is a list of equal-length lists."""
def __init__(self, rows):
self.rows = [[x % P for x in r] for r in rows]
self.height = len(rows)
self.width = len(rows[0]) if rows else 0
def row(self, i):
return list(self.rows[i])
# ============================ tree build (FieldMerkleTree::new) ============================
def build_tree(matrices):
"""Build the FieldMerkleTree digest layers for a batch of matrices (mixed heights allowed).
Mirrors p3-merkle-tree merkle_tree.rs: sort matrices tallest-first (stable), hash the tallest
matrices' rows into the first digest layer (padded to a power of two with the zero digest), then
repeatedly compress pairs up while injecting the next height's matrices at the layer whose padded
length matches. Returns (digest_layers, log_max_height). digest_layers[0] is the leaf layer.
Height property (asserted by Plonky3): matrix heights that round up to the same power of two must
be equal, so every padded-power-of-two bucket contains matrices of a single exact height. This is
what makes `height == max_height` (tree build) and `height.next_power_of_two() == curr_padded`
(verify) select the same set."""
if not matrices:
raise ValueError("no matrices")
heights = sorted(m.height for m in matrices)
for a, b in zip(heights, heights[1:]):
if a != b and _next_pow2(a) == _next_pow2(b):
raise ValueError("matrix heights that round up to the same power of two must be equal")
# stable sort tallest-first (ties keep original order, like Rust sorted_by_key(Reverse(height)))
order = sorted(range(len(matrices)), key=lambda i: -matrices[i].height)
ordered = [matrices[i] for i in order]
max_height = ordered[0].height
log_max_height = _log2_ceil(max_height)
max_height_padded = _next_pow2(max_height)
idx = 0
tallest = []
while idx < len(ordered) and ordered[idx].height == max_height:
tallest.append(ordered[idx])
idx += 1
# first digest layer: hash concatenated rows of all tallest matrices, pad with the zero digest.
layer0 = [list(DEFAULT_DIGEST) for _ in range(max_height_padded)]
for i in range(max_height):
row = []
for m in tallest:
row.extend(m.row(i))
layer0[i] = hash_iter(row)
digest_layers = [layer0]
while len(digest_layers[-1]) > 1:
prev = digest_layers[-1]
next_len_padded = len(prev) // 2
inject = []
while idx < len(ordered) and _next_pow2(ordered[idx].height) == next_len_padded:
inject.append(ordered[idx])
idx += 1
digest_layers.append(_compress_and_inject(prev, inject, next_len_padded))
return digest_layers, log_max_height
def _compress_and_inject(prev, inject, next_len_padded):
"""One layer up: compress adjacent pairs, and if there are matrices to inject at this (padded)
height, hash their rows and compress that into each node. Mirrors compress_and_inject / compress."""
out = [list(DEFAULT_DIGEST) for _ in range(next_len_padded)]
if not inject:
for i in range(next_len_padded):
out[i] = compress2to1(prev[2 * i], prev[2 * i + 1])
return out
inject_height = inject[0].height # all injected share one exact height (height property)
for i in range(inject_height):
digest = compress2to1(prev[2 * i], prev[2 * i + 1])
row = []
for m in inject:
row.extend(m.row(i))
rows_digest = hash_iter(row)
out[i] = compress2to1(digest, rows_digest)
for i in range(inject_height, next_len_padded):
digest = compress2to1(prev[2 * i], prev[2 * i + 1])
out[i] = compress2to1(digest, list(DEFAULT_DIGEST))
return out
def commit(matrices):
"""FieldMerkleTreeMmcs::commit: returns (root_digest, prover_data) where prover_data holds the
matrices and digest layers needed to open."""
digest_layers, log_max_height = build_tree(matrices)
root = digest_layers[-1][0]
return root, {"matrices": matrices, "digest_layers": digest_layers, "log_max_height": log_max_height}
def open_batch(index, prover_data):
"""FieldMerkleTreeMmcs::open_batch: returns (opened_values, proof).
opened_values are the opened rows in ORIGINAL matrix order (each matrix's row at its reduced
index); proof is the sibling digest at each layer, sibling = digest_layers[i][(index>>i) ^ 1]."""
matrices = prover_data["matrices"]
digest_layers = prover_data["digest_layers"]
log_max_height = prover_data["log_max_height"]
openings = []
for m in matrices:
log2_height = _log2_ceil(m.height)
bits_reduced = log_max_height - log2_height
reduced_index = index >> bits_reduced
openings.append(m.row(reduced_index))
proof = [digest_layers[i][(index >> i) ^ 1] for i in range(log_max_height)]
return openings, proof
# ============================ verify_batch (the on-chain-shaped check) ============================
def verify_batch(commit_root, dimensions, index, opened_values, proof):
"""FieldMerkleTreeMmcs::verify_batch: recompute the root from the opened leaf rows and the sibling
path, and compare to `commit_root`. Returns True on match, False otherwise (never raises for a
well-formed but wrong opening).
dimensions: list of (width, height) in the SAME order as opened_values (original matrix order).
Mirrors p3-merkle-tree mmcs.rs verify_batch: group opened rows by padded height, tallest first;
seed the running root with the hash of the tallest group's concatenated rows; then walk the proof,
at each step ordering (root, sibling) by the index parity, compressing, and (when the next group's
padded height is reached) compressing in that group's hashed rows."""
n = len(dimensions)
# stable sort (original index, dims) tallest-first by height
order = sorted(range(n), key=lambda i: -dimensions[i][1])
heights = [dimensions[i][1] for i in order]
ptr = 0
def take_group(padded):
nonlocal ptr
idxs = []
while ptr < n and _next_pow2(heights[ptr]) == padded:
idxs.append(order[ptr])
ptr += 1
return idxs
if n == 0:
return False
curr_height_padded = _next_pow2(heights[0])
# seed root with the tallest group's concatenated opened rows
group = take_group(curr_height_padded)
seed = []
for oi in group:
seed.extend(opened_values[oi])
root = hash_iter(seed)
idx = index
for sibling in proof:
if idx & 1 == 0:
left, right = root, sibling
else:
left, right = sibling, root
root = compress2to1(left, right)
idx >>= 1
curr_height_padded >>= 1
# if the next group is at this (padded) height, inject its hashed opened rows
if ptr < n and _next_pow2(heights[ptr]) == curr_height_padded:
grp = take_group(curr_height_padded)
row = []
for oi in grp:
row.extend(opened_values[oi])
nxt = hash_iter(row)
root = compress2to1(root, nxt)
return root == list(commit_root)
# ============================ matrix construction (shared with the extractor) ============================
def make_matrix(height, width, base):
"""Deterministic matrix identical to the Rust extractor: value[i*w+j] = base + i*w + j."""
return Matrix([[base + (i * width + j) for j in range(width)] for i in range(height)])
# ============================ CONFIRMED conformance KATs (from the pinned library) ============================
# Real known-answer vectors emitted by executing the pinned p3-merkle-tree / p3-symmetric / p3-commit
# 0.4.3-succinct crates (via the extractor). This reference reproduces every value EXACTLY, which is
# what makes the construction + outputs CONFIRMED-conformant (not merely self-consistent). PERM_ZEROS is
# the extractor's permute([0;16]); it equals the confirmed Poseidon2 "zeros" vector, proving the
# extractor's permutation is the same one this reference uses.
PERM_ZEROS = [1787823396, 953829438, 89382455, 347481625, 1754527224, 916217775, 1056029082,
410644796, 1169123478, 1854704276, 1195829987, 1485264906, 1824644035, 1948268315,
847945433, 190591038]
PRIM_KATS = {
'hash_item_5': [881553380, 703286570, 452412164, 1683859154, 394218790, 501691982, 498441819, 937656841],
'hash_slice_2': [1843359319, 912981492, 1448073574, 280410802, 1934669154, 1885461061, 224340142, 529679527],
'hash_slice_3': [1831345102, 1426305082, 956789587, 1706209078, 311264389, 552910277, 9044084, 1828165221],
'hash_slice_8': [8999572, 1033765830, 347083905, 1304769627, 1321299677, 1106238442, 1523849989, 954594225],
'hash_slice_9': [1134257664, 40304233, 1823880005, 1134792540, 180685702, 1633847360, 1460964786, 872499523],
'compress_h2_h3': [1968576159, 1450511489, 1750728079, 899535959, 1052761632, 829425096, 282044891, 1592617075],
}
CONFORMANCE_CASES = [
{
"name": 'single_8x2',
"matspec": [(8, 2, 10)],
"index": 3,
"root": [35761595, 1133593632, 733114748, 517674920, 1449914397, 74512820, 381712048, 469819303],
"openings": [[16, 17]],
"proof": [[1513856679, 1890540197, 1949407553, 586338782, 197781917, 573541314, 1955108120, 661229895], [57846071, 657849236, 262378030, 1091294242, 669362455, 676261940, 190131986, 664894526], [1143223726, 1793027126, 1353127284, 491971160, 1959272008, 1195367436, 417822678, 1372117039]],
},
{
"name": 'single_6x2',
"matspec": [(6, 2, 100)],
"index": 5,
"root": [1043564500, 1812685593, 1597353357, 1392680266, 1355213241, 1159759876, 1586103175, 799081422],
"openings": [[110, 111]],
"proof": [[1089158652, 427625262, 1157475631, 1692145862, 1671348918, 1986544368, 497315490, 1960470114], [1787823396, 953829438, 89382455, 347481625, 1754527224, 916217775, 1056029082, 410644796], [747170349, 1608481817, 1569266992, 1154307224, 561605764, 1907985030, 1454474361, 357054770]],
},
{
"name": 'single_8x1',
"matspec": [(8, 1, 200)],
"index": 6,
"root": [447902873, 1264273264, 1781377203, 392525377, 451230220, 285479002, 124104889, 737840045],
"openings": [[206]],
"proof": [[854476755, 1853946983, 1409001429, 1897109439, 1880752521, 1036730533, 1817549359, 907866104], [757904997, 571589503, 673816220, 1066413580, 473597103, 1271204909, 363424036, 1450652756], [402779725, 764084359, 1406275333, 1158469515, 32391761, 236523006, 1039588073, 264072612]],
},
{
"name": 'mixed_8x2_4x3',
"matspec": [(8, 2, 10), (4, 3, 300)],
"index": 5,
"root": [902263792, 353615665, 93922356, 1251424848, 837594048, 2005066510, 431363100, 287722769],
"openings": [[20, 21], [306, 307, 308]],
"proof": [[1732750591, 1876733121, 1364724824, 1847015379, 1792368333, 483325461, 1174939365, 1939380322], [932176417, 320077372, 1748871293, 625438914, 179521004, 655947905, 828588297, 31952751], [930401557, 471785873, 904189917, 1626475238, 550043796, 1590748862, 306781755, 496390863]],
},
{
"name": 'mixed_8x1_4x2_2x2',
"matspec": [(8, 1, 1), (4, 2, 50), (2, 2, 400)],
"index": 6,
"root": [439675894, 37405611, 255560432, 531897590, 349476430, 1463998586, 540993751, 1307259928],
"openings": [[7], [56, 57], [402, 403]],
"proof": [[723290459, 412642417, 484634914, 785252854, 182396116, 1183763863, 630430401, 175545021], [1257356383, 1204259532, 1915921675, 53621155, 1015367194, 742423503, 1998129904, 1554700909], [13474675, 1966185163, 1255599565, 471961853, 1823737756, 1607265064, 1964440250, 1219296485]],
},
{
"name": 'mixed_5x2_3x1',
"matspec": [(5, 2, 20), (3, 1, 70)],
"index": 4,
"root": [700351145, 394944774, 166238365, 1683786139, 1659356855, 1460803726, 573770743, 1196572690],
"openings": [[28, 29], [72]],
"proof": [[0, 0, 0, 0, 0, 0, 0, 0], [1714214715, 1781831455, 644202942, 667527548, 1395787195, 671219385, 1433934871, 1883643874], [1521468259, 1627990232, 1145489794, 839124709, 507242897, 777414579, 429233007, 1147797058]],
},
]
def _case_matrices(case):
return [make_matrix(h, w, b) for (h, w, b) in case["matspec"]]
def _case_dims(case):
return [(w, h) for (h, w, b) in case["matspec"]]
def conformance_kats():
"""Return (passed, total) after reproducing every pinned-library known-answer vector: the perm
sanity, the primitive hash/compress KATs, and each MMCS case (root + openings + proof + verify)."""
passed = 0
total = 0
total += 1
if permute([0] * 16) == PERM_ZEROS:
passed += 1
for name, expected in PRIM_KATS.items():
total += 1
if name == 'hash_item_5':
got = hash_item(5)
elif name == 'hash_slice_2':
got = hash_slice([1, 2])
elif name == 'hash_slice_3':
got = hash_slice([1, 2, 3])
elif name == 'hash_slice_8':
got = hash_slice(list(range(1, 9)))
elif name == 'hash_slice_9':
got = hash_slice(list(range(1, 10)))
elif name == 'compress_h2_h3':
got = compress2to1(hash_slice([1, 2]), hash_slice([1, 2, 3]))
if got == expected:
passed += 1
for case in CONFORMANCE_CASES:
mats = _case_matrices(case)
root, pdata = commit(mats)
openings, proof = open_batch(case["index"], pdata)
dims = _case_dims(case)
# root matches
total += 1
if root == case["root"]:
passed += 1
# openings match
total += 1
if openings == case["openings"]:
passed += 1
# proof matches
total += 1
if proof == case["proof"]:
passed += 1
# verify accepts the emitted opening against the emitted root
total += 1
if verify_batch(case["root"], dims, case["index"], case["openings"], case["proof"]):
passed += 1
# verify rejects a tampered proof
total += 1
tampered = [list(d) for d in case["proof"]]
tampered[0] = list(tampered[0])
tampered[0][0] = (tampered[0][0] + 1) % P
if not verify_batch(case["root"], dims, case["index"], case["openings"], tampered):
passed += 1
return passed, total
# ============================ shared cross-language vector set ============================
# A fixed set the harness runs in Python, Node, and Java, asserting byte-identical outputs AND that
# each reproduces the pinned-library conformance vectors.
def shared_vectors():
prim = {
"hash_item_5": hash_item(5),
"hash_slice_2": hash_slice([1, 2]),
"hash_slice_3": hash_slice([1, 2, 3]),
"hash_slice_8": hash_slice(list(range(1, 9))),
"hash_slice_9": hash_slice(list(range(1, 10))),
"compress_h2_h3": compress2to1(hash_slice([1, 2]), hash_slice([1, 2, 3])),
}
cases = []
for case in CONFORMANCE_CASES:
mats = _case_matrices(case)
root, pdata = commit(mats)
openings, proof = open_batch(case["index"], pdata)
dims = _case_dims(case)
cases.append({
"name": case["name"],
"index": case["index"],
"root": root,
"openings": openings,
"proof": proof,
"rootMatch": root == case["root"],
"openingsMatch": openings == case["openings"],
"proofMatch": proof == case["proof"],
"verifyOk": verify_batch(case["root"], dims, case["index"], case["openings"], case["proof"]),
})
return {
"constants": {"P": P, "width": WIDTH, "rate": RATE, "out": OUT, "digestElems": DIGEST_ELEMS},
"permZeros": permute([0] * 16),
"prim": prim,
"cases": cases,
}
if __name__ == "__main__":
import json
json.dump(shared_vectors(), sys.stdout)