#!/usr/bin/env python3 # Independent reference for the Mixed Matrix Commitment Scheme (MMCS) that Plonky3/SP1 use over # BabyBear, component (c) of the PQ STARK-verify precompile 0x0AE8 (direct SP1/Plonky3 inner FRI/STARK # verify). The MMCS is the Merkle-tree vector commitment FRI (component (d)) and the trace commitment # are built on. See docs/AERE-STARK-VERIFIER-PORT-SPEC.md section 4. # # ============================ HONEST SCOPE (read first) ============================ # This module implements the MMCS root computation and opening-verification over the CONFIRMED # Poseidon2-BabyBear permutation (component (b)), and its construction + outputs are now # CONFIRMED-FROM-SOURCE against the pinned Plonky3 revision, with a real known-answer test that PASSES. # - CONFIRMED construction (2026-07-19): the exact SP1 inner config, verbatim from p3-merkle-tree's # own mmcs.rs tests: # Perm = Poseidon2 # MyHash = PaddingFreeSponge (the leaf hasher) # MyCompress = TruncatedPermutation (the 2-to-1 node compressor) # MyMmcs = FieldMerkleTreeMmcs # So a digest is 8 BabyBear elements (32 bytes). Source: p3-merkle-tree / p3-symmetric / p3-commit # 0.4.3-succinct (the exact crates pinned in the repo Cargo.lock; checksums matched, see # spec-mmcs-babybear.md). # - CONFIRMED outputs (real KAT PASSED): a small Rust extractor depending on the pinned crates was # compiled and run; it commits to several known matrix batches and emits the Merkle root plus an # open_batch opening (opened rows + sibling path), and asserts the library's own verify_batch # accepts them. This reference reproduces every root, opening, and proof byte-for-byte (canonical # u32), and its verify_batch accepts the emitted openings and rejects tampered ones. See # conformance() / test_mmcs_babybear.py and spec-mmcs-babybear.md. # - MONTGOMERY note: field elements are serialized as canonical u32 (PrimeField32::as_canonical_u32), # the same canonical form the Poseidon2 confirmation used. The permutation carries the Montgomery # R^{-1} internal-layer factor internally (component (b)); the MMCS layer above it is pure field # data + hashing, no additional Montgomery subtlety. The extractor's permute([0;16]) is emitted as # a perm-sanity KAT and equals the confirmed Poseidon2 "zeros" vector, proving the extractor uses # the same confirmed permutation this reference does. # # What this module still does NOT do: it does not verify any STARK proof, and it is deliberately NOT # wired to make the top-level 0x0AE8 accept anything. The MMCS being confirmed is necessary but not # sufficient: the duplex-sponge challenger (component (f)), the FRI folding/consistency arithmetic # (component (d)), and the SP1 recursion-AIR constraint evaluation (component (e)) are still un-ported, # so the precompile stays fail-closed (returns EMPTY for every input). See the README and port spec. # # Source (pinned, checksum-matched to the repo Cargo.lock): # p3-merkle-tree 0.4.3-succinct d5703d9229d52a8c09970e4d722c3a8b4d37e688c306c3a1c03b872efcd204e6 # p3-symmetric 0.4.3-succinct 9047ce85c086a9b3f118e10078f10636f7bfeed5da871a04da0b61400af8793a # p3-commit 0.4.3-succinct 50acacc7219fce6c01db938f82c1b21b5e7133990b7fff861f91534aeb569419 # p3-matrix 0.4.3-succinct 75c3f150ceb90e09539413bf481e618d05ee19210b4e467d2902eb82d2e15281 # (leaf hash/compress permutation: p3-baby-bear / p3-poseidon2 0.4.3-succinct, component (b)). import os import sys HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import poseidon2_babybear_reference as P2 # the CONFIRMED Poseidon2-BabyBear permutation (component (b)) P = P2.P # BabyBear prime WIDTH = 16 # Poseidon2 state width RATE = 8 # PaddingFreeSponge rate OUT = 8 # digest size in field elements (DIGEST_ELEMS) DIGEST_ELEMS = 8 DEFAULT_DIGEST = [0] * DIGEST_ELEMS # the zero (padding) digest # ============================ primitives ============================ def permute(state16): """The CONFIRMED Poseidon2-BabyBear width-16 permutation (component (b)).""" return P2.permute(state16) def hash_iter(elems): """PaddingFreeSponge over a sequence of BabyBear elements. Overwrite-mode, padding-free sponge (p3-symmetric sponge.rs): state starts all-zero; for each chunk of RATE elements, OVERWRITE the first len(chunk) lanes (leaving the remaining lanes at their prior value), then permute; the digest is the first OUT lanes. An empty input yields the all-zero digest with no permutation (the Rust `for chunk in chunks(RATE)` loop runs zero times).""" state = [0] * WIDTH elems = [e % P for e in elems] i = 0 n = len(elems) while i < n: chunk = elems[i:i + RATE] for j in range(len(chunk)): state[j] = chunk[j] state = permute(state) i += RATE return state[:OUT] def hash_slice(elems): """CryptographicHasher::hash_slice: hash a single row of field elements.""" return hash_iter(list(elems)) def hash_item(x): """CryptographicHasher::hash_item: hash a single field element.""" return hash_iter([x]) def compress2to1(left8, right8): """TruncatedPermutation: permute(left||right) truncated to 8 lanes.""" if len(left8) != DIGEST_ELEMS or len(right8) != DIGEST_ELEMS: raise ValueError("compress inputs must be length 8") pre = [x % P for x in left8] + [x % P for x in right8] return permute(pre)[:DIGEST_ELEMS] # ============================ helpers ============================ def _log2_ceil(n): """log2_ceil_usize: smallest k with 2^k >= n (0 for n<=1). Matches p3-util.""" if n <= 1: return 0 return (n - 1).bit_length() def _next_pow2(n): if n <= 1: return 1 return 1 << ((n - 1).bit_length()) class Matrix: """A row-major matrix of BabyBear elements; rows is a list of equal-length lists.""" def __init__(self, rows): self.rows = [[x % P for x in r] for r in rows] self.height = len(rows) self.width = len(rows[0]) if rows else 0 def row(self, i): return list(self.rows[i]) # ============================ tree build (FieldMerkleTree::new) ============================ def build_tree(matrices): """Build the FieldMerkleTree digest layers for a batch of matrices (mixed heights allowed). Mirrors p3-merkle-tree merkle_tree.rs: sort matrices tallest-first (stable), hash the tallest matrices' rows into the first digest layer (padded to a power of two with the zero digest), then repeatedly compress pairs up while injecting the next height's matrices at the layer whose padded length matches. Returns (digest_layers, log_max_height). digest_layers[0] is the leaf layer. Height property (asserted by Plonky3): matrix heights that round up to the same power of two must be equal, so every padded-power-of-two bucket contains matrices of a single exact height. This is what makes `height == max_height` (tree build) and `height.next_power_of_two() == curr_padded` (verify) select the same set.""" if not matrices: raise ValueError("no matrices") heights = sorted(m.height for m in matrices) for a, b in zip(heights, heights[1:]): if a != b and _next_pow2(a) == _next_pow2(b): raise ValueError("matrix heights that round up to the same power of two must be equal") # stable sort tallest-first (ties keep original order, like Rust sorted_by_key(Reverse(height))) order = sorted(range(len(matrices)), key=lambda i: -matrices[i].height) ordered = [matrices[i] for i in order] max_height = ordered[0].height log_max_height = _log2_ceil(max_height) max_height_padded = _next_pow2(max_height) idx = 0 tallest = [] while idx < len(ordered) and ordered[idx].height == max_height: tallest.append(ordered[idx]) idx += 1 # first digest layer: hash concatenated rows of all tallest matrices, pad with the zero digest. layer0 = [list(DEFAULT_DIGEST) for _ in range(max_height_padded)] for i in range(max_height): row = [] for m in tallest: row.extend(m.row(i)) layer0[i] = hash_iter(row) digest_layers = [layer0] while len(digest_layers[-1]) > 1: prev = digest_layers[-1] next_len_padded = len(prev) // 2 inject = [] while idx < len(ordered) and _next_pow2(ordered[idx].height) == next_len_padded: inject.append(ordered[idx]) idx += 1 digest_layers.append(_compress_and_inject(prev, inject, next_len_padded)) return digest_layers, log_max_height def _compress_and_inject(prev, inject, next_len_padded): """One layer up: compress adjacent pairs, and if there are matrices to inject at this (padded) height, hash their rows and compress that into each node. Mirrors compress_and_inject / compress.""" out = [list(DEFAULT_DIGEST) for _ in range(next_len_padded)] if not inject: for i in range(next_len_padded): out[i] = compress2to1(prev[2 * i], prev[2 * i + 1]) return out inject_height = inject[0].height # all injected share one exact height (height property) for i in range(inject_height): digest = compress2to1(prev[2 * i], prev[2 * i + 1]) row = [] for m in inject: row.extend(m.row(i)) rows_digest = hash_iter(row) out[i] = compress2to1(digest, rows_digest) for i in range(inject_height, next_len_padded): digest = compress2to1(prev[2 * i], prev[2 * i + 1]) out[i] = compress2to1(digest, list(DEFAULT_DIGEST)) return out def commit(matrices): """FieldMerkleTreeMmcs::commit: returns (root_digest, prover_data) where prover_data holds the matrices and digest layers needed to open.""" digest_layers, log_max_height = build_tree(matrices) root = digest_layers[-1][0] return root, {"matrices": matrices, "digest_layers": digest_layers, "log_max_height": log_max_height} def open_batch(index, prover_data): """FieldMerkleTreeMmcs::open_batch: returns (opened_values, proof). opened_values are the opened rows in ORIGINAL matrix order (each matrix's row at its reduced index); proof is the sibling digest at each layer, sibling = digest_layers[i][(index>>i) ^ 1].""" matrices = prover_data["matrices"] digest_layers = prover_data["digest_layers"] log_max_height = prover_data["log_max_height"] openings = [] for m in matrices: log2_height = _log2_ceil(m.height) bits_reduced = log_max_height - log2_height reduced_index = index >> bits_reduced openings.append(m.row(reduced_index)) proof = [digest_layers[i][(index >> i) ^ 1] for i in range(log_max_height)] return openings, proof # ============================ verify_batch (the on-chain-shaped check) ============================ def verify_batch(commit_root, dimensions, index, opened_values, proof): """FieldMerkleTreeMmcs::verify_batch: recompute the root from the opened leaf rows and the sibling path, and compare to `commit_root`. Returns True on match, False otherwise (never raises for a well-formed but wrong opening). dimensions: list of (width, height) in the SAME order as opened_values (original matrix order). Mirrors p3-merkle-tree mmcs.rs verify_batch: group opened rows by padded height, tallest first; seed the running root with the hash of the tallest group's concatenated rows; then walk the proof, at each step ordering (root, sibling) by the index parity, compressing, and (when the next group's padded height is reached) compressing in that group's hashed rows.""" n = len(dimensions) # stable sort (original index, dims) tallest-first by height order = sorted(range(n), key=lambda i: -dimensions[i][1]) heights = [dimensions[i][1] for i in order] ptr = 0 def take_group(padded): nonlocal ptr idxs = [] while ptr < n and _next_pow2(heights[ptr]) == padded: idxs.append(order[ptr]) ptr += 1 return idxs if n == 0: return False curr_height_padded = _next_pow2(heights[0]) # seed root with the tallest group's concatenated opened rows group = take_group(curr_height_padded) seed = [] for oi in group: seed.extend(opened_values[oi]) root = hash_iter(seed) idx = index for sibling in proof: if idx & 1 == 0: left, right = root, sibling else: left, right = sibling, root root = compress2to1(left, right) idx >>= 1 curr_height_padded >>= 1 # if the next group is at this (padded) height, inject its hashed opened rows if ptr < n and _next_pow2(heights[ptr]) == curr_height_padded: grp = take_group(curr_height_padded) row = [] for oi in grp: row.extend(opened_values[oi]) nxt = hash_iter(row) root = compress2to1(root, nxt) return root == list(commit_root) # ============================ matrix construction (shared with the extractor) ============================ def make_matrix(height, width, base): """Deterministic matrix identical to the Rust extractor: value[i*w+j] = base + i*w + j.""" return Matrix([[base + (i * width + j) for j in range(width)] for i in range(height)]) # ============================ CONFIRMED conformance KATs (from the pinned library) ============================ # Real known-answer vectors emitted by executing the pinned p3-merkle-tree / p3-symmetric / p3-commit # 0.4.3-succinct crates (via the extractor). This reference reproduces every value EXACTLY, which is # what makes the construction + outputs CONFIRMED-conformant (not merely self-consistent). PERM_ZEROS is # the extractor's permute([0;16]); it equals the confirmed Poseidon2 "zeros" vector, proving the # extractor's permutation is the same one this reference uses. PERM_ZEROS = [1787823396, 953829438, 89382455, 347481625, 1754527224, 916217775, 1056029082, 410644796, 1169123478, 1854704276, 1195829987, 1485264906, 1824644035, 1948268315, 847945433, 190591038] PRIM_KATS = { 'hash_item_5': [881553380, 703286570, 452412164, 1683859154, 394218790, 501691982, 498441819, 937656841], 'hash_slice_2': [1843359319, 912981492, 1448073574, 280410802, 1934669154, 1885461061, 224340142, 529679527], 'hash_slice_3': [1831345102, 1426305082, 956789587, 1706209078, 311264389, 552910277, 9044084, 1828165221], 'hash_slice_8': [8999572, 1033765830, 347083905, 1304769627, 1321299677, 1106238442, 1523849989, 954594225], 'hash_slice_9': [1134257664, 40304233, 1823880005, 1134792540, 180685702, 1633847360, 1460964786, 872499523], 'compress_h2_h3': [1968576159, 1450511489, 1750728079, 899535959, 1052761632, 829425096, 282044891, 1592617075], } CONFORMANCE_CASES = [ { "name": 'single_8x2', "matspec": [(8, 2, 10)], "index": 3, "root": [35761595, 1133593632, 733114748, 517674920, 1449914397, 74512820, 381712048, 469819303], "openings": [[16, 17]], "proof": [[1513856679, 1890540197, 1949407553, 586338782, 197781917, 573541314, 1955108120, 661229895], [57846071, 657849236, 262378030, 1091294242, 669362455, 676261940, 190131986, 664894526], [1143223726, 1793027126, 1353127284, 491971160, 1959272008, 1195367436, 417822678, 1372117039]], }, { "name": 'single_6x2', "matspec": [(6, 2, 100)], "index": 5, "root": [1043564500, 1812685593, 1597353357, 1392680266, 1355213241, 1159759876, 1586103175, 799081422], "openings": [[110, 111]], "proof": [[1089158652, 427625262, 1157475631, 1692145862, 1671348918, 1986544368, 497315490, 1960470114], [1787823396, 953829438, 89382455, 347481625, 1754527224, 916217775, 1056029082, 410644796], [747170349, 1608481817, 1569266992, 1154307224, 561605764, 1907985030, 1454474361, 357054770]], }, { "name": 'single_8x1', "matspec": [(8, 1, 200)], "index": 6, "root": [447902873, 1264273264, 1781377203, 392525377, 451230220, 285479002, 124104889, 737840045], "openings": [[206]], "proof": [[854476755, 1853946983, 1409001429, 1897109439, 1880752521, 1036730533, 1817549359, 907866104], [757904997, 571589503, 673816220, 1066413580, 473597103, 1271204909, 363424036, 1450652756], [402779725, 764084359, 1406275333, 1158469515, 32391761, 236523006, 1039588073, 264072612]], }, { "name": 'mixed_8x2_4x3', "matspec": [(8, 2, 10), (4, 3, 300)], "index": 5, "root": [902263792, 353615665, 93922356, 1251424848, 837594048, 2005066510, 431363100, 287722769], "openings": [[20, 21], [306, 307, 308]], "proof": [[1732750591, 1876733121, 1364724824, 1847015379, 1792368333, 483325461, 1174939365, 1939380322], [932176417, 320077372, 1748871293, 625438914, 179521004, 655947905, 828588297, 31952751], [930401557, 471785873, 904189917, 1626475238, 550043796, 1590748862, 306781755, 496390863]], }, { "name": 'mixed_8x1_4x2_2x2', "matspec": [(8, 1, 1), (4, 2, 50), (2, 2, 400)], "index": 6, "root": [439675894, 37405611, 255560432, 531897590, 349476430, 1463998586, 540993751, 1307259928], "openings": [[7], [56, 57], [402, 403]], "proof": [[723290459, 412642417, 484634914, 785252854, 182396116, 1183763863, 630430401, 175545021], [1257356383, 1204259532, 1915921675, 53621155, 1015367194, 742423503, 1998129904, 1554700909], [13474675, 1966185163, 1255599565, 471961853, 1823737756, 1607265064, 1964440250, 1219296485]], }, { "name": 'mixed_5x2_3x1', "matspec": [(5, 2, 20), (3, 1, 70)], "index": 4, "root": [700351145, 394944774, 166238365, 1683786139, 1659356855, 1460803726, 573770743, 1196572690], "openings": [[28, 29], [72]], "proof": [[0, 0, 0, 0, 0, 0, 0, 0], [1714214715, 1781831455, 644202942, 667527548, 1395787195, 671219385, 1433934871, 1883643874], [1521468259, 1627990232, 1145489794, 839124709, 507242897, 777414579, 429233007, 1147797058]], }, ] def _case_matrices(case): return [make_matrix(h, w, b) for (h, w, b) in case["matspec"]] def _case_dims(case): return [(w, h) for (h, w, b) in case["matspec"]] def conformance_kats(): """Return (passed, total) after reproducing every pinned-library known-answer vector: the perm sanity, the primitive hash/compress KATs, and each MMCS case (root + openings + proof + verify).""" passed = 0 total = 0 total += 1 if permute([0] * 16) == PERM_ZEROS: passed += 1 for name, expected in PRIM_KATS.items(): total += 1 if name == 'hash_item_5': got = hash_item(5) elif name == 'hash_slice_2': got = hash_slice([1, 2]) elif name == 'hash_slice_3': got = hash_slice([1, 2, 3]) elif name == 'hash_slice_8': got = hash_slice(list(range(1, 9))) elif name == 'hash_slice_9': got = hash_slice(list(range(1, 10))) elif name == 'compress_h2_h3': got = compress2to1(hash_slice([1, 2]), hash_slice([1, 2, 3])) if got == expected: passed += 1 for case in CONFORMANCE_CASES: mats = _case_matrices(case) root, pdata = commit(mats) openings, proof = open_batch(case["index"], pdata) dims = _case_dims(case) # root matches total += 1 if root == case["root"]: passed += 1 # openings match total += 1 if openings == case["openings"]: passed += 1 # proof matches total += 1 if proof == case["proof"]: passed += 1 # verify accepts the emitted opening against the emitted root total += 1 if verify_batch(case["root"], dims, case["index"], case["openings"], case["proof"]): passed += 1 # verify rejects a tampered proof total += 1 tampered = [list(d) for d in case["proof"]] tampered[0] = list(tampered[0]) tampered[0][0] = (tampered[0][0] + 1) % P if not verify_batch(case["root"], dims, case["index"], case["openings"], tampered): passed += 1 return passed, total # ============================ shared cross-language vector set ============================ # A fixed set the harness runs in Python, Node, and Java, asserting byte-identical outputs AND that # each reproduces the pinned-library conformance vectors. def shared_vectors(): prim = { "hash_item_5": hash_item(5), "hash_slice_2": hash_slice([1, 2]), "hash_slice_3": hash_slice([1, 2, 3]), "hash_slice_8": hash_slice(list(range(1, 9))), "hash_slice_9": hash_slice(list(range(1, 10))), "compress_h2_h3": compress2to1(hash_slice([1, 2]), hash_slice([1, 2, 3])), } cases = [] for case in CONFORMANCE_CASES: mats = _case_matrices(case) root, pdata = commit(mats) openings, proof = open_batch(case["index"], pdata) dims = _case_dims(case) cases.append({ "name": case["name"], "index": case["index"], "root": root, "openings": openings, "proof": proof, "rootMatch": root == case["root"], "openingsMatch": openings == case["openings"], "proofMatch": proof == case["proof"], "verifyOk": verify_batch(case["root"], dims, case["index"], case["openings"], case["proof"]), }) return { "constants": {"P": P, "width": WIDTH, "rate": RATE, "out": OUT, "digestElems": DIGEST_ELEMS}, "permZeros": permute([0] * 16), "prim": prim, "cases": cases, } if __name__ == "__main__": import json json.dump(shared_vectors(), sys.stdout)