aere-quantum/verifiable-data/baza.mjs

// baza.mjs: the verifiable dataset (roadmap master point 20, pista B, 2026-10-02). A dataset is an append-only list of records; a
// VERSION is a commitment to it: the RFC 9162 Merkle root of the record leaves (the log, `../verify-layer/arbore-merkle.mjs`), its size,
// and for each indexed field the root of a SORTED index tree over (value, record). From a commitment anyone can verify, without the data:
//   - a record is in the dataset at a version (inclusion proof in the log, log2(n) hashes);
//   - a newer version only APPENDED (consistency proof between the two log roots): a version that is not a prefix of the next is a
//     rewritten history;
//   - a query answer (equality or range on an indexed field) is COMPLETE and CORRECT: every returned row proves itself in the log and in
//     the index, the rows are the contiguous run of the sorted index, and the two neighbours outside the range are shown with their
//     proofs, so no matching row could have been left out; the completeness rests on the index being sorted, which anyone holding the
//     data audits in full (`auditeaza`) and no single query can;
//   - a row reveals only the fields the query asked for (selective disclosure): every field is committed as
//     SHA-256(salt | field | value) in the record leaf, so an undisclosed field stays hidden and a disclosed one cannot be changed. This
//     is hash-based disclosure, NOT zero knowledge: the verifier learns the disclosed values; a proof about rows without revealing them
//     (counts, sums, membership in a set) needs a proving system and is not built here (see README).
// The commitment itself is an AIP-23 `data` envelope (proof-kinds), notarizable on AereNotary and read back under a post-quantum
// certified anchor by the reference verifier: the TIME of a version is then the chain's, not the publisher's.
// Only node:crypto and the Merkle tree of the verification layer. Records: JSON objects with a string `id` (unique) and flat fields
// whose values are strings, finite numbers, booleans or null; arrays and nested objects are refused (an index needs a total order).
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { leafHash, merkleTree, verifyInclusion, verifyConsistency } from '../verify-layer/arbore-merkle.mjs';
import { buildProof } from '../proof-kinds/proof-kinds.mjs';

export const FORMAT = 'aere-verifiable-dataset-1';
export const KIND_RASPUNS = 'aere-verifiable-query-answer';
export const KIND_INREGISTRARE = 'aere-verifiable-record';
export const KIND_CONSISTENTA = 'aere-verifiable-consistency';
const H = /^0x[0-9a-f]{64}$/;
const sha256 = (b) => '0x' + crypto.createHash('sha256').update(b).digest('hex');
const utf8 = (s) => Buffer.from(s, 'utf8');

/** Canonical JSON: keys sorted at every depth, no whitespace; refuses what JSON cannot carry faithfully. */
export function canonical(x) {
  if (x === null) return 'null';
  if (typeof x === 'string') return JSON.stringify(x);
  if (typeof x === 'boolean') return x ? 'true' : 'false';
  if (typeof x === 'number') { if (!Number.isFinite(x)) throw new Error('canonical: a number must be finite'); return JSON.stringify(x); }
  if (Array.isArray(x)) return '[' + x.map(canonical).join(',') + ']';
  if (typeof x === 'object') return '{' + Object.keys(x).sort().map((k) => { if (x[k] === undefined) throw new Error('canonical: undefined under ' + k); return JSON.stringify(k) + ':' + canonical(x[k]); }).join(',') + '}';
  throw new Error('canonical: unsupported value of type ' + typeof x);
}
const tip = (v) => v === null ? 'null' : typeof v;
const TIPURI = ['boolean', 'null', 'number', 'string'];
/** Total order on field values: by type name, then by value (numbers numerically, strings by code unit). One place, used by both sides. */
export function comparaValori(a, b) {
  const ta = tip(a), tb = tip(b);
  if (!TIPURI.includes(ta) || !TIPURI.includes(tb)) throw new Error('compare: a field value must be a string, a finite number, a boolean or null');
  if (ta !== tb) return ta < tb ? -1 : 1;
  if (ta === 'number') return a < b ? -1 : a > b ? 1 : 0;
  if (ta === 'boolean') return a === b ? 0 : a ? 1 : -1;
  const sa = String(a), sb = String(b); return sa < sb ? -1 : sa > sb ? 1 : 0;
}
export const angajamentCamp = (field, value, salt) => sha256(utf8(String(salt) + '|' + field + '|' + canonical(value)));
export const frunzaInregistrare = (id, c) => leafHash(utf8(canonical({ id, c })));
export const frunzaIndex = (field, value, i, recordLeaf) => leafHash(utf8(canonical({ field, value, i, recordLeaf })));
export const hashAngajament = (commitment) => sha256(utf8(canonical(commitment)));

function valideazaInregistrare(r) {
  if (!r || typeof r !== 'object' || Array.isArray(r)) throw new Error('a record is a JSON object');
  if (typeof r.id !== 'string' || !r.id.length || r.id.length > 256) throw new Error('a record needs a string id (1..256 chars)');
  const fields = r.fields && typeof r.fields === 'object' ? r.fields : Object.fromEntries(Object.entries(r).filter(([k]) => k !== 'id'));
  for (const [k, v] of Object.entries(fields)) {
    if (!/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(k)) throw new Error('field name "' + k + '" is not [A-Za-z_][A-Za-z0-9_]{0,63}');
    if (!TIPURI.includes(tip(v)) || (typeof v === 'number' && !Number.isFinite(v))) throw new Error('field "' + k + '" must be a string, a finite number, a boolean or null');
    if (typeof v === 'string' && v.length > 4096) throw new Error('field "' + k + '" is longer than 4096 chars');
  }
  return { id: r.id, fields };
}

/** Creates a dataset folder: set.json (name, indexed fields), an empty record file, no versions. */
export function creeaza(dir, { name, indexes = [] }) {
  if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(String(name))) throw new Error('dataset name: [a-z0-9][a-z0-9-]{0,63}');
  if (!Array.isArray(indexes) || indexes.some((f) => !/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(f))) throw new Error('indexes: a list of field names');
  fs.mkdirSync(dir, { recursive: true });
  if (fs.existsSync(path.join(dir, 'set.json'))) throw new Error('the dataset already exists at ' + dir);
  fs.writeFileSync(path.join(dir, 'set.json'), JSON.stringify({ format: FORMAT, name, indexes: [...new Set(indexes)].sort() }, null, 1));
  fs.writeFileSync(path.join(dir, 'inregistrari.ndjson'), '');
  fs.writeFileSync(path.join(dir, 'versiuni.json'), '[]');
  return deschide(dir);
}
export function deschide(dir) {
  const set = JSON.parse(fs.readFileSync(path.join(dir, 'set.json'), 'utf8'));
  if (set.format !== FORMAT) throw new Error('not a ' + FORMAT + ' dataset: ' + dir);
  const text = fs.readFileSync(path.join(dir, 'inregistrari.ndjson'), 'utf8');
  const inregistrari = text.split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const versiuni = JSON.parse(fs.readFileSync(path.join(dir, 'versiuni.json'), 'utf8'));
  return { dir, set, inregistrari, versiuni, _cache: new Map() };
}
/** Appends records (never rewrites): each gets a fresh 16-byte salt per field; ids must be new. */
export function adauga(db, records) {
  const ids = new Set(db.inregistrari.map((r) => r.id)); const noi = [];
  for (const r of records) {
    const { id, fields } = valideazaInregistrare(r);
    if (ids.has(id)) throw new Error('duplicate id "' + id + '" (a dataset appends; it never rewrites a record)');
    ids.add(id);
    const salts = Object.fromEntries(Object.keys(fields).map((k) => [k, crypto.randomBytes(16).toString('hex')]));
    noi.push({ id, fields, salts });
  }
  fs.appendFileSync(path.join(db.dir, 'inregistrari.ndjson'), noi.map((r) => JSON.stringify(r) + '\n').join(''));
  db.inregistrari.push(...noi); db._cache.clear();
  return noi.length;
}
const angajamente = (r) => Object.fromEntries(Object.keys(r.fields).sort().map((k) => [k, angajamentCamp(k, r.fields[k], r.salts[k])]));
/** The trees of a version of `size` records: the log and one sorted index per indexed field (memoized per size). */
function arbori(db, size) {
  if (db._cache.has(size)) return db._cache.get(size);
  if (!Number.isInteger(size) || size < 1 || size > db.inregistrari.length) throw new Error('version size ' + size + ' outside 1..' + db.inregistrari.length);
  const recs = db.inregistrari.slice(0, size).map((r, i) => { const c = angajamente(r); return { i, id: r.id, c, leaf: frunzaInregistrare(r.id, c) }; });
  const log = merkleTree(recs.map((r) => r.leaf));
  const index = {};
  for (const f of db.set.indexes) {
    const intrari = recs.filter((r) => f in db.inregistrari[r.i].fields).map((r) => ({ value: db.inregistrari[r.i].fields[f], i: r.i, recordLeaf: r.leaf }));
    intrari.sort((a, b) => comparaValori(a.value, b.value) || a.i - b.i);
    index[f] = { intrari, tree: merkleTree(intrari.map((e) => frunzaIndex(f, e.value, e.i, e.recordLeaf))) };
  }
  const t = { recs, log, index }; db._cache.set(size, t); return t;
}
/** Commits the current records as a new version; returns the commitment (also appended to versiuni.json). */
export function angajeaza(db, { createdAt = new Date().toISOString() } = {}) {
  const size = db.inregistrari.length; if (!size) throw new Error('nothing to commit: the dataset is empty');
  const prev = db.versiuni[db.versiuni.length - 1] || null;
  if (prev && prev.size === size) throw new Error('nothing new since version ' + prev.version + ' (' + size + ' records)');
  const t = arbori(db, size);
  const commitment = { v: 1, format: FORMAT, name: db.set.name, version: db.versiuni.length + 1, size, logRoot: t.log.root(), createdAt,
    indexes: Object.fromEntries(db.set.indexes.map((f) => [f, { root: t.index[f].tree.root(), size: t.index[f].intrari.length }])),
    previous: prev ? hashAngajament(prev) : null };
  db.versiuni.push(commitment);
  fs.writeFileSync(path.join(db.dir, 'versiuni.json'), JSON.stringify(db.versiuni, null, 1));
  return commitment;
}
export function versiune(db, n) { const c = db.versiuni.find((x) => x.version === Number(n)); if (!c) throw new Error('no version ' + n); return c; }
const fara = (fields, care) => care === '*' ? { ...fields } : Object.fromEntries(Object.entries(fields).filter(([k]) => care.includes(k)));
const dezvaluie = (r, care) => Object.fromEntries(Object.keys(fara(r.fields, care)).map((k) => [k, { value: r.fields[k], salt: r.salts[k] }]));
const dovada = (p) => ({ index: p.index, treeSize: p.treeSize, path: p.path });
/** One record at a version, with its log inclusion proof; `fields` = '*' (all) or a list of field names to disclose. */
export function dovadaInregistrare(db, n, i, { fields = '*' } = {}) {
  const c = versiune(db, n); const t = arbori(db, c.size); const r = t.recs[i]; if (!r) throw new Error('record ' + i + ' outside version ' + n + ' (' + c.size + ' records)');
  return { v: 1, kind: KIND_INREGISTRARE, name: c.name, version: c.version, commitmentHash: hashAngajament(c), i, id: r.id, c: r.c, disclosed: dezvaluie(db.inregistrari[i], fields), logProof: dovada(t.log.inclusionProof(i, c.size)) };
}
/** Equality (eq) or range (from..to, inclusive, either side open) on an indexed field; `fields` = the fields to disclose besides it. */
export function interogheaza(db, n, { field, eq, from, to, fields = [], limit = 1000 } = {}) {
  const c = versiune(db, n); if (!c.indexes[field]) throw new Error('field "' + field + '" is not indexed in ' + c.name);
  if (eq !== undefined) { from = eq; to = eq; }
  if (from !== undefined && to !== undefined && comparaValori(from, to) > 0) throw new Error('from is above to');
  const t = arbori(db, c.size); const ix = t.index[field]; const e = ix.intrari;
  let start = 0; while (start < e.length && from !== undefined && comparaValori(e[start].value, from) < 0) start++;
  let end = start; while (end < e.length && (to === undefined || comparaValori(e[end].value, to) <= 0)) end++;
  if (end - start > limit) throw new Error('the answer would have ' + (end - start) + ' rows, above the limit ' + limit);
  const care = fields === '*' ? '*' : [...new Set([field, ...fields])];
  const rows = e.slice(start, end).map((x, k) => ({ pos: start + k, i: x.i, id: t.recs[x.i].id, c: t.recs[x.i].c, disclosed: dezvaluie(db.inregistrari[x.i], care),
    logProof: dovada(t.log.inclusionProof(x.i, c.size)), indexProof: dovada(ix.tree.inclusionProof(start + k, e.length)) }));
  const vecin = (pos) => pos < 0 || pos >= e.length ? null : ({ pos, value: e[pos].value, i: e[pos].i, recordLeaf: e[pos].recordLeaf, indexProof: dovada(ix.tree.inclusionProof(pos, e.length)) });
  return { v: 1, kind: KIND_RASPUNS, name: c.name, version: c.version, commitmentHash: hashAngajament(c), field, from: from === undefined ? null : from, to: to === undefined ? null : to,
    indexSize: e.length, rows, before: vecin(start - 1), after: vecin(end) };
}
/** The log of version a is a prefix of the log of version b (RFC 9162 consistency). */
export function dovadaConsistenta(db, a, b) {
  const ca = versiune(db, a), cb = versiune(db, b); if (ca.size > cb.size) throw new Error('version ' + a + ' is larger than version ' + b);
  const t = arbori(db, cb.size);
  return { v: 1, kind: KIND_CONSISTENTA, name: cb.name, from: { version: ca.version, commitmentHash: hashAngajament(ca) }, to: { version: cb.version, commitmentHash: hashAngajament(cb) }, proof: t.log.consistencyProof(ca.size, cb.size) };
}
/** Rebuilds every version from the records and compares the roots: the full audit anyone holding the data runs. */
export function auditeaza(db) {
  const probleme = [];
  for (const c of db.versiuni) {
    try { const t = arbori(db, c.size); if (t.log.root() !== c.logRoot) probleme.push('version ' + c.version + ': log root differs');
      for (const f of Object.keys(c.indexes)) { if (!t.index[f] || t.index[f].tree.root() !== c.indexes[f].root || t.index[f].intrari.length !== c.indexes[f].size) probleme.push('version ' + c.version + ': index ' + f + ' differs'); }
    } catch (e) { probleme.push('version ' + c.version + ': ' + e.message); }
  }
  for (let k = 1; k < db.versiuni.length; k++) if (db.versiuni[k].previous !== hashAngajament(db.versiuni[k - 1])) probleme.push('version ' + db.versiuni[k].version + ' does not name version ' + db.versiuni[k - 1].version + ' as previous');
  return { ok: probleme.length === 0, probleme };
}

// ----- verification, without the data (what a reader runs on a commitment and an answer) -----
const ok = (motive, cond, m) => { if (!cond) motive.push(m); return cond; };
function verificaRand(r, commitment, field, motive, eticheta) {
  if (!r || typeof r !== 'object' || !r.c || typeof r.c !== 'object' || !r.disclosed) return ok(motive, false, eticheta + ': malformed row');
  for (const [k, d] of Object.entries(r.disclosed)) {
    if (!d || typeof d.salt !== 'string') return ok(motive, false, eticheta + ': disclosed field ' + k + ' without a salt');
    const okCamp = r.c[k] === angajamentCamp(k, d.value, d.salt) /*PAZA:camp*/;
    if (!ok(motive, okCamp, eticheta + ': the disclosed value of "' + k + '" does not match its commitment')) return false;
  }
  const leaf = frunzaInregistrare(r.id, r.c);
  if (!ok(motive, r.logProof && verifyInclusion({ ...r.logProof, leafHash: leaf, treeSize: commitment.size, rootHash: commitment.logRoot }) && r.logProof.index === r.i, eticheta + ': the record is not in the log of version ' + commitment.version)) return false;
  if (field !== null) {
    if (!ok(motive, r.disclosed[field] !== undefined, eticheta + ': the queried field "' + field + '" is not disclosed')) return false;
    const ix = commitment.indexes[field];
    const okIndex = r.indexProof && verifyInclusion({ ...r.indexProof, leafHash: frunzaIndex(field, r.disclosed[field].value, r.i, leaf), treeSize: ix.size, rootHash: ix.root }) && r.indexProof.index === r.pos;
    if (!ok(motive, okIndex, eticheta + ': the row is not at position ' + r.pos + ' of the index "' + field + '"')) return false;
  }
  return true;
}
/** A query answer against a commitment: every row proves itself, the rows are the whole contiguous run, the neighbours are outside. */
export function verificaRaspuns(answer, commitment) {
  const motive = [];
  if (!answer || answer.kind !== KIND_RASPUNS || !commitment) return { verdict: 'INVALID', motive: ['not a ' + KIND_RASPUNS], rows: 0 };
  ok(motive, answer.commitmentHash === hashAngajament(commitment) && answer.version === commitment.version && answer.name === commitment.name, 'the answer names another commitment');
  const f = answer.field; const ix = commitment.indexes && commitment.indexes[f];
  if (!ok(motive, !!ix, 'the field "' + f + '" is not indexed in the commitment')) return { verdict: 'INVALID', motive, rows: 0 };
  ok(motive, answer.indexSize === ix.size, 'the answer\'s index size differs from the commitment\'s');
  const from = answer.from, to = answer.to; const rows = Array.isArray(answer.rows) ? answer.rows : [];
  if (from !== null && to !== null) ok(motive, comparaValori(from, to) <= 0, 'from is above to');
  const inRange = (v) => (from === null || comparaValori(v, from) >= 0) && (to === null || comparaValori(v, to) <= 0);
  let start = rows.length ? rows[0].pos : (answer.before ? answer.before.pos + 1 : 0);
  for (const [k, r] of rows.entries()) {
    if (!verificaRand(r, commitment, f, motive, 'row ' + k)) continue;
    ok(motive, r.pos === start + k, 'row ' + k + ' breaks the contiguous run of the index');
    ok(motive, inRange(r.disclosed[f].value), 'row ' + k + ' is outside the queried range');
    if (k) { const okOrdine = comparaValori(rows[k - 1].disclosed[f].value, r.disclosed[f].value) < 0 || (comparaValori(rows[k - 1].disclosed[f].value, r.disclosed[f].value) === 0 && rows[k - 1].i < r.i) /*PAZA:ordine*/; ok(motive, okOrdine, 'rows ' + (k - 1) + ' and ' + k + ' are not in index order'); }
  }
  const end = start + rows.length;
  const vecin = (n, pos, cond, m) => {
    if (!n) return ok(motive, cond, m);
    const okVecin = n.pos === pos && n.indexProof && verifyInclusion({ ...n.indexProof, leafHash: frunzaIndex(f, n.value, n.i, n.recordLeaf), treeSize: ix.size, rootHash: ix.root }) && n.indexProof.index === pos;
    return ok(motive, okVecin, 'the neighbour at position ' + pos + ' does not prove itself in the index');
  };
  // completeness: the row before the run is below the range (or the run starts at 0), the row after is above it (or the run ends the index)
  const okInainte = vecin(answer.before, start - 1, start === 0, 'the run does not start at 0 and no neighbour before it is shown') && (!answer.before || (from !== null && comparaValori(answer.before.value, from) < 0)) /*PAZA:completitudine*/;
  ok(motive, okInainte, 'a row before the run could still be in the range (the answer may be incomplete)');
  const okDupa = vecin(answer.after, end, end === ix.size, 'the run does not end the index and no neighbour after it is shown') && (!answer.after || (to !== null && comparaValori(answer.after.value, to) > 0)) /*PAZA:completitudine*/;
  ok(motive, okDupa, 'a row after the run could still be in the range (the answer may be incomplete)');
  return { verdict: motive.length ? 'INVALID' : 'VALID', motive, rows: rows.length };
}
/** One record (from `dovadaInregistrare` or the serving endpoint) against a commitment. */
export function verificaInregistrare(rec, commitment) {
  const motive = [];
  if (!rec || rec.kind !== KIND_INREGISTRARE) return { ok: false, motive: ['not a ' + KIND_INREGISTRARE] };
  ok(motive, rec.commitmentHash === hashAngajament(commitment) && rec.version === commitment.version, 'the record names another commitment');
  verificaRand(rec, commitment, null, motive, 'record ' + rec.i);
  return { ok: motive.length === 0, motive };
}
/** A consistency proof between two commitments: the older log is a prefix of the newer (nothing rewritten, nothing removed). */
export function verificaConsistentaVersiuni(d, older, newer) {
  const motive = [];
  if (!d || d.kind !== KIND_CONSISTENTA || !d.proof) return { ok: false, motive: ['not a ' + KIND_CONSISTENTA] };
  ok(motive, d.from.commitmentHash === hashAngajament(older) && d.to.commitmentHash === hashAngajament(newer), 'the proof names other commitments');
  const okRadacini = d.proof.firstRoot === older.logRoot && d.proof.secondRoot === newer.logRoot && d.proof.firstSize === older.size && d.proof.secondSize === newer.size /*PAZA:consistenta*/;
  ok(motive, okRadacini, 'the proof\'s roots or sizes are not the commitments\'');
  ok(motive, verifyConsistency(d.proof), 'the older log is not a prefix of the newer (rewritten history)');
  ok(motive, newer.previous === hashAngajament(older) || newer.version !== older.version + 1, 'the newer version does not name the older as previous');
  return { ok: motive.length === 0, motive };
}
/**
 * Availability sampling: k record positions drawn from a seed (random when none; the seed is printed so the draw can be repeated), each
 * fetched from `url` + /record/<version>/<i> through `ia(url) -> JSON | null` and verified against the commitment. A record that is not
 * served, or does not verify, fails the sample. What this measures: the publisher serves what it committed to, NOW, at the sampled
 * positions; k samples bound the share of unserved records a publisher can hide only probabilistically ((1 - share)^k survive).
 */
export async function esantioneaza({ url, commitment, k = 16, seed = null, ia }) {
  const n = commitment.size; const cerute = Math.min(Math.max(Number(k) || 1, 1), n);
  const s = seed || crypto.randomBytes(16).toString('hex'); const pozitii = new Set(); let ctr = 0;
  while (pozitii.size < cerute) { const h = crypto.createHash('sha256').update(s + ':' + ctr++).digest(); pozitii.add(Number(h.readBigUInt64BE(0) % BigInt(n))); }
  const lipsa = [], motive = []; let verificate = 0;
  for (const i of [...pozitii].sort((x, y) => x - y)) {
    let rec = null; try { rec = await ia(url.replace(/\/$/, '') + '/record/' + commitment.version + '/' + i); } catch (e) { motive.push('record ' + i + ': ' + String(e.message).slice(0, 80)); }
    if (!rec) { lipsa.push(i); continue; }
    const r = verificaInregistrare(rec, commitment);
    if (r.ok && rec.i === i) verificate++; else motive.push('record ' + i + ': ' + (rec.i !== i ? 'another record was served' : r.motive.join('; ')));
  }
  const okEsantion = lipsa.length === 0 && verificate === cerute /*PAZA:esantion*/;
  if (lipsa.length) motive.push('not served: ' + lipsa.join(', '));
  return { ok: okEsantion, seed: s, cerute, verificate, lipsa, motive };
}
/** The AIP-23 `data` envelope of a commitment: sha256 = SHA-256 of the canonical commitment, name = dataset@version. */
export function plicAngajament(commitment, { createdAt = commitment.createdAt } = {}) {
  return buildProof('data', { content: utf8(canonical(commitment)), name: commitment.name + '@' + commitment.version, createdAt });
}
export function verificaPlicAngajament(env, commitment) {
  const st = env && env.statement;
  return !!st && env.kind === 'aere-proof-of-data-attestation' && st.sha256 === hashAngajament(commitment) && st.name === commitment.name + '@' + commitment.version && H.test(String(env.statementHash));
}