Reproducible code
The full test suite, exactly as run against the ZL transcription (data/voynich/ZL3b-n.txt). Pure Python stdlib — no dependencies. Run each with the parser module voynich_lib.py on the path.
scripts/voynich_lib.py
#!/usr/bin/env python3
"""Shared Voynich parsing: ZL EVA IVTFF -> tokens tagged with controls.
Each row: (folio, line, pos_in_line, currier_L, hand_H, section_I, token)."""
import re, pathlib
SRC = pathlib.Path(__file__).resolve().parent.parent / "data/voynich/ZL3b-n.txt"
_TAG = re.compile(r"<[^>]*>")
_ALT = re.compile(r"\[([^:\]]*)[:][^\]]*\]")
_HEAD = re.compile(r"^<(f[^.>]+)>\s+<!\s*(.*?)>")
_LOCUS = re.compile(r"^<(f[^.>]+)\.(\d+)[^>]*>\s*(.*)$")
def _meta(s): return dict(re.findall(r"\$([A-Z])=([^\s>]+)", s))
def clean_tokens(text):
text = _TAG.sub("", text); text = _ALT.sub(r"\1", text)
text = text.replace("!", "").replace("%", "")
out = []
for t in re.split(r"[.,]", text):
t = t.strip()
if t and re.fullmatch(r"[a-z]+", t): out.append(t)
return out
def lev1(a, b):
if a == b: return False
la, lb = len(a), len(b)
if abs(la - lb) > 1: return False
if la == lb: return sum(x != y for x, y in zip(a, b)) == 1
if la > lb: a, b, la, lb = b, a, lb, la
i = j = diff = 0
while i < la and j < lb:
if a[i] == b[j]: i += 1; j += 1
else:
diff += 1; j += 1
if diff > 1: return False
return True
def parse(path=SRC):
rows = []; cur = {"f": None, "L": "?", "H": "?", "I": "?"}
for raw in pathlib.Path(path).read_text(encoding="utf-8", errors="ignore").splitlines():
if raw.startswith("#") or not raw.strip(): continue
m = _HEAD.match(raw)
if m:
md = _meta(m.group(2))
cur = {"f": m.group(1), "L": md.get("L", "?"), "H": md.get("H", "?"), "I": md.get("I", "?")}
continue
m = _LOCUS.match(raw)
if m:
folio, line, text = m.group(1), int(m.group(2)), m.group(3)
for pos, tok in enumerate(clean_tokens(text)):
rows.append((folio, line, pos, cur["L"], cur["H"], cur["I"], tok))
return rows
scripts/voynich_parse_stats.py
#!/usr/bin/env python3
"""Milestone 1 of the Voynich controlled-structure experiment.
Parse the ZL EVA IVTFF transliteration into tokens tagged with their controls
(folio, line, in-line position, Currier language $L, scribal hand $H, section $I),
then REPRODUCE the known statistical anomalies as a pipeline sanity check:
- adjacent-word repetition rate (known: high vs natural language)
- one-edit-neighbour rate (the "drift by one glyph" signature)
- character conditional entropy h1/h2 (known: h2 ~2 bits, unusually low)
If these match the literature, the parser is trustworthy for the controlled tests.
"""
import re, math, pathlib
from collections import Counter
SRC = pathlib.Path(__file__).resolve().parent.parent / "data/voynich/ZL3b-n.txt"
TAG = re.compile(r"<[^>]*>") # inline tags: <%>, <$>, <!...>, <->, <~>
ALT = re.compile(r"\[([^:\]]*)[:][^\]]*\]") # [a:b] uncertain reading -> first option
HEAD = re.compile(r"^<(f[^.>]+)>\s+<!\s*(.*?)>") # page header w/ metadata
LOCUS = re.compile(r"^<(f[^.>]+)\.(\d+)[^>]*>\s*(.*)$") # locus line: folio.line + text
def meta(s):
return dict(re.findall(r"\$([A-Z])=([^\s>]+)", s))
def clean_tokens(text):
text = TAG.sub("", text)
text = ALT.sub(r"\1", text)
text = text.replace("!", "").replace("%", "")
toks = re.split(r"[.,]", text)
out = []
for t in toks:
t = t.strip()
if t and re.fullmatch(r"[a-z]+", t): # drop illegible/uncertain (*, ?, etc.)
out.append(t)
return out
def lev1(a, b):
"""True iff Levenshtein(a,b) == 1 (one substitution/insertion/deletion)."""
if a == b: return False
la, lb = len(a), len(b)
if abs(la - lb) > 1: return False
if la == lb:
return sum(x != y for x, y in zip(a, b)) == 1
if la > lb: a, b, la, lb = b, a, lb, la # ensure a shorter
i = j = 0; diff = 0
while i < la and j < lb:
if a[i] == b[j]: i += 1; j += 1
else:
diff += 1; j += 1
if diff > 1: return False
return True
rows = [] # (folio, line, pos, L, H, I, token)
cur = {"f": None, "L": "?", "H": "?", "I": "?"}
for raw in SRC.read_text(encoding="utf-8", errors="ignore").splitlines():
if raw.startswith("#") or not raw.strip():
continue
m = HEAD.match(raw)
if m:
md = meta(m.group(2))
cur = {"f": m.group(1), "L": md.get("L", "?"), "H": md.get("H", "?"), "I": md.get("I", "?")}
continue
m = LOCUS.match(raw)
if m:
folio, line, text = m.group(1), int(m.group(2)), m.group(3)
for pos, tok in enumerate(clean_tokens(text)):
rows.append((folio, line, pos, cur["L"], cur["H"], cur["I"], tok))
toks = [r[6] for r in rows]
N = len(toks)
print(f"parsed: {N} tokens, {len(set(toks))} types, {len(set(r[0] for r in rows))} folios")
print(f"controls present -> Currier $L: {sorted(set(r[3] for r in rows))} | "
f"hands $H: {sorted(set(r[4] for r in rows))} | sections $I: {sorted(set(r[5] for r in rows))}")
# --- adjacent repetition & one-edit-neighbour (within line, in reading order) ---
pairs = same = oneoff = 0
prev = None; prev_key = None
for folio, line, pos, L, H, I, tok in rows:
key = (folio, line)
if prev is not None and key == prev_key:
pairs += 1
if tok == prev: same += 1
elif lev1(tok, prev): oneoff += 1
prev, prev_key = tok, key
print(f"\nadjacent-word repetition: {same/pairs*100:.2f}% (natural language typically <0.5%)")
print(f"one-edit-neighbour rate : {oneoff/pairs*100:.2f}% (the 'mutate by one glyph' drift)")
print(f"either (repeat or 1-edit): {(same+oneoff)/pairs*100:.2f}% of adjacent pairs")
# --- character conditional entropy over the glyph stream (letters + word-space) ---
stream = " ".join(toks)
uni = Counter(stream); tot = len(stream)
h1 = -sum(c/tot * math.log2(c/tot) for c in uni.values())
bg = Counter(zip(stream, stream[1:])); tb = sum(bg.values())
h_joint = -sum(c/tb * math.log2(c/tb) for c in bg.values())
h2 = h_joint - h1 # H(next char | current char)
mw = sum(len(t) for t in toks) / N
print(f"\nmean word length : {mw:.2f} glyphs")
print(f"char entropy h1 : {h1:.2f} bits/char")
print(f"char conditional h2 : {h2:.2f} bits/char (Voynich known ~2.0; English letters ~3.0-3.6)")
scripts/voynich_longrange_test.py
#!/usr/bin/env python3
"""Milestone 2: controlled long-range structure test for Voynichese.
Metric: word CLUSTERING = frequency-weighted KL(occurrence-distribution || uniform)
across B equal position-bins. High = words cluster in regions (topical organization).
We compare the REAL corpus against progressively stronger nulls and two generators:
- full shuffle : destroys all order (keeps word frequencies)
- within-section shuffle: keeps each position's SECTION, destroys finer order
- within-folio shuffle : keeps each position's FOLIO, destroys finer order
- global word-Markov : order-2 Markov over whole text (local structure only)
- per-section Markov : order-1 Markov trained/generated PER SECTION
(reproduces section vocabulary by construction)
Reading: if real clustering >> within-section null, there is topical structure FINER
than sections. If real ~ per-section generator, "topical structure" reduces to
"sections use different words" (a sectioned generator suffices). z-scores vs the
Monte-Carlo null distributions quantify each.
"""
import math, random, statistics, sys
from collections import Counter, defaultdict
from voynich_lib import parse
random.seed(7)
B, MINC, M = 40, 10, 150 # bins, min count to be "eligible", Monte-Carlo shuffles
rows = parse()
toks = [r[6] for r in rows]
N = len(toks)
sec = [r[5] for r in rows]
fol = [r[0] for r in rows]
binof = [i * B // N for i in range(N)] # fixed position-bin per index
counts = Counter(toks)
eligible = {w for w, c in counts.items() if c >= MINC}
nw = {w: counts[w] for w in eligible}
totw = sum(nw.values())
print(f"corpus: {N} tokens, {len(counts)} types; eligible (>={MINC}): {len(eligible)} types "
f"covering {totw} tokens ({totw/N*100:.0f}%)\n")
def clustering(seq):
"""freq-weighted mean KL(occurrence dist || uniform) over eligible words."""
hist = defaultdict(lambda: [0]*B)
for i, w in enumerate(seq):
if w in eligible: hist[w][binof[i]] += 1
acc = 0.0
for w, h in hist.items():
n = nw[w]; kl = 0.0
for b in h:
if b:
p = b / n
kl += p * math.log2(p * B) # vs uniform 1/B
acc += n * kl
return acc / totw
C_real = clustering(toks)
def shuffled(kind):
s = toks[:]
if kind == "full":
random.shuffle(s)
else:
key = sec if kind == "section" else fol
groups = defaultdict(list)
for i, k in enumerate(key): groups[k].append(i)
for idxs in groups.values():
vals = [s[i] for i in idxs]; random.shuffle(vals)
for i, v in zip(idxs, vals): s[i] = v
return s
def null_dist(kind):
vals = [clustering(shuffled(kind)) for _ in range(M)]
return statistics.mean(vals), statistics.pstdev(vals)
def markov_global(order=2):
nxt = defaultdict(list)
for i in range(len(toks)-order):
nxt[tuple(toks[i:i+order])].append(toks[i+order])
starts = [tuple(toks[i:i+order]) for i in range(len(toks)-order)]
out = list(random.choice(starts))
while len(out) < N:
k = tuple(out[-order:]); nx = nxt.get(k)
out.append(random.choice(nx) if nx else random.choice(toks))
return out[:N]
def markov_per_section():
# contiguous section blocks in reading order; order-1 Markov within each block
out = [None]*N
blocks = [] # (indices_in_order) grouped by contiguous section runs
start = 0
for i in range(1, N+1):
if i == N or sec[i] != sec[start]:
blocks.append(list(range(start, i))); start = i
for idxs in blocks:
bt = [toks[i] for i in idxs]
nxt = defaultdict(list)
for a, b in zip(bt, bt[1:]): nxt[a].append(b)
gen = [random.choice(bt)]
while len(gen) < len(bt):
nx = nxt.get(gen[-1]); gen.append(random.choice(nx) if nx else random.choice(bt))
for i, v in zip(idxs, gen): out[i] = v
return out
def z(x, mu, sd): return (x - mu) / sd if sd else float("nan")
print(f"REAL clustering C = {C_real:.4f}\n")
print(f"{'comparison':<26}{'C':>9}{'z vs real':>12}")
for kind in ("full", "section", "folio"):
mu, sd = null_dist(kind)
print(f"{'null: '+kind+' shuffle':<26}{mu:>9.4f}{z(C_real, mu, sd):>12.1f}")
# generators: average a few samples
for name, fn in (("gen: global word-Markov", markov_global), ("gen: per-section Markov", markov_per_section)):
samp = [clustering(fn()) for _ in range(8)]
mu, sd = statistics.mean(samp), statistics.pstdev(samp)
print(f"{name:<26}{mu:>9.4f}{z(C_real, mu, sd):>12.1f}")
scripts/voynich_longrange_mi.py
#!/usr/bin/env python3
"""Milestone 3: long-range CHARACTER mutual information vs distance.
A second, independent long-range signal (Montemurro-Pury style). For glyphs
separated by distance d: I(d) = sum p(x,y;d) log2[ p(x,y;d) / (p(x)p(y)) ].
Natural language shows long-range correlations that persist (slow decay) well
above a local-Markov baseline. A purely local generator decays fast to the
shuffle floor. We compare:
- REAL (whole glyph stream, reading order, words joined by space)
- full shuffle (correlation floor / finite-size bias)
- char-Markov(3) (matched local structure; excess of REAL over this = long-range)
And the CONTROLLED version, counting only pairs inside the SAME folio, so a
real-vs-Markov excess can't be just page-to-page vocabulary drift.
"""
import math, random
from collections import Counter, defaultdict
from voynich_lib import parse
random.seed(7)
rows = parse()
# glyph stream with folio id per glyph; words joined by space
chars, fols = [], []
last_f = None
for folio, line, pos, L, Hh, I, tok in rows:
if chars: chars.append(" "); fols.append(folio) # space between tokens
for c in tok: chars.append(c); fols.append(folio)
N = len(chars)
DS = [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]
def mi(seq, d, same_folio=False, fol=None):
px = Counter(seq); tot = len(seq)
joint = Counter()
n = 0
for i in range(len(seq) - d):
if same_folio and fol[i] != fol[i+d]: continue
joint[(seq[i], seq[i+d])] += 1; n += 1
if n == 0: return 0.0
I = 0.0
for (x, y), c in joint.items():
pxy = c / n
I += pxy * math.log2(pxy / ((px[x]/tot) * (px[y]/tot)))
return I
def char_markov(order=3):
nxt = defaultdict(list)
for i in range(N - order):
nxt[tuple(chars[i:i+order])].append(chars[i+order])
out = list(chars[:order])
while len(out) < N:
k = tuple(out[-order:]); nx = nxt.get(k)
out.append(random.choice(nx) if nx else random.choice(chars))
return out[:N]
sh = chars[:]; random.shuffle(sh)
mk = char_markov(3)
print(f"glyph stream: {N} chars, alphabet {len(set(chars))}\n")
print(f"{'d':>5}{'REAL':>9}{'shuffle':>9}{'Markov3':>9}{'REAL|folio':>12}{'Mk3|folio':>11}")
for d in DS:
r = mi(chars, d)
s = mi(sh, d)
m = mi(mk, d)
rf = mi(chars, d, True, fols)
mf = mi(mk, d, True, fols)
print(f"{d:>5}{r:>9.4f}{s:>9.4f}{m:>9.4f}{rf:>12.4f}{mf:>11.4f}")
print("\nbits of mutual information. REAL>>Markov at large d => long-range structure beyond local rules.")
print("REAL|folio vs Mk3|folio isolates within-page long-range signal (controls page vocabulary drift).")
