From 2ba9293ba8fa982f315576e567ca4087bc887894 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 18:55:38 -0700 Subject: [PATCH 1/7] Add MuonClip polar quotient Gram baseline notebook --- .../notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb | 1 + 1 file changed, 1 insertion(+) create mode 100644 baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb diff --git a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb new file mode 100644 index 00000000..cbd6f30d --- /dev/null +++ b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":5,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11"}},"cells":[{"cell_type":"markdown","metadata":{},"source":["# MNIST MLP3 — MuonClip polar quotient Gram spectrum\n","\n","This notebook trains the repository's canonical `784 → 512 → 512 → 10` MLP3 on MNIST with MuonClip. At initialization and after every epoch it computes the local Jacobian of the polar quotient projection\n","\n","\\[\n","\\Pi(W)=UV^T,\\qquad J_W=D\\Pi(W),\\qquad G_W=J_W^T J_W.\n","\\]\n","\n","The spectrum is **not normalized or rebinned**. The raw positive eigenvalues of \\(G_W\\) are computed exactly from the singular values of each weight matrix and passed directly to WeightWatcher's own `WW_powerlaw.pl_fit`. The zero-mode multiplicity is recorded separately.\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["from pathlib import Path\n","import os, sys, math, random\n","import numpy as np\n","import pandas as pd\n","import torch\n","import torch.nn.functional as F\n","from torch.utils.data import DataLoader, Subset\n","from torchvision import datasets, transforms\n","import matplotlib.pyplot as plt\n","\n","ROOT = None\n","for p in [Path.cwd(), *Path.cwd().parents]:\n"," if (p / \"baseline\" / \"rg_baselines\").is_dir(): ROOT = (p / \"baseline\").resolve(); break\n"," if (p / \"rg_baselines\").is_dir(): ROOT = p.resolve(); break\n","if ROOT is None: raise RuntimeError(\"Run from CalculatedContent/rg_optimizers\")\n","sys.path.insert(0, str(ROOT))\n","from rg_baselines import MLP3, DEFAULT_BASELINE_SEEDS, MNIST_REFERENCE_SUITE_SLUG\n","from weightwatcher.WW_powerlaw import pl_fit\n","\n","DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\")\n","DATA_DIR = Path(os.environ.get(\"RG_BASELINE_DATA_DIR\", ROOT / \"data\")).expanduser().resolve()\n","RUN_ROOT = Path(os.environ.get(\"RG_BASELINE_RUN_ROOT\", ROOT / \"runs\")).expanduser().resolve() / MNIST_REFERENCE_SUITE_SLUG / \"muonclip_polar_quotient_gram\"\n","RUN_ROOT.mkdir(parents=True, exist_ok=True)\n","print(\"device:\", DEVICE)\n","print(\"output:\", RUN_ROOT)\n"]},{"cell_type":"markdown","metadata":{},"source":["## MuonClip and experiment configuration\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["EPOCHS = int(os.environ.get(\"MUONCLIP_POLAR_EPOCHS\", \"30\"))\n","BATCH_SIZE = int(os.environ.get(\"MUONCLIP_POLAR_BATCH\", \"256\"))\n","SEEDS = tuple(int(x) for x in os.environ.get(\"MUONCLIP_POLAR_SEEDS\", \",\".join(map(str, DEFAULT_BASELINE_SEEDS))).split(\",\"))\n","MATRIX_LR = float(os.environ.get(\"MUONCLIP_POLAR_LR\", \"2e-3\"))\n","AUX_LR = float(os.environ.get(\"MUONCLIP_POLAR_AUX_LR\", \"2e-3\"))\n","MOMENTUM = float(os.environ.get(\"MUONCLIP_POLAR_MOMENTUM\", \"0.95\"))\n","WEIGHT_DECAY = float(os.environ.get(\"MUONCLIP_POLAR_WEIGHT_DECAY\", \"1e-2\"))\n","RMS_SCALE = float(os.environ.get(\"MUONCLIP_POLAR_RMS_SCALE\", \"0.20\"))\n","GRAD_CLIP = float(os.environ.get(\"MUONCLIP_POLAR_GRAD_CLIP\", \"1.0\"))\n","NS_STEPS = int(os.environ.get(\"MUONCLIP_POLAR_NS_STEPS\", \"5\"))\n","\n","@torch.no_grad()\n","def zeropower(update, steps=5, eps=1e-7):\n"," transpose = update.shape[0] > update.shape[1]\n"," x = update.T if transpose else update\n"," x = x.float() / torch.linalg.vector_norm(x.float()).clamp_min(eps)\n"," a,b,c = 3.4445,-4.7750,2.0315\n"," for _ in range(steps):\n"," g = x @ x.T\n"," x = a*x + (b*g + c*(g@g)) @ x\n"," return x.T if transpose else x\n","\n","class MuonClip(torch.optim.Optimizer):\n"," def __init__(self, params):\n"," super().__init__(params, dict(lr=MATRIX_LR, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY))\n"," @torch.no_grad()\n"," def step(self):\n"," for group in self.param_groups:\n"," for p in group['params']:\n"," if p.grad is None: continue\n"," state = self.state[p]\n"," buf = state.setdefault('momentum_buffer', torch.zeros_like(p.grad))\n"," buf.mul_(group['momentum']).add_(p.grad)\n"," u = zeropower(buf, NS_STEPS).to(p.dtype)\n"," u.mul_(RMS_SCALE * math.sqrt(max(p.shape)))\n"," if group['weight_decay']:\n"," p.mul_(1.0 - group['lr'] * group['weight_decay'])\n"," p.add_(u, alpha=-group['lr'])\n"]},{"cell_type":"markdown","metadata":{},"source":["## Quotient projection Jacobian and Gram spectrum\n","\n","For a full-rank \\(m\\times n\\) matrix with singular values \\(\\sigma_1,\\ldots,\\sigma_r\\), \\(r=\\min(m,n)\\), the nonzero eigenvalues of \\(G_W=J_W^T J_W\\) are\n","\n","\\[\n","\\lambda_{ij}=\\frac{4}{(\\sigma_i+\\sigma_j)^2},\\quad i n: out += (E - U@(U.T@E)) @ V @ np.diag(1/s) @ V.T\n"," elif n > m: out += U @ np.diag(1/s) @ U.T @ E @ (np.eye(n) - V@V.T)\n"," return out\n","\n","def polar_gram_spectrum(W):\n"," W = np.asarray(W, dtype=np.float64); m,n = W.shape\n"," s = np.linalg.svd(W, compute_uv=False); r=min(m,n)\n"," rot = np.array([4.0/(s[i]+s[j])**2 for i in range(r) for j in range(i+1,r)], dtype=np.float64)\n"," trans = np.repeat(1.0/(s*s), abs(m-n)) if m != n else np.empty(0)\n"," positive = np.sort(np.concatenate([rot, trans]))\n"," return positive, int(m*n - positive.size)\n","\n","def fit_with_weightwatcher(evals):\n"," fit = pl_fit(data=np.asarray(evals, dtype=np.float64), xmin=None, xmax=None, verbose=False)\n"," return dict(alpha=float(fit.alpha), D=float(fit.D), xmin=float(fit.xmin), xmax=float(np.max(evals)), tail_evals=int(np.count_nonzero(evals >= fit.xmin)))\n","\n","# Small numerical check of D Pi(W)\n","rng=np.random.default_rng(123)\n","for shape in [(3,3),(4,3),(3,5)]:\n"," W=rng.normal(size=shape); E=rng.normal(size=shape); eps=1e-6\n"," fd=(polar_factor(W+eps*E)-polar_factor(W-eps*E))/(2*eps)\n"," an=polar_jacobian_action(W,E)\n"," err=np.linalg.norm(fd-an)/np.linalg.norm(fd)\n"," print(shape, 'Frechet relative error', err)\n"," assert err < 1e-7\n"]},{"cell_type":"markdown","metadata":{},"source":["## MNIST loaders\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])\n","full_train = datasets.MNIST(str(DATA_DIR), train=True, download=True, transform=transform)\n","test_set = datasets.MNIST(str(DATA_DIR), train=False, download=True, transform=transform)\n","g = torch.Generator().manual_seed(20_260_807)\n","perm = torch.randperm(len(full_train), generator=g).tolist()\n","val_idx, train_idx = perm[:5000], perm[5000:]\n","train_set, val_set = Subset(full_train, train_idx), Subset(full_train, val_idx)\n","\n","def loaders(seed):\n"," tg=torch.Generator().manual_seed(seed+101)\n"," common=dict(batch_size=BATCH_SIZE, num_workers=0 if DEVICE.type=='mps' else 0, pin_memory=DEVICE.type=='cuda')\n"," return (DataLoader(train_set, shuffle=True, generator=tg, **common), DataLoader(train_set, shuffle=False, **common), DataLoader(val_set, shuffle=False, **common), DataLoader(test_set, shuffle=False, **common))\n","\n","@torch.inference_mode()\n","def evaluate(model, loader):\n"," model.eval(); loss=0.0; correct=0; n=0\n"," for x,y in loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); z=model(x)\n"," loss += float(F.cross_entropy(z,y,reduction='sum').cpu()); correct += int((z.argmax(1)==y).sum().cpu()); n += y.numel()\n"," return loss/n, correct/n\n"]},{"cell_type":"markdown","metadata":{},"source":["## Train and fit the quotient-Gram spectrum with WeightWatcher\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def set_seed(seed):\n"," random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)\n"," if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)\n","\n","def analyze(model, seed, epoch):\n"," rows=[]; spectra={}\n"," for name in ('fc1','fc2','fc3'):\n"," W=getattr(model,name).weight.detach().cpu().numpy()\n"," evals, zeros = polar_gram_spectrum(W)\n"," fit=fit_with_weightwatcher(evals)\n"," rows.append(dict(seed=seed, epoch=epoch, layer=name, zero_modes=zeros, positive_modes=len(evals), **fit))\n"," spectra[f'seed_{seed}__epoch_{epoch:03d}__{name}']=evals\n"," return rows, spectra\n","\n","all_perf=[]; all_fits=[]; all_spectra={}\n","for seed in SEEDS:\n"," set_seed(seed); train_loader, train_eval, val_loader, test_loader = loaders(seed)\n"," model=MLP3().to(DEVICE)\n"," matrices=[p for p in model.parameters() if p.ndim==2]\n"," biases=[p for p in model.parameters() if p.ndim!=2]\n"," muon=MuonClip(matrices)\n"," aux=torch.optim.AdamW(biases, lr=AUX_LR, weight_decay=WEIGHT_DECAY)\n","\n"," rows,spec=analyze(model,seed,0); all_fits.extend(rows); all_spectra.update(spec)\n"," for epoch in range(1,EPOCHS+1):\n"," model.train()\n"," for x,y in train_loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); muon.zero_grad(set_to_none=True); aux.zero_grad(set_to_none=True)\n"," loss=F.cross_entropy(model(x),y); loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP); muon.step(); aux.step()\n"," tr_loss,tr_acc=evaluate(model,train_eval); va_loss,va_acc=evaluate(model,val_loader); te_loss,te_acc=evaluate(model,test_loader)\n"," all_perf.append(dict(seed=seed,epoch=epoch,train_loss=tr_loss,validation_loss=va_loss,test_loss=te_loss,train_accuracy=tr_acc,validation_accuracy=va_acc,test_accuracy=te_acc))\n"," rows,spec=analyze(model,seed,epoch); all_fits.extend(rows); all_spectra.update(spec)\n"," print(f'seed={seed} epoch={epoch}/{EPOCHS} val_loss={va_loss:.5f}')\n","\n","performance=pd.DataFrame(all_perf)\n","gram_fits=pd.DataFrame(all_fits)\n","performance.to_csv(RUN_ROOT/'performance_by_epoch_and_seed.csv',index=False)\n","gram_fits.to_csv(RUN_ROOT/'polar_quotient_gram_weightwatcher_by_epoch_layer_and_seed.csv',index=False)\n","np.savez_compressed(RUN_ROOT/'polar_quotient_gram_spectra.npz', **all_spectra)\n","display(gram_fits.tail(18))\n"]},{"cell_type":"markdown","metadata":{},"source":["## WeightWatcher alpha and fit quality\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["fig,ax=plt.subplots(figsize=(9,5))\n","for layer,frame in gram_fits.groupby('layer'):\n"," m=frame.groupby('epoch',as_index=False).alpha.mean(); ax.plot(m.epoch,m.alpha,label=layer)\n","ax.set(xlabel='Epoch',ylabel='WeightWatcher alpha',title=r'Raw spectrum of $G_W=J_W^T J_W$'); ax.grid(alpha=.25); ax.legend(frameon=False); plt.show()\n","\n","fig,ax=plt.subplots(figsize=(9,5))\n","for layer,frame in gram_fits.groupby('layer'):\n"," m=frame.groupby('epoch',as_index=False).D.mean(); ax.plot(m.epoch,m.D,label=layer)\n","ax.set(xlabel='Epoch',ylabel='WeightWatcher KS D',title=r'WeightWatcher fit quality for $G_W$'); ax.grid(alpha=.25); ax.legend(frameon=False); plt.show()\n"]},{"cell_type":"markdown","metadata":{},"source":["## Artifacts\n","\n","The principal output is `polar_quotient_gram_weightwatcher_by_epoch_layer_and_seed.csv`. It contains, for every layer and checkpoint, the raw positive-mode count, zero-mode count, and WeightWatcher `alpha`, `D`, `xmin`, `xmax`, and fitted-tail size. `polar_quotient_gram_spectra.npz` contains the exact unnormalized positive eigenvalues used in those fits.\n"]}]} \ No newline at end of file From 44a89cdbb77e6bc92ccc86133440d47b5c875c24 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 20:13:47 -0700 Subject: [PATCH 2/7] Add polar Jacobian analytic and numerical helpers --- baseline/rg_baselines/polar_jacobian.py | 108 ++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 baseline/rg_baselines/polar_jacobian.py diff --git a/baseline/rg_baselines/polar_jacobian.py b/baseline/rg_baselines/polar_jacobian.py new file mode 100644 index 00000000..932798e5 --- /dev/null +++ b/baseline/rg_baselines/polar_jacobian.py @@ -0,0 +1,108 @@ +"""Polar-projection Jacobian spectra for single matrix checkpoints.""" +from __future__ import annotations +import numpy as np + + +def polar_factor(weight: np.ndarray) -> np.ndarray: + w = np.asarray(weight, dtype=np.float64) + u, _, vh = np.linalg.svd(w, full_matrices=False) + return u @ vh + + +def frechet_action(weight: np.ndarray, perturbation: np.ndarray) -> np.ndarray: + """Closed-form D Pi(W)[E] for Pi(W)=U V^T at full-rank W.""" + w = np.asarray(weight, dtype=np.float64) + e = np.asarray(perturbation, dtype=np.float64) + u, s, vh = np.linalg.svd(w, full_matrices=False) + v = vh.T + a = u.T @ e @ v + omega = (a - a.T) / (s[:, None] + s[None, :]) + np.fill_diagonal(omega, 0.0) + out = u @ omega @ v.T + m, n = w.shape + if m > n: + out += (e - u @ (u.T @ e)) @ v @ np.diag(1.0 / s) @ v.T + elif n > m: + out += u @ np.diag(1.0 / s) @ u.T @ e @ (np.eye(n) - v @ v.T) + return out + + +def analytic_gram_spectrum(weight: np.ndarray) -> tuple[np.ndarray, int]: + """Exact positive spectrum of (D Pi)^* D Pi and zero-mode count.""" + w = np.asarray(weight, dtype=np.float64) + m, n = w.shape + s = np.linalg.svd(w, compute_uv=False) + r = min(m, n) + rot = np.fromiter( + (4.0 / (s[i] + s[j]) ** 2 for i in range(r) for j in range(i + 1, r)), + dtype=np.float64, + count=r * (r - 1) // 2, + ) + trans = np.repeat(1.0 / (s * s), abs(m - n)) if m != n else np.empty(0) + positive = np.sort(np.concatenate([rot, trans])) + return positive, int(m * n - positive.size) + + +def numerical_probe_gram_spectrum( + weight: np.ndarray, + *, + probes: int = 128, + eps_rel: float = 1e-5, + seed: int = 918273, +) -> tuple[np.ndarray, dict[str, float]]: + """Finite-difference single-checkpoint response Gram X^T X spectrum.""" + w = np.asarray(weight, dtype=np.float64) + rng = np.random.default_rng(int(seed)) + step = float(eps_rel) * max(1.0, float(np.linalg.norm(w, "fro"))) + x = np.empty((w.size, int(probes)), dtype=np.float64) + for k in range(int(probes)): + e = rng.normal(size=w.shape) + e /= np.linalg.norm(e, "fro") + response = (polar_factor(w + step * e) - polar_factor(w - step * e)) / (2.0 * step) + x[:, k] = response.reshape(-1) + gram = x.T @ x + evals = np.linalg.eigvalsh(gram) + tol = np.finfo(float).eps * max(gram.shape) * max(float(evals[-1]), 1.0) + return np.sort(evals[evals > tol]), { + "probes": int(probes), + "epsilon": float(step), + "epsilon_relative": float(eps_rel), + } + + +def weightwatcher_pl_fit(evals: np.ndarray) -> dict[str, float]: + """Use WeightWatcher's own PL fitter on raw positive eigenvalues.""" + from weightwatcher.WW_powerlaw import pl_fit + values = np.asarray(evals, dtype=np.float64) + values = values[np.isfinite(values) & (values > 0)] + fit = pl_fit(data=values, xmin=None, xmax=None, verbose=False) + return { + "alpha": float(fit.alpha), + "D": float(fit.D), + "xmin": float(fit.xmin), + "xmax": float(np.max(values)), + "tail_evals": int(np.count_nonzero(values >= fit.xmin)), + } + + +def finite_difference_error(weight: np.ndarray, perturbation: np.ndarray, eps_rel: float = 1e-6) -> float: + w = np.asarray(weight, dtype=np.float64) + e = np.asarray(perturbation, dtype=np.float64) + step = float(eps_rel) * max(1.0, float(np.linalg.norm(w, "fro"))) + fd = (polar_factor(w + step * e) - polar_factor(w - step * e)) / (2.0 * step) + exact = frechet_action(w, e) + return float(np.linalg.norm(fd - exact) / max(np.linalg.norm(fd), np.finfo(float).tiny)) + + +def probe_convergence(weight: np.ndarray, counts=(16, 32, 64, 128, 256), *, eps_rel=1e-5, seed=918273): + """Return analytic and finite-probe WeightWatcher fits for one checkpoint.""" + analytic, _ = analytic_gram_spectrum(weight) + reference = weightwatcher_pl_fit(analytic) + rows = [{"method": "analytic", "probes": np.nan, **reference}] + for count in counts: + sampled, _ = numerical_probe_gram_spectrum(weight, probes=int(count), eps_rel=eps_rel, seed=seed) + rows.append({"method": "numerical_finite_difference", "probes": int(count), **weightwatcher_pl_fit(sampled)}) + for row in rows: + row["alpha_analytic"] = reference["alpha"] + row["alpha_difference"] = row["alpha"] - reference["alpha"] + return rows From 92c90ccee80e07f39571c603a22957aa69e4e720 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 20:14:21 -0700 Subject: [PATCH 3/7] Compare analytic and numerical polar Jacobian spectra --- .../notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb index cbd6f30d..2d587c36 100644 --- a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb +++ b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb @@ -1 +1 @@ -{"nbformat":4,"nbformat_minor":5,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11"}},"cells":[{"cell_type":"markdown","metadata":{},"source":["# MNIST MLP3 — MuonClip polar quotient Gram spectrum\n","\n","This notebook trains the repository's canonical `784 → 512 → 512 → 10` MLP3 on MNIST with MuonClip. At initialization and after every epoch it computes the local Jacobian of the polar quotient projection\n","\n","\\[\n","\\Pi(W)=UV^T,\\qquad J_W=D\\Pi(W),\\qquad G_W=J_W^T J_W.\n","\\]\n","\n","The spectrum is **not normalized or rebinned**. The raw positive eigenvalues of \\(G_W\\) are computed exactly from the singular values of each weight matrix and passed directly to WeightWatcher's own `WW_powerlaw.pl_fit`. The zero-mode multiplicity is recorded separately.\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["from pathlib import Path\n","import os, sys, math, random\n","import numpy as np\n","import pandas as pd\n","import torch\n","import torch.nn.functional as F\n","from torch.utils.data import DataLoader, Subset\n","from torchvision import datasets, transforms\n","import matplotlib.pyplot as plt\n","\n","ROOT = None\n","for p in [Path.cwd(), *Path.cwd().parents]:\n"," if (p / \"baseline\" / \"rg_baselines\").is_dir(): ROOT = (p / \"baseline\").resolve(); break\n"," if (p / \"rg_baselines\").is_dir(): ROOT = p.resolve(); break\n","if ROOT is None: raise RuntimeError(\"Run from CalculatedContent/rg_optimizers\")\n","sys.path.insert(0, str(ROOT))\n","from rg_baselines import MLP3, DEFAULT_BASELINE_SEEDS, MNIST_REFERENCE_SUITE_SLUG\n","from weightwatcher.WW_powerlaw import pl_fit\n","\n","DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\")\n","DATA_DIR = Path(os.environ.get(\"RG_BASELINE_DATA_DIR\", ROOT / \"data\")).expanduser().resolve()\n","RUN_ROOT = Path(os.environ.get(\"RG_BASELINE_RUN_ROOT\", ROOT / \"runs\")).expanduser().resolve() / MNIST_REFERENCE_SUITE_SLUG / \"muonclip_polar_quotient_gram\"\n","RUN_ROOT.mkdir(parents=True, exist_ok=True)\n","print(\"device:\", DEVICE)\n","print(\"output:\", RUN_ROOT)\n"]},{"cell_type":"markdown","metadata":{},"source":["## MuonClip and experiment configuration\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["EPOCHS = int(os.environ.get(\"MUONCLIP_POLAR_EPOCHS\", \"30\"))\n","BATCH_SIZE = int(os.environ.get(\"MUONCLIP_POLAR_BATCH\", \"256\"))\n","SEEDS = tuple(int(x) for x in os.environ.get(\"MUONCLIP_POLAR_SEEDS\", \",\".join(map(str, DEFAULT_BASELINE_SEEDS))).split(\",\"))\n","MATRIX_LR = float(os.environ.get(\"MUONCLIP_POLAR_LR\", \"2e-3\"))\n","AUX_LR = float(os.environ.get(\"MUONCLIP_POLAR_AUX_LR\", \"2e-3\"))\n","MOMENTUM = float(os.environ.get(\"MUONCLIP_POLAR_MOMENTUM\", \"0.95\"))\n","WEIGHT_DECAY = float(os.environ.get(\"MUONCLIP_POLAR_WEIGHT_DECAY\", \"1e-2\"))\n","RMS_SCALE = float(os.environ.get(\"MUONCLIP_POLAR_RMS_SCALE\", \"0.20\"))\n","GRAD_CLIP = float(os.environ.get(\"MUONCLIP_POLAR_GRAD_CLIP\", \"1.0\"))\n","NS_STEPS = int(os.environ.get(\"MUONCLIP_POLAR_NS_STEPS\", \"5\"))\n","\n","@torch.no_grad()\n","def zeropower(update, steps=5, eps=1e-7):\n"," transpose = update.shape[0] > update.shape[1]\n"," x = update.T if transpose else update\n"," x = x.float() / torch.linalg.vector_norm(x.float()).clamp_min(eps)\n"," a,b,c = 3.4445,-4.7750,2.0315\n"," for _ in range(steps):\n"," g = x @ x.T\n"," x = a*x + (b*g + c*(g@g)) @ x\n"," return x.T if transpose else x\n","\n","class MuonClip(torch.optim.Optimizer):\n"," def __init__(self, params):\n"," super().__init__(params, dict(lr=MATRIX_LR, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY))\n"," @torch.no_grad()\n"," def step(self):\n"," for group in self.param_groups:\n"," for p in group['params']:\n"," if p.grad is None: continue\n"," state = self.state[p]\n"," buf = state.setdefault('momentum_buffer', torch.zeros_like(p.grad))\n"," buf.mul_(group['momentum']).add_(p.grad)\n"," u = zeropower(buf, NS_STEPS).to(p.dtype)\n"," u.mul_(RMS_SCALE * math.sqrt(max(p.shape)))\n"," if group['weight_decay']:\n"," p.mul_(1.0 - group['lr'] * group['weight_decay'])\n"," p.add_(u, alpha=-group['lr'])\n"]},{"cell_type":"markdown","metadata":{},"source":["## Quotient projection Jacobian and Gram spectrum\n","\n","For a full-rank \\(m\\times n\\) matrix with singular values \\(\\sigma_1,\\ldots,\\sigma_r\\), \\(r=\\min(m,n)\\), the nonzero eigenvalues of \\(G_W=J_W^T J_W\\) are\n","\n","\\[\n","\\lambda_{ij}=\\frac{4}{(\\sigma_i+\\sigma_j)^2},\\quad i n: out += (E - U@(U.T@E)) @ V @ np.diag(1/s) @ V.T\n"," elif n > m: out += U @ np.diag(1/s) @ U.T @ E @ (np.eye(n) - V@V.T)\n"," return out\n","\n","def polar_gram_spectrum(W):\n"," W = np.asarray(W, dtype=np.float64); m,n = W.shape\n"," s = np.linalg.svd(W, compute_uv=False); r=min(m,n)\n"," rot = np.array([4.0/(s[i]+s[j])**2 for i in range(r) for j in range(i+1,r)], dtype=np.float64)\n"," trans = np.repeat(1.0/(s*s), abs(m-n)) if m != n else np.empty(0)\n"," positive = np.sort(np.concatenate([rot, trans]))\n"," return positive, int(m*n - positive.size)\n","\n","def fit_with_weightwatcher(evals):\n"," fit = pl_fit(data=np.asarray(evals, dtype=np.float64), xmin=None, xmax=None, verbose=False)\n"," return dict(alpha=float(fit.alpha), D=float(fit.D), xmin=float(fit.xmin), xmax=float(np.max(evals)), tail_evals=int(np.count_nonzero(evals >= fit.xmin)))\n","\n","# Small numerical check of D Pi(W)\n","rng=np.random.default_rng(123)\n","for shape in [(3,3),(4,3),(3,5)]:\n"," W=rng.normal(size=shape); E=rng.normal(size=shape); eps=1e-6\n"," fd=(polar_factor(W+eps*E)-polar_factor(W-eps*E))/(2*eps)\n"," an=polar_jacobian_action(W,E)\n"," err=np.linalg.norm(fd-an)/np.linalg.norm(fd)\n"," print(shape, 'Frechet relative error', err)\n"," assert err < 1e-7\n"]},{"cell_type":"markdown","metadata":{},"source":["## MNIST loaders\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])\n","full_train = datasets.MNIST(str(DATA_DIR), train=True, download=True, transform=transform)\n","test_set = datasets.MNIST(str(DATA_DIR), train=False, download=True, transform=transform)\n","g = torch.Generator().manual_seed(20_260_807)\n","perm = torch.randperm(len(full_train), generator=g).tolist()\n","val_idx, train_idx = perm[:5000], perm[5000:]\n","train_set, val_set = Subset(full_train, train_idx), Subset(full_train, val_idx)\n","\n","def loaders(seed):\n"," tg=torch.Generator().manual_seed(seed+101)\n"," common=dict(batch_size=BATCH_SIZE, num_workers=0 if DEVICE.type=='mps' else 0, pin_memory=DEVICE.type=='cuda')\n"," return (DataLoader(train_set, shuffle=True, generator=tg, **common), DataLoader(train_set, shuffle=False, **common), DataLoader(val_set, shuffle=False, **common), DataLoader(test_set, shuffle=False, **common))\n","\n","@torch.inference_mode()\n","def evaluate(model, loader):\n"," model.eval(); loss=0.0; correct=0; n=0\n"," for x,y in loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); z=model(x)\n"," loss += float(F.cross_entropy(z,y,reduction='sum').cpu()); correct += int((z.argmax(1)==y).sum().cpu()); n += y.numel()\n"," return loss/n, correct/n\n"]},{"cell_type":"markdown","metadata":{},"source":["## Train and fit the quotient-Gram spectrum with WeightWatcher\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def set_seed(seed):\n"," random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)\n"," if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)\n","\n","def analyze(model, seed, epoch):\n"," rows=[]; spectra={}\n"," for name in ('fc1','fc2','fc3'):\n"," W=getattr(model,name).weight.detach().cpu().numpy()\n"," evals, zeros = polar_gram_spectrum(W)\n"," fit=fit_with_weightwatcher(evals)\n"," rows.append(dict(seed=seed, epoch=epoch, layer=name, zero_modes=zeros, positive_modes=len(evals), **fit))\n"," spectra[f'seed_{seed}__epoch_{epoch:03d}__{name}']=evals\n"," return rows, spectra\n","\n","all_perf=[]; all_fits=[]; all_spectra={}\n","for seed in SEEDS:\n"," set_seed(seed); train_loader, train_eval, val_loader, test_loader = loaders(seed)\n"," model=MLP3().to(DEVICE)\n"," matrices=[p for p in model.parameters() if p.ndim==2]\n"," biases=[p for p in model.parameters() if p.ndim!=2]\n"," muon=MuonClip(matrices)\n"," aux=torch.optim.AdamW(biases, lr=AUX_LR, weight_decay=WEIGHT_DECAY)\n","\n"," rows,spec=analyze(model,seed,0); all_fits.extend(rows); all_spectra.update(spec)\n"," for epoch in range(1,EPOCHS+1):\n"," model.train()\n"," for x,y in train_loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); muon.zero_grad(set_to_none=True); aux.zero_grad(set_to_none=True)\n"," loss=F.cross_entropy(model(x),y); loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP); muon.step(); aux.step()\n"," tr_loss,tr_acc=evaluate(model,train_eval); va_loss,va_acc=evaluate(model,val_loader); te_loss,te_acc=evaluate(model,test_loader)\n"," all_perf.append(dict(seed=seed,epoch=epoch,train_loss=tr_loss,validation_loss=va_loss,test_loss=te_loss,train_accuracy=tr_acc,validation_accuracy=va_acc,test_accuracy=te_acc))\n"," rows,spec=analyze(model,seed,epoch); all_fits.extend(rows); all_spectra.update(spec)\n"," print(f'seed={seed} epoch={epoch}/{EPOCHS} val_loss={va_loss:.5f}')\n","\n","performance=pd.DataFrame(all_perf)\n","gram_fits=pd.DataFrame(all_fits)\n","performance.to_csv(RUN_ROOT/'performance_by_epoch_and_seed.csv',index=False)\n","gram_fits.to_csv(RUN_ROOT/'polar_quotient_gram_weightwatcher_by_epoch_layer_and_seed.csv',index=False)\n","np.savez_compressed(RUN_ROOT/'polar_quotient_gram_spectra.npz', **all_spectra)\n","display(gram_fits.tail(18))\n"]},{"cell_type":"markdown","metadata":{},"source":["## WeightWatcher alpha and fit quality\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["fig,ax=plt.subplots(figsize=(9,5))\n","for layer,frame in gram_fits.groupby('layer'):\n"," m=frame.groupby('epoch',as_index=False).alpha.mean(); ax.plot(m.epoch,m.alpha,label=layer)\n","ax.set(xlabel='Epoch',ylabel='WeightWatcher alpha',title=r'Raw spectrum of $G_W=J_W^T J_W$'); ax.grid(alpha=.25); ax.legend(frameon=False); plt.show()\n","\n","fig,ax=plt.subplots(figsize=(9,5))\n","for layer,frame in gram_fits.groupby('layer'):\n"," m=frame.groupby('epoch',as_index=False).D.mean(); ax.plot(m.epoch,m.D,label=layer)\n","ax.set(xlabel='Epoch',ylabel='WeightWatcher KS D',title=r'WeightWatcher fit quality for $G_W$'); ax.grid(alpha=.25); ax.legend(frameon=False); plt.show()\n"]},{"cell_type":"markdown","metadata":{},"source":["## Artifacts\n","\n","The principal output is `polar_quotient_gram_weightwatcher_by_epoch_layer_and_seed.csv`. It contains, for every layer and checkpoint, the raw positive-mode count, zero-mode count, and WeightWatcher `alpha`, `D`, `xmin`, `xmax`, and fitted-tail size. `polar_quotient_gram_spectra.npz` contains the exact unnormalized positive eigenvalues used in those fits.\n"]}]} \ No newline at end of file +{"nbformat":4,"nbformat_minor":5,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11"}},"cells":[{"cell_type":"markdown","metadata":{},"source":["# MNIST MLP3 — MuonClip polar Jacobian: analytic vs numerical WeightWatcher fits\n","\n","For each matrix checkpoint \\(W=U\\Sigma V^\\top\\), define the polar projection \\(\\Pi(W)=UV^\\top\\), its Fréchet derivative \\(J_W=D\\Pi(W)\\), and the Gram operator \\(G_W=J_W^\\ast J_W\\). This notebook compares two spectra from the **same single checkpoint** and sends both raw positive spectra to WeightWatcher's own `WW_powerlaw.pl_fit`.\n","\n","**Analytic derivative.** With \\(A=U^\\top E V\\),\n","\\[\n","D\\Pi_W[E]=U\\Omega V^\\top + D\\Pi_W[E]_\\perp,\n","\\qquad\n","\\Omega_{ij}=\\frac{A_{ij}-A_{ji}}{\\sigma_i+\\sigma_j},\\;\\Omega_{ii}=0.\n","\\]\n","For \\(m>n\\), \\(D\\Pi_W[E]_\\perp=(I-UU^\\top)EV\\Sigma^{-1}V^\\top\\); for \\(n>m\\), \\(D\\Pi_W[E]_\\perp=U\\Sigma^{-1}U^\\top E(I-VV^\\top)\\). Therefore the nonzero eigenvalues of \\(G_W\\) are\n","\\[\n","\\boxed{\\lambda_{ij}^{\\rm rot}=4/(\\sigma_i+\\sigma_j)^2},\\quad ig.shape[1]; x=g.T if t else g; x=x.float()/torch.linalg.vector_norm(x.float()).clamp_min(eps); a,b,c=3.4445,-4.7750,2.0315\n"," for _ in range(steps): q=x@x.T; x=a*x+(b*q+c*(q@q))@x\n"," return x.T if t else x\n","class MuonClip(torch.optim.Optimizer):\n"," def __init__(self,params): super().__init__(params,dict(lr=MATRIX_LR,momentum=MOMENTUM,weight_decay=WEIGHT_DECAY))\n"," @torch.no_grad()\n"," def step(self):\n"," for group in self.param_groups:\n"," for p in group['params']:\n"," if p.grad is None: continue\n"," b=self.state[p].setdefault('momentum_buffer',torch.zeros_like(p.grad)); b.mul_(group['momentum']).add_(p.grad)\n"," u=zeropower(b).to(p.dtype); u.mul_(RMS_SCALE*math.sqrt(max(p.shape))); p.mul_(1-group['lr']*group['weight_decay']); p.add_(u,alpha=-group['lr'])\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Verify the analytic Fréchet formula numerically on small rectangular matrices.\n","rng=np.random.default_rng(123)\n","for shape in [(3,3),(4,3),(3,5)]:\n"," W=rng.normal(size=shape); E=rng.normal(size=shape); err=finite_difference_error(W,E)\n"," print(shape,err); assert err<1e-7\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])\n","full=datasets.MNIST(str(DATA_DIR),train=True,download=True,transform=transform); test=datasets.MNIST(str(DATA_DIR),train=False,download=True,transform=transform)\n","g=torch.Generator().manual_seed(20_260_807); perm=torch.randperm(len(full),generator=g).tolist(); val_idx,train_idx=perm[:5000],perm[5000:]; train,val=Subset(full,train_idx),Subset(full,val_idx)\n","def loaders(seed):\n"," tg=torch.Generator().manual_seed(seed+101); kw=dict(batch_size=BATCH,num_workers=0,pin_memory=DEVICE.type=='cuda')\n"," return DataLoader(train,shuffle=True,generator=tg,**kw),DataLoader(train,shuffle=False,**kw),DataLoader(val,shuffle=False,**kw),DataLoader(test,shuffle=False,**kw)\n","@torch.inference_mode()\n","def evaluate(model,loader):\n"," model.eval(); loss=correct=n=0\n"," for x,y in loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); z=model(x); loss+=float(F.cross_entropy(z,y,reduction='sum').cpu()); correct+=int((z.argmax(1)==y).sum().cpu()); n+=y.numel()\n"," return loss/n,correct/n\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def seed_all(seed): random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)\n","def numerical_due(epoch): return epoch in {0,EPOCHS} or (NUMERIC_EVERY>0 and epoch%NUMERIC_EVERY==0)\n","def analyze(model,seed,epoch):\n"," rows=[]; spectra={}\n"," for j,name in enumerate(('fc1','fc2','fc3')):\n"," W=getattr(model,name).weight.detach().cpu().numpy(); a,z=analytic_gram_spectrum(W); af=weightwatcher_pl_fit(a)\n"," rows.append(dict(seed=seed,epoch=epoch,layer=name,method='analytic',zero_modes=z,positive_modes=len(a),probes=np.nan,epsilon=np.nan,**af)); spectra[f'analytic__seed_{seed}__epoch_{epoch:03d}__{name}']=a\n"," if numerical_due(epoch):\n"," n,meta=numerical_probe_gram_spectrum(W,probes=PROBES,eps_rel=EPS,seed=918273+100000*seed+100*epoch+j); nf=weightwatcher_pl_fit(n)\n"," rows.append(dict(seed=seed,epoch=epoch,layer=name,method='numerical_finite_difference',zero_modes=np.nan,positive_modes=len(n),probes=PROBES,epsilon=meta['epsilon'],**nf)); spectra[f'numeric__seed_{seed}__epoch_{epoch:03d}__{name}']=n\n"," return rows,spectra\n","\n","perf=[]; fits=[]; spectra={}\n","for seed in SEEDS:\n"," seed_all(seed); tr,tre,va,te=loaders(seed); model=MLP3().to(DEVICE); matrices=[p for p in model.parameters() if p.ndim==2]; biases=[p for p in model.parameters() if p.ndim!=2]; muon=MuonClip(matrices); aux=torch.optim.AdamW(biases,lr=AUX_LR,weight_decay=WEIGHT_DECAY)\n"," r,s=analyze(model,seed,0); fits+=r; spectra.update(s)\n"," for epoch in range(1,EPOCHS+1):\n"," model.train()\n"," for x,y in tr:\n"," x,y=x.to(DEVICE),y.to(DEVICE); muon.zero_grad(set_to_none=True); aux.zero_grad(set_to_none=True); loss=F.cross_entropy(model(x),y); loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP); muon.step(); aux.step()\n"," tl,ta=evaluate(model,tre); vl,vaa=evaluate(model,va); ql,qa=evaluate(model,te); perf.append(dict(seed=seed,epoch=epoch,train_loss=tl,validation_loss=vl,test_loss=ql,train_accuracy=ta,validation_accuracy=vaa,test_accuracy=qa)); r,s=analyze(model,seed,epoch); fits+=r; spectra.update(s); print(seed,epoch,vl)\n","performance=pd.DataFrame(perf); gram_fits=pd.DataFrame(fits); performance.to_csv(RUN_ROOT/'performance_by_epoch_and_seed.csv',index=False); gram_fits.to_csv(RUN_ROOT/'polar_jacobian_gram_weightwatcher_analytic_vs_numeric.csv',index=False); np.savez_compressed(RUN_ROOT/'polar_jacobian_gram_spectra_analytic_vs_numeric.npz',**spectra)\n"]},{"cell_type":"markdown","metadata":{},"source":["## WeightWatcher fit comparison\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["a=gram_fits[gram_fits.method.eq('analytic')]; n=gram_fits[gram_fits.method.eq('numerical_finite_difference')]\n","cmp=n.merge(a,on=['seed','epoch','layer'],suffixes=('_numeric','_analytic'),validate='one_to_one'); cmp['alpha_difference']=cmp.alpha_numeric-cmp.alpha_analytic; cmp['alpha_abs_difference']=cmp.alpha_difference.abs(); cmp.to_csv(RUN_ROOT/'polar_jacobian_weightwatcher_fit_comparison.csv',index=False)\n","display(cmp[['seed','epoch','layer','probes_numeric','alpha_analytic','alpha_numeric','alpha_difference','D_analytic','D_numeric','xmin_analytic','xmin_numeric','tail_evals_analytic','tail_evals_numeric']])\n","if not cmp.empty:\n"," fig,ax=plt.subplots(figsize=(7,6))\n"," for layer,f in cmp.groupby('layer'): ax.scatter(f.alpha_analytic,f.alpha_numeric,label=layer)\n"," vals=np.r_[cmp.alpha_analytic,cmp.alpha_numeric]; lo,hi=np.nanmin(vals),np.nanmax(vals); ax.plot([lo,hi],[lo,hi],'--'); ax.set(xlabel='analytic WW alpha',ylabel='numerical WW alpha',title='Polar Jacobian Gram PL fit'); ax.grid(alpha=.25); ax.legend(frameon=False); plt.show()\n"]},{"cell_type":"markdown","metadata":{},"source":["## Single-checkpoint probe convergence\n","\n","For any selected checkpoint matrix `W`, use the helper below to compare the analytic WeightWatcher fit against numerical finite-difference fits with increasing probe count.\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Example:\n","# W=model.fc2.weight.detach().cpu().numpy()\n","# display(pd.DataFrame(probe_convergence(W,counts=(16,32,64,128,256),eps_rel=EPS)))\n"]},{"cell_type":"markdown","metadata":{},"source":["## Outputs\n","\n","`polar_jacobian_gram_weightwatcher_analytic_vs_numeric.csv` contains both methods' WeightWatcher fits. `polar_jacobian_gram_spectra_analytic_vs_numeric.npz` contains the raw positive eigenvalues actually passed to WeightWatcher. `polar_jacobian_weightwatcher_fit_comparison.csv` places analytic and numerical `alpha`, `D`, `xmin`, and tail counts side by side.\n"]}]} \ No newline at end of file From 7dadcb42f0d44f9894b764472be7f4d607c81142 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 22:26:10 -0700 Subject: [PATCH 4/7] Add raw ESD and MLE KS fitting --- baseline/rg_baselines/polar_jacobian.py | 55 +++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/baseline/rg_baselines/polar_jacobian.py b/baseline/rg_baselines/polar_jacobian.py index 932798e5..c308e6e9 100644 --- a/baseline/rg_baselines/polar_jacobian.py +++ b/baseline/rg_baselines/polar_jacobian.py @@ -27,6 +27,12 @@ def frechet_action(weight: np.ndarray, perturbation: np.ndarray) -> np.ndarray: return out +def raw_weight_esd(weight: np.ndarray) -> np.ndarray: + """Raw nonzero eigenvalues of W^T W, i.e. squared singular values.""" + s = np.linalg.svd(np.asarray(weight, dtype=np.float64), compute_uv=False) + return np.sort(s * s) + + def analytic_gram_spectrum(weight: np.ndarray) -> tuple[np.ndarray, int]: """Exact positive spectrum of (D Pi)^* D Pi and zero-mode count.""" w = np.asarray(weight, dtype=np.float64) @@ -70,6 +76,47 @@ def numerical_probe_gram_spectrum( } +def powerlaw_mle_grid_ks(evals: np.ndarray, *, min_tail: int = 5) -> dict[str, float]: + """Continuous Pareto MLE with an x_min grid search minimizing KS distance. + + For every candidate x_min leaving at least ``min_tail`` observations, + alpha = 1 + n / sum(log(x/x_min)). The selected candidate minimizes the + Kolmogorov-Smirnov distance between the empirical tail CDF and the fitted + continuous Pareto CDF. + """ + values = np.sort(np.asarray(evals, dtype=np.float64)) + values = values[np.isfinite(values) & (values > 0)] + if values.size < max(2, int(min_tail)): + return {"alpha": np.nan, "D": np.nan, "xmin": np.nan, "xmax": np.nan, "tail_evals": 0} + + best = None + for start in range(0, values.size - int(min_tail) + 1): + xmin = float(values[start]) + tail = values[start:] + n = int(tail.size) + denom = float(np.sum(np.log(tail / xmin))) + if not np.isfinite(denom) or denom <= 0: + continue + alpha = 1.0 + n / denom + empirical = np.arange(1, n + 1, dtype=np.float64) / n + theoretical = 1.0 - np.power(tail / xmin, 1.0 - alpha) + D = float(np.max(np.abs(empirical - theoretical))) + candidate = (D, start, alpha, xmin, n) + if best is None or candidate[0] < best[0]: + best = candidate + + if best is None: + return {"alpha": np.nan, "D": np.nan, "xmin": np.nan, "xmax": float(np.max(values)), "tail_evals": 0} + D, _, alpha, xmin, n = best + return { + "alpha": float(alpha), + "D": float(D), + "xmin": float(xmin), + "xmax": float(np.max(values)), + "tail_evals": int(n), + } + + def weightwatcher_pl_fit(evals: np.ndarray) -> dict[str, float]: """Use WeightWatcher's own PL fitter on raw positive eigenvalues.""" from weightwatcher.WW_powerlaw import pl_fit @@ -94,14 +141,14 @@ def finite_difference_error(weight: np.ndarray, perturbation: np.ndarray, eps_re return float(np.linalg.norm(fd - exact) / max(np.linalg.norm(fd), np.finfo(float).tiny)) -def probe_convergence(weight: np.ndarray, counts=(16, 32, 64, 128, 256), *, eps_rel=1e-5, seed=918273): - """Return analytic and finite-probe WeightWatcher fits for one checkpoint.""" +def probe_convergence(weight: np.ndarray, counts=(16, 32, 64, 128, 256), *, eps_rel=1e-5, seed=918273, min_tail=5): + """Return analytic and finite-probe MLE/KS fits for one checkpoint.""" analytic, _ = analytic_gram_spectrum(weight) - reference = weightwatcher_pl_fit(analytic) + reference = powerlaw_mle_grid_ks(analytic, min_tail=min_tail) rows = [{"method": "analytic", "probes": np.nan, **reference}] for count in counts: sampled, _ = numerical_probe_gram_spectrum(weight, probes=int(count), eps_rel=eps_rel, seed=seed) - rows.append({"method": "numerical_finite_difference", "probes": int(count), **weightwatcher_pl_fit(sampled)}) + rows.append({"method": "numerical_finite_difference", "probes": int(count), **powerlaw_mle_grid_ks(sampled, min_tail=min_tail)}) for row in rows: row["alpha_analytic"] = reference["alpha"] row["alpha_difference"] = row["alpha"] - reference["alpha"] From c946bf7dfc8916c25e9f626bcd28391f3c522c22 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 22:27:01 -0700 Subject: [PATCH 5/7] temporary --- .../notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp | 1 + 1 file changed, 1 insertion(+) create mode 100644 baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp diff --git a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp new file mode 100644 index 00000000..b3a42524 --- /dev/null +++ b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp @@ -0,0 +1 @@ +placeholder \ No newline at end of file From 4aceb391f22127583301d20dff7c28187c7b5673 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 22:27:33 -0700 Subject: [PATCH 6/7] Compare raw and polar spectra with MLE KS fits --- .../notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb index 2d587c36..df24a971 100644 --- a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb +++ b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.ipynb @@ -1 +1 @@ -{"nbformat":4,"nbformat_minor":5,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11"}},"cells":[{"cell_type":"markdown","metadata":{},"source":["# MNIST MLP3 — MuonClip polar Jacobian: analytic vs numerical WeightWatcher fits\n","\n","For each matrix checkpoint \\(W=U\\Sigma V^\\top\\), define the polar projection \\(\\Pi(W)=UV^\\top\\), its Fréchet derivative \\(J_W=D\\Pi(W)\\), and the Gram operator \\(G_W=J_W^\\ast J_W\\). This notebook compares two spectra from the **same single checkpoint** and sends both raw positive spectra to WeightWatcher's own `WW_powerlaw.pl_fit`.\n","\n","**Analytic derivative.** With \\(A=U^\\top E V\\),\n","\\[\n","D\\Pi_W[E]=U\\Omega V^\\top + D\\Pi_W[E]_\\perp,\n","\\qquad\n","\\Omega_{ij}=\\frac{A_{ij}-A_{ji}}{\\sigma_i+\\sigma_j},\\;\\Omega_{ii}=0.\n","\\]\n","For \\(m>n\\), \\(D\\Pi_W[E]_\\perp=(I-UU^\\top)EV\\Sigma^{-1}V^\\top\\); for \\(n>m\\), \\(D\\Pi_W[E]_\\perp=U\\Sigma^{-1}U^\\top E(I-VV^\\top)\\). Therefore the nonzero eigenvalues of \\(G_W\\) are\n","\\[\n","\\boxed{\\lambda_{ij}^{\\rm rot}=4/(\\sigma_i+\\sigma_j)^2},\\quad ig.shape[1]; x=g.T if t else g; x=x.float()/torch.linalg.vector_norm(x.float()).clamp_min(eps); a,b,c=3.4445,-4.7750,2.0315\n"," for _ in range(steps): q=x@x.T; x=a*x+(b*q+c*(q@q))@x\n"," return x.T if t else x\n","class MuonClip(torch.optim.Optimizer):\n"," def __init__(self,params): super().__init__(params,dict(lr=MATRIX_LR,momentum=MOMENTUM,weight_decay=WEIGHT_DECAY))\n"," @torch.no_grad()\n"," def step(self):\n"," for group in self.param_groups:\n"," for p in group['params']:\n"," if p.grad is None: continue\n"," b=self.state[p].setdefault('momentum_buffer',torch.zeros_like(p.grad)); b.mul_(group['momentum']).add_(p.grad)\n"," u=zeropower(b).to(p.dtype); u.mul_(RMS_SCALE*math.sqrt(max(p.shape))); p.mul_(1-group['lr']*group['weight_decay']); p.add_(u,alpha=-group['lr'])\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Verify the analytic Fréchet formula numerically on small rectangular matrices.\n","rng=np.random.default_rng(123)\n","for shape in [(3,3),(4,3),(3,5)]:\n"," W=rng.normal(size=shape); E=rng.normal(size=shape); err=finite_difference_error(W,E)\n"," print(shape,err); assert err<1e-7\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])\n","full=datasets.MNIST(str(DATA_DIR),train=True,download=True,transform=transform); test=datasets.MNIST(str(DATA_DIR),train=False,download=True,transform=transform)\n","g=torch.Generator().manual_seed(20_260_807); perm=torch.randperm(len(full),generator=g).tolist(); val_idx,train_idx=perm[:5000],perm[5000:]; train,val=Subset(full,train_idx),Subset(full,val_idx)\n","def loaders(seed):\n"," tg=torch.Generator().manual_seed(seed+101); kw=dict(batch_size=BATCH,num_workers=0,pin_memory=DEVICE.type=='cuda')\n"," return DataLoader(train,shuffle=True,generator=tg,**kw),DataLoader(train,shuffle=False,**kw),DataLoader(val,shuffle=False,**kw),DataLoader(test,shuffle=False,**kw)\n","@torch.inference_mode()\n","def evaluate(model,loader):\n"," model.eval(); loss=correct=n=0\n"," for x,y in loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); z=model(x); loss+=float(F.cross_entropy(z,y,reduction='sum').cpu()); correct+=int((z.argmax(1)==y).sum().cpu()); n+=y.numel()\n"," return loss/n,correct/n\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def seed_all(seed): random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)\n","def numerical_due(epoch): return epoch in {0,EPOCHS} or (NUMERIC_EVERY>0 and epoch%NUMERIC_EVERY==0)\n","def analyze(model,seed,epoch):\n"," rows=[]; spectra={}\n"," for j,name in enumerate(('fc1','fc2','fc3')):\n"," W=getattr(model,name).weight.detach().cpu().numpy(); a,z=analytic_gram_spectrum(W); af=weightwatcher_pl_fit(a)\n"," rows.append(dict(seed=seed,epoch=epoch,layer=name,method='analytic',zero_modes=z,positive_modes=len(a),probes=np.nan,epsilon=np.nan,**af)); spectra[f'analytic__seed_{seed}__epoch_{epoch:03d}__{name}']=a\n"," if numerical_due(epoch):\n"," n,meta=numerical_probe_gram_spectrum(W,probes=PROBES,eps_rel=EPS,seed=918273+100000*seed+100*epoch+j); nf=weightwatcher_pl_fit(n)\n"," rows.append(dict(seed=seed,epoch=epoch,layer=name,method='numerical_finite_difference',zero_modes=np.nan,positive_modes=len(n),probes=PROBES,epsilon=meta['epsilon'],**nf)); spectra[f'numeric__seed_{seed}__epoch_{epoch:03d}__{name}']=n\n"," return rows,spectra\n","\n","perf=[]; fits=[]; spectra={}\n","for seed in SEEDS:\n"," seed_all(seed); tr,tre,va,te=loaders(seed); model=MLP3().to(DEVICE); matrices=[p for p in model.parameters() if p.ndim==2]; biases=[p for p in model.parameters() if p.ndim!=2]; muon=MuonClip(matrices); aux=torch.optim.AdamW(biases,lr=AUX_LR,weight_decay=WEIGHT_DECAY)\n"," r,s=analyze(model,seed,0); fits+=r; spectra.update(s)\n"," for epoch in range(1,EPOCHS+1):\n"," model.train()\n"," for x,y in tr:\n"," x,y=x.to(DEVICE),y.to(DEVICE); muon.zero_grad(set_to_none=True); aux.zero_grad(set_to_none=True); loss=F.cross_entropy(model(x),y); loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP); muon.step(); aux.step()\n"," tl,ta=evaluate(model,tre); vl,vaa=evaluate(model,va); ql,qa=evaluate(model,te); perf.append(dict(seed=seed,epoch=epoch,train_loss=tl,validation_loss=vl,test_loss=ql,train_accuracy=ta,validation_accuracy=vaa,test_accuracy=qa)); r,s=analyze(model,seed,epoch); fits+=r; spectra.update(s); print(seed,epoch,vl)\n","performance=pd.DataFrame(perf); gram_fits=pd.DataFrame(fits); performance.to_csv(RUN_ROOT/'performance_by_epoch_and_seed.csv',index=False); gram_fits.to_csv(RUN_ROOT/'polar_jacobian_gram_weightwatcher_analytic_vs_numeric.csv',index=False); np.savez_compressed(RUN_ROOT/'polar_jacobian_gram_spectra_analytic_vs_numeric.npz',**spectra)\n"]},{"cell_type":"markdown","metadata":{},"source":["## WeightWatcher fit comparison\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["a=gram_fits[gram_fits.method.eq('analytic')]; n=gram_fits[gram_fits.method.eq('numerical_finite_difference')]\n","cmp=n.merge(a,on=['seed','epoch','layer'],suffixes=('_numeric','_analytic'),validate='one_to_one'); cmp['alpha_difference']=cmp.alpha_numeric-cmp.alpha_analytic; cmp['alpha_abs_difference']=cmp.alpha_difference.abs(); cmp.to_csv(RUN_ROOT/'polar_jacobian_weightwatcher_fit_comparison.csv',index=False)\n","display(cmp[['seed','epoch','layer','probes_numeric','alpha_analytic','alpha_numeric','alpha_difference','D_analytic','D_numeric','xmin_analytic','xmin_numeric','tail_evals_analytic','tail_evals_numeric']])\n","if not cmp.empty:\n"," fig,ax=plt.subplots(figsize=(7,6))\n"," for layer,f in cmp.groupby('layer'): ax.scatter(f.alpha_analytic,f.alpha_numeric,label=layer)\n"," vals=np.r_[cmp.alpha_analytic,cmp.alpha_numeric]; lo,hi=np.nanmin(vals),np.nanmax(vals); ax.plot([lo,hi],[lo,hi],'--'); ax.set(xlabel='analytic WW alpha',ylabel='numerical WW alpha',title='Polar Jacobian Gram PL fit'); ax.grid(alpha=.25); ax.legend(frameon=False); plt.show()\n"]},{"cell_type":"markdown","metadata":{},"source":["## Single-checkpoint probe convergence\n","\n","For any selected checkpoint matrix `W`, use the helper below to compare the analytic WeightWatcher fit against numerical finite-difference fits with increasing probe count.\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Example:\n","# W=model.fc2.weight.detach().cpu().numpy()\n","# display(pd.DataFrame(probe_convergence(W,counts=(16,32,64,128,256),eps_rel=EPS)))\n"]},{"cell_type":"markdown","metadata":{},"source":["## Outputs\n","\n","`polar_jacobian_gram_weightwatcher_analytic_vs_numeric.csv` contains both methods' WeightWatcher fits. `polar_jacobian_gram_spectra_analytic_vs_numeric.npz` contains the raw positive eigenvalues actually passed to WeightWatcher. `polar_jacobian_weightwatcher_fit_comparison.csv` places analytic and numerical `alpha`, `D`, `xmin`, and tail counts side by side.\n"]}]} \ No newline at end of file +{"nbformat":4,"nbformat_minor":5,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11"}},"cells":[{"cell_type":"markdown","metadata":{},"source":["# MNIST MLP3 — MuonClip polar Jacobian: raw W ESD vs analytic and numerical response spectra\n","\n","For each matrix checkpoint \\(W=U\\Sigma V^\\top\\), this notebook compares three **raw, unnormalized** spectra:\n","\n","1. the ordinary weight ESD \\(\\lambda(W^\\top W)=\\{\\sigma_i^2\\}\\);\n","2. the exact analytic spectrum of \\(G_W=(D\\Pi(W))^*D\\Pi(W)\\), where \\(\\Pi(W)=UV^\\top\\);\n","3. a numerical single-checkpoint finite-difference response Gram spectrum from isotropic random perturbations.\n","\n","All three are fit by the same in-notebook continuous power-law MLE with a grid search over \\(x_{\\min}\\) minimizing the Kolmogorov--Smirnov distance. No spectral normalization, log binning, KDE, or separate fitting rule is used.\n","\n","For the analytic polar derivative, with \\(A=U^\\top E V\\),\n","\\[\n","\\Omega_{ij}=\\frac{A_{ij}-A_{ji}}{\\sigma_i+\\sigma_j},\\qquad \\Omega_{ii}=0.\n","\\]\n","The nonzero Gram eigenvalues are\n","\\[\n","\\lambda_{ij}^{\\rm rot}=\\frac{4}{(\\sigma_i+\\sigma_j)^2},\\quad ig.shape[1]; x=g.T if t else g; x=x.float()/torch.linalg.vector_norm(x.float()).clamp_min(eps); a,b,c=3.4445,-4.7750,2.0315\n"," for _ in range(steps): q=x@x.T; x=a*x+(b*q+c*(q@q))@x\n"," return x.T if t else x\n","class MuonClip(torch.optim.Optimizer):\n"," def __init__(self,params): super().__init__(params,dict(lr=MATRIX_LR,momentum=MOMENTUM,weight_decay=WEIGHT_DECAY))\n"," @torch.no_grad()\n"," def step(self):\n"," for group in self.param_groups:\n"," for p in group['params']:\n"," if p.grad is None: continue\n"," b=self.state[p].setdefault('momentum_buffer',torch.zeros_like(p.grad)); b.mul_(group['momentum']).add_(p.grad)\n"," u=zeropower(b).to(p.dtype); u.mul_(RMS_SCALE*math.sqrt(max(p.shape))); p.mul_(1-group['lr']*group['weight_decay']); p.add_(u,alpha=-group['lr'])\n","\n","rng=np.random.default_rng(123)\n","for shape in [(3,3),(4,3),(3,5)]:\n"," W=rng.normal(size=shape); E=rng.normal(size=shape); err=finite_difference_error(W,E)\n"," print(shape,err); assert err<1e-7\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])\n","full=datasets.MNIST(str(DATA_DIR),train=True,download=True,transform=transform); test=datasets.MNIST(str(DATA_DIR),train=False,download=True,transform=transform)\n","g=torch.Generator().manual_seed(20_260_807); perm=torch.randperm(len(full),generator=g).tolist(); val_idx,train_idx=perm[:5000],perm[5000:]; train,val=Subset(full,train_idx),Subset(full,val_idx)\n","def loaders(seed):\n"," tg=torch.Generator().manual_seed(seed+101); kw=dict(batch_size=BATCH,num_workers=0,pin_memory=DEVICE.type=='cuda')\n"," return DataLoader(train,shuffle=True,generator=tg,**kw),DataLoader(train,shuffle=False,**kw),DataLoader(val,shuffle=False,**kw),DataLoader(test,shuffle=False,**kw)\n","@torch.inference_mode()\n","def evaluate(model,loader):\n"," model.eval(); loss=correct=n=0\n"," for x,y in loader:\n"," x,y=x.to(DEVICE),y.to(DEVICE); z=model(x); loss+=float(F.cross_entropy(z,y,reduction='sum').cpu()); correct+=int((z.argmax(1)==y).sum().cpu()); n+=y.numel()\n"," return loss/n,correct/n\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def seed_all(seed): random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)\n","def numerical_due(epoch): return epoch in {0,EPOCHS} or (NUMERIC_EVERY>0 and epoch%NUMERIC_EVERY==0)\n","def fit_row(seed,epoch,layer,method,spectrum,**extra):\n"," return dict(seed=seed,epoch=epoch,layer=layer,method=method,spectrum_size=len(spectrum),**extra,**powerlaw_mle_grid_ks(spectrum,min_tail=MIN_TAIL))\n","def analyze(model,seed,epoch):\n"," rows=[]; spectra={}\n"," for j,name in enumerate(('fc1','fc2','fc3')):\n"," W=getattr(model,name).weight.detach().cpu().numpy()\n"," raw=raw_weight_esd(W); analytic,zero_modes=analytic_gram_spectrum(W)\n"," rows.append(fit_row(seed,epoch,name,'raw_weight_WtW',raw,zero_modes=np.nan,probes=np.nan,epsilon=np.nan)); spectra[f'raw__seed_{seed}__epoch_{epoch:03d}__{name}']=raw\n"," rows.append(fit_row(seed,epoch,name,'analytic_polar_JtJ',analytic,zero_modes=zero_modes,probes=np.nan,epsilon=np.nan)); spectra[f'analytic__seed_{seed}__epoch_{epoch:03d}__{name}']=analytic\n"," if numerical_due(epoch):\n"," numeric,meta=numerical_probe_gram_spectrum(W,probes=PROBES,eps_rel=EPS,seed=918273+100000*seed+100*epoch+j)\n"," rows.append(fit_row(seed,epoch,name,'numerical_polar_JtJ',numeric,zero_modes=np.nan,probes=PROBES,epsilon=meta['epsilon'])); spectra[f'numeric__seed_{seed}__epoch_{epoch:03d}__{name}']=numeric\n"," return rows,spectra\n","\n","perf=[]; fits=[]; spectra={}\n","for seed in SEEDS:\n"," seed_all(seed); tr,tre,va,te=loaders(seed); model=MLP3().to(DEVICE); matrices=[p for p in model.parameters() if p.ndim==2]; biases=[p for p in model.parameters() if p.ndim!=2]; muon=MuonClip(matrices); aux=torch.optim.AdamW(biases,lr=AUX_LR,weight_decay=WEIGHT_DECAY)\n"," r,s=analyze(model,seed,0); fits+=r; spectra.update(s)\n"," for epoch in range(1,EPOCHS+1):\n"," model.train()\n"," for x,y in tr:\n"," x,y=x.to(DEVICE),y.to(DEVICE); muon.zero_grad(set_to_none=True); aux.zero_grad(set_to_none=True); loss=F.cross_entropy(model(x),y); loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP); muon.step(); aux.step()\n"," tl,ta=evaluate(model,tre); vl,vaa=evaluate(model,va); ql,qa=evaluate(model,te); perf.append(dict(seed=seed,epoch=epoch,train_loss=tl,validation_loss=vl,test_loss=ql,train_accuracy=ta,validation_accuracy=vaa,test_accuracy=qa)); r,s=analyze(model,seed,epoch); fits+=r; spectra.update(s); print(seed,epoch,vl)\n","performance=pd.DataFrame(perf); three_way_fits=pd.DataFrame(fits); performance.to_csv(RUN_ROOT/'performance_by_epoch_and_seed.csv',index=False); three_way_fits.to_csv(RUN_ROOT/'raw_vs_polar_mle_ks_by_epoch_layer_seed.csv',index=False); np.savez_compressed(RUN_ROOT/'raw_vs_polar_spectra.npz',**spectra)\n"]},{"cell_type":"markdown","metadata":{},"source":["## Three-way exponent comparison\n","\n","The table below reports \\(\\alpha\\), KS distance \\(D\\), \\(x_{\\min}\\), and fitted-tail size for every available method. The numerical response spectrum is a finite-probe random-subspace estimate; the analytic spectrum is the full exact positive spectrum.\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["cols=['seed','epoch','layer','method','spectrum_size','alpha','D','xmin','tail_evals','probes']\n","display(three_way_fits[cols].sort_values(['seed','epoch','layer','method']))\n","alpha_table=three_way_fits.pivot_table(index=['seed','epoch','layer'],columns='method',values='alpha',aggfunc='first').reset_index(); alpha_table.to_csv(RUN_ROOT/'alpha_three_way_comparison.csv',index=False); display(alpha_table)\n","final=three_way_fits[three_way_fits.epoch.eq(EPOCHS)].copy(); display(final[cols].sort_values(['seed','layer','method']))\n"]},{"cell_type":"markdown","metadata":{},"source":["## Single-checkpoint numerical convergence\n","\n","For a selected checkpoint matrix `W`, `probe_convergence` repeats the finite-difference calculation at increasing probe counts and compares its MLE/KS exponent with the analytic polar-Gram exponent.\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Example after training:\n","# W=model.fc2.weight.detach().cpu().numpy()\n","# display(pd.DataFrame(probe_convergence(W,counts=(16,32,64,128,256),eps_rel=EPS,min_tail=MIN_TAIL)))\n"]}]} \ No newline at end of file From 6c50343d3a525d8436f3606d01ba00c946d06b92 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Mon, 17 Aug 2026 22:27:44 -0700 Subject: [PATCH 7/7] Remove temporary notebook placeholder --- .../notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp | 1 - 1 file changed, 1 deletion(-) delete mode 100644 baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp diff --git a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp b/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp deleted file mode 100644 index b3a42524..00000000 --- a/baseline/notebooks/MNIST_MLP3_MuonClip_Polar_Jacobian_Baseline.tmp +++ /dev/null @@ -1 +0,0 @@ -placeholder \ No newline at end of file