modelcard.tex

\documentclass[10pt,a4paper]{article}

\usepackage[margin=2cm]{geometry}
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{graphicx}
\usepackage{amsmath,amssymb}
\usepackage{xcolor}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{titlesec}
\usepackage{fancyhdr}
\usepackage{colortbl}
\usepackage{multirow}
\usepackage{array}
\usepackage{float}

% Colors
\definecolor{supreme}{RGB}{30, 60, 114}
\definecolor{lightbg}{RGB}{245, 247, 250}
\definecolor{accent}{RGB}{0, 128, 128}
\definecolor{phase}{RGB}{100, 100, 100}

% Hyperlinks
\hypersetup{
    colorlinks=true,
    linkcolor=supreme,
    urlcolor=accent,
    citecolor=supreme
}

% Section styling
\titleformat{\section}{\Large\bfseries\color{supreme}}{}{0em}{}[\titlerule]
\titleformat{\subsection}{\large\bfseries\color{supreme!80}}{}{0em}{}
\titleformat{\subsubsection}{\normalsize\bfseries\color{supreme!60}}{}{0em}{}

% Header/Footer
\pagestyle{fancy}
\fancyhf{}
\fancyhead[L]{\textcolor{phase}{\small Supreme --- Model Card}}
\fancyhead[R]{\textcolor{phase}{\small v1.0 / April 2026}}
\fancyfoot[C]{\textcolor{phase}{\thepage}}
\renewcommand{\headrulewidth}{0.4pt}

\newcommand{\metricbox}[2]{%
    \begin{tabular}{@{}l@{}}
        \textcolor{phase}{\scriptsize #1} \\
        \textbf{\large #2}
    \end{tabular}
}

\begin{document}

% ============================================================
% TITLE
% ============================================================

\begin{center}
    {\Huge\bfseries\textcolor{supreme}{Supreme}}\\[6pt]
    {\Large Model Card}\\[12pt]
    {\large A World Model with HOPE Hierarchical Memory for Robotics}\\[8pt]
    {\normalsize April 2026 \quad$\cdot$\quad Version 1.0}
\end{center}

\vspace{1em}

\noindent\fcolorbox{supreme!20}{lightbg}{%
\begin{minipage}{\dimexpr\textwidth-2\fboxsep-2\fboxrule}
\centering
\vspace{6pt}
\begin{tabular}{cccc}
    \metricbox{Total Parameters}{$\sim$6.7M} &
    \metricbox{Best Real Reward}{$-$10.57} &
    \metricbox{Full Training Time}{$\sim$3 hours} &
    \metricbox{Hardware}{M4 Air 16GB}
\end{tabular}
\vspace{6pt}
\end{minipage}
}

% ============================================================
\section{Model Overview}
% ============================================================

Supreme implements the Vision--Memory--Controller (V-M-C) architecture from World Models (Ha \& Schmidhuber, 2018) with the M model replaced by HOPE (Hierarchical Optimisation through Pattern Extraction) from the Nested Learning paper (Behrouz et al., 2025). The agent learns a compressed predictive model of its environment, then trains inside its own hallucinated dreams via CMA-ES evolutionary strategy.

\subsection{Intended Use}

\begin{itemize}[nosep,leftmargin=*]
    \item Continuous control in MuJoCo robotics environments (Reacher-v5)
    \item Research into hierarchical memory for model-based reinforcement learning
    \item Dream training / imagination-based policy optimisation
    \item Studying continual learning properties of multi-frequency memory systems
\end{itemize}

% ============================================================
\section{Architecture Specifications}
% ============================================================

\subsection{V Model --- ConvVAE Encoder}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Architecture & Convolutional Variational Autoencoder \\
Input & RGB frames, $64 \times 64 \times 3$ \\
Latent dimension & 32 \\
Parameters & $\sim$4.4M (encoder + decoder) \\
KL regularisation & Free bits with $\text{kl\_tolerance} = 0.5$ per dim \\
Training data & 100,000 frames from 2,000 random rollouts \\
Training time & $\sim$2 minutes on MPS \\
KL health & 13.80 nats/sample \\
\bottomrule
\end{tabular}
\end{table}

\subsection{M Model --- HOPEMemory}

The M model predicts next-latent distributions:
$P(z_{t+1} \mid a_t, z_t, h_t) = \sum_{k=1}^{K} \pi_k \, \mathcal{N}(\mu_k, \sigma_k^2)$

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Total parameters & $\sim$2.3M \\
Input projection & $34 \to 128$ (latent 32 + action 2) \\
\addlinespace
\multicolumn{2}{@{}l}{\textit{Self-Modifying Titans}} \\
\quad Memory dimension & 128 \\
\quad Self-referential matrices & 6 ($M_k, M_v, M_q, M_\eta, M_\alpha, M_\text{memory}$) \\
\quad Update rule & L2-regression Delta Gradient Descent \\
\quad Key/query normalisation & L2 \\
\quad Local convolution window & 4 \\
\addlinespace
\multicolumn{2}{@{}l}{\textit{Continuum Memory System (CMS)}} \\
\quad Number of levels & 4 \\
\quad Frequencies & [1, 16, 1\,000\,000, 16\,000\,000] \\
\quad Active levels (50-step episodes) & L0 (freq=1), L1 (freq=16) \\
\quad Output dimension & 256 \\
\quad Level architecture & Sequential MLP chain \\
\addlinespace
\multicolumn{2}{@{}l}{\textit{MDN Head}} \\
\quad Gaussian components & 5 \\
\quad Output dimension & 32 (matching latent dim) \\
\quad Covariance & Diagonal \\
\quad Temperature range & $\tau \in [0.5, 1.3]$ \\
\addlinespace
Optimiser & AdamW throughout (M3 infrastructure validated Phase 6) \\
Loss function & MDN negative log-likelihood \\
\bottomrule
\end{tabular}
\end{table}

\subsection{C Model --- Controller}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Architecture & 1-hidden-layer MLP \\
Input dimension & 160 ($z_t = 32$, $h_t = 128$) \\
Hidden layer & 32 units, ELU activation \\
Output & 2 actions, $\tanh$ bounded to $[-1, 1]$ \\
Parameters & 5,218 \\
Training method & CMA-ES (population 64, $\sigma_0 = 0.3$) \\
Rollouts per evaluation & 8 \\
Generations & 100 \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Training Details}
% ============================================================

\subsection{Phase Summary}

\begin{table}[H]
\centering
\small
\begin{tabular}{@{}clllr@{}}
\toprule
\textbf{Phase} & \textbf{Component} & \textbf{Method} & \textbf{Key Output} & \textbf{Time} \\
\midrule
0 & Environment & MuJoCo setup & Reacher-v5 verified & --- \\
1 & V (ConvVAE) & Backprop (MSE+KL) & Frozen encoder & 2 min \\
2 & M (HOPE) & Backprop (MDN NLL) & Frozen world model & $\sim$30 min \\
3v1 & C (linear) & CMA-ES, real env & Reward: $-$16.1 & 7.4 min \\
3v2 & C (nonlinear) & CMA-ES, real env & Reward: $-$8.2 & 57.8 min \\
4 & C (dream) & CMA-ES, DreamEnv & Poor transfer & $\sim$10 min \\
5 & C (iterative) & CMA-ES + reward head & Reward: $\sim\!-$77 & $\sim$45 min \\
6 & M + C (scaled) & CMS inner-loop + ablations & Gradient starvation & $\sim$67 min \\
7 & Reward head & Decoupled training & Reward: $-$107 & $\sim$15 min \\
8A & M (fine-tune) & Multi-step + sched.~sampling & R$^2$@25: 0.99 & 21.9 min \\
8B+C & C (branched) & CMA-ES + $\lambda_u$ penalty & Dream: $-$32.0 & 7.2 min \\
8D & Transfer eval & Real environment & Reward: $-$59.7 & --- \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Data}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Environment & MuJoCo Reacher-v5 \\
Observation space & $64 \times 64$ RGB \\
Action space & 2 continuous $\in [-1, 1]$ \\
Episode length & 50 steps \\
Collection policy & Random \\
Rollouts collected & 2,000 \\
Total frames & 100,000 \\
Storage format & HDF5, uint8 \\
Storage size & $\sim$375 MB \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Hardware \& Software}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Component} & \textbf{Specification} \\
\midrule
Hardware & Apple Mac M4 Air \\
RAM & 16 GB unified memory \\
Compute backend & MPS (Metal Performance Shaders) \\
Python & 3.12 \\
PyTorch & 2.x with MPS support \\
MuJoCo & Gymnasium[mujoco] \\
Evolution strategy & \texttt{cma} Python package \\
Data storage & HDF5 via \texttt{h5py} \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Evaluation Results}
% ============================================================

\subsection{Real-Environment Transfer}

All controllers evaluated over 100 rollouts in the real MuJoCo Reacher-v5 environment:

\begin{table}[H]
\centering
\begin{tabular}{@{}lcrrl@{}}
\toprule
\textbf{Agent} & \textbf{Training Mode} & \textbf{Reward} & \textbf{$\pm$ Std} & \textbf{Notes} \\
\midrule
Random & --- & $-$43.17 & 3.44 & No learning \\
Phase 3v1 (linear) & Real env & $-$16.1 & --- & 322 params \\
\rowcolor{lightbg}
Phase 3v2 (nonlinear) & Real env & $\mathbf{-10.57}$ & 3.08 & \textbf{Gold standard} \\
Phase 5 (iterative) & Dream & $\sim\!-$77 & --- & Reward head added \\
Phase 7 (calibrated) & Dream & $-$107.02 & 2.18 & Full 50-step dreams \\
\rowcolor{lightbg}
Phase 8 (branched) & Dream + buffer & $\mathbf{-59.71}$ & 2.19 & \textbf{Best dream-trained} \\
\bottomrule
\end{tabular}
\end{table}

\subsection{World Model Fidelity (Phase 8A)}

\begin{table}[H]
\centering
\begin{tabular}{@{}lcc@{}}
\toprule
\textbf{Metric} & \textbf{Before Fine-Tuning} & \textbf{After Fine-Tuning} \\
\midrule
R$^2$ at horizon 25 & 0.9735 & 0.9902 \\
Single-step MDN loss & $-$119.95 & $-$138.00 \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Dream Training (Phase 8B+C)}

\begin{table}[H]
\centering
\begin{tabular}{@{}lr@{}}
\toprule
\textbf{Detail} & \textbf{Value} \\
\midrule
Real-state buffer size & 24,500 transitions (500 episodes) \\
Branch curriculum & $k = 5 \to 10 \to 15$ steps \\
MDN variance penalty $\lambda_u$ & 0.5 \\
CMA-ES convergence & Generation 52 (early stopped) \\
Best dream reward & $-$31.99 \\
Training time & 7.2 minutes \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{HOPE-Specific Properties}
% ============================================================

\subsection{CMS Frequency Separation (Validated Phase 6)}

\begin{table}[H]
\centering
\begin{tabular}{@{}llll@{}}
\toprule
\textbf{CMS Level} & \textbf{Frequency} & \textbf{Update Cadence (50-step ep.)} & \textbf{Drift During Training} \\
\midrule
Level 0 (fast) & 1 & Every step (50$\times$) & High (as expected) \\
Level 1 (medium) & 16 & Every 16 steps ($\sim$3$\times$) & Moderate \\
Level 2 (slow) & 1\,000\,000 & Never (frozen) & Zero \\
Level 3 (permanent) & 16\,000\,000 & Never (frozen) & Zero \\
\bottomrule
\end{tabular}
\end{table}

\subsection{CMS Reset Levels for Dream Training}

\begin{table}[H]
\centering
\begin{tabular}{@{}llll@{}}
\toprule
\textbf{reset\_level} & \textbf{Resets} & \textbf{Persists} & \textbf{Use Case} \\
\midrule
\texttt{all} & Titans + all CMS & Nothing & Standard (matches original paper) \\
\texttt{fast} & Titans + L0 & L1, L2 & Cross-episode memory \\
\texttt{fast+mid} & Titans + L0 + L1 & L2 only & Strongest persistence \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Limitations \& Known Issues}
% ============================================================

\begin{enumerate}[nosep]
    \item \textbf{Dream-to-real gap.} The best dream-trained controller ($-$59.7) still trails the real-environment-trained controller ($-$10.6) by a significant margin. Compounding prediction error remains the primary bottleneck despite multi-step fine-tuning and branched rollouts.

    \item \textbf{Single environment.} Validated only on Reacher-v5 (2-DOF, simple visuals). Scaling to harder tasks (pick-and-place, dexterous manipulation) is untested.

    \item \textbf{Reward head fragility.} Joint training of the reward head with the MDN loss causes gradient starvation. Decoupled training is required.

    \item \textbf{CMS frequency levels are task-specific.} The [1, 16, 1M, 16M] schedule from the NL paper means only L0 and L1 actively update in 50-step episodes; longer horizons would activate L2/L3.

    \item \textbf{MPS backend limitations.} Some PyTorch operations fall back to CPU on MPS. CUDA would likely provide 2--3$\times$ speedup.

    \item \textbf{No sim-to-real transfer tested.} All evaluations are in simulation. Physical robot deployment is out of scope.
\end{enumerate}

% ============================================================
\section{Key Design Decisions}
% ============================================================

\begin{enumerate}[nosep]
    \item \textbf{Tiny controller:} 5,218 parameters keeps CMA-ES tractable. All intelligence resides in V and M.
    \item \textbf{Fixed-dim HOPE projection:} Memory matrix projected to 256-dim $h_t$ for CMA-ES compatibility.
    \item \textbf{CMS frequencies from NL paper:} [1, 16, 1M, 16M] naturally creates a fast/slow hierarchy where L0--L1 adapt per-episode while L2--L3 serve as frozen knowledge anchors.
    \item \textbf{Branched rollouts over full dreams:} MBPO-style $k$-step branches from real states cap compounding error.
    \item \textbf{Scheduled sampling for M fine-tuning:} Gradually shifts from teacher-forced to autoregressive training.
    \item \textbf{MDN variance as uncertainty signal:} Free uncertainty estimate for reward penalty without ensembles.
\end{enumerate}

% ============================================================
\section{References}
% ============================================================

\begin{enumerate}[nosep,label={[\arabic*]},leftmargin=*]
    \item Ha, D.\ \& Schmidhuber, J.\ (2018). World Models. \textit{arXiv:1803.10122}.
    \item Behrouz, A.\ et al.\ (2025). Nested Learning. (HOPE/NL paper).
    \item Janner, M.\ et al.\ (2019). When to Trust Your Model: Model-Based Policy Optimization. \textit{NeurIPS}.
    \item Yu, T.\ et al.\ (2020). MOPO: Model-Based Offline Policy Optimization. \textit{NeurIPS}.
    \item Hansen, N.\ (2016). The CMA Evolution Strategy: A Tutorial. \textit{arXiv:1604.00772}.
\end{enumerate}

\vfill

\begin{center}
\textcolor{phase}{\rule{0.5\textwidth}{0.5pt}}\\[6pt]
\textcolor{phase}{\small Supreme v1.0 --- Built on Mac M4 Air --- April 2026}
\end{center}

\end{document}