modelcard2.tex

\documentclass[10pt,a4paper]{article}

\usepackage[margin=2cm]{geometry}
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{graphicx}
\usepackage{amsmath,amssymb}
\usepackage{xcolor}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage{titlesec}
\usepackage{fancyhdr}
\usepackage{colortbl}
\usepackage{multirow}
\usepackage{array}
\usepackage{float}

% Colors
\definecolor{supreme}{RGB}{30, 60, 114}
\definecolor{lightbg}{RGB}{245, 247, 250}
\definecolor{accent}{RGB}{0, 128, 128}
\definecolor{phase}{RGB}{100, 100, 100}

% Hyperlinks
\hypersetup{
    colorlinks=true,
    linkcolor=supreme,
    urlcolor=accent,
    citecolor=supreme
}

% Section styling
\titleformat{\section}{\Large\bfseries\color{supreme}}{}{0em}{}[\titlerule]
\titleformat{\subsection}{\large\bfseries\color{supreme!80}}{}{0em}{}
\titleformat{\subsubsection}{\normalsize\bfseries\color{supreme!60}}{}{0em}{}

% Header/Footer
\pagestyle{fancy}
\fancyhf{}
\fancyhead[L]{\textcolor{phase}{\small Supreme --- Model Card}}
\fancyhead[R]{\textcolor{phase}{\small v1.0 / April 2026}}
\fancyfoot[C]{\textcolor{phase}{\thepage}}
\renewcommand{\headrulewidth}{0.4pt}

\newcommand{\metricbox}[2]{%
    \begin{tabular}{@{}l@{}}
        \textcolor{phase}{\scriptsize #1} \\
        \textbf{\large #2}
    \end{tabular}
}

\begin{document}

% ============================================================
% TITLE
% ============================================================

\begin{center}
    {\Huge\bfseries\textcolor{supreme}{Supreme}}\\[6pt]
    {\Large Model Card}\\[12pt]
    {\large A World Model with HOPE Hierarchical Memory for Robotics}\\[8pt]
    {\normalsize April 2026 \quad$\cdot$\quad Version 1.0}
\end{center}

\vspace{1em}

\noindent\fcolorbox{supreme!20}{lightbg}{%
\begin{minipage}{\dimexpr\textwidth-2\fboxsep-2\fboxrule}
\centering
\vspace{6pt}
\begin{tabular}{cccc}
    \metricbox{Total Parameters}{$\sim$6.7M} &
    \metricbox{Best Result}{629.6 (Walker2d)} &
    \metricbox{Benchmarks}{4 suites, 15 tasks} &
    \metricbox{Hardware}{M2 Air 8GB}
\end{tabular}
\vspace{6pt}
\end{minipage}
}

% ============================================================
\section{Model Overview}
% ============================================================

Supreme implements the Vision--Memory--Controller (V-M-C) architecture from World Models (Ha \& Schmidhuber, 2018) with the M model replaced by HOPE (Hierarchical Optimisation through Pattern Extraction) from the Nested Learning paper (Behrouz et al., 2025). The agent learns a compressed predictive model of its environment, then trains inside its own hallucinated dreams via CMA-ES evolutionary strategy.

\subsection{Intended Use}

\begin{itemize}[nosep,leftmargin=*]
    \item Continuous locomotion in MuJoCo and DMControl environments
    \item Model-based RL with hierarchical memory on consumer hardware
    \item Compact world models ($\sim$6.7M params) with evolutionary controller search
    \item Research into multi-frequency memory systems for predictive state representations
\end{itemize}

% ============================================================
\section{Architecture Specifications}
% ============================================================

\subsection{V Model --- ConvVAE Encoder}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Architecture & Convolutional Variational Autoencoder \\
Input & RGB frames, $64 \times 64 \times 3$ \\
Latent dimension & 32 \\
Parameters & $\sim$4.4M (encoder + decoder) \\
KL regularisation & Free bits with $\text{kl\_tolerance} = 0.5$ per dim \\
Training data & 100,000 frames from 2,000 random rollouts \\
Training time & $\sim$2 minutes on MPS \\
KL health & 13.80 nats/sample \\
\bottomrule
\end{tabular}
\end{table}

\subsection{M Model --- HOPEMemory}

The M model predicts next-latent distributions:
$P(z_{t+1} \mid a_t, z_t, h_t) = \sum_{k=1}^{K} \pi_k \, \mathcal{N}(\mu_k, \sigma_k^2)$

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Total parameters & $\sim$2.3M \\
Input projection & $34 \to 128$ (latent 32 + action 2) \\
\addlinespace
\multicolumn{2}{@{}l}{\textit{Self-Modifying Titans}} \\
\quad Memory dimension & 128 \\
\quad Self-referential matrices & 6 ($M_k, M_v, M_q, M_\eta, M_\alpha, M_\text{memory}$) \\
\quad Update rule & L2-regression Delta Gradient Descent \\
\quad Key/query normalisation & L2 \\
\quad Local convolution window & 4 \\
\addlinespace
\multicolumn{2}{@{}l}{\textit{Continuum Memory System (CMS)}} \\
\quad Number of levels & 4 \\
\quad Frequencies & [1, 16, 1\,000\,000, 16\,000\,000] \\
\quad Active levels (50-step episodes) & L0 (freq=1), L1 (freq=16) \\
\quad Output dimension & 256 \\
\quad Level architecture & Sequential MLP chain \\
\addlinespace
\multicolumn{2}{@{}l}{\textit{MDN Head}} \\
\quad Gaussian components & 5 \\
\quad Output dimension & 32 (matching latent dim) \\
\quad Covariance & Diagonal \\
\quad Temperature range & $\tau \in [0.5, 1.3]$ \\
\addlinespace
Optimiser & AdamW throughout (M3 infrastructure validated Phase 6) \\
Loss function & MDN negative log-likelihood \\
\bottomrule
\end{tabular}
\end{table}

\subsection{C Model --- Controller}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Architecture & 1-hidden-layer MLP \\
Input dimension & 160 ($z_t = 32$, $h_t = 128$) \\
Hidden layer & 32 units, ELU activation \\
Output & 2 actions, $\tanh$ bounded to $[-1, 1]$ \\
Parameters & 5,218 \\
Training method & CMA-ES (population 64, $\sigma_0 = 0.3$) \\
Rollouts per evaluation & 8 \\
Generations & 100 \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Training Details}
% ============================================================

\subsection{Phase Summary}

\begin{table}[H]
\centering
\small
\begin{tabular}{@{}clllr@{}}
\toprule
\textbf{Phase} & \textbf{Component} & \textbf{Method} & \textbf{Key Output} & \textbf{Time} \\
\midrule
0 & Environment & MuJoCo setup & Reacher-v5 verified & --- \\
1 & V (ConvVAE) & Backprop (MSE+KL) & Frozen encoder & 2 min \\
2 & M (HOPE) & Backprop (MDN NLL) & Frozen world model & $\sim$30 min \\
3v1 & C (linear) & CMA-ES, real env & Reward: $-$16.1 & 7.4 min \\
3v2 & C (nonlinear) & CMA-ES, real env & Reward: $-$8.2 & 57.8 min \\
4 & C (dream) & CMA-ES, DreamEnv & Poor transfer & $\sim$10 min \\
5 & C (iterative) & CMA-ES + reward head & Reward: $\sim\!-$77 & $\sim$45 min \\
6 & M + C (scaled) & CMS inner-loop + ablations & Gradient starvation & $\sim$67 min \\
7 & Reward head & Decoupled training & Reward: $-$107 & $\sim$15 min \\
8A & M (fine-tune) & Multi-step + sched.~sampling & R$^2$@25: 0.99 & 21.9 min \\
8B+C & C (branched) & CMA-ES + $\lambda_u$ penalty & Dream: $-$32.0 & 7.2 min \\
8D & Transfer eval & Real environment & Reward: $-$59.7 & --- \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Data}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Property} & \textbf{Value} \\
\midrule
Environment & MuJoCo Reacher-v5 \\
Observation space & $64 \times 64$ RGB \\
Action space & 2 continuous $\in [-1, 1]$ \\
Episode length & 50 steps \\
Collection policy & Random \\
Rollouts collected & 2,000 \\
Total frames & 100,000 \\
Storage format & HDF5, uint8 \\
Storage size & $\sim$375 MB \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Hardware \& Software}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Component} & \textbf{Specification} \\
\midrule
Development hardware & Apple Mac M4 Air, 16\,GB \\
Benchmark hardware & Apple Mac M2 Air, 8\,GB \\
Compute backend & MPS (Metal Performance Shaders) \\
Python & 3.12 \\
PyTorch & 2.x with MPS support \\
MuJoCo & Gymnasium[mujoco] \\
Evolution strategy & \texttt{cma} Python package \\
Data storage & HDF5 via \texttt{h5py} \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Multi-Environment Benchmark Results}
% ============================================================

Supreme was evaluated across \textbf{4 benchmark suites (15 tasks)} using real-environment CMA-ES for continuous control and dream-based CMA-ES for Atari 100k. All experiments ran on a MacBook M2 Air with 8\,GB unified memory---demonstrating that the full V-M-C pipeline is tractable on consumer hardware.

\subsection{Continuous Locomotion --- MuJoCo}

\begin{table}[H]
\centering
\begin{tabular}{@{}lcrrrr@{}}
\toprule
\textbf{Task} & \textbf{Supreme (HOPE)} & \textbf{vs Random} & \textbf{DreamerV3} & \textbf{TD-MPC2} & \textbf{SAC} \\
\midrule
\rowcolor{lightbg}
\textbf{Walker2d-v5} & $\mathbf{629.6 \pm 0.0}$ & $\mathbf{393\times}$ & 3\,822 & 5\,108 & 5\,200 \\
Hopper-v5 & $84.9 \pm 0.0$ & $5.9\times$ & 1\,282 & 3\,216 & 3\,200 \\
HalfCheetah-v5 & $-517.0 \pm 0.0$ & --- & 1\,200 & 8\,540 & 11\,500 \\
\bottomrule
\end{tabular}
\caption{MuJoCo results. Random baselines: Walker2d 1.6, Hopper 14.5, HalfCheetah $-$282.}
\end{table}

\noindent\textbf{Walker2d-v5} is Supreme's strongest result: \textbf{393$\boldsymbol{\times}$ the random baseline}. With only 5,350 controller parameters and 80 CMA-ES generations (2,560 total evaluation episodes), the agent discovers sustained bipedal forward walking---a non-trivial emergent behaviour for an evolutionary strategy operating on a learned latent state. This demonstrates that the HOPE memory's hierarchical representation provides a sufficiently rich encoding of dynamics for CMA-ES to exploit, even without gradient-based policy optimisation.

\noindent\textbf{Hopper-v5} reaches $5.9\times$ random, showing the pipeline generalises to single-leg balancing. HalfCheetah scores below random due to a known limitation of the 200-step CMA evaluation cap favouring short-horizon gaits (see Section~\ref{sec:limitations}).

\subsection{Continuous Locomotion --- DMControl}

\begin{table}[H]
\centering
\begin{tabular}{@{}lcrrrrr@{}}
\toprule
\textbf{Task} & \textbf{Supreme (HOPE)} & \textbf{vs Random} & \textbf{Random} & \textbf{DreamerV3} & \textbf{DrQ-v2} & \textbf{TDMPC} \\
\midrule
\rowcolor{lightbg}
cartpole-swingup & $\mathbf{220.9 \pm 23.4}$ & $\mathbf{7.4\times}$ & 30 & 840 & 855 & 863 \\
\rowcolor{lightbg}
cheetah-run & $\mathbf{166.7 \pm 65.3}$ & $\mathbf{5.6\times}$ & 30 & 795 & 810 & 844 \\
\rowcolor{lightbg}
walker-walk & $\mathbf{159.7 \pm 43.8}$ & $\mathbf{1.6\times}$ & 100 & 945 & 955 & 967 \\
\bottomrule
\end{tabular}
\caption{DMControl results. All three tasks exceed their random baselines.}
\end{table}

\noindent\textbf{All three DMControl tasks surpass their random baselines}, confirming that the V-M-C pipeline transfers meaningfully across continuous locomotion domains with no task-specific modifications to V or M. The \textbf{cartpole-swingup} result ($7.4\times$ random) is particularly notable: the agent learns the swing-and-balance strategy purely through evolutionary search over the HOPE latent space. Across both MuJoCo and DMControl, Supreme achieves above-random locomotion in \textbf{5 out of 6 continuous control tasks}---using a single, unified architecture.

\subsection{Atari 100k (Dream Training)}

Under the Atari 100k protocol (100k real environment steps only), controllers are trained entirely inside hallucinated rollouts from the learned world model:

\begin{table}[H]
\centering
\small
\begin{tabular}{@{}lcrrcr@{}}
\toprule
\textbf{Task} & \textbf{Supreme} & \textbf{Random} & \textbf{Human} & \textbf{HN\%} & \textbf{DreamerV3} \\
\midrule
Boxing & $-23.0$ & 0.1 & 12.1 & $-192.5\%$ & 75.7 \\
Breakout & $0.0$ & 1.7 & 30.5 & $-5.9\%$ & 18.7 \\
Pong & $-21.0$ & $-20.7$ & 14.6 & $-0.8\%$ & 18.2 \\
Qbert & $0.0$ & 163.9 & 13\,455 & $-1.2\%$ & 1\,042 \\
RoadRunner & $0.0$ & 11.5 & 7\,845 & $-0.1\%$ & 5\,661 \\
\bottomrule
\end{tabular}
\caption{Atari 100k. HN\% = human-normalised score. Dream-only training does not yet transfer.}
\end{table}

\noindent Atari 100k remains an open challenge. The dream-only training paradigm does not produce transferable policies for discrete-action, visually complex environments at current model scale, suggesting HOPE's predictive fidelity is better suited to continuous dynamics.

\subsection{MetaWorld MT10}

Four MetaWorld manipulation tasks were evaluated; all returned 0\% success rate. These tasks require precise multi-step object manipulation that the current CMA-ES search budget (80 generations, pop=16) does not reliably discover within the limited controller capacity.

\subsection{Benchmark Configuration}

\begin{table}[H]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
\textbf{Parameter} & \textbf{Value} \\
\midrule
Hardware & MacBook M2 Air, 8\,GB unified memory \\
Compute backend & MPS (Metal Performance Shaders) \\
Locomotion training & Real-environment CMA-ES \\
Atari training & Dream-based CMA-ES \\
CMA-ES population & 16 \\
CMA-ES generations & 80 \\
CMA-ES $\sigma_0$ & 0.5 \\
Rollouts per candidate & 2 \\
CMA eval max steps & 200 (full-length for final evaluation) \\
Controller hidden units & 32 (ELU activation) \\
Final evaluation episodes & 30 \\
Avg.\ time per task & $\sim$78 minutes \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Development History (Reacher-v5)}
% ============================================================

The architecture was developed iteratively on MuJoCo Reacher-v5 (M4 Air 16\,GB). This section preserves the full phase-by-phase progression.

\subsection{Reacher-v5 Transfer Results}

All controllers evaluated over 100 rollouts in the real Reacher-v5 environment:

\begin{table}[H]
\centering
\begin{tabular}{@{}lcrrl@{}}
\toprule
\textbf{Agent} & \textbf{Training Mode} & \textbf{Reward} & \textbf{$\pm$ Std} & \textbf{Notes} \\
\midrule
Random & --- & $-$43.17 & 3.44 & No learning \\
Phase 3v1 (linear) & Real env & $-$16.1 & --- & 322 params \\
\rowcolor{lightbg}
Phase 3v2 (nonlinear) & Real env & $\mathbf{-10.57}$ & 3.08 & \textbf{Gold standard} \\
Phase 5 (iterative) & Dream & $\sim\!-$77 & --- & Reward head added \\
Phase 7 (calibrated) & Dream & $-$107.02 & 2.18 & Full 50-step dreams \\
\rowcolor{lightbg}
Phase 8 (branched) & Dream + buffer & $\mathbf{-59.71}$ & 2.19 & \textbf{Best dream-trained} \\
\bottomrule
\end{tabular}
\end{table}

\subsection{World Model Fidelity (Phase 8A)}

\begin{table}[H]
\centering
\begin{tabular}{@{}lcc@{}}
\toprule
\textbf{Metric} & \textbf{Before Fine-Tuning} & \textbf{After Fine-Tuning} \\
\midrule
R$^2$ at horizon 25 & 0.9735 & 0.9902 \\
Single-step MDN loss & $-$119.95 & $-$138.00 \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{HOPE-Specific Properties}
% ============================================================

\subsection{CMS Frequency Separation (Validated Phase 6)}

\begin{table}[H]
\centering
\begin{tabular}{@{}llll@{}}
\toprule
\textbf{CMS Level} & \textbf{Frequency} & \textbf{Update Cadence (50-step ep.)} & \textbf{Drift During Training} \\
\midrule
Level 0 (fast) & 1 & Every step (50$\times$) & High (as expected) \\
Level 1 (medium) & 16 & Every 16 steps ($\sim$3$\times$) & Moderate \\
Level 2 (slow) & 1\,000\,000 & Never (frozen) & Zero \\
Level 3 (permanent) & 16\,000\,000 & Never (frozen) & Zero \\
\bottomrule
\end{tabular}
\end{table}

\subsection{CMS Reset Levels for Dream Training}

\begin{table}[H]
\centering
\begin{tabular}{@{}llll@{}}
\toprule
\textbf{reset\_level} & \textbf{Resets} & \textbf{Persists} & \textbf{Use Case} \\
\midrule
\texttt{all} & Titans + all CMS & Nothing & Standard (matches original paper) \\
\texttt{fast} & Titans + L0 & L1, L2 & Cross-episode memory \\
\texttt{fast+mid} & Titans + L0 + L1 & L2 only & Strongest persistence \\
\bottomrule
\end{tabular}
\end{table}

% ============================================================
\section{Limitations \& Known Issues}
% ============================================================

\label{sec:limitations}
\begin{enumerate}[nosep]
    \item \textbf{Absolute performance gap with gradient-based SOTA.} Supreme's best MuJoCo result (Walker2d 629.6) is $\sim$12\% of DreamerV3 (3,822). The evolutionary CMA-ES controller search, while hardware-efficient, cannot yet match gradient-based methods that train for millions of steps with replay buffers.

    \item \textbf{Episode-length truncation trade-off.} The 200-step CMA evaluation cap accelerates search 3--5$\times$ but biases controllers toward short-horizon gaits. HalfCheetah ($-$517) suffers most: the controller collapses over full 1,000-step episodes. Adaptive horizon scheduling may mitigate this.

    \item \textbf{Dream-to-real gap.} Dream-only training (Atari 100k, Reacher Phase 8) still trails real-environment training significantly. Compounding prediction error remains the primary bottleneck for imagination-based policy optimisation.

    \item \textbf{Discrete-action and manipulation tasks.} Atari 100k (all below random) and MetaWorld MT10 (0\% success) indicate the current V-M-C pipeline does not yet generalise to discrete-action or contact-rich manipulation domains.

    \item \textbf{CMS frequency levels are task-specific.} The [1, 16, 1M, 16M] schedule means only L0 and L1 actively update in typical episode lengths ($\leq$1000 steps); L2/L3 remain frozen knowledge anchors.

    \item \textbf{MPS backend limitations.} Some PyTorch operations fall back to CPU on Apple Silicon. CUDA hardware would likely provide 2--3$\times$ speedup.

    \item \textbf{No sim-to-real transfer tested.} All evaluations are in simulation.
\end{enumerate}

% ============================================================
\section{Key Design Decisions}
% ============================================================

\begin{enumerate}[nosep]
    \item \textbf{Tiny controller:} $\sim$5,350 parameters keeps CMA-ES tractable. All intelligence resides in V and M.
    \item \textbf{Real-env CMA-ES for locomotion:} Bypasses the dream-to-real gap entirely for continuous control; dream training reserved for sample-limited regimes (Atari 100k).
    \item \textbf{Fixed-dim HOPE projection:} Memory matrix projected to 256-dim $h_t$ for CMA-ES compatibility.
    \item \textbf{CMS frequencies from NL paper:} [1, 16, 1M, 16M] naturally creates a fast/slow hierarchy where L0--L1 adapt per-episode while L2--L3 serve as frozen knowledge anchors.
    \item \textbf{CMA eval step cap:} 200-step episodes during CMA-ES search give 3--5$\times$ speedup; full-length evaluation for final scoring.
    \item \textbf{V+M caching:} Pre-trained vision and memory checkpoints are reused across controller re-training, enabling rapid iteration.
    \item \textbf{MDN variance as uncertainty signal:} Free uncertainty estimate for reward penalty without ensembles.
\end{enumerate}

% ============================================================
\section{References}
% ============================================================

\begin{enumerate}[nosep,label={[\arabic*]},leftmargin=*]
    \item Ha, D.\ \& Schmidhuber, J.\ (2018). World Models. \textit{arXiv:1803.10122}.
    \item Behrouz, A.\ et al.\ (2025). Nested Learning. (HOPE/NL paper).
    \item Janner, M.\ et al.\ (2019). When to Trust Your Model: Model-Based Policy Optimization. \textit{NeurIPS}.
    \item Yu, T.\ et al.\ (2020). MOPO: Model-Based Offline Policy Optimization. \textit{NeurIPS}.
    \item Hansen, N.\ (2016). The CMA Evolution Strategy: A Tutorial. \textit{arXiv:1604.00772}.
\end{enumerate}

\vfill

\begin{center}
\textcolor{phase}{\rule{0.5\textwidth}{0.5pt}}\\[6pt]
\textcolor{phase}{\small Supreme v1.0 --- Benchmarked on Mac M2 Air 8\,GB --- April 2026}
\end{center}

\end{document}