% sections/model_selection.tex

\section{Semantic Model Selection}
\label{sec:model_selection}

The core routing innovation is \emph{semantic model selection}: once the decision engine matches a routing decision $d^*$, the system analyzes the request's semantic content---its embedding, domain, complexity, and interaction history---to select the most cost-effective model from the decision's candidate set.
Unlike static routing or single-criterion difficulty classifiers, semantic selection operates over the full signal context produced by the signal engine, enabling cost-quality optimization that respects per-decision privacy and safety constraints.

We integrate thirteen algorithms within a unified interface, enabling systematic comparison and hybrid combinations across deployment scenarios.

\subsection{Problem Setting}

Given query embedding $\mathbf{e}_q \in \mathbb{R}^d$, domain category $z \in \{1, \ldots, C\}$, candidate models $\mathcal{M}_{d^*} = \{m_1, \ldots, m_K\}$ with associated costs $\{c_1, \ldots, c_K\}$, and quality estimators, the semantic selection problem is:
\begin{equation}
  m^* = \arg\max_{m_k \in \mathcal{M}_{d^*}} \; \text{Quality}(\mathbf{e}_q, z, m_k; \Theta) - \lambda \cdot \text{Cost}(m_k)
\end{equation}
where $\lambda \geq 0$ is a cost-sensitivity parameter and $\Theta$ represents algorithm-specific parameters.
The per-decision candidate set $\mathcal{M}_{d^*}$ is critical: privacy-constrained decisions restrict candidates to compliant models, while cost-optimized decisions include a broader pool with aggressive cost weighting.
We categorize algorithms into families based on their selection mechanism (\Cref{fig:selection_taxonomy}).

\begin{figure}[t]
\centering
\resizebox{\linewidth}{!}{%
\begin{tikzpicture}[
    node distance=0.2cm,
    fam/.style={rectangle, draw, rounded corners=2pt, fill=blue!6,
                minimum height=0.45cm, minimum width=1.6cm,
                align=left, font=\scriptsize, inner sep=3pt},
    alg/.style={rectangle, draw, rounded corners=1pt, fill=black!4,
                minimum height=0.38cm, align=left,
                font=\scriptsize, inner sep=2pt},
    root/.style={rectangle, draw, rounded corners=3pt, fill=orange!5,
                 minimum height=0.5cm, align=center,
                 font=\scriptsize\bfseries, inner sep=4pt},
    edge from parent/.style={draw, thick, ->, >=stealth},
    level 1/.style={sibling distance=1.05cm, level distance=2.6cm},
  ]
  % Root
  \node[root] (rt) at (0, 0) {Semantic\\[-1pt]Selection};

  % Families (y positions)
  \node[fam] (f1) at (3.0, 3.15) {Rating-Based};
  \node[fam] (f2) at (3.0, 2.1) {Embedding};
  \node[fam] (f3) at (3.0, 1.25) {Cascading};
  \node[fam] (f4) at (3.0, 0.0) {Classical ML};
  \node[fam] (f5) at (3.0, -1.25) {RL};
  \node[fam] (f6) at (3.0, -2.1) {Latency};
  \node[fam] (f7) at (3.0, -3.15) {Multi-Round};

  % Algorithms
  \node[alg] (a1) at (6.0, 3.45) {Static};
  \node[alg] (a2) at (6.0, 2.85) {Elo Rating};
  \node[alg] (a3) at (6.0, 2.4) {RouterDC};
  \node[alg] (a4) at (6.0, 1.8) {Hybrid};
  \node[alg] (a5) at (6.0, 1.25) {AutoMix};
  \node[alg] (a6) at (6.0, 0.6) {KNN};
  \node[alg] (a7) at (6.0, 0.2) {KMeans};
  \node[alg] (a8) at (6.0, -0.2) {SVM};
  \node[alg] (a9) at (6.0, -0.6) {MLP};
  \node[alg] (a10) at (6.0, -0.95) {Thompson};
  \node[alg] (a11) at (6.0, -1.55) {GMTRouter};
  \node[alg] (a12) at (6.0, -2.1) {Latency-Aware};
  \node[alg] (a13) at (6.0, -3.15) {ReMoM};

  % Root → Families
  \foreach \f in {f1,f2,f3,f4,f5,f6,f7}
    \draw[thick, ->, >=stealth] (rt.east) -- (\f.west);

  % Families → Algorithms
  \draw[thick, ->, >=stealth] (f1.east) -- (a1.west);
  \draw[thick, ->, >=stealth] (f1.east) -- (a2.west);
  \draw[thick, ->, >=stealth] (f2.east) -- (a3.west);
  \draw[thick, ->, >=stealth] (f2.east) -- (a4.west);
  \draw[thick, ->, >=stealth] (f3.east) -- (a5.west);
  \draw[thick, ->, >=stealth] (f4.east) -- (a6.west);
  \draw[thick, ->, >=stealth] (f4.east) -- (a7.west);
  \draw[thick, ->, >=stealth] (f4.east) -- (a8.west);
  \draw[thick, ->, >=stealth] (f4.east) -- (a9.west);
  \draw[thick, ->, >=stealth] (f5.east) -- (a10.west);
  \draw[thick, ->, >=stealth] (f5.east) -- (a11.west);
  \draw[thick, ->, >=stealth] (f6.east) -- (a12.west);
  \draw[thick, ->, >=stealth] (f7.east) -- (a13.west);
\end{tikzpicture}
}
\caption{Taxonomy of thirteen semantic model selection algorithms organized by selection mechanism.  Families span from lightweight rating-based methods (Static, Elo) to learned approaches (RouterDC, Classical ML, RL), adaptive cascading (AutoMix), real-time latency tracking, and multi-round synthesis (ReMoM).}
\label{fig:selection_taxonomy}
\end{figure}

\subsection{Rating-Based Selection}

\noindent\textbf{Static.}
Each model carries a pre-configured quality score $s_k$; selection is $m^* = \arg\max_k s_k$.
Serves as a deterministic baseline.

\noindent\textbf{Elo Rating} (adapted from RouteLLM~\cite{ong2024routellm}).
Models maintain Elo ratings $R_k$ updated from pairwise user preference feedback.
Selection probability follows the Bradley-Terry model:
\begin{equation}
  P(m_i \succ m_j) = \frac{1}{1 + 10^{(R_j - R_i)/400}}
\end{equation}
Models are sampled proportional to their expected win rate against the candidate pool.
Ratings are updated online as user feedback arrives.

\subsection{Embedding-Based Selection}

\noindent\textbf{RouterDC}~\cite{chen2024routerdc}.
Dual contrastive learning trains query and model encoders to produce embeddings in a shared space.
Selection maximizes cosine similarity:
\begin{equation}
  m^* = \arg\max_{m_k \in \mathcal{M}_{d^*}} \cos(\mathbf{e}_q, \mathbf{e}_{m_k})
\end{equation}
The contrastive training objective encourages queries to be close to their best-performing model's embedding and distant from poorly-performing models.

\noindent\textbf{Hybrid}~\cite{hu2024routerbench}.
Combines Elo ratings, embedding similarity, and cost in a weighted score:
\begin{equation}
  \text{score}(m_k) = \alpha \cdot \tilde{R}_k + \beta \cdot \cos(\mathbf{e}_q, \mathbf{e}_{m_k}) + \gamma \cdot (1 - \tilde{c}_k)
\end{equation}
where $\tilde{R}_k$ and $\tilde{c}_k$ are normalized ratings and costs, and $\alpha + \beta + \gamma = 1$ are configurable weights.

\subsection{Cascading Selection}

\noindent\textbf{AutoMix}~\cite{aggarwal2023automix}.
Formulated as a Partially Observable Markov Decision Process (POMDP).
Models are ordered by capability $m_1 \prec m_2 \prec \cdots \prec m_K$.
The cascade:
\begin{enumerate}
  \item Generate response $a_k$ with current model $m_k$ (starting from $k=1$, the cheapest).
  \item Self-verify: estimate response quality $\hat{q}_k$ using $m_k$ itself.
  \item If $\hat{q}_k \geq \tau_k$, accept $a_k$; otherwise, escalate to $m_{k+1}$.
\end{enumerate}
The expected cost is:
\begin{equation}
  \mathbb{E}[C] = \sum_{k=1}^{K} C_k \cdot \prod_{j=1}^{k-1}(1 - P(\hat{q}_j \geq \tau_j))
\end{equation}
where $P(\hat{q}_j \geq \tau_j)$ is the probability that model $m_j$ passes self-verification.
This naturally trades off cost against quality.

\subsection{Classical ML Selection}

These methods train on routing records $\{(\mathbf{e}_q^{(i)}, z^{(i)}, m^{*(i)}, q^{(i)})\}$ where $q^{(i)}$ is a quality score.
Feature vectors combine embeddings and domain information:
\begin{equation}
  \mathbf{f} = [\mathbf{e}_q \in \mathbb{R}^d; \; \text{onehot}(z) \in \{0,1\}^C]
\end{equation}

\noindent\textbf{KNN.}
$k$-nearest neighbor search with Ball Tree indexing.
Quality-weighted majority voting:
\begin{equation}
  m^* = \arg\max_m \sum_{i \in \text{kNN}(\mathbf{f})} \mathbf{1}[m^{*(i)} = m] \cdot q^{(i)}
\end{equation}

\noindent\textbf{KMeans.}
Assigns queries to pre-computed clusters; selects the best model for the assigned cluster based on a combined quality-latency score:
\begin{equation}
  m^* = \arg\max_m \bigl(\alpha \cdot \text{quality}(m, z_\text{cluster}) - (1-\alpha) \cdot \text{latency}(m)\bigr)
\end{equation}

\noindent\textbf{SVM.}
Multi-class SVM with RBF or linear kernel, trained to classify feature vectors directly into model selections.

\noindent\textbf{MLP.}
A feed-forward neural network (two hidden layers with ReLU activation) mapping $\mathbf{f}$ to a softmax distribution over candidate models:
\begin{equation}
  P(m_k \mid \mathbf{f}) = \text{softmax}\bigl(W_2 \cdot \text{ReLU}(W_1 \mathbf{f} + b_1) + b_2\bigr)_k
\end{equation}
The MLP is implemented in the GPU-accelerated Candle runtime for low-latency inference.

\subsection{Reinforcement Learning Selection}

\noindent\textbf{Thompson Sampling}~\cite{thompson1933likelihood}.
Each model maintains a Beta prior:
\begin{equation}
  \theta_k \sim \text{Beta}(\alpha_k, \beta_k)
\end{equation}
Selection samples from each posterior and picks the maximum: $m^* = \arg\max_k \theta_k$.
Parameters $(\alpha_k, \beta_k)$ are updated from user preference feedback, naturally balancing exploration and exploitation.

\noindent\textbf{GMTRouter}~\cite{xie2025gmtrouter}.
Models multi-turn user-query-model interactions as a heterogeneous graph.
Graph neural network message passing captures complex interaction patterns:
\begin{equation}
  \mathbf{h}_v^{(l+1)} = \text{AGG}\bigl(\{\mathbf{h}_u^{(l)} \mid u \in \mathcal{N}(v)\}\bigr)
\end{equation}
where nodes represent users, queries, and models, and edges encode historical routing outcomes.

\subsection{Latency-Aware Selection}

\noindent\textbf{Latency-Aware.}
Selects the model with the best observed latency using percentile-based Time-per-Output-Token (TPOT) and Time-to-First-Token (TTFT) statistics collected at runtime.
For each candidate model $m_k$, the selector computes a normalized latency score:
\begin{equation}
  s_k = \frac{1}{|P|} \sum_{p \in P} \frac{\text{perc}_p(m_k)}{\min_{j}\, \text{perc}_p(m_j)}
\end{equation}
where $P \subseteq \{\text{TPOT}, \text{TTFT}\}$ is the set of configured performance metrics and $\text{perc}_p(m_k)$ is the observed percentile value for model $m_k$ on metric $p$.
Selection minimizes this score: $m^* = \arg\min_k s_k$.
This enables adaptive routing that responds to real-time backend performance degradation without requiring explicit latency thresholds as signal conditions.

\subsection{Multi-Round Reasoning (ReMoM)}

The ReMoM (Reasoning for Mixture of Models) strategy extends single-shot selection to multi-round parallel reasoning with LLM-driven synthesis.
Inspired by PaCoRe~\cite{pacore2025} but generalized to heterogeneous model pools, ReMoM executes a \emph{breadth schedule} of decreasing parallelism across rounds, where each subsequent round synthesizes the outputs of the previous round via prompted LLM calls (\Cref{fig:remom_flow}).

\begin{figure}[t]
\centering
\resizebox{\linewidth}{!}{%
\begin{tikzpicture}[
    node distance=0.3cm,
    call/.style={rectangle, draw, rounded corners=2pt,
                 minimum height=0.48cm, minimum width=1.35cm,
                 align=center, font=\scriptsize, inner sep=2pt},
    synth/.style={trapezium, draw, trapezium angle=75, rounded corners=1pt,
                  fill=orange!8, minimum height=0.5cm,
                  align=center, font=\scriptsize, inner sep=3pt},
    query/.style={rectangle, draw, rounded corners=2pt, fill=black!4,
                  font=\scriptsize, inner sep=3pt, align=center},
    outbox/.style={rectangle, draw, rounded corners=3pt, fill=green!8,
                   font=\scriptsize\bfseries, inner sep=4pt, align=center},
    roundlbl/.style={font=\scriptsize\bfseries, text=gray!70},
    arr/.style={->, >=stealth, thick},
    arrs/.style={->, >=stealth, semithick, gray!60},
  ]

  % === Query ===
  \node[query] (q) at (0, 0) {Query $q$};

  % === Round 1: b_1 = 4 ===
  \node[roundlbl] at (2.5, 1.55) {Round 1 ($b_1{=}4$)};
  \node[call, fill=blue!10]   (r1c1) at (2.5, 1.0)  {\footnotesize $m_A$};
  \node[call, fill=blue!10]   (r1c2) at (2.5, 0.4)  {\footnotesize $m_B$};
  \node[call, fill=blue!10]   (r1c3) at (2.5, -0.2) {\footnotesize $m_A$};
  \node[call, fill=yellow!15] (r1c4) at (2.5, -0.8) {\footnotesize $m_C$};

  % Parallel bracket
  \draw[decorate, decoration={brace, amplitude=4pt, mirror}, thick, gray!50]
    (1.7, 1.25) -- (1.7, -1.05) node[midway, left=5pt, font=\tiny, text=gray] {parallel};

  % Query → Round 1
  \draw[arr] (q.east) -- ++(0.4,0) |- (r1c1.west);
  \draw[arr] (q.east) -- ++(0.4,0) |- (r1c2.west);
  \draw[arr] (q.east) -- ++(0.4,0) |- (r1c3.west);
  \draw[arr] (q.east) -- ++(0.4,0) |- (r1c4.west);

  % === Synthesis 1 ===
  \node[synth] (s1) at (4.7, 0.1) {Synthesis\\[-1pt]Prompt};

  % Round 1 → Synthesis 1
  \draw[arrs] (r1c1.east) -- ++(0.3,0) |- (s1.west);
  \draw[arrs] (r1c2.east) -- (s1.west);
  \draw[arrs] (r1c3.east) -- ++(0.3,0) |- (s1.west);
  \draw[arrs] (r1c4.east) -- ++(0.3,0) |- (s1.west);

  % === Round 2: b_2 = 2 ===
  \node[roundlbl] at (6.8, 0.95) {Round 2 ($b_2{=}2$)};
  \node[call, fill=blue!10]   (r2c1) at (6.8, 0.4)  {\footnotesize $m_B$};
  \node[call, fill=yellow!15] (r2c2) at (6.8, -0.2) {\footnotesize $m_C$};

  % Synthesis 1 → Round 2
  \draw[arr] (s1.east) -- ++(0.2,0) |- (r2c1.west);
  \draw[arr] (s1.east) -- ++(0.2,0) |- (r2c2.west);

  % === Synthesis 2 ===
  \node[synth] (s2) at (8.8, 0.1) {Synthesis\\[-1pt]Prompt};

  % Round 2 → Synthesis 2
  \draw[arrs] (r2c1.east) -- ++(0.15,0) |- (s2.west);
  \draw[arrs] (r2c2.east) -- ++(0.15,0) |- (s2.west);

  % === Round 3: b_3 = 1 (final) ===
  \node[roundlbl] at (10.7, 0.55) {Final ($b_3{=}1$)};
  \node[call, fill=blue!10] (r3c1) at (10.7, 0.1) {\footnotesize $m_A$};

  % Synthesis 2 → Round 3
  \draw[arr] (s2.east) -- (r3c1.west);

  % === Output ===
  \node[outbox] (out) at (12.6, 0.1) {Output};
  \draw[arr] (r3c1.east) -- (out.west);

  % === Compaction annotation ===
  \node[font=\tiny, text=gray, anchor=north] at (4.7, -1.25)
    {optionally compact responses};
  \draw[densely dotted, gray!40] (4.7, -1.15) -- (s1.south);
  \draw[densely dotted, gray!40] (8.8, -1.15) -- (s2.south);
  \node[font=\tiny, text=gray, anchor=north] at (8.8, -1.25)
    {before prompt construction};

\end{tikzpicture}
}
\caption{ReMoM execution flow with breadth schedule $\mathbf{b} = [4, 2]$.
Round~1 distributes 4~parallel calls across models ($m_A$, $m_B$, $m_C$); responses are (optionally compacted and) assembled into a synthesis prompt.
Round~2 sends 2~parallel synthesis calls.
A final round ($b_3 = 1$, auto-appended) produces the single output.
Each synthesis prompt includes the original query and all previous-round responses as numbered references, delegating quality judgment to the synthesizing LLM.}
\label{fig:remom_flow}
\end{figure}

\noindent\textbf{Breadth schedule.}
The operator specifies a breadth schedule $\mathbf{b} = [b_1, b_2, \ldots, b_R]$ defining the number of parallel model calls per round.
A final synthesis round with $b_{R+1} = 1$ is automatically appended, yielding a total of $R+1$ rounds.
For example, $\mathbf{b} = [32, 4]$ produces three rounds: 32 parallel calls, then 4 parallel calls each synthesizing the 32 responses, then a single final call synthesizing the 4 responses.

\noindent\textbf{Model distribution.}
At each round, $b_r$ calls are distributed among the candidate models $\mathcal{M}_{d^*}$ according to one of three strategies:
(1)~\emph{equal}: calls are distributed evenly across all candidates with round-robin remainder allocation;
(2)~\emph{weighted}: calls are distributed proportionally to model weights (currently equivalent to equal distribution);
(3)~\emph{first\_only}: all $b_r$ calls target a single model with different random seeds, providing PaCoRe-compatible single-model diversity.
Calls within each round execute concurrently with configurable concurrency limits.

\noindent\textbf{LLM-driven synthesis.}
After collecting responses from round $r$, the system constructs a synthesis prompt for round $r+1$ using a configurable Go \texttt{text/template}.
The default template presents the original query alongside all previous-round responses as numbered references, instructing the next-round model(s) to \emph{``analyze these references and provide your own comprehensive solution.''}
When reasoning content is available (e.g., from models supporting extended thinking), the template additionally includes each reference's chain-of-thought reasoning.
This approach delegates quality judgment entirely to the synthesizing LLM rather than relying on explicit scoring or weighted aggregation.

\noindent\textbf{Response compaction.}
To manage prompt length across rounds, responses can be compacted before inclusion in synthesis prompts.
Two strategies are supported: \emph{full} (no compaction, the default) and \emph{last\_n\_tokens} (retaining only the final $N$ tokens, estimated at $\sim$4 characters per token).
This is particularly important for high-breadth schedules where concatenating all responses would exceed context limits.

\noindent\textbf{Execution flow.}
The complete algorithm proceeds as:
\begin{enumerate}
  \item \textbf{Schedule construction}: Append $[1]$ to the user-specified breadth schedule $\mathbf{b}$.
  \item \textbf{Round~1 (parallel generation)}: Distribute $b_1$ calls across candidate models; execute concurrently with temperature $T$ (default 1.0) for response diversity.
  \item \textbf{Rounds~$2 \ldots R{+}1$ (synthesis)}: For each subsequent round, build a synthesis prompt from the previous round's (optionally compacted) responses, distribute $b_r$ calls, and execute concurrently.
  \item \textbf{Final output}: Return the single response from the final round ($b_{R+1} = 1$).
\end{enumerate}

ReMoM is particularly effective when model capabilities are uncertain or when the task benefits from diverse perspectives (e.g., complex reasoning, multi-faceted analysis).
The breadth schedule provides fine-grained control over the cost--quality tradeoff: higher initial breadth increases diversity at the cost of additional LLM calls, while the funneling structure ensures convergence to a single synthesized answer.

\subsection{Unified Selection Interface}

All thirteen algorithms implement a common interface:
\begin{equation}
  \text{Select}: (\mathbf{e}_q, z, \mathcal{M}, \Theta) \to (m^*, c)
\end{equation}
returning the selected model and a confidence score.
This uniformity enables:
(1)~per-decision algorithm selection---different routing decisions can use different selection algorithms, allowing cost-optimized decisions to use cascading (AutoMix) while quality-sensitive decisions use embedding-based (RouterDC) selection;
(2)~A/B testing across algorithms on live traffic;
(3)~ensemble methods that combine multiple selectors.

\subsection{Cost-Aware Selection in Multi-Provider Settings}

In multi-provider deployments where the same logical model may be served by different providers at different price points, the selection algorithms operate in conjunction with the configured provider/backend topology (\Cref{subsec:provider_backend}).
The selection algorithm chooses the best \emph{model} based on semantic analysis, and the routing configuration binds that choice to a provider-backed target.
This two-stage process separates quality optimization (which model is best for this query?) from deployment optimization (which configured backend target should serve this model?), while endpoint health, load balancing, and recovery stay in the gateway or orchestration layer.
