<!DOCTYPE html><html lang="en">
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>Emo: Pretraining Mixture of Experts for Emergent Modularity</title>
<!--Generated on Thu May  7 17:57:11 2026 by LaTeXML (version 0.8.8) http://dlmf.nist.gov/LaTeXML/.-->
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link rel="stylesheet" href="/static/browse/0.3.4/css/arxiv-html-papers-20260131.css" type="text/css">
<script src="/static/browse/0.3.4/js/arxiv-html-papers-20260131.js"></script>
<script>
  // Restore the saved color scheme preference, or
  // enact the browser preference if "automatic", 
  // without expecting DOM load to have completed.
  //
  // Also restore any saved readingmode and ToC display preferences.
  function initializeReadingPreferences() {
    let saved_theme = localStorage.getItem("ar5iv_theme") || "automatic";
    if (saved_theme === "automatic") {
      if (window.matchMedia("(prefers-color-scheme: dark)").matches) {
        saved_theme = "dark";
      }
    }
    if (saved_theme == "dark") {
      document.documentElement.setAttribute("data-theme", "dark");
    } else {
      document.documentElement.setAttribute("data-theme", "light");
    }

    const tocDisplay = localStorage.getItem('arxiv_html_paper_toc_display');
    if (tocDisplay) {
      document.documentElement.setAttribute("data-toc-display", tocDisplay);
    }
    const readingMode = localStorage.getItem('arxiv_html_paper_reading_mode');
    if (readingMode) {
      document.documentElement.setAttribute("data-reading-mode", readingMode);
    }
  }
  // Run as soon as JS starts, to minimize repainting
  initializeReadingPreferences();
</script>
<link rel="apple-touch-icon" sizes="180x180"
  href="/static/browse/0.3.4/images/icons/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32"
  href="/static/browse/0.3.4/images/icons/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16"
  href="/static/browse/0.3.4/images/icons/favicon-16x16.png">
<link rel="manifest" href="/static/browse/0.3.4/images/icons/site.webmanifest">
<link rel="mask-icon" href="/static/browse/0.3.4/images/icons/safari-pinned-tab.svg" color="#5bbad5">
<link rel="stylesheet" type="text/css" media="screen" href="https://use.typekit.net/utz6mli.css">
</head>
<body>
<dialog id="modal-form" aria-labelledby="modal-title" closedby="any">
  <form id="modal-form-content" method="dialog" enctype="multipart/form-data">
    <header class="modal-header">
      <h5 id="modal-title" class="modal-title">Report GitHub Issue</h5>
      <button type="submit" formnovalidate class="modal-close" aria-label="Close">×</button>
    </header>

    <div class="modal-body">
      <label for="form_title">Title:</label>
      <input class="form-control" id="form_title" name="form_title" required placeholder="Enter title">

      <p id="selectedTextModalDescription" hidden>Content selection saved. Describe the issue below:</p>

      <label for="description">Description:</label>
      <textarea class="form-control" id="description" name="description" required maxlength="500"
        placeholder="500 characters maximum"></textarea>
    </div>

    <footer class="modal-footer">
      <button type="submit" value="internal-report" class="sr-only modal-submit">Submit without GitHub</button>
      <button type="submit" value="github-report" class="modal-submit">Submit in GitHub</button>
    </footer>
  </form>
</dialog>

<header class="arxiv-html-header">
  <div class="html-header-logo">
    <a href="/">
      <img alt="arXiv logo" class="logo desktop-only" width="100"
        src="/static/browse/0.3.4/images/arxiv-logo-one-color-white.svg">
      <span class="sr-only">Back to arXiv</span>
    </a>
  </div>
  <!--TOC, dark mode, links-->
  <nav class="html-header-nav">
    <a class="header-button hover-effect desktop-only" href="https://info.arxiv.org/about/accessible_HTML.html"
      target="_blank">Why HTML?</a>
    <a class="header-button" title="Report an Issue" href="#" title="Report an issue"
      onclick="event.preventDefault(); showModalForm();">
      <svg role="presentation" class="mobile-only toggle-icon" aria-hidden="true" height="1.25rem"
        viewBox="0 0 640 640">
        <path
          d="M224 160C224 107 267 64 320 64C373 64 416 107 416 160L416 163.6C416 179.3 403.3 192 387.6 192L252.5 192C236.8 192 224.1 179.3 224.1 163.6L224.1 160zM569.6 172.8C580.2 186.9 577.3 207 563.2 217.6L465.4 290.9C470.7 299.8 474.7 309.6 477.2 320L576 320C593.7 320 608 334.3 608 352C608 369.7 593.7 384 576 384L480 384L480 416C480 418.6 479.9 421.3 479.8 423.9L563.2 486.4C577.3 497 580.2 517.1 569.6 531.2C559 545.3 538.9 548.2 524.8 537.6L461.7 490.3C438.5 534.5 395.2 566.5 344 574.2L344 344C344 330.7 333.3 320 320 320C306.7 320 296 330.7 296 344L296 574.2C244.8 566.5 201.5 534.5 178.3 490.3L115.2 537.6C101.1 548.2 81 545.3 70.4 531.2C59.8 517.1 62.7 497 76.8 486.4L160.2 423.9C160.1 421.3 160 418.7 160 416L160 384L64 384C46.3 384 32 369.7 32 352C32 334.3 46.3 320 64 320L162.8 320C165.3 309.6 169.3 299.8 174.6 290.9L76.8 217.6C62.7 207 59.8 186.9 70.4 172.8C81 158.7 101.1 155.8 115.2 166.4L224 248C236.3 242.9 249.8 240 264 240L376 240C390.2 240 403.7 242.8 416 248L524.8 166.4C538.9 155.8 559 158.7 569.6 172.8z" />
      </svg>
      <span class="desktop-only">Report Issue</span></a>
    <!--back to abstract-->
    <a class="header-button" title="Back to abstract page" aria-label="Back to abstract page"
      href="/abs/2605.06663v1">
      <svg class="mobile-only toggle-icon" role="presentation" height="1.25rem" viewBox="0 0 512 512" fill="#ffffff"
        aria-hidden="true">
        <path
          d="M502.6 278.6c12.5-12.5 12.5-32.8 0-45.3l-128-128c-12.5-12.5-32.8-12.5-45.3 0s-12.5 32.8 0 45.3L402.7 224 192 224c-17.7 0-32 14.3-32 32s14.3 32 32 32l210.7 0-73.4 73.4c-12.5 12.5-12.5 32.8 0 45.3s32.8 12.5 45.3 0l128-128zM160 96c17.7 0 32-14.3 32-32s-14.3-32-32-32L96 32C43 32 0 75 0 128L0 384c0 53 43 96 96 96l64 0c17.7 0 32-14.3 32-32s-14.3-32-32-32l-64 0c-17.7 0-32-14.3-32-32l0-256c0-17.7 14.3-32 32-32l64 0z">
        </path>
      </svg>
      <span class="desktop-only">Back to Abstract</span>
    </a>
    <!-- PDF download link -->
    <a class="header-button" title="Download PDF" href="/pdf/2605.06663v1"
      target="_blank">
      <svg class="mobile-only toggle-icon" role="presentation" height="1.25rem" viewBox="0 0 576 542">
        <path
          d="M208 48L96 48c-8.8 0-16 7.2-16 16l0 384c0 8.8 7.2 16 16 16l80 0 0 48-80 0c-35.3 0-64-28.7-64-64L32 64C32 28.7 60.7 0 96 0L229.5 0c17 0 33.3 6.7 45.3 18.7L397.3 141.3c12 12 18.7 28.3 18.7 45.3l0 149.5-48 0 0-128-88 0c-39.8 0-72-32.2-72-72l0-88zM348.1 160L256 67.9 256 136c0 13.3 10.7 24 24 24l68.1 0zM240 380l32 0c33.1 0 60 26.9 60 60s-26.9 60-60 60l-12 0 0 28c0 11-9 20-20 20s-20-9-20-20l0-128c0-11 9-20 20-20zm32 80c11 0 20-9 20-20s-9-20-20-20l-12 0 0 40 12 0zm96-80l32 0c28.7 0 52 23.3 52 52l0 64c0 28.7-23.3 52-52 52l-32 0c-11 0-20-9-20-20l0-128c0-11 9-20 20-20zm32 128c6.6 0 12-5.4 12-12l0-64c0-6.6-5.4-12-12-12l-12 0 0 88 12 0zm76-108c0-11 9-20 20-20l48 0c11 0 20 9 20 20s-9 20-20 20l-28 0 0 24 28 0c11 0 20 9 20 20s-9 20-20 20l-28 0 0 44c0 11-9 20-20 20s-20-9-20-20l0-128z" />
      </svg>
      <span class="desktop-only">Download PDF</span></a>
    <!-- navigational table of contents toggle -->
    <a class="header-button toggle-icon" href="javascript:toggleNavTOC();" title="Toggle navigation"
      aria-label="Toggle navigation">
      <svg height="1.25rem" role="presentation" viewBox="0 0 512 512">
        <path
          d="M40 48C26.7 48 16 58.7 16 72v48c0 13.3 10.7 24 24 24H88c13.3 0 24-10.7 24-24V72c0-13.3-10.7-24-24-24H40zM192 64c-17.7 0-32 14.3-32 32s14.3 32 32 32H480c17.7 0 32-14.3 32-32s-14.3-32-32-32H192zm0 160c-17.7 0-32 14.3-32 32s14.3 32 32 32H480c17.7 0 32-14.3 32-32s-14.3-32-32-32H192zm0 160c-17.7 0-32 14.3-32 32s14.3 32 32 32H480c17.7 0 32-14.3 32-32s-14.3-32-32-32H192zM16 232v48c0 13.3 10.7 24 24 24H88c13.3 0 24-10.7 24-24V232c0-13.3-10.7-24-24-24H40c-13.3 0-24 10.7-24 24zM40 368c-13.3 0-24 10.7-24 24v48c0 13.3 10.7 24 24 24H88c13.3 0 24-10.7 24-24V392c0-13.3-10.7-24-24-24H40z">
        </path>
      </svg>
    </a>
    <!--- collapsable header / reading mode toggle -->
    <a class="header-button toggle-icon" href="javascript:toggleReadingMode();"
      title="Disable reading mode, show header and footer">
      <svg role="presentation" height="1.25rem"
        viewBox="0 0 448 512"><!--!Font Awesome Free v7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2026 Fonticons, Inc.-->
        <path
          d="M32 32C14.3 32 0 46.3 0 64l0 96c0 17.7 14.3 32 32 32s32-14.3 32-32l0-64 64 0c17.7 0 32-14.3 32-32s-14.3-32-32-32L32 32zM64 352c0-17.7-14.3-32-32-32S0 334.3 0 352l0 96c0 17.7 14.3 32 32 32l96 0c17.7 0 32-14.3 32-32s-14.3-32-32-32l-64 0 0-64zM320 32c-17.7 0-32 14.3-32 32s14.3 32 32 32l64 0 0 64c0 17.7 14.3 32 32 32s32-14.3 32-32l0-96c0-17.7-14.3-32-32-32l-96 0zM448 352c0-17.7-14.3-32-32-32s-32 14.3-32 32l0 64-64 0c-17.7 0-32 14.3-32 32s14.3 32 32 32l96 0c17.7 0 32-14.3 32-32l0-96z" />
      </svg>
    </a>
    <!--- colored theme toggle -->
    <button type="button" class="header-button color-tog" onclick="toggleColorScheme();" title="Toggle dark/light mode" aria-label="Toggle color scheme">
      <span class="toggle-icon automatic-tog" aria-hidden="true">
        <svg role="presentation" height="1.25rem" viewBox="0 0 24 24">
          <path
            d="m14.3 16-.7-2h-3.2l-.7 2H7.8L11 7h2l3.2 9h-1.9M20 8.69V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69m-9.15 3.96h2.3L12 9l-1.15 3.65Z">
          </path>
        </svg>
      </span>
      <span class="toggle-icon light-tog" aria-hidden="true">
        <svg role="presentation" height="1.25rem" viewBox="0 0 24 24">
          <path
            d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z">
          </path>
        </svg>
      </span>
      <span class="toggle-icon dark-tog" aria-hidden="true">
        <svg role="presentation" height="1.25rem" viewBox="0 0 24 24">
          <path
            d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z">
          </path>
        </svg>
      </span>
    </button>
  </nav>
</header><nav class="ltx_page_navbar">
<nav class="ltx_TOC">
<ol class="ltx_toclist">
<li class="ltx_tocentry ltx_tocentry_abstract"><a href="#abstract1" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Abstract</span></a></li>
<li class="ltx_tocentry ltx_tocentry_section"><a href="#S1" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">1 </span>Introduction</span></a></li>
<li class="ltx_tocentry ltx_tocentry_section">
<a href="#S2" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">2 </span>Related Work</span></a>
<ol class="ltx_toclist ltx_toclist_section">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S2.SS0.SSS0.Px1" title="In 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Mixture-of-Experts as Scalable Architectures.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S2.SS0.SSS0.Px2" title="In 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Specialization and Modularity of Existing MoEs.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S2.SS0.SSS0.Px3" title="In 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Training MoEs with Structured or Specialized Experts.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_section">
<a href="#S3" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">3 </span>Modular Mixture of Experts (<span class="ltx_text ltx_font_smallcaps">Emo</span>)</span></a>
<ol class="ltx_toclist ltx_toclist_section">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S3.SS0.SSS0.Px1" title="In 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Naive Approach.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S3.SS0.SSS0.Px2" title="In 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span>’s Approach.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#S3.SS1" title="In 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">3.1 </span>Preliminary: Mixture of Experts Architecture</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#S3.SS2" title="In 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">3.2 </span><span class="ltx_text ltx_font_smallcaps">Emo</span>: An Objective to Induce Modularity</span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S3.SS2.SSS0.Px1" title="In 3.2 Emo: An Objective to Induce Modularity ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Formulation.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#S3.SS3" title="In 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">3.3 </span>Key Technical Considerations</span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S3.SS3.SSS0.Px1" title="In 3.3 Key Technical Considerations ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Consideration 1. Load Balancing.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S3.SS3.SSS0.Px2" title="In 3.3 Key Technical Considerations ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Consideration 2. Choosing Expert Pool Size.</span></a></li>
</ol>
</li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_section">
<a href="#S4" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">4 </span>Experimental Setup</span></a>
<ol class="ltx_toclist ltx_toclist_section">
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#S4.SS1" title="In 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">4.1 </span>Architecture &amp; Training Details</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#S4.SS2" title="In 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">4.2 </span>Evaluation</span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S4.SS2.SSS0.Px1" title="In 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Full-model Evaluation.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S4.SS2.SSS0.Px2" title="In 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Selective Expert Use.</span></a></li>
</ol>
</li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_section">
<a href="#S5" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">5 </span>Results and Analysis</span></a>
<ol class="ltx_toclist ltx_toclist_section">
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#S5.SS1" title="In 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">5.1 </span>Full-Model Evaluation</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#S5.SS2" title="In 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">5.2 </span>Selective Expert Use</span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S5.SS2.SSS0.Px1" title="In 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Standard MoEs (Green) Degrade Sharply.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S5.SS2.SSS0.Px2" title="In 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> (Purple) Enables Modular Use.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S5.SS2.SSS0.Px3" title="In 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Expert Subsets of <span class="ltx_text ltx_font_smallcaps">Emo</span> Outperform Memory-matched Models Trained from Scratch.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S5.SS2.SSS0.Px4" title="In 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> is Robust to Expert Selection Schemes.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#S5.SS3" title="In 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">5.3 </span>Semantic Specialization Emerge in <span class="ltx_text ltx_font_smallcaps">Emo</span></span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S5.SS3.SSS0.Px1" title="In 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> Clusters Align with Semantic Themes.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S5.SS3.SSS0.Px2" title="In 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> Expert Activation Patterns across Domains are Distinct and Matches Human Intuition.</span></a></li>
</ol>
</li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_section">
<a href="#S6" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">6 </span>Future Directions</span></a>
<ol class="ltx_toclist ltx_toclist_section">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S6.SS0.SSS0.Px1" title="In 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Accessible Deployment of Large Sparse MoEs.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S6.SS0.SSS0.Px2" title="In 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Fine-grained Control.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S6.SS0.SSS0.Px3" title="In 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Modular Development and Maintenance.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#S6.SS0.SSS0.Px4" title="In 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Higher Degrees of Monitorability.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_section"><a href="#S7" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">7 </span>Conclusion</span></a></li>
<li class="ltx_tocentry ltx_tocentry_bibliography"><a href="#bib" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">References</span></a></li>
<li class="ltx_tocentry ltx_tocentry_appendix">
<a href="#A1" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">A </span>Architectural &amp; Training Details and Ablations</span></a>
<ol class="ltx_toclist ltx_toclist_appendix">
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A1.SS1" title="In Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">A.1 </span>Load Balancing</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A1.SS2" title="In Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">A.2 </span>How to Choose <math class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math></span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A1.SS3" title="In Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">A.3 </span>Shared Experts</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#A1.SS4" title="In Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">A.4 </span>Tuning LR and LB</span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A1.SS4.SSS0.Px1" title="In A.4 Tuning LR and LB ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Standard MoE Hyperparameter Ablations.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A1.SS4.SSS0.Px2" title="In A.4 Tuning LR and LB ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> Hyperparameter Ablations.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A1.SS5" title="In Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">A.5 </span>Prenorm vs ReorderedNorm</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_appendix">
<a href="#A2" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">B </span>Selective expert use Details</span></a>
<ol class="ltx_toclist ltx_toclist_appendix">
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A2.SS1" title="In Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">B.1 </span>130B Token Experiments</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection">
<a href="#A2.SS2" title="In Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">B.2 </span>Ablations on expert subset initialization</span></a>
<ol class="ltx_toclist ltx_toclist_subsection">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A2.SS2.SSS0.Px1" title="In B.2 Ablations on expert subset initialization ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> is Sample-efficient in Expert-selection.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A2.SS2.SSS0.Px2" title="In B.2 Ablations on expert subset initialization ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> Depends on Validation Data.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A2.SS3" title="In Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">B.3 </span>Selective Expert Use on "Other" Category in MMLU and MMLU Pro</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A2.SS4" title="In Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">B.4 </span>Annealing Standard MoEs to be Modular.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_subsection"><a href="#A2.SS5" title="In Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">B.5 </span>Generations from Selective Expert Use on GSM8K</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_appendix">
<a href="#A3" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">C </span>Evaluation Details</span></a>
<ol class="ltx_toclist ltx_toclist_appendix">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A3.SS0.SSS0.Px1" title="In Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">MMLU and MMLU-Pro Setup.</span></a></li>
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A3.SS0.SSS0.Px2" title="In Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title">Evaluation and Finetuning Protocol.</span></a></li>
</ol>
</li>
<li class="ltx_tocentry ltx_tocentry_appendix">
<a href="#A4" title="In Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_tag ltx_tag_ref">D </span>Token Clustering Details</span></a>
<ol class="ltx_toclist ltx_toclist_appendix">
<li class="ltx_tocentry ltx_tocentry_paragraph"><a href="#A4.SS0.SSS0.Px1" title="In Appendix D Token Clustering Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_title"><span class="ltx_text ltx_font_smallcaps">Emo</span> forms a Distinct Cluster for the First Token of Each Document.</span></a></li>
</ol>
</li>
</ol></nav>
</nav>
<div class="ltx_page_main">
<div id="infobox" class="infobox">
  <a id="license-tr" href="https://info.arxiv.org/help/license/index.html#licenses-available">
    License: CC BY 4.0
  </a>
  <div id="watermark-tr">
arXiv:2605.06663v1 [cs.CL] 07 May 2026</div>
</div><div class="ltx_page_content">
<article class="ltx_document ltx_authors_1line">
<h1 class="ltx_title ltx_title_document">
<span id="11.1" class="ltx_text ltx_font_smallcaps">Emo</span>: Pretraining Mixture of Experts
<br class="ltx_break">for Emergent Modularity
</h1>
<div class="ltx_authors">
<span class="ltx_creator ltx_role_author">
<span class="ltx_personname">Ryan Wang<math id="1.m1" class="ltx_Math" alttext="\hskip 1.00006pt{}^{{\color[rgb]{0.14453125,0.30859375,0.7890625}\definecolor[named]{pgfstrokecolor}{rgb}{0.14453125,0.30859375,0.7890625}\boldsymbol{\alpha}}{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}" display="inline" intent=":literal"><semantics><msup><mi></mi><mrow><mi style="--ltx-fg-color:#254FC9;" mathcolor="#254FC9">𝜶</mi><mo lspace="0em" rspace="0em">​</mo><mi style="--ltx-fg-color:#F0539B;" mathcolor="#F0539B">𝜷</mi></mrow></msup><annotation encoding="application/x-tex">\hskip 1.00006pt{}^{{\color[rgb]{0.14453125,0.30859375,0.7890625}\definecolor[named]{pgfstrokecolor}{rgb}{0.14453125,0.30859375,0.7890625}\boldsymbol{\alpha}}{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}</annotation></semantics></math>   Akshita Bhagia<math id="id2.2.m2" class="ltx_Math" alttext="\hskip 1.00006pt{}^{{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}" display="inline" intent=":literal"><semantics><msup><mi></mi><mi style="--ltx-fg-color:#F0539B;" mathcolor="#F0539B">𝜷</mi></msup><annotation encoding="application/x-tex">\hskip 1.00006pt{}^{{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}</annotation></semantics></math>  Sewon Min<math id="id3.3.m3" class="ltx_Math" alttext="\hskip 1.00006pt{}^{{\color[rgb]{0.14453125,0.30859375,0.7890625}\definecolor[named]{pgfstrokecolor}{rgb}{0.14453125,0.30859375,0.7890625}\boldsymbol{\alpha}}{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}" display="inline" intent=":literal"><semantics><msup><mi></mi><mrow><mi style="--ltx-fg-color:#254FC9;" mathcolor="#254FC9">𝜶</mi><mo lspace="0em" rspace="0em">​</mo><mi style="--ltx-fg-color:#F0539B;" mathcolor="#F0539B">𝜷</mi></mrow></msup><annotation encoding="application/x-tex">\hskip 1.00006pt{}^{{\color[rgb]{0.14453125,0.30859375,0.7890625}\definecolor[named]{pgfstrokecolor}{rgb}{0.14453125,0.30859375,0.7890625}\boldsymbol{\alpha}}{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}</annotation></semantics></math>  
<br class="ltx_break"><math id="id4.4.m4" class="ltx_Math" alttext="\hskip 1.00006pt{}^{{\color[rgb]{0.14453125,0.30859375,0.7890625}\definecolor[named]{pgfstrokecolor}{rgb}{0.14453125,0.30859375,0.7890625}\boldsymbol{\alpha}}}" display="inline" intent=":literal"><semantics><msup><mi></mi><mi style="--ltx-fg-color:#254FC9;" mathcolor="#254FC9">𝜶</mi></msup><annotation encoding="application/x-tex">\hskip 1.00006pt{}^{{\color[rgb]{0.14453125,0.30859375,0.7890625}\definecolor[named]{pgfstrokecolor}{rgb}{0.14453125,0.30859375,0.7890625}\boldsymbol{\alpha}}}</annotation></semantics></math>University of California, Berkeley
 <math id="id5.5.m5" class="ltx_Math" alttext="\hskip 1.00006pt{}^{{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}" display="inline" intent=":literal"><semantics><msup><mi></mi><mi style="--ltx-fg-color:#F0539B;" mathcolor="#F0539B">𝜷</mi></msup><annotation encoding="application/x-tex">\hskip 1.00006pt{}^{{\color[rgb]{0.94140625,0.32421875,0.609375}\definecolor[named]{pgfstrokecolor}{rgb}{0.94140625,0.32421875,0.609375}\boldsymbol{\beta}}}</annotation></semantics></math>Allen Institute for AI
 
<br class="ltx_break"> 
<span id="12.7.1" class="ltx_text ltx_font_typewriter">ryanyxw@berkeley.edu</span>
 <span id="13.8.2" class="ltx_text ltx_font_typewriter">akshitab@allenai.org</span>
 <span id="14.9.3" class="ltx_text ltx_font_typewriter">sewonm@berkeley.edu</span> 
<br class="ltx_break">
</span></span>
</div>

<div id="abstract1" class="ltx_abstract">
<h6 class="ltx_title ltx_title_abstract">Abstract</h6>
<p class="ltx_p">Large language models are typically deployed as monolithic systems, requiring the full model even when applications need only a narrow subset of capabilities, e.g., code, math, or domain-specific knowledge. Mixture-of-Experts (MoEs) seemingly offer a potential alternative by activating only a subset of experts per input, but in practice, restricting inference to a subset of experts for a given domain leads to severe performance degradation.
This limits their practicality in memory-constrained settings, especially as models grow larger and sparser.
We introduce <span class="ltx_text ltx_font_bold ltx_font_smallcaps">Emo</span>, an MoE designed for modularity—the independent use and composition of expert subsets—without requiring human-defined priors. Our key idea is to encourage tokens from similar domains to rely on similar experts. Since tokens within a document often share a domain, <span class="ltx_text ltx_font_smallcaps">Emo</span> restricts them to select experts from a shared pool, while allowing different documents to use different pools. This simple constraint enables coherent expert groupings to emerge during pretraining using document boundaries alone.
We pretrain a 1B-active, 14B-total <span class="ltx_text ltx_font_smallcaps">Emo</span> on 1T tokens. As a full model, it matches standard MoE performance. Crucially, it enables selective expert use: retaining only 25% (12.5%) of experts incurs just a 1% (3%) absolute drop, whereas standard MoEs break under the same setting. We further find that expert subsets in <span class="ltx_text ltx_font_smallcaps">Emo</span> specialize at semantic levels (e.g., domains such as math or code), in contrast to the low-level syntactic specialization observed in standard MoEs. Altogether, our results demonstrate a path toward modular, memory-efficient deployment of large, sparse models and open new opportunities for composable architectures.</p>
</div>
<div id="id10" class="ltx_para">
<table id="id10.4" class="ltx_tabular ltx_centering ltx_align_middle">
<tbody class="ltx_tbody">
<tr id="id7.1.1" class="ltx_tr">
<td id="id7.1.1.1" class="ltx_td ltx_align_right" style="padding-bottom:1.84995pt;"><span id="id7.1.1.1.1" class="ltx_text" style="font-size:90%;position:relative; bottom:-1.5pt;"><img src="2605.06663v1/x1.png" id="id7.1.1.1.1.g1" class="ltx_graphics ltx_img_square" width="20" height="19" alt="[Uncaptioned image]"></span></td>
<td id="id7.1.1.2" class="ltx_td ltx_align_left" style="padding-bottom:1.84995pt;"><span id="id7.1.1.2.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Model</span></td>
<td id="id7.1.1.3" class="ltx_td ltx_align_left" style="padding-bottom:1.84995pt;"><a href="https://huggingface.co/allenai/EMO" title="" class="ltx_ref ltx_href" style="font-size:90%;"><span class="ltx_ref ltx_nolink ltx_path ltx_font_typewriter">hf.co/allenai/EMO</span></a></td>
</tr>
<tr id="id8.2.2" class="ltx_tr">
<td id="id8.2.2.1" class="ltx_td ltx_align_right" style="padding-bottom:1.84995pt;"><span id="id8.2.2.1.1" class="ltx_text" style="font-size:90%;position:relative; bottom:-1.5pt;"><img src="2605.06663v1/x2.png" id="id8.2.2.1.1.g1" class="ltx_graphics ltx_img_square" width="19" height="19" alt="[Uncaptioned image]"></span></td>
<td id="id8.2.2.2" class="ltx_td ltx_align_left" style="padding-bottom:1.84995pt;"><span id="id8.2.2.2.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Code</span></td>
<td id="id8.2.2.3" class="ltx_td ltx_align_left" style="padding-bottom:1.84995pt;"><a href="https://github.com/allenai/EMO" title="" class="ltx_ref ltx_href" style="font-size:90%;"><span class="ltx_ref ltx_nolink ltx_path ltx_font_typewriter">github.com/allenai/EMO</span></a></td>
</tr>
<tr id="id9.3.3" class="ltx_tr">
<td id="id9.3.3.1" class="ltx_td ltx_align_right" style="padding-bottom:1.84995pt;"><span id="id9.3.3.1.1" class="ltx_text" style="font-size:90%;position:relative; bottom:-1.5pt;"><img src="2605.06663v1/figures/blog.png" id="id9.3.3.1.1.g1" class="ltx_graphics ltx_img_square" width="14" height="14" alt="[Uncaptioned image]"></span></td>
<td id="id9.3.3.2" class="ltx_td ltx_align_left" style="padding-bottom:1.84995pt;"><span id="id9.3.3.2.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Blog</span></td>
<td id="id9.3.3.3" class="ltx_td ltx_align_left" style="padding-bottom:1.84995pt;"><a href="https://allenai.org/blog/emo" title="" class="ltx_ref ltx_href" style="font-size:90%;"><span class="ltx_ref ltx_nolink ltx_path ltx_font_typewriter">allenai.org/blog/emo</span></a></td>
</tr>
<tr id="id10.4.4" class="ltx_tr">
<td id="id10.4.4.1" class="ltx_td ltx_align_right"><span id="id10.4.4.1.1" class="ltx_text" style="font-size:90%;position:relative; bottom:-1.5pt;"><img src="2605.06663v1/figures/search.png" id="id10.4.4.1.1.g1" class="ltx_graphics ltx_img_square" width="14" height="14" alt="[Uncaptioned image]"></span></td>
<td id="id10.4.4.2" class="ltx_td ltx_align_left"><span id="id10.4.4.2.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Visualization</span></td>
<td id="id10.4.4.3" class="ltx_td ltx_align_left"><a href="https://emovisualization.netlify.app" title="" class="ltx_ref ltx_href" style="font-size:90%;"><span class="ltx_ref ltx_nolink ltx_path ltx_font_typewriter">https://emovisualization.netlify.app</span></a></td>
</tr>
</tbody>
</table>
</div>
<section id="S1" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">1 </span>Introduction</h2>

<div id="S1.p1" class="ltx_para">
<p id="S1.p1.1" class="ltx_p">Large language models (LLMs) are typically trained and deployed as monolithic systems: a single model is pretrained, finetuned, and served as one unified entity <cite class="ltx_cite ltx_citemacro_cite">Olmo<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib36" title="Olmo 3" class="ltx_ref">2026</a>); DeepSeek-AI<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib9" title="DeepSeek-v3 technical report" class="ltx_ref">2025</a>); Yang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib10" title="Qwen3 technical report" class="ltx_ref">2025a</a>)</cite>. While effective, this paradigm becomes increasingly restrictive as models scale. In many deployment settings, applications require only a narrow subset of capabilities—such as code generation, mathematical reasoning, or domain-specific knowledge—but must still serve the full model, incurring unnecessary computational cost and memory use. Moreover, the monolithic design prevents isolating, updating, or improving specific capabilities without retraining and redeploying the entire system.</p>
</div>
<div id="S1.p2" class="ltx_para">
<p id="S1.p2.1" class="ltx_p">Mixture-of-Experts (MoE) models appear to offer a natural path toward relaxing this constraint, as they consist of many small FFNs (<span id="S1.p2.1.1" class="ltx_text ltx_font_italic">experts</span>), of which only a small subset is activated for each input token <cite class="ltx_cite ltx_citemacro_cite">DeepSeek-AI<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib9" title="DeepSeek-v3 technical report" class="ltx_ref">2025</a>, <a href="#bib.bib8" title="DeepSeek-v2: a strong, economical, and efficient mixture-of-experts language model" class="ltx_ref">2024</a>)</cite>. However, existing MoEs still require the full model for any task: tokens within the same input activate different experts, causing most or all experts to be used over the course of a task. As we show, this behavior—partially driven by experts specializing in low-level lexical patterns (e.g., prepositions, punctuation)—prevents subsets of the model from being usable independently, limiting the deployability of MoEs in memory-constrained settings, an issue that becomes increasingly important as models grow larger and sparser <cite class="ltx_cite ltx_citemacro_cite">Dai<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib7" title="DeepSeekMoE: towards ultimate expert specialization in mixture-of-experts language models" class="ltx_ref">2024</a>); DeepSeek-AI<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib9" title="DeepSeek-v3 technical report" class="ltx_ref">2025</a>); Yang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib10" title="Qwen3 technical report" class="ltx_ref">2025a</a>)</cite>.</p>
</div>
<figure id="S1.F1" class="ltx_figure"><img src="2605.06663v1/x3.png" id="S1.F1.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="250" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 1: </span>
<span id="S1.F1.6.1" class="ltx_text ltx_font_bold">(Left)</span>
<span id="S1.F1.7.2" class="ltx_text ltx_font_smallcaps">Emo</span> is an MoE trained with modularity as a first-class objective.
For a given domain (e.g., math, code, biomedical), users can select a small subset of experts of any size and retain near full-model performance. This turns a single model into a composable architecture, enabling flexible deployment with improved memory-accuracy tradeoffs for large, sparse MoEs.
<span id="S1.F1.8.3" class="ltx_text ltx_font_bold">(Right)</span>
Averaged performance over 16 MMLU categories across different memory budgets. <span id="S1.F1.9.4" class="ltx_text ltx_font_smallcaps">Emo</span> (purple) and Reg. MoE (green) are single models evaluated with expert subsets of different sizes. <span id="S1.F1.10.5" class="ltx_text ltx_font_smallcaps">Emo</span> expert subsets push the Pareto frontier in memory-accuracy trade-off, outperforming standard MoEs and even fixed-budget models trained from scratch.
</figcaption>
</figure>
<div id="S1.p3" class="ltx_para">
<p id="S1.p3.1" class="ltx_p">We instead seek to train MoE models in which experts organize into coherent groups that can be selectively used and composed. Concretely, we train an MoE model to be <span id="S1.p3.1.1" class="ltx_text ltx_font_italic">modular</span>, i.e., to support (1) the independent use of expert subsets and (2) their composition into a strong general-purpose model. Achieving this in practice, however, is challenging. Prior work has explored partitioning training data into predefined domains (e.g., math, coding) and training separate experts <cite class="ltx_cite ltx_citemacro_cite">Sukhbaatar<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib21" title="Branch-train-MiX: mixing expert LLMs into a mixture-of-experts LLM" class="ltx_ref">2024</a>); Shi<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib20" title="FlexOlmo: open language models for flexible data use" class="ltx_ref">2025</a>)</cite>, but this is too restricted for model’s learning and limits the model’s overall performance.</p>
</div>
<div id="S1.p4" class="ltx_para">
<p id="S1.p4.1" class="ltx_p">In this work, we propose to train MoE models in which modular structure emerges directly from the data, without relying on human-defined prior, and introduce <span id="S1.p4.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps">Emo</span>, an MoE that follows this approach. Our key intuition is that tokens from the similar domains should activate similar subsets of experts.
Assuming that tokens within a document tend to share a domain, we enforce this structure by restricting all tokens in a document to select their active experts from a shared pool. For example, in an MoE with 128 total and 8 active experts, all tokens from a document select their active subset from a shared pool of 32 experts. Different documents may use different expert pools, allowing the model to learn recurring expert subsets across the training corpus. Importantly, <span id="S1.p4.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> does not require predefined task or domain labels: expert subsets emerge in a self-supervised way, using document boundaries as the only grouping signal.</p>
</div>
<div id="S1.p5" class="ltx_para">
<p id="S1.p5.1" class="ltx_p">We train a 1B-active, 14B-total parameter <span id="S1.p5.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> model on 1 trillion tokens. As a full model, <span id="S1.p5.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> matches the overall performance of a standard MoE. More importantly, however, it enables effective composition of expert subsets, which standard MoEs fail to support. Across domain-specific subsets of MMLU and MMLU-Pro (e.g., math, physics, biology, social sciences), identifying and deploying only the most relevant experts largely preserve performance, e.g., 1% absolute performance drop when retaining 25% of experts, and 3% when retaining 12.5%. This is in contrast to standard MoEs that see severe degradation under the same constraint, e.g., 10% and 15% drops, respectively. These results show that <span id="S1.p5.1.3" class="ltx_text ltx_font_smallcaps">Emo</span> makes MoEs significantly more practical and accessible: instead of loading the full model, one can serve only a small subset of experts relevant to a given task or domain (Figure <a href="#S1.F1" title="Figure 1 ‣ 1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a>), which has important implications for deployment in memory-constrained settings <cite class="ltx_cite ltx_citemacro_cite">Song<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib37" title="BlockFFN: towards end-side acceleration-friendly mixture-of-experts with chunk-level activation sparsity" class="ltx_ref">2025</a>); Shen and Henderson (<a href="#bib.bib47" title="Temporally extended mixture-of-experts models" class="ltx_ref">2026</a>); Tairin<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib48" title="EMoE: task-aware memory efficient mixture-of-experts-based (moe) model inference" class="ltx_ref">2025</a>)</cite>.</p>
</div>
<div id="S1.p6" class="ltx_para">
<p id="S1.p6.1" class="ltx_p">We further analyze routing patterns and find that expert subsets specialize at higher-level semantic granularity, such as domains and topics (e.g., math, code), which is in contrast to experts in standard MoEs that specialize in lower-level syntactic patterns (e.g., prepositions, punctuation). This difference suggests that expert specialization in <span id="S1.p6.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is qualitatively distinct and underlies its modularity.</p>
</div>
<div id="S1.p7" class="ltx_para">
<p id="S1.p7.1" class="ltx_p">Together, these results demonstrate that modularity can be built into large language models, opening a path for broader functionalities, such as targeted extension training or more interpretable and debuggable components to better regulate model behavior.
We release both <span id="S1.p7.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> and a matched baseline trained on the same data to support reproducibility and further study.</p>
</div>
</section>
<section id="S2" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">2 </span>Related Work</h2>

<section id="S2.SS0.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Mixture-of-Experts as Scalable Architectures.</h4>

<div id="S2.SS0.SSS0.Px1.p1" class="ltx_para">
<p id="S2.SS0.SSS0.Px1.p1.1" class="ltx_p">Mixture-of-Experts (MoE) architectures introduce sparsity into Transformers by activating only a subset of experts per input, enabling efficient scaling to very large models <cite class="ltx_cite ltx_citemacro_cite">Shazeer<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib5" title="Outrageously large neural networks: the sparsely-gated mixture-of-experts layer" class="ltx_ref">2017</a>); Lepikhin<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib33" title="GShard: scaling giant models with conditional computation and automatic sharding" class="ltx_ref">2021</a>); Fedus<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib34" title="Switch transformers: scaling to trillion parameter models with simple and efficient sparsity" class="ltx_ref">2022</a>)</cite>. Recent systems push this paradigm further by increasing both the number of experts and the degree of sparsity—for example, DeepSeek-V3 <cite class="ltx_cite ltx_citemacro_cite">DeepSeek-AI<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib9" title="DeepSeek-v3 technical report" class="ltx_ref">2025</a>)</cite> employs hundreds of experts per layer while activating only a small subset per token—allowing models to reach scales of hundreds of billions of parameters.</p>
</div>
<div id="S2.SS0.SSS0.Px1.p2" class="ltx_para">
<p id="S2.SS0.SSS0.Px1.p2.1" class="ltx_p">As MoEs grow larger and sparser, memory bottlenecks become a central challenge: even inactive experts need to reside in VRAM at inference time. This has motivated a line of work such as memory-constrained scaling laws <cite class="ltx_cite ltx_citemacro_cite">Li<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib49" title="Can mixture-of-experts surpass dense llms under strictly equal resources?" class="ltx_ref">2026</a>)</cite>, memory-efficient serving <cite class="ltx_cite ltx_citemacro_cite">Song<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib37" title="BlockFFN: towards end-side acceleration-friendly mixture-of-experts with chunk-level activation sparsity" class="ltx_ref">2025</a>); Shen and Henderson (<a href="#bib.bib47" title="Temporally extended mixture-of-experts models" class="ltx_ref">2026</a>); Tairin<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib48" title="EMoE: task-aware memory efficient mixture-of-experts-based (moe) model inference" class="ltx_ref">2025</a>)</cite>, and expert pruning for a general purpose model that removes redundant experts <cite class="ltx_cite ltx_citemacro_cite">Lu<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib18" title="Not all experts are equal: efficient expert pruning and skipping for mixture-of-experts large language models" class="ltx_ref">2024</a>)</cite>.</p>
</div>
<div id="S2.SS0.SSS0.Px1.p3" class="ltx_para">
<p id="S2.SS0.SSS0.Px1.p3.1" class="ltx_p">This work introduces an MoE that enables selective use of expert subsets for a given downstream task. Among its benefits, this provides a new way to alleviate memory bottlenecks in large, sparse MoEs, complementary to prior approaches.</p>
</div>
</section>
<section id="S2.SS0.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Specialization and Modularity of Existing MoEs.</h4>

<div id="S2.SS0.SSS0.Px2.p1" class="ltx_para">
<p id="S2.SS0.SSS0.Px2.p1.1" class="ltx_p">A growing body of work studies the extent to which specialization emerges in MoE models.
Prior work finds that specialization is often driven by surface-level patterns (e.g., token ID that is context-independent) or low-level lexical cues (e.g., prepositions, punctuations) <cite class="ltx_cite ltx_citemacro_cite">Jiang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib52" title="Mixtral of experts" class="ltx_ref">2024</a>); Muennighoff<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">2025</a>)</cite>, while other works find that specialization is confined to only a tiny subset of experts <cite class="ltx_cite ltx_citemacro_cite">Chaudhari<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib11" title="MoE lens – an expert is all you need" class="ltx_ref">2026</a>)</cite>.
Other work suggests that apparent expert specialization may largely reflect geometric properties of the representation space that is difficult to interpret <cite class="ltx_cite ltx_citemacro_cite">Wang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib45" title="The myth of expert specialization in moes: why routing reflects geometry, not necessarily domain expertise" class="ltx_ref">2026</a>)</cite>.
In parallel, several works attempt to exploit these patterns for efficiency, for example by pruning experts for a given task <cite class="ltx_cite ltx_citemacro_cite">jie hu<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib16" title="Quantifying expert specialization for effective pruning in mixture-of-experts models" class="ltx_ref">2025</a>); Dong<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib17" title="Domain-specific pruning of large mixture-of-experts models with few-shot demonstrations" class="ltx_ref">2025</a>); Lu<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib18" title="Not all experts are equal: efficient expert pruning and skipping for mixture-of-experts large language models" class="ltx_ref">2024</a>); Chen<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib19" title="Task-specific expert pruning for sparse mixture-of-experts" class="ltx_ref">2022</a>); Huang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib44" title="Discovering important experts for mixture-of-experts models pruning through a theoretical perspective" class="ltx_ref">2026</a>)</cite>.</p>
</div>
<div id="S2.SS0.SSS0.Px2.p2" class="ltx_para">
<p id="S2.SS0.SSS0.Px2.p2.1" class="ltx_p">In this work, we show that standard MoEs trained with conventional objectives do not support meaningful use of small expert subsets for downstream domains, and instead advocate for training an MoE with modularity as a first-class objective. When training accordingly, MoEs naturally support selective use of expert subsets, and this behavior is robust across different subset selection methods.</p>
</div>
</section>
<section id="S2.SS0.SSS0.Px3" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Training MoEs with Structured or Specialized Experts.</h4>

<div id="S2.SS0.SSS0.Px3.p1" class="ltx_para">
<p id="S2.SS0.SSS0.Px3.p1.1" class="ltx_p">Prior work has explored training MoEs with more structured or specialized experts. One line of work promotes interpretability or diversity across experts, primarily to reduce redundancy <cite class="ltx_cite ltx_citemacro_cite">Yang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib12" title="Mixture of experts made intrinsically interpretable" class="ltx_ref">2025b</a>); Park<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib13" title="Monet: mixture of monosemantic experts for transformers" class="ltx_ref">2025</a>); Hu<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib14" title="Improving moe performance and efficiency with plug-and-play intra-layer specialization and cross-layer coupling losses" class="ltx_ref">2026</a>); Guo<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib15" title="Advancing expert specialization for better MoE" class="ltx_ref">2025</a>)</cite>, but such approaches do not ensure that expert subsets are usable in isolation. Another line of work explicitly partitions training data into predefined domains (e.g., math, biomedical), train separate experts, and merge them into a single MoE <cite class="ltx_cite ltx_citemacro_cite">Shi<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib20" title="FlexOlmo: open language models for flexible data use" class="ltx_ref">2025</a>); Sukhbaatar<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib21" title="Branch-train-MiX: mixing expert LLMs into a mixture-of-experts LLM" class="ltx_ref">2024</a>); Li<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib38" title="Branch-train-merge: embarrassingly parallel training of expert language models" class="ltx_ref">2022</a>)</cite>. While this enables standalone use of expert subsets, it relies on fixed, human-defined priors, which restricts flexibility and limits overall model performance. In contrast, we train an MoE end-to-end with modularity as a first-class objective, allowing expert structure to emerge without requiring predefined domains or human priors.</p>
</div>
<div id="S2.SS0.SSS0.Px3.p2" class="ltx_para">
<p id="S2.SS0.SSS0.Px3.p2.1" class="ltx_p">The closest line of work is ModuleFormer <cite class="ltx_cite ltx_citemacro_cite">Shen<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib35" title="ModuleFormer: modularity emerges from mixture-of-experts" class="ltx_ref">2023</a>)</cite>, which shares our goal of training a modular MoE that supports standalone use of expert subsets. It introduces an objective that maximizes mutual information between tokens and experts. However, they evaluate only against dense models, without standard MoEs. We attempted to reproduce ModuleFormer and found that they do not perform better than standard MoEs, and degrades significantly when less than 40% of experts are retained, which is consistent with their reported results. <span id="S2.SS0.SSS0.Px3.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> largely shares the motivation with ModuleFormer but proposed a more effective training objective that significantly outperforms standard MoEs and other parameter-matched and memory-matched baselines, showing minimal degradation even with an expert subset size of just 12.5%.</p>
</div>
</section>
</section>
<section id="S3" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">3 </span>Modular Mixture of Experts (<span id="S3.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>)</h2>

<figure id="S3.F2" class="ltx_figure"><img src="2605.06663v1/x4.png" id="S3.F2.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="358" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 2: </span>
Comparison of training of a standard MoE and <span id="S3.F2.9.1" class="ltx_text ltx_font_smallcaps">Emo</span> (<math id="S3.F2.3.m1" class="ltx_Math" alttext="k=2,n=10" display="inline" intent=":literal"><semantics><mrow><mrow><mi>k</mi><mo>=</mo><mn>2</mn></mrow><mo>,</mo><mrow><mi>n</mi><mo>=</mo><mn>10</mn></mrow></mrow><annotation encoding="application/x-tex">k=2,n=10</annotation></semantics></math>, shared experts omitted for simplicity).
<span id="S3.F2.10.2" class="ltx_text ltx_font_bold">(Left)</span> In a standard MoE, each token independently selects its top-<math id="S3.F2.4.m2" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math> experts. <span id="S3.F2.11.3" class="ltx_text ltx_font_bold">(Right)</span> In <span id="S3.F2.12.4" class="ltx_text ltx_font_smallcaps">Emo</span>, the router first selects a subset of experts for each document, and all tokens are constrained to route within this subset. This enforces consistent expert usage across the document, encouraging groups of experts to form domain specialization.
</figcaption>
</figure>
<div id="S3.p1" class="ltx_para">
<p id="S3.p1.1" class="ltx_p">The goal of <span id="S3.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is to pre-train an MoE with modularity as the first-class objective, i.e., (1) expert subsets should be usable in isolation for a particular downstream domain, and (2) their composition—the full model—remains a strong general-purpose model.</p>
</div>
<section id="S3.SS0.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Naive Approach.</h4>

<div id="S3.SS0.SSS0.Px1.p1" class="ltx_para">
<p id="S3.SS0.SSS0.Px1.p1.1" class="ltx_p">A straightforward approach to develop modularity is to enforce expert specialization in MoEs by routing tokens to experts based on predefined semantic domains (e.g., math, biology, code). Methods such as FlexOlmo <cite class="ltx_cite ltx_citemacro_cite">Shi<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib20" title="FlexOlmo: open language models for flexible data use" class="ltx_ref">2025</a>)</cite> and BTX <cite class="ltx_cite ltx_citemacro_cite">Sukhbaatar<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib21" title="Branch-train-MiX: mixing expert LLMs into a mixture-of-experts LLM" class="ltx_ref">2024</a>)</cite> instantiate this idea. However, this formulation requires domain labels across pretraining data, which can be ambigious, difficult to obtain, and injects human biases. Having fixed domains also restricts flexibility, making it difficult for the model to be applied to new domains during inference.</p>
</div>
</section>
<section id="S3.SS0.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="S3.SS0.SSS0.Px2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>’s Approach.</h4>

<div id="S3.SS0.SSS0.Px2.p1" class="ltx_para">
<p id="S3.SS0.SSS0.Px2.p1.1" class="ltx_p">Instead, we induce modular structure without explicit domain labels (Figure <a href="#S3.F2" title="Figure 2 ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>). Our key observation is that tokens within the same document usually come from the same domain. We therefore treat <em id="S3.SS0.SSS0.Px2.p1.1.1" class="ltx_emph ltx_font_italic">document boundaries</em> as a <em id="S3.SS0.SSS0.Px2.p1.1.2" class="ltx_emph ltx_font_italic">weak supervisory signal</em>: for each document, the router selects a shared expert pool, and all tokens in that document choose their active experts only from this pool. Different documents can use different pools, allowing modular expert subsets to emerge directly from the training data.</p>
</div>
<div id="S3.SS0.SSS0.Px2.p2" class="ltx_para">
<p id="S3.SS0.SSS0.Px2.p2.1" class="ltx_p">In the rest of the section, we first describe the standard MoE architecture and objective (§<a href="#S3.SS1" title="3.1 Preliminary: Mixture of Experts Architecture ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.1</span></a>), then describe <span id="S3.SS0.SSS0.Px2.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>’s training objective (§<a href="#S3.SS2" title="3.2 Emo: An Objective to Induce Modularity ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.2</span></a>).</p>
</div>
</section>
<section id="S3.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">3.1 </span>Preliminary: Mixture of Experts Architecture</h3>

<div id="S3.SS1.p1" class="ltx_para">
<p id="S3.SS1.p1.4" class="ltx_p">Mixture-of-Experts (MoE) models are decoder-only Transformer language models <cite class="ltx_cite ltx_citemacro_citep">(Vaswani<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib2" title="Attention is all you need" class="ltx_ref">2017</a>)</cite> in which the feedforward sublayer is replaced by a sparse mixture of expert networks. Let the model contain <math id="S3.SS1.p1.1.m1" class="ltx_Math" alttext="n" display="inline" intent=":literal"><semantics><mi>n</mi><annotation encoding="application/x-tex">n</annotation></semantics></math> total experts, consisting of <math id="S3.SS1.p1.2.m2" class="ltx_Math" alttext="n_{r}" display="inline" intent=":literal"><semantics><msub><mi>n</mi><mi>r</mi></msub><annotation encoding="application/x-tex">n_{r}</annotation></semantics></math> routed experts and <math id="S3.SS1.p1.3.m3" class="ltx_Math" alttext="n_{s}" display="inline" intent=":literal"><semantics><msub><mi>n</mi><mi>s</mi></msub><annotation encoding="application/x-tex">n_{s}</annotation></semantics></math> shared experts (<math id="S3.SS1.p1.4.m4" class="ltx_Math" alttext="n=n_{r}+n_{s}" display="inline" intent=":literal"><semantics><mrow><mi>n</mi><mo>=</mo><mrow><msub><mi>n</mi><mi>r</mi></msub><mo>+</mo><msub><mi>n</mi><mi>s</mi></msub></mrow></mrow><annotation encoding="application/x-tex">n=n_{r}+n_{s}</annotation></semantics></math>). Routed experts are selected dynamically on a per-token basis, while shared experts are always active.</p>
</div>
<div id="S3.SS1.p2" class="ltx_para">
<p id="S3.SS1.p2.2" class="ltx_p">Given the hidden state <math id="S3.SS1.p2.1.m1" class="ltx_Math" alttext="x_{t}" display="inline" intent=":literal"><semantics><msub><mi>x</mi><mi>t</mi></msub><annotation encoding="application/x-tex">x_{t}</annotation></semantics></math> at token position <math id="S3.SS1.p2.2.m2" class="ltx_Math" alttext="t" display="inline" intent=":literal"><semantics><mi>t</mi><annotation encoding="application/x-tex">t</annotation></semantics></math>, a router produces logits over the routed experts,</p>
<table id="S3.Ex1" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex1.m1" class="ltx_Math" alttext="r(x_{t})\in\mathbb{R}^{n_{r}},\qquad p_{t}=\operatorname{softmax}(r(x_{t}))." display="block" intent=":literal"><semantics><mrow><mrow><mrow><mrow><mi>r</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow><mo>∈</mo><msup><mi>ℝ</mi><msub><mi>n</mi><mi>r</mi></msub></msup></mrow><mo rspace="2.167em">,</mo><mrow><msub><mi>p</mi><mi>t</mi></msub><mo>=</mo><mrow><mi>softmax</mi><mo>⁡</mo><mrow><mo stretchy="false">(</mo><mrow><mi>r</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow><mo stretchy="false">)</mo></mrow></mrow></mrow></mrow><mo lspace="0em">.</mo></mrow><annotation encoding="application/x-tex">r(x_{t})\in\mathbb{R}^{n_{r}},\qquad p_{t}=\operatorname{softmax}(r(x_{t})).</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS1.p2.5" class="ltx_p">Let <math id="S3.SS1.p2.3.m1" class="ltx_Math" alttext="\mathcal{K}_{t}=\operatorname{Top}\text{-}K(p_{t},k)\subseteq\{1,\dots,n_{r}\}" display="inline" intent=":literal"><semantics><mrow><msub><mi class="ltx_font_mathcaligraphic">𝒦</mi><mi>t</mi></msub><mo>=</mo><mrow><mrow><mi>Top</mi><mo lspace="0.167em">⁡</mo><mrow><mtext>-</mtext><mo lspace="0em" rspace="0em">​</mo><mi>K</mi></mrow></mrow><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>p</mi><mi>t</mi></msub><mo>,</mo><mi>k</mi><mo stretchy="false">)</mo></mrow></mrow><mo>⊆</mo><mrow><mo stretchy="false">{</mo><mn>1</mn><mo>,</mo><mi mathvariant="normal">…</mi><mo>,</mo><msub><mi>n</mi><mi>r</mi></msub><mo stretchy="false">}</mo></mrow></mrow><annotation encoding="application/x-tex">\mathcal{K}_{t}=\operatorname{Top}\text{-}K(p_{t},k)\subseteq\{1,\dots,n_{r}\}</annotation></semantics></math> denote the indices of the top-<math id="S3.SS1.p2.4.m2" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math> routed experts selected for token <math id="S3.SS1.p2.5.m3" class="ltx_Math" alttext="t" display="inline" intent=":literal"><semantics><mi>t</mi><annotation encoding="application/x-tex">t</annotation></semantics></math>. The MoE feedforward output is then</p>
<table id="S3.Ex2" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex2.m1" class="ltx_Math" alttext="\operatorname{FFN}_{\mathrm{out}}(x_{t})=\sum_{i\in\mathcal{K}_{t}}(p_{t})_{i}\,E_{i}(x_{t})\;+\;\sum_{j=1}^{n_{s}}E^{(s)}_{j}(x_{t})," display="block" intent=":literal"><semantics><mrow><mrow><mrow><msub><mi>FFN</mi><mi>out</mi></msub><mo>⁡</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow><mo rspace="0.111em">=</mo><mrow><mrow><munder><mo movablelimits="false" rspace="0em">∑</mo><mrow><mi>i</mi><mo>∈</mo><msub><mi class="ltx_font_mathcaligraphic">𝒦</mi><mi>t</mi></msub></mrow></munder><mrow><msub><mrow><mo stretchy="false">(</mo><msub><mi>p</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow><mi>i</mi></msub><mo lspace="0em" rspace="0em">​</mo><msub><mi>E</mi><mi>i</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo rspace="0.280em" stretchy="false">)</mo></mrow></mrow></mrow><mo rspace="0.335em">+</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>j</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>s</mi></msub></munderover><mrow><msubsup><mi>E</mi><mi>j</mi><mrow><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo></mrow></msubsup><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow></mrow></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\operatorname{FFN}_{\mathrm{out}}(x_{t})=\sum_{i\in\mathcal{K}_{t}}(p_{t})_{i}\,E_{i}(x_{t})\;+\;\sum_{j=1}^{n_{s}}E^{(s)}_{j}(x_{t}),</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS1.p2.9" class="ltx_p">where <math id="S3.SS1.p2.6.m1" class="ltx_Math" alttext="E_{i}" display="inline" intent=":literal"><semantics><msub><mi>E</mi><mi>i</mi></msub><annotation encoding="application/x-tex">E_{i}</annotation></semantics></math> denotes the <math id="S3.SS1.p2.7.m2" class="ltx_Math" alttext="i" display="inline" intent=":literal"><semantics><mi>i</mi><annotation encoding="application/x-tex">i</annotation></semantics></math>-th routed expert and <math id="S3.SS1.p2.8.m3" class="ltx_Math" alttext="E^{(s)}_{j}" display="inline" intent=":literal"><semantics><msubsup><mi>E</mi><mi>j</mi><mrow><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo></mrow></msubsup><annotation encoding="application/x-tex">E^{(s)}_{j}</annotation></semantics></math> denotes the <math id="S3.SS1.p2.9.m4" class="ltx_Math" alttext="j" display="inline" intent=":literal"><semantics><mi>j</mi><annotation encoding="application/x-tex">j</annotation></semantics></math>-th shared expert.</p>
</div>
<div id="S3.SS1.p3" class="ltx_para">
<p id="S3.SS1.p3.1" class="ltx_p">The resulting <math id="S3.SS1.p3.1.m1" class="ltx_Math" alttext="\operatorname{FFN}_{\mathrm{out}}(x_{t})" display="inline" intent=":literal"><semantics><mrow><msub><mi>FFN</mi><mi>out</mi></msub><mo>⁡</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow><annotation encoding="application/x-tex">\operatorname{FFN}_{\mathrm{out}}(x_{t})</annotation></semantics></math> is used throughout the forward pass of the model to compute token probabilities. We train the model using the standard autoregressive language modeling objective:</p>
<table id="S3.Ex3" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex3.m1" class="ltx_Math" alttext="\mathcal{L}_{\mathrm{CE}}=-\sum_{t=1}^{T}\log P(x_{t}\mid x_{&lt;t})," display="block" intent=":literal"><semantics><mrow><mrow><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>CE</mi></msub><mo>=</mo><mrow><mo>−</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>t</mi><mo>=</mo><mn>1</mn></mrow><mi>T</mi></munderover><mrow><mrow><mi>log</mi><mo lspace="0.167em">⁡</mo><mi>P</mi></mrow><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><mrow><msub><mi>x</mi><mi>t</mi></msub><mo>∣</mo><msub><mi>x</mi><mrow><mi></mi><mo>&lt;</mo><mi>t</mi></mrow></msub></mrow><mo stretchy="false">)</mo></mrow></mrow></mrow></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\mathcal{L}_{\mathrm{CE}}=-\sum_{t=1}^{T}\log P(x_{t}\mid x_{&lt;t}),</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS1.p3.2" class="ltx_p">where the conditional probabilities <math id="S3.SS1.p3.2.m1" class="ltx_Math" alttext="P(x_{t}\mid x_{&lt;t})" display="inline" intent=":literal"><semantics><mrow><mi>P</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><mrow><msub><mi>x</mi><mi>t</mi></msub><mo>∣</mo><msub><mi>x</mi><mrow><mi></mi><mo>&lt;</mo><mi>t</mi></mrow></msub></mrow><mo stretchy="false">)</mo></mrow></mrow><annotation encoding="application/x-tex">P(x_{t}\mid x_{&lt;t})</annotation></semantics></math> are computed using the MoE layer defined above.</p>
</div>
<div id="S3.SS1.p4" class="ltx_para">
<p id="S3.SS1.p4.1" class="ltx_p">In addition to the cross entropy, MoE training includes auxiliary losses such as the load balancing loss <math id="S3.SS1.p4.1.m1" class="ltx_Math" alttext="\mathcal{L}_{\mathrm{LB}}" display="inline" intent=":literal"><semantics><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>LB</mi></msub><annotation encoding="application/x-tex">\mathcal{L}_{\mathrm{LB}}</annotation></semantics></math> to encourage uniform expert utilization:</p>
<table id="S3.Ex4" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex4.m1" class="ltx_Math" alttext="\mathcal{L}_{\mathrm{LB}}=n_{r}\sum_{i=1}^{n_{r}}\bar{f}_{i}\cdot\bar{P}_{i}," display="block" intent=":literal"><semantics><mrow><mrow><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>LB</mi></msub><mo>=</mo><mrow><msub><mi>n</mi><mi>r</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>r</mi></msub></munderover><mrow><msub><mover accent="true"><mi>f</mi><mo>¯</mo></mover><mi>i</mi></msub><mo lspace="0.222em" rspace="0.222em">⋅</mo><msub><mover accent="true"><mi>P</mi><mo>¯</mo></mover><mi>i</mi></msub></mrow></mrow></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\mathcal{L}_{\mathrm{LB}}=n_{r}\sum_{i=1}^{n_{r}}\bar{f}_{i}\cdot\bar{P}_{i},</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS1.p4.5" class="ltx_p">where <math id="S3.SS1.p4.2.m1" class="ltx_Math" alttext="\bar{f}_{i}" display="inline" intent=":literal"><semantics><msub><mover accent="true"><mi>f</mi><mo>¯</mo></mover><mi>i</mi></msub><annotation encoding="application/x-tex">\bar{f}_{i}</annotation></semantics></math> is the fraction of tokens routed to expert <math id="S3.SS1.p4.3.m2" class="ltx_Math" alttext="i" display="inline" intent=":literal"><semantics><mi>i</mi><annotation encoding="application/x-tex">i</annotation></semantics></math> and <math id="S3.SS1.p4.4.m3" class="ltx_Math" alttext="\bar{P}_{i}" display="inline" intent=":literal"><semantics><msub><mover accent="true"><mi>P</mi><mo>¯</mo></mover><mi>i</mi></msub><annotation encoding="application/x-tex">\bar{P}_{i}</annotation></semantics></math> is the average routing probability of expert <math id="S3.SS1.p4.5.m4" class="ltx_Math" alttext="i" display="inline" intent=":literal"><semantics><mi>i</mi><annotation encoding="application/x-tex">i</annotation></semantics></math> across all tokens. The full objective is</p>
<table id="S3.Ex5" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex5.m1" class="ltx_Math" alttext="\mathcal{L}=\mathcal{L}_{\mathrm{CE}}+\alpha\mathcal{L}_{\mathrm{LB}}+\beta\mathcal{L}_{\mathrm{RZ}}," display="block" intent=":literal"><semantics><mrow><mrow><mi class="ltx_font_mathcaligraphic">ℒ</mi><mo>=</mo><mrow><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>CE</mi></msub><mo>+</mo><mrow><mi>α</mi><mo lspace="0em" rspace="0em">​</mo><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>LB</mi></msub></mrow><mo>+</mo><mrow><mi>β</mi><mo lspace="0em" rspace="0em">​</mo><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>RZ</mi></msub></mrow></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\mathcal{L}=\mathcal{L}_{\mathrm{CE}}+\alpha\mathcal{L}_{\mathrm{LB}}+\beta\mathcal{L}_{\mathrm{RZ}},</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS1.p4.8" class="ltx_p">where <math id="S3.SS1.p4.6.m1" class="ltx_Math" alttext="\mathcal{L}_{\mathrm{RZ}}" display="inline" intent=":literal"><semantics><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>RZ</mi></msub><annotation encoding="application/x-tex">\mathcal{L}_{\mathrm{RZ}}</annotation></semantics></math> regularizes router logits, and <math id="S3.SS1.p4.7.m2" class="ltx_Math" alttext="\alpha" display="inline" intent=":literal"><semantics><mi>α</mi><annotation encoding="application/x-tex">\alpha</annotation></semantics></math> and <math id="S3.SS1.p4.8.m3" class="ltx_Math" alttext="\beta" display="inline" intent=":literal"><semantics><mi>β</mi><annotation encoding="application/x-tex">\beta</annotation></semantics></math> control auxiliary loss weights.</p>
</div>
</section>
<section id="S3.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">3.2 </span><span id="S3.SS2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>: An Objective to Induce Modularity</h3>

<div id="S3.SS2.p1" class="ltx_para">
<p id="S3.SS2.p1.1" class="ltx_p">The goal of <span id="S3.SS2.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is to induce modularity by leveraging <em id="S3.SS2.p1.1.2" class="ltx_emph ltx_font_italic">document boundaries</em> as a weak supervisory signal. <span id="S3.SS2.p1.1.3" class="ltx_text ltx_font_smallcaps">Emo</span> achieves this by selecting a <em id="S3.SS2.p1.1.4" class="ltx_emph ltx_font_italic">document expert pool</em> for each document and constrains all tokens in the document to route within this pool during training (Figure <a href="#S3.F2" title="Figure 2 ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>).</p>
</div>
<section id="S3.SS2.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Formulation.</h4>

<div id="S3.SS2.SSS0.Px1.p1" class="ltx_para">
<p id="S3.SS2.SSS0.Px1.p1.3" class="ltx_p">Recall that <math id="S3.SS2.SSS0.Px1.p1.1.m1" class="ltx_Math" alttext="p_{t}=\operatorname{softmax}(r(x_{t}))\in\mathbb{R}^{n_{r}}" display="inline" intent=":literal"><semantics><mrow><msub><mi>p</mi><mi>t</mi></msub><mo>=</mo><mrow><mi>softmax</mi><mo>⁡</mo><mrow><mo stretchy="false">(</mo><mrow><mi>r</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow><mo stretchy="false">)</mo></mrow></mrow><mo>∈</mo><msup><mi>ℝ</mi><msub><mi>n</mi><mi>r</mi></msub></msup></mrow><annotation encoding="application/x-tex">p_{t}=\operatorname{softmax}(r(x_{t}))\in\mathbb{R}^{n_{r}}</annotation></semantics></math> denotes the routing distribution for token <math id="S3.SS2.SSS0.Px1.p1.2.m2" class="ltx_Math" alttext="t" display="inline" intent=":literal"><semantics><mi>t</mi><annotation encoding="application/x-tex">t</annotation></semantics></math>.
We define the document expert pool <math id="S3.SS2.SSS0.Px1.p1.3.m3" class="ltx_Math" alttext="\mathcal{D}" display="inline" intent=":literal"><semantics><mi class="ltx_font_mathcaligraphic">𝒟</mi><annotation encoding="application/x-tex">\mathcal{D}</annotation></semantics></math> based on the average routing distribution across tokens:</p>
<table id="S3.Ex6" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex6.m1" class="ltx_Math" alttext="\mathcal{D}=\operatorname{Top}\text{-}K\!\left(\frac{1}{T}\sum_{t=1}^{T}p_{t},\,d\right)\subseteq\{1,\dots,n_{r}\}." display="block" intent=":literal"><semantics><mrow><mrow><mi class="ltx_font_mathcaligraphic">𝒟</mi><mo>=</mo><mrow><mrow><mi>Top</mi><mo lspace="0.167em">⁡</mo><mrow><mtext>-</mtext><mo lspace="0em" rspace="0em">​</mo><mpadded style="width:0.751em;" width="0.751em"><mi>K</mi></mpadded></mrow></mrow><mo lspace="0em" rspace="0em">​</mo><mrow><mo>(</mo><mrow><mfrac><mn>1</mn><mi>T</mi></mfrac><mo lspace="0em" rspace="0em">​</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>t</mi><mo>=</mo><mn>1</mn></mrow><mi>T</mi></munderover><msub><mi>p</mi><mi>t</mi></msub></mrow></mrow><mo rspace="0.337em">,</mo><mi>d</mi><mo>)</mo></mrow></mrow><mo>⊆</mo><mrow><mo stretchy="false">{</mo><mn>1</mn><mo>,</mo><mi mathvariant="normal">…</mi><mo>,</mo><msub><mi>n</mi><mi>r</mi></msub><mo stretchy="false">}</mo></mrow></mrow><mo lspace="0em">.</mo></mrow><annotation encoding="application/x-tex">\mathcal{D}=\operatorname{Top}\text{-}K\!\left(\frac{1}{T}\sum_{t=1}^{T}p_{t},\,d\right)\subseteq\{1,\dots,n_{r}\}.</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS2.SSS0.Px1.p1.4" class="ltx_p">Routing is then restricted to <math id="S3.SS2.SSS0.Px1.p1.4.m1" class="ltx_Math" alttext="\mathcal{D}" display="inline" intent=":literal"><semantics><mi class="ltx_font_mathcaligraphic">𝒟</mi><annotation encoding="application/x-tex">\mathcal{D}</annotation></semantics></math> via a masked and renormalized distribution</p>
<table id="S3.Ex7" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex7.m1" class="ltx_Math" alttext="\hat{p}_{t}(i)=\begin{cases}\dfrac{p_{t}(i)}{\sum_{j\in\mathcal{D}}p_{t}(j)}&amp;\text{if }i\in\mathcal{D},\\[8.0pt]
0&amp;\text{otherwise}.\end{cases}" display="block" intent=":literal"><semantics><mrow><mrow><msub><mover accent="true"><mi>p</mi><mo>^</mo></mover><mi>t</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><mi>i</mi><mo stretchy="false">)</mo></mrow></mrow><mo>=</mo><mrow><mo>{</mo><mtable columnspacing="5pt" displaystyle="true" rowspacing="0pt"><mtr><mtd class="ltx_align_left" columnalign="left"><mfrac><mrow><msub><mi>p</mi><mi>t</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><mi>i</mi><mo stretchy="false">)</mo></mrow></mrow><mrow><msub><mo>∑</mo><mrow><mi>j</mi><mo>∈</mo><mi class="ltx_font_mathcaligraphic">𝒟</mi></mrow></msub><mrow><msub><mi>p</mi><mi>t</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><mi>j</mi><mo stretchy="false">)</mo></mrow></mrow></mrow></mfrac></mtd><mtd class="ltx_align_left" columnalign="left"><mrow><mrow><mrow><mtext>if </mtext><mo lspace="0em" rspace="0em">​</mo><mi>i</mi></mrow><mo>∈</mo><mi class="ltx_font_mathcaligraphic">𝒟</mi></mrow><mo>,</mo></mrow></mtd></mtr><mtr><mtd class="ltx_align_left" columnalign="left"><mn>0</mn></mtd><mtd class="ltx_align_left" columnalign="left"><mrow><mtext>otherwise</mtext><mo lspace="0em">.</mo></mrow></mtd></mtr></mtable></mrow></mrow><annotation encoding="application/x-tex">\hat{p}_{t}(i)=\begin{cases}\dfrac{p_{t}(i)}{\sum_{j\in\mathcal{D}}p_{t}(j)}&amp;\text{if }i\in\mathcal{D},\\[8.0pt]
0&amp;\text{otherwise}.\end{cases}</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS2.SSS0.Px1.p1.5" class="ltx_p">The routed experts are then</p>
<table id="S3.Ex8" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex8.m1" class="ltx_Math" alttext="\mathcal{R}_{t}=\operatorname{Top}\text{-}K(\hat{p}_{t},k)." display="block" intent=":literal"><semantics><mrow><mrow><msub><mi class="ltx_font_mathcaligraphic">ℛ</mi><mi>t</mi></msub><mo>=</mo><mrow><mrow><mi>Top</mi><mo lspace="0.167em">⁡</mo><mrow><mtext>-</mtext><mo lspace="0em" rspace="0em">​</mo><mi>K</mi></mrow></mrow><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mover accent="true"><mi>p</mi><mo>^</mo></mover><mi>t</mi></msub><mo>,</mo><mi>k</mi><mo stretchy="false">)</mo></mrow></mrow></mrow><mo lspace="0em">.</mo></mrow><annotation encoding="application/x-tex">\mathcal{R}_{t}=\operatorname{Top}\text{-}K(\hat{p}_{t},k).</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
</div>
<div id="S3.SS2.SSS0.Px1.p2" class="ltx_para">
<p id="S3.SS2.SSS0.Px1.p2.3" class="ltx_p">The resulting feedforward output is</p>
<table id="S3.Ex9" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex9.m1" class="ltx_Math" alttext="\operatorname{FFN}_{\mathrm{out}}(x_{t})=\sum_{i\in\mathcal{R}_{t}}(\hat{p}_{t})_{i}\,E_{i}(x_{t})\;+\;\sum_{j=1}^{n_{s}}E^{(s)}_{j}(x_{t})," display="block" intent=":literal"><semantics><mrow><mrow><mrow><msub><mi>FFN</mi><mi>out</mi></msub><mo>⁡</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow><mo rspace="0.111em">=</mo><mrow><mrow><munder><mo movablelimits="false" rspace="0em">∑</mo><mrow><mi>i</mi><mo>∈</mo><msub><mi class="ltx_font_mathcaligraphic">ℛ</mi><mi>t</mi></msub></mrow></munder><mrow><msub><mrow><mo stretchy="false">(</mo><msub><mover accent="true"><mi>p</mi><mo>^</mo></mover><mi>t</mi></msub><mo stretchy="false">)</mo></mrow><mi>i</mi></msub><mo lspace="0em" rspace="0em">​</mo><msub><mi>E</mi><mi>i</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo rspace="0.280em" stretchy="false">)</mo></mrow></mrow></mrow><mo rspace="0.335em">+</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>j</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>s</mi></msub></munderover><mrow><msubsup><mi>E</mi><mi>j</mi><mrow><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo></mrow></msubsup><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><msub><mi>x</mi><mi>t</mi></msub><mo stretchy="false">)</mo></mrow></mrow></mrow></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\operatorname{FFN}_{\mathrm{out}}(x_{t})=\sum_{i\in\mathcal{R}_{t}}(\hat{p}_{t})_{i}\,E_{i}(x_{t})\;+\;\sum_{j=1}^{n_{s}}E^{(s)}_{j}(x_{t}),</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS2.SSS0.Px1.p2.2" class="ltx_p">where <math id="S3.SS2.SSS0.Px1.p2.1.m1" class="ltx_Math" alttext="E_{i}" display="inline" intent=":literal"><semantics><msub><mi>E</mi><mi>i</mi></msub><annotation encoding="application/x-tex">E_{i}</annotation></semantics></math> denotes routed experts and <math id="S3.SS2.SSS0.Px1.p2.2.m2" class="ltx_Math" alttext="E^{(s)}_{j}" display="inline" intent=":literal"><semantics><msubsup><mi>E</mi><mi>j</mi><mrow><mo stretchy="false">(</mo><mi>s</mi><mo stretchy="false">)</mo></mrow></msubsup><annotation encoding="application/x-tex">E^{(s)}_{j}</annotation></semantics></math> denotes shared experts.</p>
</div>
<div id="S3.SS2.SSS0.Px1.p3" class="ltx_para">
<p id="S3.SS2.SSS0.Px1.p3.5" class="ltx_p">The hyperparameter <math id="S3.SS2.SSS0.Px1.p3.1.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> controls subset granularity: smaller <math id="S3.SS2.SSS0.Px1.p3.2.m2" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> enforces highly specialized expert subsets with limited expressivity (e.g., <math id="S3.SS2.SSS0.Px1.p3.3.m3" class="ltx_Math" alttext="d=k" display="inline" intent=":literal"><semantics><mrow><mi>d</mi><mo>=</mo><mi>k</mi></mrow><annotation encoding="application/x-tex">d=k</annotation></semantics></math> forces all tokens in a document to use the same experts), while larger <math id="S3.SS2.SSS0.Px1.p3.4.m4" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> increases flexibility at the cost of weaker modular structure (e.g., <math id="S3.SS2.SSS0.Px1.p3.5.m5" class="ltx_Math" alttext="d=n_{r}" display="inline" intent=":literal"><semantics><mrow><mi>d</mi><mo>=</mo><msub><mi>n</mi><mi>r</mi></msub></mrow><annotation encoding="application/x-tex">d=n_{r}</annotation></semantics></math> recovers the standard MoE).</p>
</div>
</section>
</section>
<section id="S3.SS3" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">3.3 </span>Key Technical Considerations</h3>

<div id="S3.SS3.p1" class="ltx_para">
<p id="S3.SS3.p1.1" class="ltx_p">Several technical choices were important for effective training
of <span id="S3.SS3.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> (see §<a href="#A1" title="Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A</span></a> for details).</p>
</div>
<section id="S3.SS3.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Consideration 1. Load Balancing.</h4>

<div id="S3.SS3.SSS0.Px1.p1" class="ltx_para">
<p id="S3.SS3.SSS0.Px1.p1.1" class="ltx_p">A central challenge is that load balancing and document-level routing appear to impose opposing pressures. This conflict arises under standard micro-batch load balancing, where the load-balancing loss is computed over only a few documents. While this local implementation reduces cross-device communication and simplifies distributed training, it also encourages tokens from the same document to spread across many experts, directly opposing the shared-pool constraint and causing unstable training.</p>
</div>
<div id="S3.SS3.SSS0.Px1.p2" class="ltx_para">
<p id="S3.SS3.SSS0.Px1.p2.1" class="ltx_p">We address this by adopting global load balancing <cite class="ltx_cite ltx_citemacro_cite">Qiu<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib4" title="Demons in the detail: on implementing load balancing loss for training specialized mixture-of-expert models" class="ltx_ref">2025</a>)</cite>, aggregating routing statistics across data-parallel groups. Applied over a larger and more diverse set of documents, load balancing encourages uniform utilization of experts <span id="S3.SS3.SSS0.Px1.p2.1.1" class="ltx_text ltx_font_italic">across</span> documents, while our routing constraint enforces expert consistency <span id="S3.SS3.SSS0.Px1.p2.1.2" class="ltx_text ltx_font_italic">within</span> each document, making the two objectives largely complementary. Empirically, this is important for stable training: see Figure <a href="#A1.F7" title="Figure 7 ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">7</span></a> in §<a href="#A1" title="Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A</span></a>.</p>
</div>
</section>
<section id="S3.SS3.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Consideration 2. Choosing Expert Pool Size.</h4>

<div id="S3.SS3.SSS0.Px2.p1" class="ltx_para">
<p id="S3.SS3.SSS0.Px2.p1.3" class="ltx_p">Fixing a single expert pool size <math id="S3.SS3.SSS0.Px2.p1.1.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> works well during training but limits inference-time flexibility. The model "overfits" only to expert sets of size <math id="S3.SS3.SSS0.Px2.p1.2.m2" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> and performs poorly when deployed as expert subsets that isn’t of size <math id="S3.SS3.SSS0.Px2.p1.3.m3" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math>.</p>
</div>
<div id="S3.SS3.SSS0.Px2.p2" class="ltx_para">
<p id="S3.SS3.SSS0.Px2.p2.1" class="ltx_p">To enable the model to support expert subsets of all sizes, we treat <math id="S3.SS3.SSS0.Px2.p2.1.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> as a random variable and sample it independently for each document during pretraining:</p>
<table id="S3.Ex10" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S3.Ex10.m1" class="ltx_Math" alttext="d\sim\mathcal{U}\{k,\dots,n_{r}\}." display="block" intent=":literal"><semantics><mrow><mrow><mi>d</mi><mo>∼</mo><mrow><mi class="ltx_font_mathcaligraphic">𝒰</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">{</mo><mi>k</mi><mo>,</mo><mi mathvariant="normal">…</mi><mo>,</mo><msub><mi>n</mi><mi>r</mi></msub><mo stretchy="false">}</mo></mrow></mrow></mrow><mo lspace="0em">.</mo></mrow><annotation encoding="application/x-tex">d\sim\mathcal{U}\{k,\dots,n_{r}\}.</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="S3.SS3.SSS0.Px2.p2.3" class="ltx_p">where <math id="S3.SS3.SSS0.Px2.p2.2.m1" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math> is the number of active experts per token and <math id="S3.SS3.SSS0.Px2.p2.3.m2" class="ltx_Math" alttext="n_{r}" display="inline" intent=":literal"><semantics><msub><mi>n</mi><mi>r</mi></msub><annotation encoding="application/x-tex">n_{r}</annotation></semantics></math> is the total number of routable experts. This exposes the model to a range of expert pool sizes during training, enabling it to support expert subsets of varying capacities for selective expert use.</p>
</div>
</section>
</section>
</section>
<section id="S4" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">4 </span>Experimental Setup</h2>

<section id="S4.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">4.1 </span>Architecture &amp; Training Details</h3>

<div id="S4.SS1.p1" class="ltx_para">
<p id="S4.SS1.p1.4" class="ltx_p">We consider an MoE with 1B active and 14B total parameters, consisting of <math id="S4.SS1.p1.1.m1" class="ltx_Math" alttext="n=128" display="inline" intent=":literal"><semantics><mrow><mi>n</mi><mo>=</mo><mn>128</mn></mrow><annotation encoding="application/x-tex">n=128</annotation></semantics></math> experts (<math id="S4.SS1.p1.2.m2" class="ltx_Math" alttext="n_{r}=127" display="inline" intent=":literal"><semantics><mrow><msub><mi>n</mi><mi>r</mi></msub><mo>=</mo><mn>127</mn></mrow><annotation encoding="application/x-tex">n_{r}=127</annotation></semantics></math> routed, <math id="S4.SS1.p1.3.m3" class="ltx_Math" alttext="n_{s}=1" display="inline" intent=":literal"><semantics><mrow><msub><mi>n</mi><mi>s</mi></msub><mo>=</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">n_{s}=1</annotation></semantics></math> shared), with <math id="S4.SS1.p1.4.m4" class="ltx_Math" alttext="k=8" display="inline" intent=":literal"><semantics><mrow><mi>k</mi><mo>=</mo><mn>8</mn></mrow><annotation encoding="application/x-tex">k=8</annotation></semantics></math> experts activated per token.
The baseline MoE and <span id="S4.SS1.p1.4.1" class="ltx_text ltx_font_smallcaps">Emo</span> share the same architecture; they differ only in their training objectives, as described in §<a href="#S3.SS2" title="3.2 Emo: An Objective to Induce Modularity ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.2</span></a>.</p>
</div>
<div id="S4.SS1.p2" class="ltx_para">
<p id="S4.SS1.p2.1" class="ltx_p">We train both the baseline MoE and <span id="S4.SS1.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> from scratch on 1 trillion tokens from the OLMoE pretraining corpus <cite class="ltx_cite ltx_citemacro_citep">(Muennighoff<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">2025</a>)</cite>, followed by an additional 50B-token linear annealing phase.
For ablations, we additionally train models on 130B tokens and include comparison to dense baselines and smaller MoEs.</p>
</div>
<div id="S4.SS1.p3" class="ltx_para">
<p id="S4.SS1.p3.1" class="ltx_p">Our architecture largely follows that of OLMoE <cite class="ltx_cite ltx_citemacro_citep">(Muennighoff<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">2025</a>)</cite>, with several key improvements: (1) adding a shared expert, (2) using pre-norm instead of post-norm, and (3) removing QK-norm; see §<a href="#A1" title="Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A</span></a> for details on the improvements introduced by these changes.
These modifications make our baseline MoEs significantly more competitive: as shown in §<a href="#S5.SS1" title="5.1 Full-Model Evaluation ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.1</span></a>, our baseline MoE trained on 1T tokens consistently outperforms OLMoE trained on 5T tokens despite being trained on the same data.</p>
</div>
</section>
<section id="S4.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">4.2 </span>Evaluation</h3>

<div id="S4.SS2.p1" class="ltx_para">
<p id="S4.SS2.p1.1" class="ltx_p">We evaluate our models under two settings: (1) <span id="S4.SS2.p1.1.1" class="ltx_text ltx_font_bold">full-model evaluation</span>, reflecting the standard use case in which a pretrained model is deployed for a broad set of tasks, and (2) <span id="S4.SS2.p1.1.2" class="ltx_text ltx_font_bold">selective expert use</span>, where only a task-specific subset of experts is activated for a particular task or domain. Additional details on evaluation tasks and settings are provided in §<a href="#A3" title="Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">C</span></a>.</p>
</div>
<section id="S4.SS2.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Full-model Evaluation.</h4>

<div id="S4.SS2.SSS0.Px1.p1" class="ltx_para">
<p id="S4.SS2.SSS0.Px1.p1.1" class="ltx_p">We first evaluate the full model under zero-shot settings. We report results on five evaluation suites:
(1) <span id="S4.SS2.SSS0.Px1.p1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps">MC9</span>, an average over nine multiple-choice benchmarks including ARC-Easy <cite class="ltx_cite ltx_citemacro_citep">(Clark<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib22" title="Think you have solved question answering? try arc, the ai2 reasoning challenge" class="ltx_ref">2018</a>)</cite>, ARC-Challenge <cite class="ltx_cite ltx_citemacro_citep">(Clark<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib22" title="Think you have solved question answering? try arc, the ai2 reasoning challenge" class="ltx_ref">2018</a>)</cite>, BoolQ <cite class="ltx_cite ltx_citemacro_citep">(Clark<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib23" title="BoolQ: exploring the surprising difficulty of natural yes/no questions" class="ltx_ref">2019</a>)</cite>, CSQA <cite class="ltx_cite ltx_citemacro_citep">(Talmor<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib24" title="CommonsenseQA: a question answering challenge targeting commonsense knowledge" class="ltx_ref">2019</a>)</cite>, HellaSwag <cite class="ltx_cite ltx_citemacro_citep">(Zellers<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib25" title="HellaSwag: can a machine really finish your sentence?" class="ltx_ref">2019</a>)</cite>, OpenBookQA <cite class="ltx_cite ltx_citemacro_citep">(Mihaylov<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib26" title="Can a suit of armor conduct electricity? a new dataset for open book question answering" class="ltx_ref">2018</a>)</cite>, PIQA <cite class="ltx_cite ltx_citemacro_citep">(Bisk<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib27" title="PIQA: reasoning about physical commonsense in natural language" class="ltx_ref">2020</a>)</cite>, SocialIQa <cite class="ltx_cite ltx_citemacro_citep">(Sap<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib28" title="SocialIQA: commonsense reasoning about social interactions" class="ltx_ref">2019</a>)</cite>, and WinoGrande <cite class="ltx_cite ltx_citemacro_citep">(Sakaguchi<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib29" title="WinoGrande: an adversarial winograd schema challenge at scale" class="ltx_ref">2020</a>)</cite>;
(2) <span id="S4.SS2.SSS0.Px1.p1.1.2" class="ltx_text ltx_font_bold">Gen5</span>, an average over five generative tasks including CoQA <cite class="ltx_cite ltx_citemacro_citep">(Reddy<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib39" title="CoQA: a conversational question answering challenge" class="ltx_ref">2019</a>)</cite>, SQuAD <cite class="ltx_cite ltx_citemacro_citep">(Rajpurkar<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib40" title="SQuAD: 100,000+ questions for machine comprehension of text" class="ltx_ref">2016</a>)</cite>, Natural Questions <cite class="ltx_cite ltx_citemacro_citep">(Kwiatkowski<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib41" title="Natural questions: a benchmark for question answering research" class="ltx_ref">2019</a>)</cite>, TriviaQA <cite class="ltx_cite ltx_citemacro_citep">(Joshi<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib42" title="TriviaQA: a large scale distantly supervised challenge dataset for reading comprehension" class="ltx_ref">2017</a>)</cite>, and DROP <cite class="ltx_cite ltx_citemacro_citep">(Dua<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib43" title="DROP: a reading comprehension benchmark requiring discrete reasoning over paragraphs" class="ltx_ref">2019</a>)</cite>;
(3) <span id="S4.SS2.SSS0.Px1.p1.1.3" class="ltx_text ltx_font_bold">MMLU</span> <cite class="ltx_cite ltx_citemacro_citep">(Hendrycks<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib30" title="Measuring massive multitask language understanding" class="ltx_ref">2021</a>)</cite><span id="footnote1" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">1</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">1</sup><span class="ltx_tag ltx_tag_note">1</span>Aggregated results exclude the “other” category; see §<a href="#A3" title="Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">C</span></a> and <a href="#A2.SS3" title='B.3 Selective Expert Use on "Other" Category in MMLU and MMLU Pro ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity' class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.3</span></a> for details.</span></span></span>;
(4) <span id="S4.SS2.SSS0.Px1.p1.1.4" class="ltx_text ltx_font_bold">MMLU-Pro</span> <cite class="ltx_cite ltx_citemacro_citep">(Wang<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib31" title="Mmlu-pro: a more robust and challenging multi-task language understanding benchmark" class="ltx_ref">2024</a>)</cite><a href="#footnote1" title="footnote 1 ‣ Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a>; and
(5) <span id="S4.SS2.SSS0.Px1.p1.1.5" class="ltx_text ltx_font_bold ltx_font_smallcaps">GSM8K</span> <cite class="ltx_cite ltx_citemacro_citep">(Cobbe<span class="ltx_text ltx_bib_etal"> et al.</span>, <a href="#bib.bib32" title="Training verifiers to solve math word problems" class="ltx_ref">2021</a>)</cite>.</p>
</div>
</section>
<section id="S4.SS2.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Selective Expert Use.</h4>

<div id="S4.SS2.SSS0.Px2.p1" class="ltx_para">
<p id="S4.SS2.SSS0.Px2.p1.1" class="ltx_p">We next evaluate whether models can be deployed using only a subset of experts for each downstream domain (Figure <a href="#S1.F1" title="Figure 1 ‣ 1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a>).
We consider coarse-grained domain grouping of MMLU and MMLU-Pro, e.g., math, physics, health, philosophy, history, which contain 16<a href="#footnote1" title="footnote 1 ‣ Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> and 13<a href="#footnote1" title="footnote 1 ‣ Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> domains, respectively, as well as <span id="S4.SS2.SSS0.Px2.p1.1.1" class="ltx_text ltx_font_smallcaps">GSM8K</span>.</p>
</div>
<div id="S4.SS2.SSS0.Px2.p2" class="ltx_para">
<p id="S4.SS2.SSS0.Px2.p2.2" class="ltx_p">For each domain, we assume access to a small validation set to identify relevant experts. In §<a href="#A2.SS2" title="B.2 Ablations on expert subset initialization ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.2</span></a>, we show that this validation set can be extremely small: even a single few-shot example is sufficient to select an effective expert subset. We consider two selection methods: (1) a simple approach that aggregates routing probabilities across tokens and ranks experts by their average routing probability, and (2) Easy-EP <cite class="ltx_cite ltx_citemacro_cite">Dong<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib17" title="Domain-specific pruning of large mixture-of-experts models with few-shot demonstrations" class="ltx_ref">2025</a>)</cite>, a more computationally expensive, state-of-the-art expert selection method.
We then retain the top-<math id="S4.SS2.SSS0.Px2.p2.1.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> experts in each layer and discard the rest, producing a domain-specific subset of experts that can be used as a standalone model.
We vary <math id="S4.SS2.SSS0.Px2.p2.2.m2" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> to measure how performance changes as fewer experts are retained. We report both zero-shot performance and performance after finetuning.
More evaluation details can be found in §<a href="#A3" title="Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">C</span></a>.</p>
</div>
</section>
</section>
</section>
<section id="S5" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">5 </span>Results and Analysis</h2>

<section id="S5.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">5.1 </span>Full-Model Evaluation</h3>

<figure id="S5.T1" class="ltx_table">
<table id="S5.T1.1" class="ltx_tabular ltx_centering ltx_guessed_headers ltx_align_middle">
<tbody class="ltx_tbody">
<tr id="S5.T1.1.2.1" class="ltx_tr">
<th id="S5.T1.1.2.1.1" class="ltx_td ltx_th ltx_th_row ltx_border_tt"></th>
<th id="S5.T1.1.2.1.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_tt"><span id="S5.T1.1.2.1.2.1" class="ltx_text" style="font-size:80%;"># train tokens</span></th>
<td id="S5.T1.1.2.1.3" class="ltx_td ltx_align_right ltx_border_tt"><span id="S5.T1.1.2.1.3.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MC9</span></td>
<td id="S5.T1.1.2.1.4" class="ltx_td ltx_align_right ltx_border_tt"><span id="S5.T1.1.2.1.4.1" class="ltx_text ltx_font_bold" style="font-size:80%;">Gen5</span></td>
<td id="S5.T1.1.2.1.5" class="ltx_td ltx_align_right ltx_border_tt"><span id="S5.T1.1.2.1.5.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MMLU</span></td>
<td id="S5.T1.1.2.1.6" class="ltx_td ltx_align_right ltx_border_tt"><span id="S5.T1.1.2.1.6.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MMLU Pro</span></td>
<td id="S5.T1.1.2.1.7" class="ltx_td ltx_align_right ltx_border_tt"><span id="S5.T1.1.2.1.7.1" class="ltx_text ltx_font_bold" style="font-size:80%;">GSM8K</span></td>
</tr>
<tr id="S5.T1.1.1" class="ltx_tr">
<th id="S5.T1.1.1.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t">
<span id="S5.T1.1.1.1.1" class="ltx_text" style="font-size:80%;">OLMoE</span><sup id="S5.T1.1.1.1.2" class="ltx_sup"><span id="S5.T1.1.1.1.2.1" class="ltx_text" style="font-size:80%;">†</span></sup>
</th>
<th id="S5.T1.1.1.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t"><span id="S5.T1.1.1.2.1" class="ltx_text" style="font-size:80%;">5T</span></th>
<td id="S5.T1.1.1.3" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.1.3.1" class="ltx_text" style="font-size:80%;">63.5</span></td>
<td id="S5.T1.1.1.4" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.1.4.1" class="ltx_text" style="font-size:80%;">57.6</span></td>
<td id="S5.T1.1.1.5" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.1.5.1" class="ltx_text" style="font-size:80%;">42.8</span></td>
<td id="S5.T1.1.1.6" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.1.6.1" class="ltx_text" style="font-size:80%;">18.7</span></td>
<td id="S5.T1.1.1.7" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.1.7.1" class="ltx_text" style="font-size:80%;">13.7</span></td>
</tr>
<tr id="S5.T1.1.3.2" class="ltx_tr">
<th id="S5.T1.1.3.2.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t"><span id="S5.T1.1.3.2.1.1" class="ltx_text" style="font-size:80%;">Reg. MoE</span></th>
<th id="S5.T1.1.3.2.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t"><span id="S5.T1.1.3.2.2.1" class="ltx_text" style="font-size:80%;">1T</span></th>
<td id="S5.T1.1.3.2.3" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.3.2.3.1" class="ltx_text" style="font-size:80%;">63.9</span></td>
<td id="S5.T1.1.3.2.4" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.3.2.4.1" class="ltx_text" style="font-size:80%;">59.7</span></td>
<td id="S5.T1.1.3.2.5" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.3.2.5.1" class="ltx_text" style="font-size:80%;">42.4</span></td>
<td id="S5.T1.1.3.2.6" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.3.2.6.1" class="ltx_text" style="font-size:80%;">19.3</span></td>
<td id="S5.T1.1.3.2.7" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.3.2.7.1" class="ltx_text" style="font-size:80%;">13.9</span></td>
</tr>
<tr id="S5.T1.1.4.3" class="ltx_tr">
<th id="S5.T1.1.4.3.1" class="ltx_td ltx_align_left ltx_th ltx_th_row">
<span id="S5.T1.1.4.3.1.1" class="ltx_text ltx_font_smallcaps" style="font-size:80%;">Emo</span><span id="S5.T1.1.4.3.1.2" class="ltx_text" style="font-size:80%;"> (Ours)</span>
</th>
<th id="S5.T1.1.4.3.2" class="ltx_td ltx_align_left ltx_th ltx_th_row"><span id="S5.T1.1.4.3.2.1" class="ltx_text" style="font-size:80%;">1T</span></th>
<td id="S5.T1.1.4.3.3" class="ltx_td ltx_align_right"><span id="S5.T1.1.4.3.3.1" class="ltx_text" style="font-size:80%;">63.1</span></td>
<td id="S5.T1.1.4.3.4" class="ltx_td ltx_align_right"><span id="S5.T1.1.4.3.4.1" class="ltx_text" style="font-size:80%;">57.9</span></td>
<td id="S5.T1.1.4.3.5" class="ltx_td ltx_align_right"><span id="S5.T1.1.4.3.5.1" class="ltx_text" style="font-size:80%;">42.8</span></td>
<td id="S5.T1.1.4.3.6" class="ltx_td ltx_align_right"><span id="S5.T1.1.4.3.6.1" class="ltx_text" style="font-size:80%;">18.5</span></td>
<td id="S5.T1.1.4.3.7" class="ltx_td ltx_align_right"><span id="S5.T1.1.4.3.7.1" class="ltx_text" style="font-size:80%;">12.0</span></td>
</tr>
<tr id="S5.T1.1.5.4" class="ltx_tr">
<th id="S5.T1.1.5.4.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t"><span id="S5.T1.1.5.4.1.1" class="ltx_text" style="font-size:80%;">Dense</span></th>
<th id="S5.T1.1.5.4.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t"><span id="S5.T1.1.5.4.2.1" class="ltx_text" style="font-size:80%;">130B</span></th>
<td id="S5.T1.1.5.4.3" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.5.4.3.1" class="ltx_text" style="font-size:80%;">54.1</span></td>
<td id="S5.T1.1.5.4.4" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.5.4.4.1" class="ltx_text" style="font-size:80%;">41.5</span></td>
<td id="S5.T1.1.5.4.5" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.5.4.5.1" class="ltx_text" style="font-size:80%;">33.0</span></td>
<td id="S5.T1.1.5.4.6" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.5.4.6.1" class="ltx_text" style="font-size:80%;">12.2</span></td>
<td id="S5.T1.1.5.4.7" class="ltx_td ltx_align_right ltx_border_t"><span id="S5.T1.1.5.4.7.1" class="ltx_text" style="font-size:80%;">2.7</span></td>
</tr>
<tr id="S5.T1.1.6.5" class="ltx_tr">
<th id="S5.T1.1.6.5.1" class="ltx_td ltx_align_left ltx_th ltx_th_row"><span id="S5.T1.1.6.5.1.1" class="ltx_text" style="font-size:80%;">Reg. MoE</span></th>
<th id="S5.T1.1.6.5.2" class="ltx_td ltx_align_left ltx_th ltx_th_row"><span id="S5.T1.1.6.5.2.1" class="ltx_text" style="font-size:80%;">130B</span></th>
<td id="S5.T1.1.6.5.3" class="ltx_td ltx_align_right"><span id="S5.T1.1.6.5.3.1" class="ltx_text" style="font-size:80%;">60.1</span></td>
<td id="S5.T1.1.6.5.4" class="ltx_td ltx_align_right"><span id="S5.T1.1.6.5.4.1" class="ltx_text" style="font-size:80%;">51.0</span></td>
<td id="S5.T1.1.6.5.5" class="ltx_td ltx_align_right"><span id="S5.T1.1.6.5.5.1" class="ltx_text" style="font-size:80%;">37.5</span></td>
<td id="S5.T1.1.6.5.6" class="ltx_td ltx_align_right"><span id="S5.T1.1.6.5.6.1" class="ltx_text" style="font-size:80%;">15.8</span></td>
<td id="S5.T1.1.6.5.7" class="ltx_td ltx_align_right"><span id="S5.T1.1.6.5.7.1" class="ltx_text" style="font-size:80%;">5.2</span></td>
</tr>
<tr id="S5.T1.1.7.6" class="ltx_tr">
<th id="S5.T1.1.7.6.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_bb">
<span id="S5.T1.1.7.6.1.1" class="ltx_text ltx_font_smallcaps" style="font-size:80%;">Emo</span><span id="S5.T1.1.7.6.1.2" class="ltx_text" style="font-size:80%;"> (Ours)</span>
</th>
<th id="S5.T1.1.7.6.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_bb"><span id="S5.T1.1.7.6.2.1" class="ltx_text" style="font-size:80%;">130B</span></th>
<td id="S5.T1.1.7.6.3" class="ltx_td ltx_align_right ltx_border_bb"><span id="S5.T1.1.7.6.3.1" class="ltx_text" style="font-size:80%;">59.1</span></td>
<td id="S5.T1.1.7.6.4" class="ltx_td ltx_align_right ltx_border_bb"><span id="S5.T1.1.7.6.4.1" class="ltx_text" style="font-size:80%;">49.2</span></td>
<td id="S5.T1.1.7.6.5" class="ltx_td ltx_align_right ltx_border_bb"><span id="S5.T1.1.7.6.5.1" class="ltx_text" style="font-size:80%;">38.1</span></td>
<td id="S5.T1.1.7.6.6" class="ltx_td ltx_align_right ltx_border_bb"><span id="S5.T1.1.7.6.6.1" class="ltx_text" style="font-size:80%;">15.5</span></td>
<td id="S5.T1.1.7.6.7" class="ltx_td ltx_align_right ltx_border_bb"><span id="S5.T1.1.7.6.7.1" class="ltx_text" style="font-size:80%;">4.2</span></td>
</tr>
</tbody>
</table>
<figcaption class="ltx_caption ltx_centering" style="font-size:80%;"><span class="ltx_tag ltx_tag_table">Table 1: </span>
<span id="S5.T1.12.1" class="ltx_text ltx_font_bold">Full-model Evaluation (§<a href="#S5.SS1" title="5.1 Full-Model Evaluation ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.1</span></a>)</span>. All models are trained on the same data mixture, and activate the same number of parameters (1B). <span id="S5.T1.13.2" class="ltx_text ltx_font_smallcaps">Emo</span> matches the performance of a standard MoE. 
<br class="ltx_break"><math id="S5.T1.3.m1" class="ltx_Math" alttext="\dagger" display="inline" intent=":literal"><semantics><mo mathsize="1.130em">†</mo><annotation encoding="application/x-tex">\dagger</annotation></semantics></math><span id="S5.T1.14.3" class="ltx_text" style="font-size:113%;">: Use the outdated architecture (no pre-norm, use QK-norm, no shared expert, micro-batch load balancing) and has 64 total experts instead of 128.</span>
</figcaption>
</figure>
<div id="S5.SS1.p1" class="ltx_para">
<p id="S5.SS1.p1.1" class="ltx_p">Table <a href="#S5.T1" title="Table 1 ‣ 5.1 Full-Model Evaluation ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> reports full-model performance for models trained on the same data with the same number of active parameters (1B).
First, our baseline MoE is competitive, outperforming <span id="S5.SS1.p1.1.1" class="ltx_text ltx_font_smallcaps">OLMoE</span> <cite class="ltx_cite ltx_citemacro_cite">Muennighoff<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">2025</a>)</cite> trained on 5T tokens despite using only 1T tokens. Nonetheless, <span id="S5.SS1.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> matches the performance of this standard MoE.</p>
</div>
<div id="S5.SS1.p2" class="ltx_para">
<p id="S5.SS1.p2.1" class="ltx_p">The trend holds in the 130B-token setting: both our baseline MoE and <span id="S5.SS1.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> significantly outperform a dense model with matched active parameters, demonstrating the benefits of sparsity. <span id="S5.SS1.p2.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> remains comparable to the standard MoE.</p>
</div>
</section>
<section id="S5.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">5.2 </span>Selective Expert Use</h3>

<figure id="S5.F3" class="ltx_figure"><img src="2605.06663v1/x5.png" id="S5.F3.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="475" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 3: </span><span id="S5.F3.10.1" class="ltx_text ltx_font_bold">Selective Expert Use</span> of MoEs trained on <em id="S5.F3.11.2" class="ltx_emph ltx_font_bold ltx_font_italic">1T</em> tokens (§<a href="#S5.SS2" title="5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.2</span></a>). Results are shown both without fine-tuning (<em id="S5.F3.12.3" class="ltx_emph ltx_font_italic">top</em>) and with fine-tuning (<em id="S5.F3.13.4" class="ltx_emph ltx_font_italic">bottom</em>).
For MMLU and MMLU-Pro, each domain selects a corresponding expert subset as described in §<a href="#S4.SS2" title="4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.2</span></a>, and we report macro-averaged results across domains (16 for MMLU and 13 for MMLU-Pro).
The baseline MoE degrades sharply under subset restriction, whereas <span id="S5.F3.14.5" class="ltx_text ltx_font_smallcaps">Emo</span> remains robust, with <math id="S5.F3.3.m1" class="ltx_Math" alttext="\approx 1" display="inline" intent=":literal"><semantics><mrow><mi></mi><mo>≈</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">\approx 1</annotation></semantics></math>% drop at 25% parameters and <math id="S5.F3.4.m2" class="ltx_Math" alttext="\approx 3" display="inline" intent=":literal"><semantics><mrow><mi></mi><mo>≈</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">\approx 3</annotation></semantics></math>% drop at 12.5%, in both without and with fine-tuning.
</figcaption>
</figure>
<div id="S5.SS2.p1" class="ltx_para">
<p id="S5.SS2.p1.1" class="ltx_p">We evaluate whether expert subsets in <span id="S5.SS2.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> can retain full-model performance for a given domain (Figure <a href="#S5.F3" title="Figure 3 ‣ 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3</span></a> for 1T tokens, Figure <a href="#A2.F11" title="Figure 11 ‣ B.1 130B Token Experiments ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">11</span></a> for 130B tokens).</p>
</div>
<section id="S5.SS2.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Standard MoEs (Green) Degrade Sharply.</h4>

<div id="S5.SS2.SSS0.Px1.p1" class="ltx_para">
<p id="S5.SS2.SSS0.Px1.p1.1" class="ltx_p">Restricting to expert subsets leads to large performance drops—for instance, over 10% when retaining 25% of experts (128<math id="S5.SS2.SSS0.Px1.p1.1.m1" class="ltx_Math" alttext="\rightarrow" display="inline" intent=":literal"><semantics><mo stretchy="false">→</mo><annotation encoding="application/x-tex">\rightarrow</annotation></semantics></math>32), and below a dense model with matched active parameters (Figure <a href="#A2.F11" title="Figure 11 ‣ B.1 130B Token Experiments ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">11</span></a>).
This trend is consistent with and without fine-tuning. These results show that standard MoEs do not support modular use: even when only a narrow set of capabilities is required, restricting to expert subsets causes performance to break down.</p>
</div>
</section>
<section id="S5.SS2.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="S5.SS2.SSS0.Px2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> (Purple) Enables Modular Use.</h4>

<div id="S5.SS2.SSS0.Px2.p1" class="ltx_para">
<p id="S5.SS2.SSS0.Px2.p1.1" class="ltx_p">In contrast, <span id="S5.SS2.SSS0.Px2.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> retains performance under subset deployment. Performance drops are minimal, e.g., about 1% at 25% expert retention and 3% at 12.5%, and the model continues to outperform dense baselines, even in an extreme case where only 6.2% of experts are retained. This trend persists after fine-tuning, e.g., notably, on GSM8K, subsets with up to 12.5% of experts perfectly recover full-model performance. These results indicate that <span id="S5.SS2.SSS0.Px2.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> supports modular use by identifying domain-relevant expert subsets, enabling significant memory savings by avoiding the need to load the full model.</p>
</div>
<div id="S5.SS2.SSS0.Px2.p2" class="ltx_para">
<p id="S5.SS2.SSS0.Px2.p2.1" class="ltx_p">Notably, we show in §<a href="#A2.SS2" title="B.2 Ablations on expert subset initialization ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.2</span></a> that selecting relevant expert subsets is sample efficient, needing as few as five examples. We also provide examples of GSM8K generations in §<a href="#A2.SS5" title="B.5 Generations from Selective Expert Use on GSM8K ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.5</span></a>, demonstrating qualitative differences in generation quality between expert subsets of <span id="S5.SS2.SSS0.Px2.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> and those for standard MoEs.</p>
</div>
</section>
<section id="S5.SS2.SSS0.Px3" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Expert Subsets of <span id="S5.SS2.SSS0.Px3.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> Outperform Memory-matched Models Trained from Scratch.</h4>

<div id="S5.SS2.SSS0.Px3.p1" class="ltx_para">
<p id="S5.SS2.SSS0.Px3.p1.1" class="ltx_p">Figure <a href="#S1.F1" title="Figure 1 ‣ 1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> (right) compares <span id="S5.SS2.SSS0.Px3.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> expert subsets against memory-matched models trained from scratch, including a standard MoE with 32 experts and a dense model. A full set of results can be found in Figure <a href="#A2.F11" title="Figure 11 ‣ B.1 130B Token Experiments ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">11</span></a> in §<a href="#A2.SS1" title="B.1 130B Token Experiments ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.1</span></a>. Despite using only a subset of the experts from a larger pretrained model, the 32-expert and 8-expert subsets of <span id="S5.SS2.SSS0.Px3.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> match or outperform these memory-matched baselines. These results suggest that expert subsets from a single <span id="S5.SS2.SSS0.Px3.p1.1.3" class="ltx_text ltx_font_smallcaps">Emo</span> model offer a stronger memory-accuracy trade-off than models trained from scratch under fixed memory budgets, forming a new Pareto frontier across memory regimes.</p>
</div>
</section>
<section id="S5.SS2.SSS0.Px4" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="S5.SS2.SSS0.Px4.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is Robust to Expert Selection Schemes.</h4>

<figure id="S5.F4" class="ltx_figure"><img src="2605.06663v1/x6.png" id="S5.F4.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="297" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 4: </span>
<span id="S5.F4.4.1" class="ltx_text ltx_font_bold">Expert Selection Methods</span> in selective expert use of MoEs trained on <em id="S5.F4.5.2" class="ltx_emph ltx_font_italic">1T</em> tokens (§<a href="#S5.SS2" title="5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.2</span></a>). Results are without fine-tuning.
For MMLU and MMLU-Pro, each domain selects a corresponding expert subset as described in §<a href="#S4.SS2" title="4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.2</span></a>.
<span id="S5.F4.6.3" class="ltx_text ltx_font_smallcaps">Emo</span> expert subsets maintain high performance compared to regular MoEs across both router-based and Easy-EP expert-selection strategies. Random expert selection converges quickly to random performance as expert subsets shrink.
</figcaption>
</figure>
<div id="S5.SS2.SSS0.Px4.p1" class="ltx_para">
<p id="S5.SS2.SSS0.Px4.p1.1" class="ltx_p">In addition to router-based selection, we evaluate <span id="S5.SS2.SSS0.Px4.p1.1.1" class="ltx_text ltx_font_smallcaps">Easy-EP</span> <cite class="ltx_cite ltx_citemacro_cite">Dong<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib17" title="Domain-specific pruning of large mixture-of-experts models with few-shot demonstrations" class="ltx_ref">2025</a>)</cite>, the state-of-the-art expert pruning method as an alternative selection strategy (Figure <a href="#S5.F4" title="Figure 4 ‣ Emo is Robust to Expert Selection Schemes. ‣ 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4</span></a>). As a sanity check, we also compare with random selection.</p>
</div>
<div id="S5.SS2.SSS0.Px4.p2" class="ltx_para">
<p id="S5.SS2.SSS0.Px4.p2.1" class="ltx_p">For a standard MoE, Easy-EP consistently outperforms router-based selection when larger subsets are retained, although performance still degrades sharply as the subset size decreases, consistent with observations from the original paper <cite class="ltx_cite ltx_citemacro_cite">Dong<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib17" title="Domain-specific pruning of large mixture-of-experts models with few-shot demonstrations" class="ltx_ref">2025</a>)</cite>. This suggests that even state-of-the-art selection methods cannot overcome the lack of localized domain-specific capabilities.</p>
</div>
<div id="S5.SS2.SSS0.Px4.p3" class="ltx_para">
<p id="S5.SS2.SSS0.Px4.p3.1" class="ltx_p">In contrast, <span id="S5.SS2.SSS0.Px4.p3.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> achieves strong performance under both router-based and Easy-EP selection methods. Performance is largely insensitive to the choice of selection scheme, and remains robust even with small expert subsets.
This highlights that modularity must be learned during training, rather than recovered through post hoc expert selection.</p>
</div>
<div id="S5.SS2.SSS0.Px4.p4" class="ltx_para">
<p id="S5.SS2.SSS0.Px4.p4.1" class="ltx_p">In §<a href="#A2.SS4" title="B.4 Annealing Standard MoEs to be Modular. ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.4</span></a>, we also test whether modularity can emerge after pre-training by annealing a standard MoE with the document-level expert pool objective (§<a href="#S3.SS2" title="3.2 Emo: An Objective to Induce Modularity ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.2</span></a>); while training <span id="S5.SS2.SSS0.Px4.p4.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> from scratch performs best, the annealed model shows signs of modularity, which we leave for future work to investigate.</p>
</div>
</section>
</section>
<section id="S5.SS3" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">5.3 </span>Semantic Specialization Emerge in <span id="S5.SS3.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>
</h3>

<figure id="S5.F5" class="ltx_figure"><img src="2605.06663v1/x7.png" id="S5.F5.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="605" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 5: </span>
<span id="S5.F5.4.1" class="ltx_text ltx_font_bold">Token Clusters</span> of pretraining data on MoEs trained on <em id="S5.F5.5.2" class="ltx_emph ltx_font_italic">1T</em> tokens, clustered according to the process described in §<a href="#S5.SS3" title="5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.3</span></a>. Claude Code was used to assign a representative short description for each cluster.
<span id="S5.F5.6.3" class="ltx_text ltx_font_smallcaps">Emo</span> clusters correspond to semantically meaningful domains, with tokens from the same document largely grouped together. Standard MoE training produces clusters of surface-level or syntactic features, with document tokens dispersed across multiple clusters.</figcaption>
</figure>
<figure id="S5.F6" class="ltx_figure"><img src="2605.06663v1/x8.png" id="S5.F6.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="414" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 6: </span>
<span id="S5.F6.8.1" class="ltx_text ltx_font_bold">Domain Similarity</span> of WebOrganizer documents on MoEs trained on <em id="S5.F6.9.2" class="ltx_emph ltx_font_italic">1T</em> tokens (§<a href="#S5.SS3" title="5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.3</span></a>). Expert utilization is highly similar (similarity <math id="S5.F6.3.m1" class="ltx_Math" alttext="&gt;0.6" display="inline" intent=":literal"><semantics><mrow><mi></mi><mo>&gt;</mo><mn>0.6</mn></mrow><annotation encoding="application/x-tex">&gt;0.6</annotation></semantics></math> for most domain pairs) in regular MoEs, while they are much more distinguishable (similarity <math id="S5.F6.4.m2" class="ltx_Math" alttext="&lt;0.4" display="inline" intent=":literal"><semantics><mrow><mi></mi><mo>&lt;</mo><mn>0.4</mn></mrow><annotation encoding="application/x-tex">&lt;0.4</annotation></semantics></math>) in <span id="S5.F6.10.3" class="ltx_text ltx_font_smallcaps">Emo</span>, especially in later layers.
</figcaption>
</figure>
<div id="S5.SS3.p1" class="ltx_para">
<p id="S5.SS3.p1.1" class="ltx_p">We analyze how functional modularity emerges in <span id="S5.SS3.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> by examining expert specialization. We find a qualitative shift in behavior: while standard MoEs specialize at the lexical level, <span id="S5.SS3.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> induces specialization at the level of domains and topics.</p>
</div>
<div id="S5.SS3.p2" class="ltx_para">
<p id="S5.SS3.p2.1" class="ltx_p">To study this, we cluster pretraining tokens based on their routing behavior. Specifically, we sample the first 100 tokens from 12K documents and extract routing probabilities across experts. We project these representations using PCA (retaining 95% variance), apply L2 normalization, and cluster them using spherical k-means with 32 clusters.</p>
</div>
<section id="S5.SS3.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="S5.SS3.SSS0.Px1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> Clusters Align with Semantic Themes.</h4>

<div id="S5.SS3.SSS0.Px1.p1" class="ltx_para">
<p id="S5.SS3.SSS0.Px1.p1.1" class="ltx_p">An interactive visualization is available at <a href="https://emovisualization.netlify.app" title="" class="ltx_ref ltx_href ltx_font_typewriter">emovisualization.netlify.app</a>;</p>
</div>
<div id="S5.SS3.SSS0.Px1.p2" class="ltx_para">
<p id="S5.SS3.SSS0.Px1.p2.1" class="ltx_p">Figure <a href="#S5.F5" title="Figure 5 ‣ 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5</span></a> shows representative cluster descriptions, assigned by Claude Code. First, in a standard MoE, clusters correspond to low-level lexical categories, such as “prepositions”, “proper names”, “copula verbs”, or “definite articles”, consistent with observations from prior work <cite class="ltx_cite ltx_citemacro_cite">Jiang<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib52" title="Mixtral of experts" class="ltx_ref">2024</a>); Muennighoff<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">2025</a>)</cite>. As a result, tokens within a single document are dispersed across many clusters.</p>
</div>
<div id="S5.SS3.SSS0.Px1.p3" class="ltx_para">
<p id="S5.SS3.SSS0.Px1.p3.1" class="ltx_p">In contrast, <span id="S5.SS3.SSS0.Px1.p3.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> produces clusters aligned with high-level semantics and domains, such as “film, music, TV &amp; book reviews”, “health, medical &amp; wellness”, “news reporting”, and “US. politics &amp; elections”. As a result, tokens within the same document are typically assigned to the same cluster, indicating consistent expert use.</p>
</div>
<div id="S5.SS3.SSS0.Px1.p4" class="ltx_para">
<p id="S5.SS3.SSS0.Px1.p4.1" class="ltx_p">This reveals two key insights. First, domain-level specialization emerges in <span id="S5.SS3.SSS0.Px1.p4.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>, even with no explicit supervision. Second, this specialization directly enables modularity: tokens from the same domain share routing patterns, allowing computation to be localized to a small, coherent subset of experts. As a result, expert subsets can be used effectively for downstream domains.</p>
</div>
</section>
<section id="S5.SS3.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="S5.SS3.SSS0.Px2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> Expert Activation Patterns across Domains are Distinct and Matches Human Intuition.</h4>

<div id="S5.SS3.SSS0.Px2.p1" class="ltx_para">
<p id="S5.SS3.SSS0.Px2.p1.1" class="ltx_p">We next ask whether domain-level expert activation patterns reflect human-interpretable domain similarity. We find that <span id="S5.SS3.SSS0.Px2.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> groups conceptually related domains while separating unrelated ones, a structure much less pronounced in standard MoEs.</p>
</div>
<div id="S5.SS3.SSS0.Px2.p2" class="ltx_para">
<p id="S5.SS3.SSS0.Px2.p2.1" class="ltx_p">To measure this, we use a random sample of 20 million documents from WebOrganizer <cite class="ltx_cite ltx_citemacro_cite">Wettig<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib46" title="Organize the web: constructing domains enhances pre-training data curation" class="ltx_ref">2025</a>)</cite>, which assigns each document to one of 24 human-labeled domains. For each domain, we construct a domain-level expert activation vector by first averaging router activations across tokens within each document, and then averaging these document-level vectors across all documents assigned to that domain. We measure similarity between domains using cosine similarity over the resulting domain-level expert activation vectors.</p>
</div>
<div id="S5.SS3.SSS0.Px2.p3" class="ltx_para">
<p id="S5.SS3.SSS0.Px2.p3.1" class="ltx_p">As shown in Figure <a href="#S5.F6" title="Figure 6 ‣ 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">6</span></a>, <span id="S5.SS3.SSS0.Px2.p3.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> produces domain-level similarity patterns that better match semantic relationships between human-labeled domains. Related domains exhibit higher expert-activation similarity, while unrelated domains are more clearly separated. In contrast, the standard MoE shows more diffuse similarities across domains, suggesting that its expert activations are less aligned with human-interpretable domain structure.</p>
</div>
<div id="S5.SS3.SSS0.Px2.p4" class="ltx_para">
<p id="S5.SS3.SSS0.Px2.p4.1" class="ltx_p">Across both models, early layers show limited domain structure, while later layers exhibit clearer alignment with human-labeled domains. This suggests that domain-level expert specialization emerges progressively in deeper layers, potentially motivating future work on inducing modularity in a layer-wise manner.</p>
</div>
</section>
</section>
</section>
<section id="S6" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">6 </span>Future Directions</h2>

<div id="S6.p1" class="ltx_para">
<p id="S6.p1.1" class="ltx_p"><span id="S6.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is among the first MoE models with emergent modularity. While this work primarily focuses on modularity for efficient deployment, it also points to several broader opportunities.</p>
</div>
<section id="S6.SS0.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Accessible Deployment of Large Sparse MoEs.</h4>

<div id="S6.SS0.SSS0.Px1.p1" class="ltx_para">
<p id="S6.SS0.SSS0.Px1.p1.1" class="ltx_p">As MoE models scale to trillions of parameters <cite class="ltx_cite ltx_citemacro_cite">DeepSeek-AI (<a href="#bib.bib50" title="DeepSeek-v4: towards highly efficient million-token context intelligence" class="ltx_ref">2026</a>); Team<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib51" title="Kimi k2: open agentic intelligence" class="ltx_ref">2026</a>)</cite>, deploying or adapting them becomes increasingly resource-intensive. Prior work addresses this through memory-constrained scaling and optimized serving <cite class="ltx_cite ltx_citemacro_cite">Li<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib49" title="Can mixture-of-experts surpass dense llms under strictly equal resources?" class="ltx_ref">2026</a>); Song<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib37" title="BlockFFN: towards end-side acceleration-friendly mixture-of-experts with chunk-level activation sparsity" class="ltx_ref">2025</a>); Shen and Henderson (<a href="#bib.bib47" title="Temporally extended mixture-of-experts models" class="ltx_ref">2026</a>); Tairin<span class="ltx_text ltx_bib_etal"> et al.</span> (<a href="#bib.bib48" title="EMoE: task-aware memory efficient mixture-of-experts-based (moe) model inference" class="ltx_ref">2025</a>)</cite>. Modularity offers an orthogonal path: selectively using small subsets of experts for a given domain, enabling more accessible deployment and adaptation, particularly well-suited for large, highly sparse models.</p>
</div>
</section>
<section id="S6.SS0.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Fine-grained Control.</h4>

<div id="S6.SS0.SSS0.Px2.p1" class="ltx_para">
<p id="S6.SS0.SSS0.Px2.p1.1" class="ltx_p">Modularity can enable finer-grained control at inference time. Since <span id="S6.SS0.SSS0.Px2.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> organizes experts along semantic domains, subsets could be selectively enabled or disabled depending on the application.
For example, clusters associated with spam, gambling, or adult content (Figure <a href="#S5.F5" title="Figure 5 ‣ 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5</span></a>) can be excluded in child-facing applications. Similarly, specialized domains, e.g., biomedical knowledge, may be valuable for benign use but risky in misuse scenarios, and could be conditionally exposed.</p>
</div>
<div id="S6.SS0.SSS0.Px2.p2" class="ltx_para">
<p id="S6.SS0.SSS0.Px2.p2.1" class="ltx_p">This suggests a potential alternative to dataset-level filtering: isolating and managing capabilities at inference time depending on scenarios.</p>
</div>
</section>
<section id="S6.SS0.SSS0.Px3" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Modular Development and Maintenance.</h4>

<div id="S6.SS0.SSS0.Px3.p1" class="ltx_para">
<p id="S6.SS0.SSS0.Px3.p1.1" class="ltx_p">Modularity can also open up a new paradigm for model updates.
Current language models are usually trained and maintained as one large system, requiring the full model, data, and compute to be available at once. A modular model could instead support modular pretraining: training task or domain-specific subset of experts and then reintegrating those experts into the full model.</p>
</div>
<div id="S6.SS0.SSS0.Px3.p2" class="ltx_para">
<p id="S6.SS0.SSS0.Px3.p2.1" class="ltx_p">As a preliminary test, we finetune a 32-expert subset from <span id="S6.SS0.SSS0.Px3.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>, then inserted it back into the original 128-expert model by replacing the corresponding experts.
The resulting model improves over the original full model, though it does not yet match the performance of the standalone subset. This provides early evidence that expert subsets can be trained independently and later integrated, which we leave to future work.</p>
</div>
</section>
<section id="S6.SS0.SSS0.Px4" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Higher Degrees of Monitorability.</h4>

<div id="S6.SS0.SSS0.Px4.p1" class="ltx_para">
<p id="S6.SS0.SSS0.Px4.p1.1" class="ltx_p">Finally, modularity can also make models easier to monitor and audit. Expert activations provide a structured signal of which parts of the model are being used for a given input. For example, if a model answers a math question while strongly activating an expert subset associated with creative writing or low-quality web content, that mismatch may warrant closer inspection. This gives model developers a more structured interface for understanding and debugging model behavior.</p>
</div>
</section>
</section>
<section id="S7" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">7 </span>Conclusion</h2>

<div id="S7.p1" class="ltx_para">
<p id="S7.p1.1" class="ltx_p">We introduced <span id="S7.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>, a mixture-of-experts model designed to make modularity emerge during pretraining. By constraining tokens within the same document to route through a shared expert pool, <span id="S7.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> induces expert subsets that specialize to high-level tasks and capabilities without relying on human-defined domains or task labels. Our results show that this structure does not come at the cost of general performance: as a full model, <span id="S7.p1.1.3" class="ltx_text ltx_font_smallcaps">Emo</span> matches standard MoEs, while its extracted expert subset remain effective even when only a small fraction of experts are retained. Beyond efficient deployment, our analyses show that <span id="S7.p1.1.4" class="ltx_text ltx_font_smallcaps">Emo</span> learns expert subsets aligned with semantic domains rather than surface-level token patterns, suggesting a qualitatively different form of specialization. Together, these results demonstrate that large language models need not remain monolithic systems. Modularity can be built into pretraining itself, opening a path toward models that are easier to deploy, adapt, inspect, and compose.</p>
</div>
</section>
<section id="Sx1" class="ltx_section">
<h2 class="ltx_title ltx_title_section">Acknowledgement</h2>

<div id="Sx1.p1" class="ltx_para">
<p id="Sx1.p1.1" class="ltx_p">We thank Prasann Singhal, Gustavo Lucas Carvalho, Weijia Shi, Jagdeep Bhatia, Colin Raffel, Berkeley AI Research members, the Sky computing lab members, and Ai2 members for valuable discussion and feedback.</p>
</div>
<div id="Sx1.p2" class="ltx_para">
<p id="Sx1.p2.1" class="ltx_p">This research was supported in part by ONR (N00014-26-1-2233), the NVIDIA Academic Grant Program, and gifts from Ai2 and Apple.
Ryan Wang was supported by the National Science Foundation Graduate Research Fellowship Program.</p>
</div>
</section>
<section id="bib" class="ltx_bibliography">
<h2 class="ltx_title ltx_title_bibliography">References</h2>

<ul id="bib.L1" class="ltx_biblist">
<li id="bib.bib27" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[1]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">Y. Bisk, R. Zellers, R. Le Bras, J. Gao, and Y. Choi</span><span class="ltx_text ltx_bib_year"> (2020)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">PIQA: reasoning about physical commonsense in natural language</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Association for the Advancement of Artificial Intelligence</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/1911.11641" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib11" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[2]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">M. Chaudhari, I. Gulati, N. Hundia, P. Karra, and S. Raval</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">MoE lens – an expert is all you need</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2603.05806</span>,
<a href="https://arxiv.org/abs/2603.05806" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib19" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[3]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">T. Chen, S. Huang, Y. Xie, B. Jiao, D. Jiang, H. Zhou, J. Li, and F. Wei</span><span class="ltx_text ltx_bib_year"> (2022)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Task-specific expert pruning for sparse mixture-of-experts</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2206.00277</span>,
<a href="https://arxiv.org/abs/2206.00277" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib23" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[4]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">C. Clark, K. Lee, M. Chang, T. Kwiatkowski, M. Collins, and K. Toutanova</span><span class="ltx_text ltx_bib_year"> (2019-06)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">BoolQ: exploring the surprising difficulty of natural yes/no questions</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)</span>,  <span class="ltx_text ltx_bib_editor">J. Burstein, C. Doran, and T. Solorio (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Minneapolis, Minnesota</span>, <span class="ltx_text ltx_bib_pages"> pp. 2924–2936</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/N19-1300/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/N19-1300" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib22" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[5]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">P. Clark, I. Cowhey, O. Etzioni, T. Khot, A. Sabharwal, C. Schoenick, and O. Tafjord</span><span class="ltx_text ltx_bib_year"> (2018)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Think you have solved question answering? try arc, the ai2 reasoning challenge</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">arXiv:1803.05457v1</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib32" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[6]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">K. Cobbe, V. Kosaraju, M. Bavarian, M. Chen, H. Jun, L. Kaiser, M. Plappert, J. Tworek, J. Hilton, R. Nakano, C. Hesse, and J. Schulman</span><span class="ltx_text ltx_bib_year"> (2021)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Training verifiers to solve math word problems</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2110.14168</span>,
<a href="https://arxiv.org/abs/2110.14168" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib7" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[7]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">D. Dai, C. Deng, C. Zhao, R.x. Xu, H. Gao, D. Chen, J. Li, W. Zeng, X. Yu, Y. Wu, Z. Xie, Y.k. Li, P. Huang, F. Luo, C. Ruan, Z. Sui, and W. Liang</span><span class="ltx_text ltx_bib_year"> (2024-08)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">DeepSeekMoE: towards ultimate expert specialization in mixture-of-experts language models</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)</span>,  <span class="ltx_text ltx_bib_editor">L. Ku, A. Martins, and V. Srikumar (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Bangkok, Thailand</span>, <span class="ltx_text ltx_bib_pages"> pp. 1280–1297</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/2024.acl-long.70/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/2024.acl-long.70" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#A1.SS3.p1.1" title="A.3 Shared Experts ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§A.3</span></a>,
<a href="#S1.p2.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>.
</span>
</li>
<li id="bib.bib8" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[8]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">DeepSeek-AI, A. Liu, B. Feng, B. Wang, B. Wang, B. Liu, C. Zhao, C. Dengr, C. Ruan, D. Dai, D. Guo, D. Yang, D. Chen, D. Ji, E. Li, F. Lin, F. Luo, G. Hao, G. Chen, G. Li, H. Zhang, H. Xu, H. Yang, H. Zhang, H. Ding, H. Xin, H. Gao, H. Li, H. Qu, J. L. Cai, J. Liang, J. Guo, J. Ni, J. Li, J. Chen, J. Yuan, J. Qiu, J. Song, K. Dong, K. Gao, K. Guan, L. Wang, L. Zhang, L. Xu, L. Xia, L. Zhao, L. Zhang, M. Li, M. Wang, M. Zhang, M. Zhang, M. Tang, M. Li, N. Tian, P. Huang, P. Wang, P. Zhang, Q. Zhu, Q. Chen, Q. Du, R. J. Chen, R. L. Jin, R. Ge, R. Pan, R. Xu, R. Chen, S. S. Li, S. Lu, S. Zhou, S. Chen, S. Wu, S. Ye, S. Ma, S. Wang, S. Zhou, S. Yu, S. Zhou, S. Zheng, T. Wang, T. Pei, T. Yuan, T. Sun, W. L. Xiao, W. Zeng, W. An, W. Liu, W. Liang, W. Gao, W. Zhang, X. Q. Li, X. Jin, X. Wang, X. Bi, X. Liu, X. Wang, X. Shen, X. Chen, X. Chen, X. Nie, X. Sun, X. Wang, X. Liu, X. Xie, X. Yu, X. Song, X. Zhou, X. Yang, X. Lu, X. Su, Y. Wu, Y. K. Li, Y. X. Wei, Y. X. Zhu, Y. Xu, Y. Huang, Y. Li, Y. Zhao, Y. Sun, Y. Li, Y. Wang, Y. Zheng, Y. Zhang, Y. Xiong, Y. Zhao, Y. He, Y. Tang, Y. Piao, Y. Dong, Y. Tan, Y. Liu, Y. Wang, Y. Guo, Y. Zhu, Y. Wang, Y. Zou, Y. Zha, Y. Ma, Y. Yan, Y. You, Y. Liu, Z. Z. Ren, Z. Ren, Z. Sha, Z. Fu, Z. Huang, Z. Zhang, Z. Xie, Z. Hao, Z. Shao, Z. Wen, Z. Xu, Z. Zhang, Z. Li, Z. Wang, Z. Gu, Z. Li, and Z. Xie</span><span class="ltx_text ltx_bib_year"> (2024)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">DeepSeek-v2: a strong, economical, and efficient mixture-of-experts language model</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2405.04434</span>,
<a href="https://arxiv.org/abs/2405.04434" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p2.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>.
</span>
</li>
<li id="bib.bib9" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[9]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">DeepSeek-AI, A. Liu, B. Feng, B. Xue, B. Wang, B. Wu, C. Lu, C. Zhao, C. Deng, C. Zhang, C. Ruan, D. Dai, D. Guo, D. Yang, D. Chen, D. Ji, E. Li, F. Lin, F. Dai, F. Luo, G. Hao, G. Chen, G. Li, H. Zhang, H. Bao, H. Xu, H. Wang, H. Zhang, H. Ding, H. Xin, H. Gao, H. Li, H. Qu, J. L. Cai, J. Liang, J. Guo, J. Ni, J. Li, J. Wang, J. Chen, J. Chen, J. Yuan, J. Qiu, J. Li, J. Song, K. Dong, K. Hu, K. Gao, K. Guan, K. Huang, K. Yu, L. Wang, L. Zhang, L. Xu, L. Xia, L. Zhao, L. Wang, L. Zhang, M. Li, M. Wang, M. Zhang, M. Zhang, M. Tang, M. Li, N. Tian, P. Huang, P. Wang, P. Zhang, Q. Wang, Q. Zhu, Q. Chen, Q. Du, R. J. Chen, R. L. Jin, R. Ge, R. Zhang, R. Pan, R. Wang, R. Xu, R. Zhang, R. Chen, S. S. Li, S. Lu, S. Zhou, S. Chen, S. Wu, S. Ye, S. Ye, S. Ma, S. Wang, S. Zhou, S. Yu, S. Zhou, S. Pan, T. Wang, T. Yun, T. Pei, T. Sun, W. L. Xiao, W. Zeng, W. Zhao, W. An, W. Liu, W. Liang, W. Gao, W. Yu, W. Zhang, X. Q. Li, X. Jin, X. Wang, X. Bi, X. Liu, X. Wang, X. Shen, X. Chen, X. Zhang, X. Chen, X. Nie, X. Sun, X. Wang, X. Cheng, X. Liu, X. Xie, X. Liu, X. Yu, X. Song, X. Shan, X. Zhou, X. Yang, X. Li, X. Su, X. Lin, Y. K. Li, Y. Q. Wang, Y. X. Wei, Y. X. Zhu, Y. Zhang, Y. Xu, Y. Xu, Y. Huang, Y. Li, Y. Zhao, Y. Sun, Y. Li, Y. Wang, Y. Yu, Y. Zheng, Y. Zhang, Y. Shi, Y. Xiong, Y. He, Y. Tang, Y. Piao, Y. Wang, Y. Tan, Y. Ma, Y. Liu, Y. Guo, Y. Wu, Y. Ou, Y. Zhu, Y. Wang, Y. Gong, Y. Zou, Y. He, Y. Zha, Y. Xiong, Y. Ma, Y. Yan, Y. Luo, Y. You, Y. Liu, Y. Zhou, Z. F. Wu, Z. Z. Ren, Z. Ren, Z. Sha, Z. Fu, Z. Xu, Z. Huang, Z. Zhang, Z. Xie, Z. Zhang, Z. Hao, Z. Gou, Z. Ma, Z. Yan, Z. Shao, Z. Xu, Z. Wu, Z. Zhang, Z. Li, Z. Gu, Z. Zhu, Z. Liu, Z. Li, Z. Xie, Z. Song, Z. Gao, and Z. Pan</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">DeepSeek-v3 technical report</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2412.19437</span>,
<a href="https://arxiv.org/abs/2412.19437" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p1.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S1.p2.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S2.SS0.SSS0.Px1.p1.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib50" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[10]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">DeepSeek-AI</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">DeepSeek-v4: towards highly efficient million-token context intelligence</span>.
</span>
<span class="ltx_bibblock">Note: <span class="ltx_text ltx_bib_note">Technical reportAvailable at Hugging Face</span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S6.SS0.SSS0.Px1.p1.1" title="Accessible Deployment of Large Sparse MoEs. ‣ 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§6</span></a>.
</span>
</li>
<li id="bib.bib17" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[11]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">Z. Dong, H. Peng, P. Liu, W. X. Zhao, D. Wu, F. Xiao, and Z. Wang</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Domain-specific pruning of large mixture-of-experts models with few-shot demonstrations</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of Advances in Neural Information Processing Systems</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2504.06792" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S4.SS2.SSS0.Px2.p2.2" title="Selective Expert Use. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>,
<a href="#S5.SS2.SSS0.Px4.p1.1" title="Emo is Robust to Expert Selection Schemes. ‣ 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§5.2</span></a>,
<a href="#S5.SS2.SSS0.Px4.p2.1" title="Emo is Robust to Expert Selection Schemes. ‣ 5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§5.2</span></a>.
</span>
</li>
<li id="bib.bib43" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[12]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">D. Dua, Y. Wang, P. Dasigi, G. Stanovsky, S. Singh, and M. Gardner</span><span class="ltx_text ltx_bib_year"> (2019-06)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">DROP: a reading comprehension benchmark requiring discrete reasoning over paragraphs</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)</span>,  <span class="ltx_text ltx_bib_editor">J. Burstein, C. Doran, and T. Solorio (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Minneapolis, Minnesota</span>, <span class="ltx_text ltx_bib_pages"> pp. 2368–2378</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/N19-1246/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/N19-1246" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib34" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[13]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">W. Fedus, B. Zoph, and N. Shazeer</span><span class="ltx_text ltx_bib_year"> (2022)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Switch transformers: scaling to trillion parameter models with simple and efficient sparsity</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Journal of Machine Learning Research</span> <span class="ltx_text ltx_bib_volume">23</span> (<span class="ltx_text ltx_bib_number">120</span>), <span class="ltx_text ltx_bib_pages"> pp. 1–39</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px1.p1.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib15" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[14]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">H. Guo, H. Lu, G. Nan, B. Chu, J. Zhuang, Y. Yang, W. Che, X. Cao, S. Leng, Q. Cui, and X. Jiang</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Advancing expert specialization for better MoE</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Advances in Neural Information Processing Systems</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2505.22323" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib30" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[15]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. Song, and J. Steinhardt</span><span class="ltx_text ltx_bib_year"> (2021)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Measuring massive multitask language understanding</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Proceedings of the International Conference on Learning Representations (ICLR)</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#A3.SS0.SSS0.Px1.p1.1" title="MMLU and MMLU-Pro Setup. ‣ Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">Appendix C</span></a>,
<a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib14" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[16]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">R. Hu, Y. Cao, B. Kong, M. Sun, and K. Yuan</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Improving moe performance and efficiency with plug-and-play intra-layer specialization and cross-layer coupling losses</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://openreview.net/forum?id=nY91ZOfB5M" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib44" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[17]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">W. Huang, Y. Zhang, X. Zheng, F. Chao, R. Ji, and L. Cao</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Discovering important experts for mixture-of-experts models pruning through a theoretical perspective</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">The Thirty-ninth Annual Conference on Neural Information Processing Systems</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://openreview.net/forum?id=7kQjbCQwtT" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib52" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[18]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">A. Q. Jiang, A. Sablayrolles, A. Roux, A. Mensch, B. Savary, C. Bamford, D. S. Chaplot, D. d. l. Casas, E. B. Hanna, F. Bressand, <span class="ltx_text ltx_bib_etal">et al.</span></span><span class="ltx_text ltx_bib_year"> (2024)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Mixtral of experts</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">arXiv preprint arXiv:2401.04088</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S5.SS3.SSS0.Px1.p2.1" title="Emo Clusters Align with Semantic Themes. ‣ 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§5.3</span></a>.
</span>
</li>
<li id="bib.bib16" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[19]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">jie hu, J. Hou, and X. Li</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Quantifying expert specialization for effective pruning in mixture-of-experts models</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://openreview.net/forum?id=9lOqGhCjtL" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib42" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[20]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">M. Joshi, E. Choi, D. Weld, and L. Zettlemoyer</span><span class="ltx_text ltx_bib_year"> (2017-07)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">TriviaQA: a large scale distantly supervised challenge dataset for reading comprehension</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)</span>,  <span class="ltx_text ltx_bib_editor">R. Barzilay and M. Kan (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Vancouver, Canada</span>, <span class="ltx_text ltx_bib_pages"> pp. 1601–1611</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/P17-1147/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/P17-1147" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib41" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[21]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">T. Kwiatkowski, J. Palomaki, O. Redfield, M. Collins, A. Parikh, C. Alberti, D. Epstein, I. Polosukhin, J. Devlin, K. Lee, K. Toutanova, L. Jones, M. Kelcey, M. Chang, A. M. Dai, J. Uszkoreit, Q. Le, and S. Petrov</span><span class="ltx_text ltx_bib_year"> (2019)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Natural questions: a benchmark for question answering research</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Transactions of the Association for Computational Linguistics</span> <span class="ltx_text ltx_bib_volume">7</span>, <span class="ltx_text ltx_bib_pages"> pp. 452–466</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/Q19-1026/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.1162/tacl%5Fa%5F00276" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib33" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[22]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">D. Lepikhin, H. Lee, Y. Xu, D. Chen, O. Firat, Y. Huang, M. Krikun, N. Shazeer, and Z. Chen</span><span class="ltx_text ltx_bib_year"> (2021)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">GShard: scaling giant models with conditional computation and automatic sharding</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the International Conference on Learning Representations</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2006.16668" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px1.p1.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib49" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[23]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">H. Li, K. M. Lo, Z. Wang, Z. Wang, W. Zheng, S. Zhou, X. Zhang, and D. Jiang</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Can mixture-of-experts surpass dense llms under strictly equal resources?</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">ICLR</span>,
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px1.p2.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S6.SS0.SSS0.Px1.p1.1" title="Accessible Deployment of Large Sparse MoEs. ‣ 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§6</span></a>.
</span>
</li>
<li id="bib.bib38" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[24]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">M. Li, S. Gururangan, T. Dettmers, M. Lewis, T. Althoff, N. A. Smith, and L. Zettlemoyer</span><span class="ltx_text ltx_bib_year"> (2022)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Branch-train-merge: embarrassingly parallel training of expert language models</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2208.03306</span>,
<a href="https://arxiv.org/abs/2208.03306" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib18" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[25]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">X. Lu, Q. Liu, Y. Xu, A. Zhou, S. Huang, B. Zhang, J. Yan, and H. Li</span><span class="ltx_text ltx_bib_year"> (2024-08)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Not all experts are equal: efficient expert pruning and skipping for mixture-of-experts large language models</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)</span>,  <span class="ltx_text ltx_bib_editor">L. Ku, A. Martins, and V. Srikumar (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Bangkok, Thailand</span>, <span class="ltx_text ltx_bib_pages"> pp. 6159–6172</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/2024.acl-long.334/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/2024.acl-long.334" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px1.p2.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib26" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[26]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">T. Mihaylov, P. Clark, T. Khot, and A. Sabharwal</span><span class="ltx_text ltx_bib_year"> (2018)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Can a suit of armor conduct electricity? a new dataset for open book question answering</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">EMNLP</span>,
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib3" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[27]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">N. Muennighoff, L. Soldaini, D. Groeneveld, K. Lo, J. Morrison, S. Min, W. Shi, P. Walsh, O. Tafjord, N. Lambert, <span class="ltx_text ltx_bib_etal">et al.</span></span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Olmoe: open mixture-of-experts language models</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Proceedings of the International Conference on Learning Representations</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#A1.F10" title="In A.5 Prenorm vs ReorderedNorm ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">Figure 10</span></a>,
<a href="#A1.SS4.SSS0.Px1.p1.9" title="Standard MoE Hyperparameter Ablations. ‣ A.4 Tuning LR and LB ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§A.4</span></a>,
<a href="#A1.SS5.p1.1" title="A.5 Prenorm vs ReorderedNorm ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§A.5</span></a>,
<a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S4.SS1.p2.1" title="4.1 Architecture &amp; Training Details ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.1</span></a>,
<a href="#S4.SS1.p3.1" title="4.1 Architecture &amp; Training Details ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.1</span></a>,
<a href="#S5.SS1.p1.1" title="5.1 Full-Model Evaluation ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§5.1</span></a>,
<a href="#S5.SS3.SSS0.Px1.p2.1" title="Emo Clusters Align with Semantic Themes. ‣ 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§5.3</span></a>.
</span>
</li>
<li id="bib.bib36" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[28]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">T. Olmo, :, A. Ettinger, A. Bertsch, B. Kuehl, D. Graham, D. Heineman, D. Groeneveld, F. Brahman, F. Timbers, H. Ivison, J. Morrison, J. Poznanski, K. Lo, L. Soldaini, M. Jordan, M. Chen, M. Noukhovitch, N. Lambert, P. Walsh, P. Dasigi, R. Berry, S. Malik, S. Shah, S. Geng, S. Arora, S. Gupta, T. Anderson, T. Xiao, T. Murray, T. Romero, V. Graf, A. Asai, A. Bhagia, A. Wettig, A. Liu, A. Rangapur, C. Anastasiades, C. Huang, D. Schwenk, H. Trivedi, I. Magnusson, J. Lochner, J. Liu, L. J. V. Miranda, M. Sap, M. Morgan, M. Schmitz, M. Guerquin, M. Wilson, R. Huff, R. L. Bras, R. Xin, R. Shao, S. Skjonsberg, S. Z. Shen, S. S. Li, T. Wilde, V. Pyatkin, W. Merrill, Y. Chang, Y. Gu, Z. Zeng, A. Sabharwal, L. Zettlemoyer, P. W. Koh, A. Farhadi, N. A. Smith, and H. Hajishirzi</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Olmo 3</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2512.13961</span>,
<a href="https://arxiv.org/abs/2512.13961" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p1.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>.
</span>
</li>
<li id="bib.bib13" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[29]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">J. Park, Y. J. Ahn, K. Kim, and J. Kang</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Monet: mixture of monosemantic experts for transformers</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Proceedings of the International Conference on Learning Representations</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib4" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[30]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">Z. Qiu, Z. Huang, B. Zheng, K. Wen, Z. Wang, R. Men, I. Titov, D. Liu, J. Zhou, and J. Lin</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Demons in the detail: on implementing load balancing loss for training specialized mixture-of-expert models</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_pages"> pp. 5005–5018</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#A1.SS1.p4.1" title="A.1 Load Balancing ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§A.1</span></a>,
<a href="#S3.SS3.SSS0.Px1.p2.1" title="Consideration 1. Load Balancing. ‣ 3.3 Key Technical Considerations ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§3.3</span></a>.
</span>
</li>
<li id="bib.bib40" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[31]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">P. Rajpurkar, J. Zhang, K. Lopyrev, and P. Liang</span><span class="ltx_text ltx_bib_year"> (2016-11)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">SQuAD: 100,000+ questions for machine comprehension of text</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing</span>,  <span class="ltx_text ltx_bib_editor">J. Su, K. Duh, and X. Carreras (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Austin, Texas</span>, <span class="ltx_text ltx_bib_pages"> pp. 2383–2392</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/D16-1264/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/D16-1264" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib39" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[32]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">S. Reddy, D. Chen, and C. D. Manning</span><span class="ltx_text ltx_bib_year"> (2019)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">CoQA: a conversational question answering challenge</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Transactions of the Association for Computational Linguistics</span> <span class="ltx_text ltx_bib_volume">7</span>, <span class="ltx_text ltx_bib_pages"> pp. 249–266</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/Q19-1016/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.1162/tacl%5Fa%5F00266" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib29" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[33]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">K. Sakaguchi, R. Le Bras, C. Bhagavatula, and Y. Choi</span><span class="ltx_text ltx_bib_year"> (2020)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">WinoGrande: an adversarial winograd schema challenge at scale</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Association for the Advancement of Artificial Intelligence</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/1907.10641" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib28" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[34]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">M. Sap, H. Rashkin, D. Chen, R. Le Bras, and Y. Choi</span><span class="ltx_text ltx_bib_year"> (2019)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">SocialIQA: commonsense reasoning about social interactions</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of Empirical Methods in Natural Language Processing</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/1904.09728" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib5" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[35]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">N. Shazeer, A. Mirhoseini, K. Maziarz, A. Davis, Q. Le, G. Hinton, and J. Dean</span><span class="ltx_text ltx_bib_year"> (2017)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Outrageously large neural networks: the sparsely-gated mixture-of-experts layer</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the International Conference on Learning Representations</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/1701.06538" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px1.p1.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib35" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[36]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">Y. Shen, Z. Zhang, T. Cao, S. Tan, Z. Chen, and C. Gan</span><span class="ltx_text ltx_bib_year"> (2023)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">ModuleFormer: modularity emerges from mixture-of-experts</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2306.04640</span>,
<a href="https://arxiv.org/abs/2306.04640" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px3.p2.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib47" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[37]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">Z. Shen and P. Henderson</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Temporally extended mixture-of-experts models</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">arXiv preprint arXiv:2604.20156</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p5.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S2.SS0.SSS0.Px1.p2.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S6.SS0.SSS0.Px1.p1.1" title="Accessible Deployment of Large Sparse MoEs. ‣ 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§6</span></a>.
</span>
</li>
<li id="bib.bib20" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[38]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">W. Shi, A. Bhagia, K. Farhat, N. Muennighoff, P. Walsh, J. Morrison, D. Schwenk, S. Longpre, J. Poznanski, A. Ettinger, D. Liu, M. Li, D. Groeneveld, M. Lewis, W. Yih, L. Soldaini, K. Lo, N. A. Smith, L. Zettlemoyer, P. W. Koh, H. Hajishirzi, A. Farhadi, and S. Min</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">FlexOlmo: open language models for flexible data use</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of Advances in Neural Information Processing Systems</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2507.07024" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p3.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S3.SS0.SSS0.Px1.p1.1" title="Naive Approach. ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§3</span></a>.
</span>
</li>
<li id="bib.bib37" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[39]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">C. Song, W. Zhao, X. Han, C. Xiao, Y. Chen, Y. Li, Z. Liu, and M. Sun</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">BlockFFN: towards end-side acceleration-friendly mixture-of-experts with chunk-level activation sparsity</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2507.08771</span>,
<a href="https://arxiv.org/abs/2507.08771" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p5.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S2.SS0.SSS0.Px1.p2.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S6.SS0.SSS0.Px1.p1.1" title="Accessible Deployment of Large Sparse MoEs. ‣ 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§6</span></a>.
</span>
</li>
<li id="bib.bib21" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[40]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">S. Sukhbaatar, O. Golovneva, V. Sharma, H. Xu, X. V. Lin, B. Rozière, J. Kahn, D. Li, W. Yih, J. Weston, and X. Li</span><span class="ltx_text ltx_bib_year"> (2024)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Branch-train-MiX: mixing expert LLMs into a mixture-of-experts LLM</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Conference on Language Modeling</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2403.07816" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p3.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S3.SS0.SSS0.Px1.p1.1" title="Naive Approach. ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§3</span></a>.
</span>
</li>
<li id="bib.bib48" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[41]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">S. Tairin, S. Mahmud, H. Shen, and A. Iyer</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">EMoE: task-aware memory efficient mixture-of-experts-based (moe) model inference</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">arXiv preprint arXiv:2503.06823</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S1.p5.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S2.SS0.SSS0.Px1.p2.1" title="Mixture-of-Experts as Scalable Architectures. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>,
<a href="#S6.SS0.SSS0.Px1.p1.1" title="Accessible Deployment of Large Sparse MoEs. ‣ 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§6</span></a>.
</span>
</li>
<li id="bib.bib24" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[42]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">A. Talmor, J. Herzig, N. Lourie, and J. Berant</span><span class="ltx_text ltx_bib_year"> (2019-06)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">CommonsenseQA: a question answering challenge targeting commonsense knowledge</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)</span>,  <span class="ltx_text ltx_bib_editor">J. Burstein, C. Doran, and T. Solorio (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Minneapolis, Minnesota</span>, <span class="ltx_text ltx_bib_pages"> pp. 4149–4158</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/N19-1421/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/N19-1421" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib51" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[43]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">K. Team, Y. Bai, Y. Bao, Y. Charles, C. Chen, G. Chen, H. Chen, H. Chen, J. Chen, N. Chen, R. Chen, Y. Chen, Y. Chen, Y. Chen, Z. Chen, J. Cui, H. Ding, M. Dong, A. Du, C. Du, D. Du, Y. Du, Y. Fan, Y. Feng, K. Fu, B. Gao, C. Gao, H. Gao, P. Gao, T. Gao, Y. Ge, S. Geng, Q. Gu, X. Gu, L. Guan, H. Guo, J. Guo, X. Hao, T. He, W. He, W. He, Y. He, C. Hong, H. Hu, Y. Hu, Z. Hu, W. Huang, Z. Huang, Z. Huang, T. Jiang, Z. Jiang, X. Jin, Y. Kang, G. Lai, C. Li, F. Li, H. Li, M. Li, W. Li, Y. Li, Y. Li, Y. Li, Z. Li, Z. Li, H. Lin, X. Lin, Z. Lin, C. Liu, C. Liu, H. Liu, J. Liu, J. Liu, L. Liu, S. Liu, T. Y. Liu, T. Liu, W. Liu, Y. Liu, Y. Liu, Y. Liu, Y. Liu, Z. Liu, E. Lu, H. Lu, L. Lu, Y. Luo, S. Ma, X. Ma, Y. Ma, S. Mao, J. Mei, X. Men, Y. Miao, S. Pan, Y. Peng, R. Qin, Z. Qin, B. Qu, Z. Shang, L. Shi, S. Shi, F. Song, J. Su, Z. Su, L. Sui, X. Sun, F. Sung, Y. Tai, H. Tang, J. Tao, Q. Teng, C. Tian, C. Wang, D. Wang, F. Wang, H. Wang, H. Wang, J. Wang, J. Wang, J. Wang, S. Wang, S. Wang, S. Wang, X. Wang, Y. Wang, Y. Wang, Y. Wang, Y. Wang, Y. Wang, Z. Wang, Z. Wang, Z. Wang, Z. Wang, C. Wei, Q. Wei, H. Wu, W. Wu, X. Wu, Y. Wu, C. Xiao, J. Xie, X. Xie, W. Xiong, B. Xu, J. Xu, L. H. Xu, L. Xu, S. Xu, W. Xu, X. Xu, Y. Xu, Z. Xu, J. Xu, J. Xu, J. Yan, Y. Yan, H. Yang, X. Yang, Y. Yang, Y. Yang, Z. Yang, Z. Yang, Z. Yang, H. Yao, X. Yao, W. Ye, Z. Ye, B. Yin, L. Yu, E. Yuan, H. Yuan, M. Yuan, S. Yuan, H. Zhan, D. Zhang, H. Zhang, W. Zhang, X. Zhang, Y. Zhang, Y. Zhang, Y. Zhang, Y. Zhang, Y. Zhang, Y. Zhang, Y. Zhang, Y. Zhang, Z. Zhang, H. Zhao, Y. Zhao, Z. Zhao, H. Zheng, S. Zheng, L. Zhong, J. Zhou, X. Zhou, Z. Zhou, J. Zhu, Z. Zhu, W. Zhuang, and X. Zu</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Kimi k2: open agentic intelligence</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2507.20534</span>,
<a href="https://arxiv.org/abs/2507.20534" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S6.SS0.SSS0.Px1.p1.1" title="Accessible Deployment of Large Sparse MoEs. ‣ 6 Future Directions ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§6</span></a>.
</span>
</li>
<li id="bib.bib2" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[44]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">A. Vaswani, N. Shazeer, N. Parmar, J. Uszkoreit, L. Jones, A. N. Gomez, Ł. Kaiser, and I. Polosukhin</span><span class="ltx_text ltx_bib_year"> (2017)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Attention is all you need</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Advances in neural information processing systems</span> <span class="ltx_text ltx_bib_volume">30</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S3.SS1.p1.4" title="3.1 Preliminary: Mixture of Experts Architecture ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§3.1</span></a>.
</span>
</li>
<li id="bib.bib45" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[45]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">X. Wang, S. Hayou, and E. Nalisnick</span><span class="ltx_text ltx_bib_year"> (2026)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">The myth of expert specialization in moes: why routing reflects geometry, not necessarily domain expertise</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">arXiv preprint arXiv:2604.09780</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px2.p1.1" title="Specialization and Modularity of Existing MoEs. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib31" class="ltx_bibitem ltx_bib_article">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[46]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">Y. Wang, X. Ma, G. Zhang, Y. Ni, A. Chandra, S. Guo, W. Ren, A. Arulraj, X. He, Z. Jiang, <span class="ltx_text ltx_bib_etal">et al.</span></span><span class="ltx_text ltx_bib_year"> (2024)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Mmlu-pro: a more robust and challenging multi-task language understanding benchmark</span>.
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_journal">Advances in Neural Information Processing Systems</span> <span class="ltx_text ltx_bib_volume">37</span>, <span class="ltx_text ltx_bib_pages"> pp. 95266–95290</span>.
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
<li id="bib.bib46" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[47]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">A. Wettig, K. Lo, S. Min, H. Hajishirzi, D. Chen, and L. Soldaini</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Organize the web: constructing domains enhances pre-training data curation</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the International Conference of Machine Learning</span>,
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2502.10341" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S5.SS3.SSS0.Px2.p2.1" title="Emo Expert Activation Patterns across Domains are Distinct and Matches Human Intuition. ‣ 5.3 Semantic Specialization Emerge in Emo ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§5.3</span></a>.
</span>
</li>
<li id="bib.bib10" class="ltx_bibitem ltx_bib_misc">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[48]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">A. Yang, A. Li, B. Yang, B. Zhang, B. Hui, B. Zheng, B. Yu, C. Gao, C. Huang, C. Lv, C. Zheng, D. Liu, F. Zhou, F. Huang, F. Hu, H. Ge, H. Wei, H. Lin, J. Tang, J. Yang, J. Tu, J. Zhang, J. Yang, J. Yang, J. Zhou, J. Zhou, J. Lin, K. Dang, K. Bao, K. Yang, L. Yu, L. Deng, M. Li, M. Xue, M. Li, P. Zhang, P. Wang, Q. Zhu, R. Men, R. Gao, S. Liu, S. Luo, T. Li, T. Tang, W. Yin, X. Ren, X. Wang, X. Zhang, X. Ren, Y. Fan, Y. Su, Y. Zhang, Y. Zhang, Y. Wan, Y. Liu, Z. Wang, Z. Cui, Z. Zhang, Z. Zhou, and Z. Qiu</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Qwen3 technical report</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><span class="ltx_text ltx_bib_external">2505.09388</span>,
<a href="https://arxiv.org/abs/2505.09388" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#A1.SS1.p4.1" title="A.1 Load Balancing ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§A.1</span></a>,
<a href="#S1.p1.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>,
<a href="#S1.p2.1" title="1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§1</span></a>.
</span>
</li>
<li id="bib.bib12" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[49]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">X. Yang, C. Venhoff, A. Khakzar, C. S. de Witt, P. K. Dokania, A. Bibi, and P. Torr</span><span class="ltx_text ltx_bib_year"> (2025)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">Mixture of experts made intrinsically interpretable</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the International Conference of Machine Learning</span>,
</span>
<span class="ltx_bibblock">Note: <span class="ltx_text ltx_bib_note">Poster</span>
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://arxiv.org/abs/2503.07639" title="" class="ltx_ref ltx_bib_external">Link</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S2.SS0.SSS0.Px3.p1.1" title="Training MoEs with Structured or Specialized Experts. ‣ 2 Related Work ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§2</span></a>.
</span>
</li>
<li id="bib.bib25" class="ltx_bibitem ltx_bib_inproceedings">
<span class="ltx_tag ltx_bib_key ltx_role_refnum ltx_tag_bibitem">[50]</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_author">R. Zellers, A. Holtzman, Y. Bisk, A. Farhadi, and Y. Choi</span><span class="ltx_text ltx_bib_year"> (2019-07)</span>
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_title">HellaSwag: can a machine really finish your sentence?</span>.
</span>
<span class="ltx_bibblock">In <span class="ltx_text ltx_bib_inbook">Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics</span>,  <span class="ltx_text ltx_bib_editor">A. Korhonen, D. Traum, and L. Màrquez (Eds.)</span>,
</span>
<span class="ltx_bibblock"><span class="ltx_text ltx_bib_place">Florence, Italy</span>, <span class="ltx_text ltx_bib_pages"> pp. 4791–4800</span>.
</span>
<span class="ltx_bibblock">External Links: <span class="ltx_text ltx_bib_links"><a href="https://aclanthology.org/P19-1472/" title="" class="ltx_ref ltx_bib_external">Link</a>,
<a href="https://dx.doi.org/10.18653/v1/P19-1472" title="" class="ltx_ref doi ltx_bib_external">Document</a></span>
</span>
<span class="ltx_bibblock ltx_bib_cited">Cited by: <a href="#S4.SS2.SSS0.Px1.p1.1" title="Full-model Evaluation. ‣ 4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">§4.2</span></a>.
</span>
</li>
</ul>
</section>
<div class="ltx_pagination ltx_role_newpage"></div>
<section id="A1" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix A </span>Architectural &amp; Training Details and Ablations</h2>

<figure id="A1.F7" class="ltx_figure"><img src="2605.06663v1/x9.png" id="A1.F7.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="498" height="295" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 7: </span><span id="A1.F7.2.1" class="ltx_text ltx_font_bold">Global vs Local Load Balancing</span> and its effects on training stability (Appendix <a href="#A1.SS1" title="A.1 Load Balancing ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A.1</span></a>). Using global load balancing leads to more stable pre-training runs with less gradient norm spikes. </figcaption>
</figure>
<section id="A1.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">A.1 </span>Load Balancing</h3>

<div id="A1.SS1.p1" class="ltx_para">
<p id="A1.SS1.p1.1" class="ltx_p">We begin by making explicit how the simplified formulation of load balancing as described in Section <a href="#S3.SS1" title="3.1 Preliminary: Mixture of Experts Architecture ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.1</span></a> is implemented in practice.</p>
</div>
<div id="A1.SS1.p2" class="ltx_para">
<p id="A1.SS1.p2.8" class="ltx_p">Under standard implementations, the statistics <math id="A1.SS1.p2.1.m1" class="ltx_Math" alttext="\bar{f}_{i}" display="inline" intent=":literal"><semantics><msub><mover accent="true"><mi>f</mi><mo>¯</mo></mover><mi>i</mi></msub><annotation encoding="application/x-tex">\bar{f}_{i}</annotation></semantics></math> and <math id="A1.SS1.p2.2.m2" class="ltx_Math" alttext="\bar{P}_{i}" display="inline" intent=":literal"><semantics><msub><mover accent="true"><mi>P</mi><mo>¯</mo></mover><mi>i</mi></msub><annotation encoding="application/x-tex">\bar{P}_{i}</annotation></semantics></math> are computed independently within each data parallel group. Let there be <math id="A1.SS1.p2.3.m3" class="ltx_Math" alttext="n_{p}" display="inline" intent=":literal"><semantics><msub><mi>n</mi><mi>p</mi></msub><annotation encoding="application/x-tex">n_{p}</annotation></semantics></math> data parallel groups<span id="footnote2" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">2</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">2</sup><span class="ltx_tag ltx_tag_note">2</span>In this work, we pre-train with data parallelism only.</span></span></span>. For each group <math id="A1.SS1.p2.4.m4" class="ltx_Math" alttext="j" display="inline" intent=":literal"><semantics><mi>j</mi><annotation encoding="application/x-tex">j</annotation></semantics></math>, let <math id="A1.SS1.p2.5.m5" class="ltx_Math" alttext="f_{i}^{j}" display="inline" intent=":literal"><semantics><msubsup><mi>f</mi><mi>i</mi><mi>j</mi></msubsup><annotation encoding="application/x-tex">f_{i}^{j}</annotation></semantics></math> denote the fraction of tokens in its micro-batch routed to expert <math id="A1.SS1.p2.6.m6" class="ltx_Math" alttext="i" display="inline" intent=":literal"><semantics><mi>i</mi><annotation encoding="application/x-tex">i</annotation></semantics></math>, and let <math id="A1.SS1.p2.7.m7" class="ltx_Math" alttext="P_{i}^{j}" display="inline" intent=":literal"><semantics><msubsup><mi>P</mi><mi>i</mi><mi>j</mi></msubsup><annotation encoding="application/x-tex">P_{i}^{j}</annotation></semantics></math> denote the average routing probability assigned to expert <math id="A1.SS1.p2.8.m8" class="ltx_Math" alttext="i" display="inline" intent=":literal"><semantics><mi>i</mi><annotation encoding="application/x-tex">i</annotation></semantics></math> across tokens in that micro-batch. The load balancing loss is then defined as</p>
<table id="A1.Ex11" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="A1.Ex11.m1" class="ltx_Math" alttext="\mathcal{L}_{\mathrm{LB}}=\frac{1}{n_{p}}\sum_{j=1}^{n_{p}}\left[n_{r}\sum_{i=1}^{n_{r}}f_{i}^{j}\cdot P_{i}^{j}\right]," display="block" intent=":literal"><semantics><mrow><mrow><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>LB</mi></msub><mo>=</mo><mrow><mfrac><mn>1</mn><msub><mi>n</mi><mi>p</mi></msub></mfrac><mo lspace="0em" rspace="0em">​</mo><mrow><munderover><mo movablelimits="false" rspace="0em">∑</mo><mrow><mi>j</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>p</mi></msub></munderover><mrow><mo>[</mo><mrow><msub><mi>n</mi><mi>r</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>r</mi></msub></munderover><mrow><msubsup><mi>f</mi><mi>i</mi><mi>j</mi></msubsup><mo lspace="0.222em" rspace="0.222em">⋅</mo><msubsup><mi>P</mi><mi>i</mi><mi>j</mi></msubsup></mrow></mrow></mrow><mo>]</mo></mrow></mrow></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\mathcal{L}_{\mathrm{LB}}=\frac{1}{n_{p}}\sum_{j=1}^{n_{p}}\left[n_{r}\sum_{i=1}^{n_{r}}f_{i}^{j}\cdot P_{i}^{j}\right],</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="A1.SS1.p2.9" class="ltx_p">which corresponds to computing the simplified objective separately within each group and averaging the results.</p>
</div>
<div id="A1.SS1.p3" class="ltx_para">
<p id="A1.SS1.p3.2" class="ltx_p">In this paper, we instead modify the load balancing objective to operate over aggregated routing statistics across data parallel groups. Specifically, we perform an all-reduce over data parallel groups to obtain the aggregated routing frequency</p>
<table id="A1.Ex12" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="A1.Ex12.m1" class="ltx_Math" alttext="\bar{f}_{i}=\sum_{j=1}^{n_{p}}f_{i}^{j}," display="block" intent=":literal"><semantics><mrow><mrow><msub><mover accent="true"><mi>f</mi><mo>¯</mo></mover><mi>i</mi></msub><mo rspace="0.111em">=</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>j</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>p</mi></msub></munderover><msubsup><mi>f</mi><mi>i</mi><mi>j</mi></msubsup></mrow></mrow><mo>,</mo></mrow><annotation encoding="application/x-tex">\bar{f}_{i}=\sum_{j=1}^{n_{p}}f_{i}^{j},</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="A1.SS1.p3.1" class="ltx_p">while retaining the per-group routing probabilities <math id="A1.SS1.p3.1.m1" class="ltx_Math" alttext="P_{i}^{j}" display="inline" intent=":literal"><semantics><msubsup><mi>P</mi><mi>i</mi><mi>j</mi></msubsup><annotation encoding="application/x-tex">P_{i}^{j}</annotation></semantics></math>. The load balancing loss is then computed as</p>
<table id="A1.Ex13" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="A1.Ex13.m1" class="ltx_Math" alttext="\mathcal{L}_{\mathrm{LB}}=\frac{1}{n_{p}}\sum_{j=1}^{n_{p}}\left[n_{r}\sum_{i=1}^{n_{r}}\bar{f}_{i}\cdot P_{i}^{j}\right]." display="block" intent=":literal"><semantics><mrow><mrow><msub><mi class="ltx_font_mathcaligraphic">ℒ</mi><mi>LB</mi></msub><mo>=</mo><mrow><mfrac><mn>1</mn><msub><mi>n</mi><mi>p</mi></msub></mfrac><mo lspace="0em" rspace="0em">​</mo><mrow><munderover><mo movablelimits="false" rspace="0em">∑</mo><mrow><mi>j</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>p</mi></msub></munderover><mrow><mo>[</mo><mrow><msub><mi>n</mi><mi>r</mi></msub><mo lspace="0em" rspace="0em">​</mo><mrow><munderover><mo movablelimits="false">∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><msub><mi>n</mi><mi>r</mi></msub></munderover><mrow><msub><mover accent="true"><mi>f</mi><mo>¯</mo></mover><mi>i</mi></msub><mo lspace="0.222em" rspace="0.222em">⋅</mo><msubsup><mi>P</mi><mi>i</mi><mi>j</mi></msubsup></mrow></mrow></mrow><mo>]</mo></mrow></mrow></mrow></mrow><mo lspace="0em">.</mo></mrow><annotation encoding="application/x-tex">\mathcal{L}_{\mathrm{LB}}=\frac{1}{n_{p}}\sum_{j=1}^{n_{p}}\left[n_{r}\sum_{i=1}^{n_{r}}\bar{f}_{i}\cdot P_{i}^{j}\right].</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
</div>
<div id="A1.SS1.p4" class="ltx_para">
<p id="A1.SS1.p4.1" class="ltx_p">This formulation, proposed and used in Qwen 3 <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib10" title="Qwen3 technical report" class="ltx_ref">48</a>, <a href="#bib.bib4" title="Demons in the detail: on implementing load balancing loss for training specialized mixture-of-expert models" class="ltx_ref">30</a>]</cite>, replaces local (micro-batch-level) estimates of <math id="A1.SS1.p4.1.m1" class="ltx_Math" alttext="\bar{f}_{i}" display="inline" intent=":literal"><semantics><msub><mover accent="true"><mi>f</mi><mo>¯</mo></mover><mi>i</mi></msub><annotation encoding="application/x-tex">\bar{f}_{i}</annotation></semantics></math> with aggregated statistics across data parallel groups, yielding a closer approximation to global routing behavior. In our setting, where training uses data parallelism only, this corresponds to computing load balancing over a larger set of sequences (i.e., the global batch up to gradient accumulation).</p>
</div>
<div id="A1.SS1.p5" class="ltx_para">
<p id="A1.SS1.p5.1" class="ltx_p">We find that global load balancing is critical for stable training in <span id="A1.SS1.p5.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>. As shown in Figure <a href="#A1.F7" title="Figure 7 ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">7</span></a>, models trained with standard micro-batch-level load balancing exhibit unstable behavior, while global load balancing leads to consistent and reliable training dynamics. This difference arises because micro-batch-level load balancing conflicts with our document-level routing constraint by enforcing uniform expert usage within each micro-batch. In contrast, global load balancing aggregates routing statistics across data-parallel groups, allowing expert utilization to diversify across documents while preserving consistent routing within each document.</p>
</div>
<figure id="A1.T2" class="ltx_table">
<table id="A1.T2.4" class="ltx_tabular ltx_centering ltx_guessed_headers ltx_align_middle">
<thead class="ltx_thead">
<tr id="A1.T2.4.5.1" class="ltx_tr">
<th id="A1.T2.4.5.1.1" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_th_row ltx_border_tt" style="padding-left:4.5pt;padding-right:4.5pt;" colspan="3"><span id="A1.T2.4.5.1.1.1" class="ltx_text ltx_font_bold" style="font-size:80%;">Configuration</span></th>
<th id="A1.T2.4.5.1.2" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_tt" style="padding-left:4.5pt;padding-right:4.5pt;" colspan="5"><span id="A1.T2.4.5.1.2.1" class="ltx_text ltx_font_bold" style="font-size:80%;">Inference</span></th>
<th id="A1.T2.4.5.1.3" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_tt" style="padding-left:4.5pt;padding-right:4.5pt;" colspan="5"><span id="A1.T2.4.5.1.3.1" class="ltx_text ltx_font_bold" style="font-size:80%;">Fine-tuning</span></th>
</tr>
<tr id="A1.T2.3.3" class="ltx_tr">
<th id="A1.T2.1.1.1" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_th_row ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><math id="A1.T2.1.1.1.m1" class="ltx_Math" alttext="n_{r}" display="inline" intent=":literal"><semantics><msub><mi mathsize="0.800em">n</mi><mi mathsize="0.800em">r</mi></msub><annotation encoding="application/x-tex">n_{r}</annotation></semantics></math></th>
<th id="A1.T2.2.2.2" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_th_row ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><math id="A1.T2.2.2.2.m1" class="ltx_Math" alttext="n_{s}" display="inline" intent=":literal"><semantics><msub><mi mathsize="0.800em">n</mi><mi mathsize="0.800em">s</mi></msub><annotation encoding="application/x-tex">n_{s}</annotation></semantics></math></th>
<th id="A1.T2.3.3.3" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_th_row ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><math id="A1.T2.3.3.3.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi mathsize="0.800em">d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math></th>
<th id="A1.T2.3.3.4" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.4.1" class="ltx_text ltx_font_bold" style="font-size:80%;">8</span></th>
<th id="A1.T2.3.3.5" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.5.1" class="ltx_text ltx_font_bold" style="font-size:80%;">16</span></th>
<th id="A1.T2.3.3.6" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.6.1" class="ltx_text ltx_font_bold" style="font-size:80%;">32</span></th>
<th id="A1.T2.3.3.7" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.7.1" class="ltx_text ltx_font_bold" style="font-size:80%;">64</span></th>
<th id="A1.T2.3.3.8" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.8.1" class="ltx_text ltx_font_bold" style="font-size:80%;">128</span></th>
<th id="A1.T2.3.3.9" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.9.1" class="ltx_text ltx_font_bold" style="font-size:80%;">8</span></th>
<th id="A1.T2.3.3.10" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.10.1" class="ltx_text ltx_font_bold" style="font-size:80%;">16</span></th>
<th id="A1.T2.3.3.11" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.11.1" class="ltx_text ltx_font_bold" style="font-size:80%;">32</span></th>
<th id="A1.T2.3.3.12" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.12.1" class="ltx_text ltx_font_bold" style="font-size:80%;">64</span></th>
<th id="A1.T2.3.3.13" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.3.3.13.1" class="ltx_text ltx_font_bold" style="font-size:80%;">128</span></th>
</tr>
</thead>
<tbody class="ltx_tbody">
<tr id="A1.T2.4.6.1" class="ltx_tr">
<th id="A1.T2.4.6.1.1" class="ltx_td ltx_align_center ltx_th ltx_th_row ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.1.1" class="ltx_text" style="font-size:80%;">128</span></th>
<th id="A1.T2.4.6.1.2" class="ltx_td ltx_align_center ltx_th ltx_th_row ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.2.1" class="ltx_text" style="font-size:80%;">0</span></th>
<th id="A1.T2.4.6.1.3" class="ltx_td ltx_align_center ltx_th ltx_th_row ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.3.1" class="ltx_text" style="font-size:80%;">32</span></th>
<td id="A1.T2.4.6.1.4" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.4.1" class="ltx_text" style="font-size:80%;">30.2</span></td>
<td id="A1.T2.4.6.1.5" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.5.1" class="ltx_text" style="font-size:80%;">35.3</span></td>
<td id="A1.T2.4.6.1.6" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.6.1" class="ltx_text" style="font-size:80%;">37.2</span></td>
<td id="A1.T2.4.6.1.7" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.7.1" class="ltx_text" style="font-size:80%;">36.0</span></td>
<td id="A1.T2.4.6.1.8" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.8.1" class="ltx_text" style="font-size:80%;">31.9</span></td>
<td id="A1.T2.4.6.1.9" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.9.1" class="ltx_text" style="font-size:80%;">31.7</span></td>
<td id="A1.T2.4.6.1.10" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.10.1" class="ltx_text" style="font-size:80%;">36.2</span></td>
<td id="A1.T2.4.6.1.11" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.11.1" class="ltx_text" style="font-size:80%;">37.8</span></td>
<td id="A1.T2.4.6.1.12" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.12.1" class="ltx_text" style="font-size:80%;">38.4</span></td>
<td id="A1.T2.4.6.1.13" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.6.1.13.1" class="ltx_text" style="font-size:80%;">37.3</span></td>
</tr>
<tr id="A1.T2.4.7.2" class="ltx_tr">
<th id="A1.T2.4.7.2.1" class="ltx_td ltx_align_center ltx_th ltx_th_row" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.1.1" class="ltx_text" style="font-size:80%;">127</span></th>
<th id="A1.T2.4.7.2.2" class="ltx_td ltx_align_center ltx_th ltx_th_row" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.2.1" class="ltx_text" style="font-size:80%;">1</span></th>
<th id="A1.T2.4.7.2.3" class="ltx_td ltx_align_center ltx_th ltx_th_row" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.3.1" class="ltx_text" style="font-size:80%;">32</span></th>
<td id="A1.T2.4.7.2.4" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.4.1" class="ltx_text" style="font-size:80%;">29.6</span></td>
<td id="A1.T2.4.7.2.5" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.5.1" class="ltx_text" style="font-size:80%;">34.4</span></td>
<td id="A1.T2.4.7.2.6" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.6.1" class="ltx_text" style="font-size:80%;">36.6</span></td>
<td id="A1.T2.4.7.2.7" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.7.1" class="ltx_text" style="font-size:80%;">35.6</span></td>
<td id="A1.T2.4.7.2.8" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.8.1" class="ltx_text" style="font-size:80%;">33.6</span></td>
<td id="A1.T2.4.7.2.9" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.9.1" class="ltx_text" style="font-size:80%;">31.7</span></td>
<td id="A1.T2.4.7.2.10" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.10.1" class="ltx_text" style="font-size:80%;">36.0</span></td>
<td id="A1.T2.4.7.2.11" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.11.1" class="ltx_text" style="font-size:80%;">38.3</span></td>
<td id="A1.T2.4.7.2.12" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.12.1" class="ltx_text" style="font-size:80%;">38.5</span></td>
<td id="A1.T2.4.7.2.13" class="ltx_td ltx_align_center" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.7.2.13.1" class="ltx_text" style="font-size:80%;">37.4</span></td>
</tr>
<tr id="A1.T2.4.4" class="ltx_tr">
<th id="A1.T2.4.4.2" class="ltx_td ltx_align_center ltx_th ltx_th_row ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.2.1" class="ltx_text" style="font-size:80%;">127</span></th>
<th id="A1.T2.4.4.3" class="ltx_td ltx_align_center ltx_th ltx_th_row ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.3.1" class="ltx_text" style="font-size:80%;">1</span></th>
<th id="A1.T2.4.4.1" class="ltx_td ltx_align_center ltx_th ltx_th_row ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><math id="A1.T2.4.4.1.m1" class="ltx_Math" alttext="U(8,128)" display="inline" intent=":literal"><semantics><mrow><mi mathsize="0.800em">U</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo maxsize="0.800em" minsize="0.800em">(</mo><mn mathsize="0.800em">8</mn><mo mathsize="0.800em">,</mo><mn mathsize="0.800em">128</mn><mo maxsize="0.800em" minsize="0.800em">)</mo></mrow></mrow><annotation encoding="application/x-tex">U(8,128)</annotation></semantics></math></th>
<td id="A1.T2.4.4.4" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.4.1" class="ltx_text" style="font-size:80%;">33.7</span></td>
<td id="A1.T2.4.4.5" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.5.1" class="ltx_text" style="font-size:80%;">36.4</span></td>
<td id="A1.T2.4.4.6" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.6.1" class="ltx_text" style="font-size:80%;">37.0</span></td>
<td id="A1.T2.4.4.7" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.7.1" class="ltx_text" style="font-size:80%;">37.7</span></td>
<td id="A1.T2.4.4.8" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.8.1" class="ltx_text" style="font-size:80%;">38.1</span></td>
<td id="A1.T2.4.4.9" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.9.1" class="ltx_text" style="font-size:80%;">34.5</span></td>
<td id="A1.T2.4.4.10" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.10.1" class="ltx_text" style="font-size:80%;">37.5</span></td>
<td id="A1.T2.4.4.11" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.11.1" class="ltx_text" style="font-size:80%;">38.5</span></td>
<td id="A1.T2.4.4.12" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.12.1" class="ltx_text" style="font-size:80%;">39.7</span></td>
<td id="A1.T2.4.4.13" class="ltx_td ltx_align_center ltx_border_bb" style="padding-left:4.5pt;padding-right:4.5pt;"><span id="A1.T2.4.4.13.1" class="ltx_text" style="font-size:80%;">39.8</span></td>
</tr>
</tbody>
</table>
<figcaption class="ltx_caption ltx_centering" style="font-size:80%;"><span class="ltx_tag ltx_tag_table">Table 2: </span><span id="A1.T2.17.1" class="ltx_text ltx_font_bold">Ablating Shared Experts and Document Pool Size</span>. Evaluation on MMLU across different architectural configurations, comparing the effects of shared experts and dynamic expert subset sizes. We compare models without a shared expert (<math id="A1.T2.8.m1" class="ltx_Math" alttext="n_{s}=0" display="inline" intent=":literal"><semantics><mrow><msub><mi>n</mi><mi>s</mi></msub><mo>=</mo><mn>0</mn></mrow><annotation encoding="application/x-tex">n_{s}=0</annotation></semantics></math>), with a shared expert and fixed expert subset size (<math id="A1.T2.9.m2" class="ltx_Math" alttext="d=32" display="inline" intent=":literal"><semantics><mrow><mi>d</mi><mo>=</mo><mn>32</mn></mrow><annotation encoding="application/x-tex">d=32</annotation></semantics></math>), and with a shared expert and dynamic expert subset size (<math id="A1.T2.10.m3" class="ltx_Math" alttext="U(8,128)" display="inline" intent=":literal"><semantics><mrow><mi>U</mi><mo lspace="0em" rspace="0em">​</mo><mrow><mo stretchy="false">(</mo><mn>8</mn><mo>,</mo><mn>128</mn><mo stretchy="false">)</mo></mrow></mrow><annotation encoding="application/x-tex">U(8,128)</annotation></semantics></math>). Using a shared expert improves performance, while dynamic expert-subset-size training enables more flexible expert selection across different budgets. </figcaption>
</figure>
</section>
<section id="A1.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">A.2 </span>How to Choose <math id="A1.SS2.1.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math>
</h3>

<div id="A1.SS2.p1" class="ltx_para">
<p id="A1.SS2.p1.3" class="ltx_p">We find that varying the size <math id="A1.SS2.p1.1.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> during training is important for enabling flexible selective expert use usage at inference time. As shown in Table <a href="#A1.T2" title="Table 2 ‣ A.1 Load Balancing ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>, models trained with a fixed <math id="A1.SS2.p1.2.m2" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> perform well at that specific expert subset size but degrade when evaluated at other subset sizes. In contrast, training with a distribution over <math id="A1.SS2.p1.3.m3" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math> yields robust performance across a wide range of expert subset sizes.</p>
</div>
</section>
<section id="A1.SS3" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">A.3 </span>Shared Experts</h3>

<div id="A1.SS3.p1" class="ltx_para">
<p id="A1.SS3.p1.1" class="ltx_p">In Table <a href="#A1.T2" title="Table 2 ‣ A.1 Load Balancing ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>, we find that incorporating shared experts improves performance in <span id="A1.SS3.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>. This is consistent with prior observations in DeepSeek-MoE <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib7" title="DeepSeekMoE: towards ultimate expert specialization in mixture-of-experts language models" class="ltx_ref">7</a>]</cite>.</p>
</div>
</section>
<section id="A1.SS4" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">A.4 </span>Tuning LR and LB</h3>

<figure id="A1.F8" class="ltx_figure"><img src="2605.06663v1/x10.png" id="A1.F8.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="499" height="295" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 8: </span><span id="A1.F8.10.1" class="ltx_text ltx_font_bold">Standard MoE LR and LB Ablations</span>. We ablate standard MoEs across learning rates and load balancing (Appendix <a href="#A1.SS4" title="A.4 Tuning LR and LB ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A.4</span></a>). We identify the best configuration as lr = <math id="A1.F8.5.m1" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> and lb = <math id="A1.F8.6.m2" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math>. For load balancing coefficient, we do not observe significant differences between <math id="A1.F8.7.m3" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math> and <math id="A1.F8.8.m4" class="ltx_Math" alttext="1e-2" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>2</mn></mrow><annotation encoding="application/x-tex">1e-2</annotation></semantics></math>, and choose the former because it had slightly higher training stability. </figcaption>
</figure>
<div id="A1.SS4.p1" class="ltx_para">
<p id="A1.SS4.p1.1" class="ltx_p">Due to limited compute budget, we perform ablations first on learning rate, then on load balancing in a sequential manner. Furthermore, we ablate the coefficients of each in increments of 10x.</p>
</div>
<section id="A1.SS4.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Standard MoE Hyperparameter Ablations.</h4>

<div id="A1.SS4.SSS0.Px1.p1" class="ltx_para">
<p id="A1.SS4.SSS0.Px1.p1.9" class="ltx_p">In Figure <a href="#A1.F8" title="Figure 8 ‣ A.4 Tuning LR and LB ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">8</span></a>, we initialize with the default lr = <math id="A1.SS4.SSS0.Px1.p1.1.m1" class="ltx_Math" alttext="4e-4" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>4</mn></mrow><annotation encoding="application/x-tex">4e-4</annotation></semantics></math>, lb = <math id="A1.SS4.SSS0.Px1.p1.2.m2" class="ltx_Math" alttext="1e-2" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>2</mn></mrow><annotation encoding="application/x-tex">1e-2</annotation></semantics></math> configurations following OLMoE <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">27</a>]</cite>. We first ablate the learning rate by increasing it to <math id="A1.SS4.SSS0.Px1.p1.3.m3" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> and <math id="A1.SS4.SSS0.Px1.p1.4.m4" class="ltx_Math" alttext="4e-2" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>2</mn></mrow><annotation encoding="application/x-tex">4e-2</annotation></semantics></math> while keeping the load balancing coefficient fixed. In our results, using a learning rate of <math id="A1.SS4.SSS0.Px1.p1.5.m5" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> led to the best results. We then ablated the load balancing coefficient from <math id="A1.SS4.SSS0.Px1.p1.6.m6" class="ltx_Math" alttext="1e-2" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>2</mn></mrow><annotation encoding="application/x-tex">1e-2</annotation></semantics></math> to <math id="A1.SS4.SSS0.Px1.p1.7.m7" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math>, which helped training stablility. We ended up selecting the hyperparameter configurations of lr = <math id="A1.SS4.SSS0.Px1.p1.8.m8" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> and lb = <math id="A1.SS4.SSS0.Px1.p1.9.m9" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math>.</p>
</div>
<figure id="A1.F9" class="ltx_figure"><img src="2605.06663v1/x11.png" id="A1.F9.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="499" height="295" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 9: </span><span id="A1.F9.9.1" class="ltx_text ltx_font_bold ltx_font_smallcaps">Emo<span id="A1.F9.9.1.1" class="ltx_text ltx_font_upright"> ablations over LR</span></span>. Ablations of <span id="A1.F9.10.2" class="ltx_text ltx_font_smallcaps">Emo</span> across learning rates. Due to limited compute resources, we fix lb = <math id="A1.F9.4.m1" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math> and only ablate the learning rate, finding that lr = <math id="A1.F9.5.m2" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> offered the best training loss. Minor training loss spikes in lr = <math id="A1.F9.6.m3" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> were resolved by implementing load balancing over global batches. </figcaption>
</figure>
</section>
<section id="A1.SS4.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="A1.SS4.SSS0.Px2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> Hyperparameter Ablations. </h4>

<div id="A1.SS4.SSS0.Px2.p1" class="ltx_para">
<p id="A1.SS4.SSS0.Px2.p1.8" class="ltx_p">In Figure <a href="#A1.F9" title="Figure 9 ‣ Standard MoE Hyperparameter Ablations. ‣ A.4 Tuning LR and LB ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">9</span></a>, we train <span id="A1.SS4.SSS0.Px2.p1.8.1" class="ltx_text ltx_font_smallcaps">Emo</span> immediately using lb = <math id="A1.SS4.SSS0.Px2.p1.1.m1" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math> and ablated learning rate across lr = <math id="A1.SS4.SSS0.Px2.p1.2.m2" class="ltx_Math" alttext="4e-4" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>4</mn></mrow><annotation encoding="application/x-tex">4e-4</annotation></semantics></math> (OLMoE default), <math id="A1.SS4.SSS0.Px2.p1.3.m3" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math>, and <math id="A1.SS4.SSS0.Px2.p1.4.m4" class="ltx_Math" alttext="4e-2" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>2</mn></mrow><annotation encoding="application/x-tex">4e-2</annotation></semantics></math>. We noticed that lr = <math id="A1.SS4.SSS0.Px2.p1.5.m5" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> led to the strongest performance by 3000 training steps and decided to move forwards with the final configuration of lr = <math id="A1.SS4.SSS0.Px2.p1.6.m6" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math> and lb = <math id="A1.SS4.SSS0.Px2.p1.7.m7" class="ltx_Math" alttext="1e-1" display="inline" intent=":literal"><semantics><mrow><mrow><mn>1</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>1</mn></mrow><annotation encoding="application/x-tex">1e-1</annotation></semantics></math>. We noticed small spikes in the loss during training for lr = <math id="A1.SS4.SSS0.Px2.p1.8.m8" class="ltx_Math" alttext="4e-3" display="inline" intent=":literal"><semantics><mrow><mrow><mn>4</mn><mo lspace="0em" rspace="0em">​</mo><mi>e</mi></mrow><mo>−</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">4e-3</annotation></semantics></math>, which were reduced when we used a global version of load balancing, see <a href="#A1.SS1" title="A.1 Load Balancing ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A.1</span></a>.</p>
</div>
</section>
</section>
<section id="A1.SS5" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">A.5 </span>Prenorm vs ReorderedNorm</h3>

<figure id="A1.F10" class="ltx_figure"><img src="2605.06663v1/x12.png" id="A1.F10.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="499" height="295" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 10: </span>
<span id="A1.F10.5.1" class="ltx_text ltx_font_bold">Prenorm w. No QK Norm Ablations</span>. Ablations on ReorderedNorm from <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">27</a>]</cite> versus Prenorm with removed QK-norm (Appendix <a href="#A1.SS5" title="A.5 Prenorm vs ReorderedNorm ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">A.5</span></a>). On both standard MoEs and <span id="A1.F10.6.2" class="ltx_text ltx_font_smallcaps">Emo</span>, using Prenorm with removed QK-norm achieves lower loss than ReorderedNorm. These experiments were conducted without applying global load balancing, shared experts, and dynamic <math id="A1.F10.2.m1" class="ltx_Math" alttext="d" display="inline" intent=":literal"><semantics><mi>d</mi><annotation encoding="application/x-tex">d</annotation></semantics></math>. </figcaption>
</figure>
<div id="A1.SS5.p1" class="ltx_para">
<p id="A1.SS5.p1.1" class="ltx_p">Both <span id="A1.SS5.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> and the standard MoE in this work implemented Prenorm with removed QK-norm instead of the default ReorderedNorm implementation in OLMoE <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib3" title="Olmoe: open mixture-of-experts language models" class="ltx_ref">27</a>]</cite>. We ran ablations that consistently showed that our new implementation achieves a lower training loss, see Figure <a href="#A1.F10" title="Figure 10 ‣ A.5 Prenorm vs ReorderedNorm ‣ Appendix A Architectural &amp; Training Details and Ablations ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">10</span></a>.</p>
</div>
</section>
</section>
<section id="A2" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix B </span>Selective expert use Details</h2>

<section id="A2.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">B.1 </span>130B Token Experiments</h3>

<figure id="A2.F11" class="ltx_figure"><img src="2605.06663v1/x13.png" id="A2.F11.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="475" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 11: </span>
<span id="A2.F11.11.1" class="ltx_text ltx_font_bold">Selective Expert Use</span> of MoEs trained on <em id="A2.F11.12.2" class="ltx_emph ltx_font_bold ltx_font_italic">130B</em> tokens (§<a href="#S5.SS2" title="5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.2</span></a>).
We report performance before fine-tuning (<em id="A2.F11.13.3" class="ltx_emph ltx_font_italic">top</em>) and after fine-tuning (<em id="A2.F11.14.4" class="ltx_emph ltx_font_italic">bottom</em>).
For MMLU and MMLU-Pro, each domain selects a corresponding expert subset as described in §<a href="#S4.SS2" title="4.2 Evaluation ‣ 4 Experimental Setup ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.2</span></a>, and we report macro-averaged results across domains (17 for MMLU and 14 for MMLU-Pro).
“Trained baseline @<math id="A2.F11.3.m1" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math>” denotes a model trained from scratch with a parameter count matched to a <math id="A2.F11.4.m2" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math>-expert subset.
Across all tasks, the <span id="A2.F11.15.5" class="ltx_text ltx_font_smallcaps">Emo</span> 32-expert subset and 8-expert subset match or outperform the corresponding Reg. MoE @ 32 and Dense @ 8 trained models, with the 8-expert subset of <span id="A2.F11.16.6" class="ltx_text ltx_font_smallcaps">Emo</span> nearly doubling the performance of the Dense @ 8 on GSM8k, both before and after fine-tuning.
</figcaption>
</figure>
<div id="A2.SS1.p1" class="ltx_para">
<p id="A2.SS1.p1.1" class="ltx_p">Following §<a href="#S5.SS2" title="5.2 Selective Expert Use ‣ 5 Results and Analysis ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5.2</span></a>, we show the results for selective expert useacross MMLU, MMLU Pro, and GSM8K for models trained on 130B tokens. Additionally, we compare against memory-matched models traine from scratch, including a standard MoE with 32 experts and a dense model. A subset of these results are presented in Figure <a href="#S1.F1" title="Figure 1 ‣ 1 Introduction ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> (right). <span id="A2.SS1.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> match or outperform all baselines in selective expert use, pushing the pareto-frontier in memory-accuracy trade-off.</p>
</div>
</section>
<section id="A2.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">B.2 </span>Ablations on expert subset initialization</h3>

<figure id="A2.F12" class="ltx_figure"><img src="2605.06663v1/x14.png" id="A2.F12.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="280" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 12: </span>Effects of <span id="A2.F12.7.1" class="ltx_text ltx_font_bold">validation data quantity</span> (<math id="A2.F12.2.m1" class="ltx_Math" alttext="n" display="inline" intent=":literal"><semantics><mi>n</mi><annotation encoding="application/x-tex">n</annotation></semantics></math>) and the presence of <span id="A2.F12.8.2" class="ltx_text ltx_font_bold">few-shot demonstrations</span> (on both the data used to select experts and the actual test-set queries) on <span id="A2.F12.9.3" class="ltx_text ltx_font_smallcaps">Emo</span> expert subset performance, without any subsequent fine-tuning. By default MMLU/MMLU Pro have 5-shot demonstrations and GSM8K has 8-shot demonstrations. For GSM8K, we run with three random seeds for n=1, 5, and 10. <span id="A2.F12.10.4" class="ltx_text ltx_font_smallcaps">Emo</span> is sample-efficient in expert selection, and using few-shot demonstrations during both expert selection and evaluation brings large performance gains. </figcaption>
</figure>
<div id="A2.SS2.p1" class="ltx_para">
<p id="A2.SS2.p1.1" class="ltx_p">We now investigate how validation data affects expert selection. To study this, we vary three factors in Figure <a href="#A2.F12" title="Figure 12 ‣ B.2 Ablations on expert subset initialization ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">12</span></a>: (1) the number of validation examples used for expert selection, (2) validation set data format (few-shot vs. zero-shot prompts), and (3) the test-set data format (few-shot vs. zero-shot prompts).</p>
</div>
<section id="A2.SS2.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="A2.SS2.SSS0.Px1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is Sample-efficient in Expert-selection. </h4>

<div id="A2.SS2.SSS0.Px1.p1" class="ltx_para">
<p id="A2.SS2.SSS0.Px1.p1.1" class="ltx_p">We first consider the default setting used in this work, when both validation and test set use few-shot prompts. In this setting, <span id="A2.SS2.SSS0.Px1.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> shows little degradation as validation set size decreases—even down to a single example (red in Figure <a href="#A2.F12" title="Figure 12 ‣ B.2 Ablations on expert subset initialization ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">12</span></a>. We hypothesize this robustness arises because of the presence of few-shot demonstrations in each validation datapoint, which may provide sufficient token-level signals.</p>
</div>
<div id="A2.SS2.SSS0.Px1.p2" class="ltx_para">
<p id="A2.SS2.SSS0.Px1.p2.1" class="ltx_p">We then investigate using zero-shot prompts for validation and evaluation (gray), performance degrades slightly as the validation set size decreases, but the drop remains modest even with only 5 validation examples. Overall, this demonstrations that <span id="A2.SS2.SSS0.Px1.p2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> remains effective even in highly data-constrained settings.</p>
</div>
</section>
<section id="A2.SS2.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="A2.SS2.SSS0.Px2.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> Depends on Validation Data.</h4>

<div id="A2.SS2.SSS0.Px2.p1" class="ltx_para">
<p id="A2.SS2.SSS0.Px2.p1.1" class="ltx_p">While <span id="A2.SS2.SSS0.Px2.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is sample-efficient, the relationship between validation data and expert subset performance is nuanced and task-dependent. On GSM8K, for example, performance improves as the validation set size decreases. One possible explanation is that smaller validation sets produce more focused estimates of expert relevance, whereas aggregating across multiple examples can smooth these signals and yield less specialized expert subsets.</p>
</div>
<div id="A2.SS2.SSS0.Px2.p2" class="ltx_para">
<p id="A2.SS2.SSS0.Px2.p2.1" class="ltx_p">We also find that validation format sometimes matters independently of evaluation format: selecting experts using few-shot prompts can outperform zero-shot prompts, even when test examples are evaluated using zero-shot formats. This suggests that both the content and structure of validation data play a key role in shaping expert selection, an interaction we leave to future work.</p>
</div>
</section>
</section>
<section id="A2.SS3" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">B.3 </span>Selective Expert Use on "Other" Category in MMLU and MMLU Pro</h3>

<figure id="A2.F13" class="ltx_figure"><img src="2605.06663v1/x15.png" id="A2.F13.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="830" height="269" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure">Figure 13: </span>
<span id="A2.F13.13.1" class="ltx_text ltx_font_bold">Selective expert use on Other category</span> on <span id="A2.F13.14.2" class="ltx_text ltx_font_smallcaps">Emo</span> and standard MoEs trained on <em id="A2.F13.15.3" class="ltx_emph ltx_font_italic">130B</em> tokens (Appendix <a href="#A2.SS3" title='B.3 Selective Expert Use on "Other" Category in MMLU and MMLU Pro ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity' class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.3</span></a>). Results are without fine-tuning. “Trained baseline @<math id="A2.F13.5.m1" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math>” denotes a model trained from scratch with a parameter count matched to a <math id="A2.F13.6.m2" class="ltx_Math" alttext="k" display="inline" intent=":literal"><semantics><mi>k</mi><annotation encoding="application/x-tex">k</annotation></semantics></math>-expert subset.
On tasks that are general, <span id="A2.F13.16.4" class="ltx_text ltx_font_smallcaps">Emo</span> expert subsets of sizes 32 and 8 performs worse compared to “Reg MoE @<math id="A2.F13.7.m3" class="ltx_Math" alttext="32" display="inline" intent=":literal"><semantics><mn>32</mn><annotation encoding="application/x-tex">32</annotation></semantics></math>” and “Dense @<math id="A2.F13.8.m4" class="ltx_Math" alttext="8" display="inline" intent=":literal"><semantics><mn>8</mn><annotation encoding="application/x-tex">8</annotation></semantics></math>” that are trained from scratch. </figcaption>
</figure>
<div id="A2.SS3.p1" class="ltx_para">
<p id="A2.SS3.p1.1" class="ltx_p">When the deployment task is general (e.g MMLU other and MMLU Pro categories, which serve as a "catch-all" for MMLU subjects), <span id="A2.SS3.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> expert subsets of size 32 and 8 experts struggle to match the Reg MoE @ 32 and Dense @8 baseline models trained from scratch (Figure <a href="#A2.F13" title='Figure 13 ‣ B.3 Selective Expert Use on "Other" Category in MMLU and MMLU Pro ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity' class="ltx_ref"><span class="ltx_text ltx_ref_tag">13</span></a>). We view this phenomenon as a property of modular models, and believe it provides concrete evidence that <span id="A2.SS3.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> works in selective expert use because it has groups of experts that have localize capabilities. When reporting aggregate metrics of MMLU and MMLU Pro, we intentionally exclude including the "other" category, as it is not an example of selective expert use on a specific task.</p>
</div>
</section>
<section id="A2.SS4" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">B.4 </span>Annealing Standard MoEs to be Modular.</h3>

<figure id="A2.T3" class="ltx_table">
<div id="A2.T3.1" class="ltx_inline-block ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:162.9pt;vertical-align:-79.2pt;"><span class="ltx_transformed_inner" style="transform:translate(21.9pt,-8.2pt) scale(1.11242783758895,1.11242783758895) ;">
<table id="A2.T3.1.1" class="ltx_tabular ltx_guessed_headers ltx_align_middle">
<thead class="ltx_thead">
<tr id="A2.T3.1.1.1.1" class="ltx_tr">
<th id="A2.T3.1.1.1.1.1" class="ltx_td ltx_th ltx_th_row ltx_border_tt" style="padding-left:3.5pt;padding-right:3.5pt;"></th>
<th id="A2.T3.1.1.1.1.2" class="ltx_td ltx_align_left ltx_th ltx_th_column ltx_th_row ltx_border_tt" style="padding-left:3.5pt;padding-right:3.5pt;" rowspan="2"><span id="A2.T3.1.1.1.1.2.1" class="ltx_text" style="font-size:80%;"># Experts</span></th>
<th id="A2.T3.1.1.1.1.3" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_tt" style="padding-left:3.5pt;padding-right:3.5pt;" colspan="3"><span id="A2.T3.1.1.1.1.3.1" class="ltx_text ltx_font_bold" style="font-size:80%;">Inference</span></th>
<th id="A2.T3.1.1.1.1.4" class="ltx_td ltx_align_center ltx_th ltx_th_column ltx_border_tt" style="padding-left:3.5pt;padding-right:3.5pt;" colspan="3"><span id="A2.T3.1.1.1.1.4.1" class="ltx_text ltx_font_bold" style="font-size:80%;">Fine-tuning</span></th>
</tr>
<tr id="A2.T3.1.1.2.2" class="ltx_tr">
<th id="A2.T3.1.1.2.2.1" class="ltx_td ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"></th>
<th id="A2.T3.1.1.2.2.2" class="ltx_td ltx_align_right ltx_th ltx_th_column ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.2.2.2.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MMLU</span></th>
<th id="A2.T3.1.1.2.2.3" class="ltx_td ltx_align_right ltx_th ltx_th_column ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.2.2.3.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MMLU-Pro</span></th>
<th id="A2.T3.1.1.2.2.4" class="ltx_td ltx_align_right ltx_th ltx_th_column ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.2.2.4.1" class="ltx_text ltx_font_bold" style="font-size:80%;">GSM8K</span></th>
<th id="A2.T3.1.1.2.2.5" class="ltx_td ltx_align_right ltx_th ltx_th_column ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.2.2.5.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MMLU</span></th>
<th id="A2.T3.1.1.2.2.6" class="ltx_td ltx_align_right ltx_th ltx_th_column ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.2.2.6.1" class="ltx_text ltx_font_bold" style="font-size:80%;">MMLU-Pro</span></th>
<th id="A2.T3.1.1.2.2.7" class="ltx_td ltx_align_right ltx_th ltx_th_column ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.2.2.7.1" class="ltx_text ltx_font_bold" style="font-size:80%;">GSM8K</span></th>
</tr>
</thead>
<tbody class="ltx_tbody">
<tr id="A2.T3.1.1.3.1" class="ltx_tr">
<th id="A2.T3.1.1.3.1.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;" rowspan="5"><span id="A2.T3.1.1.3.1.1.1" class="ltx_text" style="font-size:80%;"><span id="A2.T3.1.1.3.1.1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>-anneal</span></th>
<th id="A2.T3.1.1.3.1.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.2.1" class="ltx_text" style="font-size:80%;">8</span></th>
<td id="A2.T3.1.1.3.1.3" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.3.1" class="ltx_text" style="font-size:80%;">32.1</span></td>
<td id="A2.T3.1.1.3.1.4" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.4.1" class="ltx_text" style="font-size:80%;">13.1</span></td>
<td id="A2.T3.1.1.3.1.5" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.5.1" class="ltx_text" style="font-size:80%;">7.3</span></td>
<td id="A2.T3.1.1.3.1.6" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.6.1" class="ltx_text" style="font-size:80%;">33.5</span></td>
<td id="A2.T3.1.1.3.1.7" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.7.1" class="ltx_text" style="font-size:80%;">14.2</span></td>
<td id="A2.T3.1.1.3.1.8" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.3.1.8.1" class="ltx_text" style="font-size:80%;">22.6</span></td>
</tr>
<tr id="A2.T3.1.1.4.2" class="ltx_tr">
<th id="A2.T3.1.1.4.2.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.1.1" class="ltx_text" style="font-size:80%;">16</span></th>
<td id="A2.T3.1.1.4.2.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.2.1" class="ltx_text" style="font-size:80%;">35.4</span></td>
<td id="A2.T3.1.1.4.2.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.3.1" class="ltx_text" style="font-size:80%;">14.8</span></td>
<td id="A2.T3.1.1.4.2.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.4.1" class="ltx_text" style="font-size:80%;">9.9</span></td>
<td id="A2.T3.1.1.4.2.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.5.1" class="ltx_text" style="font-size:80%;">37.3</span></td>
<td id="A2.T3.1.1.4.2.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.6.1" class="ltx_text" style="font-size:80%;">16.3</span></td>
<td id="A2.T3.1.1.4.2.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.4.2.7.1" class="ltx_text" style="font-size:80%;">25.3</span></td>
</tr>
<tr id="A2.T3.1.1.5.3" class="ltx_tr">
<th id="A2.T3.1.1.5.3.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.1.1" class="ltx_text" style="font-size:80%;">32</span></th>
<td id="A2.T3.1.1.5.3.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.2.1" class="ltx_text" style="font-size:80%;">38.8</span></td>
<td id="A2.T3.1.1.5.3.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.3.1" class="ltx_text" style="font-size:80%;">16.5</span></td>
<td id="A2.T3.1.1.5.3.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.4.1" class="ltx_text" style="font-size:80%;">11.3</span></td>
<td id="A2.T3.1.1.5.3.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.5.1" class="ltx_text" style="font-size:80%;">39.9</span></td>
<td id="A2.T3.1.1.5.3.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.6.1" class="ltx_text" style="font-size:80%;">18.3</span></td>
<td id="A2.T3.1.1.5.3.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.5.3.7.1" class="ltx_text" style="font-size:80%;">26.8</span></td>
</tr>
<tr id="A2.T3.1.1.6.4" class="ltx_tr">
<th id="A2.T3.1.1.6.4.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.1.1" class="ltx_text" style="font-size:80%;">64</span></th>
<td id="A2.T3.1.1.6.4.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.2.1" class="ltx_text" style="font-size:80%;">41.3</span></td>
<td id="A2.T3.1.1.6.4.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.3.1" class="ltx_text" style="font-size:80%;">17.4</span></td>
<td id="A2.T3.1.1.6.4.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.4.1" class="ltx_text" style="font-size:80%;">12.8</span></td>
<td id="A2.T3.1.1.6.4.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.5.1" class="ltx_text" style="font-size:80%;">42.7</span></td>
<td id="A2.T3.1.1.6.4.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.6.1" class="ltx_text" style="font-size:80%;">19.5</span></td>
<td id="A2.T3.1.1.6.4.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.6.4.7.1" class="ltx_text" style="font-size:80%;">27.7</span></td>
</tr>
<tr id="A2.T3.1.1.7.5" class="ltx_tr">
<th id="A2.T3.1.1.7.5.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.1.1" class="ltx_text" style="font-size:80%;">128 (trained)</span></th>
<td id="A2.T3.1.1.7.5.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.2.1" class="ltx_text" style="font-size:80%;">42.3</span></td>
<td id="A2.T3.1.1.7.5.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.3.1" class="ltx_text" style="font-size:80%;">18.2</span></td>
<td id="A2.T3.1.1.7.5.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.4.1" class="ltx_text" style="font-size:80%;">13.0</span></td>
<td id="A2.T3.1.1.7.5.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.5.1" class="ltx_text" style="font-size:80%;">43.7</span></td>
<td id="A2.T3.1.1.7.5.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.6.1" class="ltx_text" style="font-size:80%;">20.1</span></td>
<td id="A2.T3.1.1.7.5.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.7.5.7.1" class="ltx_text" style="font-size:80%;">27.2</span></td>
</tr>
<tr id="A2.T3.1.1.8.6" class="ltx_tr">
<th id="A2.T3.1.1.8.6.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_bb ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;" rowspan="5"><span id="A2.T3.1.1.8.6.1.1" class="ltx_text ltx_font_smallcaps" style="font-size:80%;">Emo</span></th>
<th id="A2.T3.1.1.8.6.2" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.2.1" class="ltx_text" style="font-size:80%;">8</span></th>
<td id="A2.T3.1.1.8.6.3" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.3.1" class="ltx_text" style="font-size:80%;">36.1</span></td>
<td id="A2.T3.1.1.8.6.4" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.4.1" class="ltx_text" style="font-size:80%;">14.7</span></td>
<td id="A2.T3.1.1.8.6.5" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.5.1" class="ltx_text" style="font-size:80%;">6.9</span></td>
<td id="A2.T3.1.1.8.6.6" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.6.1" class="ltx_text" style="font-size:80%;">37.3</span></td>
<td id="A2.T3.1.1.8.6.7" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.7.1" class="ltx_text" style="font-size:80%;">15.6</span></td>
<td id="A2.T3.1.1.8.6.8" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.8.6.8.1" class="ltx_text" style="font-size:80%;">23.3</span></td>
</tr>
<tr id="A2.T3.1.1.9.7" class="ltx_tr">
<th id="A2.T3.1.1.9.7.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.1.1" class="ltx_text" style="font-size:80%;">16</span></th>
<td id="A2.T3.1.1.9.7.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.2.1" class="ltx_text" style="font-size:80%;">39.9</span></td>
<td id="A2.T3.1.1.9.7.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.3.1" class="ltx_text" style="font-size:80%;">16.6</span></td>
<td id="A2.T3.1.1.9.7.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.4.1" class="ltx_text" style="font-size:80%;">12.2</span></td>
<td id="A2.T3.1.1.9.7.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.5.1" class="ltx_text" style="font-size:80%;">40.1</span></td>
<td id="A2.T3.1.1.9.7.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.6.1" class="ltx_text" style="font-size:80%;">17.5</span></td>
<td id="A2.T3.1.1.9.7.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.9.7.7.1" class="ltx_text" style="font-size:80%;">28.3</span></td>
</tr>
<tr id="A2.T3.1.1.10.8" class="ltx_tr">
<th id="A2.T3.1.1.10.8.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.1.1" class="ltx_text" style="font-size:80%;">32</span></th>
<td id="A2.T3.1.1.10.8.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.2.1" class="ltx_text" style="font-size:80%;">41.4</span></td>
<td id="A2.T3.1.1.10.8.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.3.1" class="ltx_text" style="font-size:80%;">17.6</span></td>
<td id="A2.T3.1.1.10.8.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.4.1" class="ltx_text" style="font-size:80%;">11.7</span></td>
<td id="A2.T3.1.1.10.8.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.5.1" class="ltx_text" style="font-size:80%;">41.7</span></td>
<td id="A2.T3.1.1.10.8.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.6.1" class="ltx_text" style="font-size:80%;">19.5</span></td>
<td id="A2.T3.1.1.10.8.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.10.8.7.1" class="ltx_text" style="font-size:80%;">27.5</span></td>
</tr>
<tr id="A2.T3.1.1.11.9" class="ltx_tr">
<th id="A2.T3.1.1.11.9.1" class="ltx_td ltx_align_left ltx_th ltx_th_row" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.1.1" class="ltx_text" style="font-size:80%;">64</span></th>
<td id="A2.T3.1.1.11.9.2" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.2.1" class="ltx_text" style="font-size:80%;">42.5</span></td>
<td id="A2.T3.1.1.11.9.3" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.3.1" class="ltx_text" style="font-size:80%;">18.2</span></td>
<td id="A2.T3.1.1.11.9.4" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.4.1" class="ltx_text" style="font-size:80%;">11.0</span></td>
<td id="A2.T3.1.1.11.9.5" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.5.1" class="ltx_text" style="font-size:80%;">43.3</span></td>
<td id="A2.T3.1.1.11.9.6" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.6.1" class="ltx_text" style="font-size:80%;">20.0</span></td>
<td id="A2.T3.1.1.11.9.7" class="ltx_td ltx_align_right" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.11.9.7.1" class="ltx_text" style="font-size:80%;">27.1</span></td>
</tr>
<tr id="A2.T3.1.1.12.10" class="ltx_tr">
<th id="A2.T3.1.1.12.10.1" class="ltx_td ltx_align_left ltx_th ltx_th_row ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.1.1" class="ltx_text" style="font-size:80%;">128 (trained)</span></th>
<td id="A2.T3.1.1.12.10.2" class="ltx_td ltx_align_right ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.2.1" class="ltx_text" style="font-size:80%;">42.8</span></td>
<td id="A2.T3.1.1.12.10.3" class="ltx_td ltx_align_right ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.3.1" class="ltx_text" style="font-size:80%;">18.5</span></td>
<td id="A2.T3.1.1.12.10.4" class="ltx_td ltx_align_right ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.4.1" class="ltx_text" style="font-size:80%;">12.0</span></td>
<td id="A2.T3.1.1.12.10.5" class="ltx_td ltx_align_right ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.5.1" class="ltx_text" style="font-size:80%;">43.6</span></td>
<td id="A2.T3.1.1.12.10.6" class="ltx_td ltx_align_right ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.6.1" class="ltx_text" style="font-size:80%;">20.4</span></td>
<td id="A2.T3.1.1.12.10.7" class="ltx_td ltx_align_right ltx_border_bb" style="padding-left:3.5pt;padding-right:3.5pt;"><span id="A2.T3.1.1.12.10.7.1" class="ltx_text" style="font-size:80%;">27.8</span></td>
</tr>
</tbody>
</table>
</span></div>
<figcaption class="ltx_caption ltx_centering" style="font-size:80%;"><span class="ltx_tag ltx_tag_table">Table 3: </span>
<span id="A2.T3.15.1" class="ltx_text ltx_font_bold">Inducing Modularity during Annealing Only.</span> We compare <span id="A2.T3.16.2" class="ltx_text ltx_font_smallcaps">Emo</span>, trained from scratch on 1T tokens and annealed on 50B tokens with document-level expert pool constraint (§<a href="#S3.SS2" title="3.2 Emo: An Objective to Induce Modularity ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.2</span></a>), against <span id="A2.T3.17.3" class="ltx_text ltx_font_smallcaps">Emo</span>-anneal, a model trained on 1T tokens as a standard MoE, but annealed on 50B tokens with the document-level expert pool constraint. Across most tasks and expert subset sizes, <span id="A2.T3.18.4" class="ltx_text ltx_font_smallcaps">Emo</span> improves over <span id="A2.T3.19.5" class="ltx_text ltx_font_smallcaps">Emo</span>-anneal, indicating that pre-training from scratch is important in realizing gains during selective expert use.
</figcaption>
</figure>
<div id="A2.SS4.p1" class="ltx_para">
<p id="A2.SS4.p1.1" class="ltx_p">We investigate whether modularity requires applying the document-level expert pool objective (§<a href="#S3.SS2" title="3.2 Emo: An Objective to Induce Modularity ‣ 3 Modular Mixture of Experts (Emo) ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.2</span></a>) throughout pre-training, or whether a standard MoE can be made modular after pre-training. To test this, we take a standard MoE pretrained on 1T tokens and anneal it using the document-level expert pool objective instead of the standard MoE training objective. Denoted as <span id="A2.SS4.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span>-anneal in Table <a href="#A2.T3" title="Table 3 ‣ B.4 Annealing Standard MoEs to be Modular. ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3</span></a>, this model underperforms <span id="A2.SS4.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> on most benchmarks across most expert subset sizes. However, <span id="A2.SS4.p1.1.3" class="ltx_text ltx_font_smallcaps">Emo</span>-anneal still trains successfully and exhibits signs of modularity, suggesting that post-training applications of the document-level expert pool objective may be a promising direction for future work.</p>
</div>
</section>
<section id="A2.SS5" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">B.5 </span>Generations from Selective Expert Use on GSM8K</h3>

<div id="A2.SS5.p1" class="ltx_para">
<p id="A2.SS5.p1.1" class="ltx_p">We now demonstrate how small expert subsets of <span id="A2.SS5.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> is qualitatively better than that of regular MoEs. We give examples of GSM8K generations of <span id="A2.SS5.p1.1.2" class="ltx_text ltx_font_smallcaps">Emo</span> and Regular MoE trained on 1T tokens under selective expert use across different expert subset sizes. No finetuning was performed (expert subsets are evaluated zero-shot). We note that 8-expert subsets of <span id="A2.SS5.p1.1.3" class="ltx_text ltx_font_smallcaps">Emo</span> consistently produces coherent outputs while regular MoEs subsets deteriorate.</p>
</div>
<div id="A2.SS5.p2" class="ltx_para ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p2.pic1" class="ltx_picture" height="91.4" overflow="visible" version="1.1" viewBox="0 0 600 91.4" width="600"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,91.4) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#7373FF;" fill="#7373FF" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.46 L 0 87.94 C 0 89.85 1.55 91.4 3.46 91.4 L 596.54 91.4 C 598.45 91.4 600 89.85 600 87.94 L 600 3.46 C 600 1.55 598.45 0 596.54 0 L 3.46 0 C 1.55 0 0 1.55 0 3.46 Z"></path></g><g style="--ltx-fill-color:#F7F7FF;" fill="#F7F7FF" fill-opacity="1.0"><path style="stroke:none" d="M 0.69 3.46 L 0.69 87.94 C 0.69 89.46 1.93 90.7 3.46 90.7 L 596.54 90.7 C 598.07 90.7 599.31 89.46 599.31 87.94 L 599.31 3.46 C 599.31 1.93 598.07 0.69 596.54 0.69 L 3.46 0.69 C 1.93 0.69 0.69 1.93 0.69 3.46 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 15.7 10.16)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:41.09em;--ltx-fo-height:5.14em;--ltx-fo-depth:0em;" width="568.6" height="71.07" transform="matrix(1 0 0 -1 0 71.07)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p2.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:41.09em;">
<span id="A2.SS5.p2.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p2.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold">Q.</span> Harry slept 9 hours last night. His friend James slept only 2/3 of what Harry slept. How many more hours did Harry sleep than James?</span>
<span id="A2.SS5.p2.pic1.1.1.1.1.1.2" class="ltx_p"><span id="A2.SS5.p2.pic1.1.1.1.1.1.2.1" class="ltx_text ltx_font_bold">Gold:</span> <span id="A2.SS5.p2.pic1.1.1.1.1.1.2.2" class="ltx_text ltx_font_bold" style="--ltx-fg-color:#1F6E7C;">3</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p3" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p3.pic1" class="ltx_picture" height="65.98" overflow="visible" style="vertical-align:-32.99px" version="1.1" viewBox="0 0 296.54 65.98" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,65.98) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 62.66 C 0 64.49 1.49 65.98 3.32 65.98 L 293.22 65.98 C 295.05 65.98 296.54 64.49 296.54 62.66 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 45.79 L 295.99 45.79 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 46.34 L 0.55 62.66 C 0.55 64.18 1.79 65.42 3.32 65.42 L 293.22 65.42 C 294.75 65.42 295.99 64.18 295.99 62.66 L 295.99 46.34 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 52.77)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p3.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p3.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p3.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 8-expert subset <span id="A2.SS5.p3.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.75)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:1.87em;--ltx-fo-depth:0.22em;" width="270.96" height="29.06" transform="matrix(1 0 0 -1 0 25.94)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p3.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p3.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p3.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Olivia $2005 2005 2005 2005 2005 2005 2005 [+162 more] 200</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p3.pic2" class="ltx_picture" height="116.9" overflow="visible" style="vertical-align:-58.45px" version="1.1" viewBox="0 0 296.54 116.9" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,116.9) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 113.58 C 0 115.41 1.49 116.9 3.32 116.9 L 293.22 116.9 C 295.05 116.9 296.54 115.41 296.54 113.58 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 59.28 L 295.99 59.28 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 59.83 L 0.55 113.58 C 0.55 115.1 1.79 116.34 3.32 116.34 L 293.22 116.34 C 294.75 116.34 295.99 115.1 295.99 113.58 L 295.99 59.83 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 66.19)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.17em;" width="270.96" height="48.64" transform="matrix(1 0 0 -1 0 46.22)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p3.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p3.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p3.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p3.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 8-expert subset <span id="A2.SS5.p3.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:3.07em;--ltx-fo-depth:0em;" width="270.96" height="42.55" transform="matrix(1 0 0 -1 0 42.55)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p3.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p3.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p3.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Harry slept 9 hours. His friend James slept 2/3 of 9. So 2/3 of 9 is 6. So the answer is 6 more hours. 9 - 6 = 3. So the answer is 3.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p4" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p4.pic1" class="ltx_picture" height="62.17" overflow="visible" style="vertical-align:-31.09px" version="1.1" viewBox="0 0 296.54 62.17" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,62.17) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 58.85 C 0 60.68 1.49 62.17 3.32 62.17 L 293.22 62.17 C 295.05 62.17 296.54 60.68 296.54 58.85 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 41.98 L 295.99 41.98 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 42.54 L 0.55 58.85 C 0.55 60.38 1.79 61.62 3.32 61.62 L 293.22 61.62 C 294.75 61.62 295.99 60.38 295.99 58.85 L 295.99 42.54 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 48.96)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p4.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p4.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p4.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 32-expert subset <span id="A2.SS5.p4.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:1.82em;--ltx-fo-depth:0em;" width="270.96" height="25.25" transform="matrix(1 0 0 -1 0 25.25)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p4.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p4.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p4.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Harry slept 9 hours last night. So the answer is 9 - 2 = 7. So the answer is 7.</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p4.pic2" class="ltx_picture" height="119.32" overflow="visible" style="vertical-align:-59.66px" version="1.1" viewBox="0 0 296.54 119.32" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,119.32) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 116 C 0 117.83 1.49 119.32 3.32 119.32 L 293.22 119.32 C 295.05 119.32 296.54 117.83 296.54 116 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 61.7 L 295.99 61.7 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 62.25 L 0.55 116 C 0.55 117.53 1.79 118.76 3.32 118.76 L 293.22 118.76 C 294.75 118.76 295.99 117.53 295.99 116 L 295.99 62.25 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 68.61)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.17em;" width="270.96" height="48.64" transform="matrix(1 0 0 -1 0 46.22)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p4.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p4.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p4.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p4.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 32-expert subset <span id="A2.SS5.p4.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:3.07em;--ltx-fo-depth:0.17em;" width="270.96" height="44.97" transform="matrix(1 0 0 -1 0 42.55)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p4.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p4.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p4.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Harry slept 9 hours. James slept 2/3 of 9 hours. So James slept 9 * 2/3 = 6 hours. So Harry slept 9 - 6 = 3 hours more than James.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p5" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p5.pic1" class="ltx_picture" height="79.33" overflow="visible" style="vertical-align:-39.66px" version="1.1" viewBox="0 0 296.54 79.33" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,79.33) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 76.01 C 0 77.84 1.49 79.33 3.32 79.33 L 293.22 79.33 C 295.05 79.33 296.54 77.84 296.54 76.01 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 59.28 L 295.99 59.28 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 59.83 L 0.55 76.01 C 0.55 77.54 1.79 78.78 3.32 78.78 L 293.22 78.78 C 294.75 78.78 295.99 77.54 295.99 76.01 L 295.99 59.83 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 66.19)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.62em;--ltx-fo-depth:0.17em;" width="270.96" height="11.07" transform="matrix(1 0 0 -1 0 8.65)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p5.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p5.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p5.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 128-expert subset <span id="A2.SS5.p5.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:3.07em;--ltx-fo-depth:0em;" width="270.96" height="42.55" transform="matrix(1 0 0 -1 0 42.55)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p5.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p5.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p5.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Harry slept 9 hours. James slept 2/3 of that. So James slept 9 * 2/3 = 6 hours. So Harry slept 9 - 6 = 3 hours more than James.</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p5.pic2" class="ltx_picture" height="119.32" overflow="visible" style="vertical-align:-59.66px" version="1.1" viewBox="0 0 296.54 119.32" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,119.32) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 116 C 0 117.83 1.49 119.32 3.32 119.32 L 293.22 119.32 C 295.05 119.32 296.54 117.83 296.54 116 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 61.7 L 295.99 61.7 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 62.25 L 0.55 116 C 0.55 117.53 1.79 118.76 3.32 118.76 L 293.22 118.76 C 294.75 118.76 295.99 117.53 295.99 116 L 295.99 62.25 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 68.61)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.17em;" width="270.96" height="48.64" transform="matrix(1 0 0 -1 0 46.22)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p5.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p5.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p5.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p5.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 128-expert subset <span id="A2.SS5.p5.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:3.07em;--ltx-fo-depth:0.17em;" width="270.96" height="44.97" transform="matrix(1 0 0 -1 0 42.55)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p5.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p5.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p5.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Harry slept 9 hours. James slept 2/3 of 9 hours. So James slept 9 * 2/3 = 6 hours. So Harry slept 9 - 6 = 3 hours more than James.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p6" class="ltx_para ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p6.pic1" class="ltx_picture" height="74.79" overflow="visible" version="1.1" viewBox="0 0 600 74.79" width="600"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,74.79) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#7373FF;" fill="#7373FF" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.46 L 0 71.33 C 0 73.24 1.55 74.79 3.46 74.79 L 596.54 74.79 C 598.45 74.79 600 73.24 600 71.33 L 600 3.46 C 600 1.55 598.45 0 596.54 0 L 3.46 0 C 1.55 0 0 1.55 0 3.46 Z"></path></g><g style="--ltx-fill-color:#F7F7FF;" fill="#F7F7FF" fill-opacity="1.0"><path style="stroke:none" d="M 0.69 3.46 L 0.69 71.33 C 0.69 72.86 1.93 74.1 3.46 74.1 L 596.54 74.1 C 598.07 74.1 599.31 72.86 599.31 71.33 L 599.31 3.46 C 599.31 1.93 598.07 0.69 596.54 0.69 L 3.46 0.69 C 1.93 0.69 0.69 1.93 0.69 3.46 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 15.7 10.16)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:41.09em;--ltx-fo-height:3.94em;--ltx-fo-depth:0em;" width="568.6" height="54.46" transform="matrix(1 0 0 -1 0 54.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p6.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:41.09em;">
<span id="A2.SS5.p6.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p6.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold">Q.</span> Tom gets 4 car washes a month. If each car wash costs $15 how much does he pay in a year?</span>
<span id="A2.SS5.p6.pic1.1.1.1.1.1.2" class="ltx_p"><span id="A2.SS5.p6.pic1.1.1.1.1.1.2.1" class="ltx_text ltx_font_bold">Gold:</span> <span id="A2.SS5.p6.pic1.1.1.1.1.1.2.2" class="ltx_text ltx_font_bold" style="--ltx-fg-color:#1F6E7C;">720</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p7" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p7.pic1" class="ltx_picture" height="95.38" overflow="visible" style="vertical-align:-47.69px" version="1.1" viewBox="0 0 296.54 95.38" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,95.38) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 92.06 C 0 93.89 1.49 95.38 3.32 95.38 L 293.22 95.38 C 295.05 95.38 296.54 93.89 296.54 92.06 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 75.19 L 295.99 75.19 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 75.74 L 0.55 92.06 C 0.55 93.59 1.79 94.83 3.32 94.83 L 293.22 94.83 C 294.75 94.83 295.99 93.59 295.99 92.06 L 295.99 75.74 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 82.17)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p7.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p7.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p7.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 8-expert subset <span id="A2.SS5.p7.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0em;" width="270.96" height="58.46" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p7.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p7.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p7.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">3 4 ? 1000 3 = 9, 5 5 4 + 4 = 9 so 1000 9 = 3 4 = 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = 5 4 = …</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p7.pic2" class="ltx_picture" height="116.34" overflow="visible" style="vertical-align:-58.17px" version="1.1" viewBox="0 0 296.54 116.34" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,116.34) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 113.02 C 0 114.86 1.49 116.34 3.32 116.34 L 293.22 116.34 C 295.05 116.34 296.54 114.86 296.54 113.02 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 58.59 L 295.99 58.59 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 59.14 L 0.55 113.02 C 0.55 114.55 1.79 115.79 3.32 115.79 L 293.22 115.79 C 294.75 115.79 295.99 114.55 295.99 113.02 L 295.99 59.14 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 65.57)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.18em;" width="270.96" height="48.78" transform="matrix(1 0 0 -1 0 46.28)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p7.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p7.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p7.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p7.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 8-expert subset <span id="A2.SS5.p7.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:3.02em;--ltx-fo-depth:0em;" width="270.96" height="41.86" transform="matrix(1 0 0 -1 0 41.86)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p7.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p7.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p7.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Tom started with 4 car washes. Each car wash costs $15. So 4 * 15 = 60 dollars. So the answer is 60 dollars.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p8" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p8.pic1" class="ltx_picture" height="62.86" overflow="visible" style="vertical-align:-31.43px" version="1.1" viewBox="0 0 296.54 62.86" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,62.86) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 59.54 C 0 61.38 1.49 62.86 3.32 62.86 L 293.22 62.86 C 295.05 62.86 296.54 61.38 296.54 59.54 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 42.67 L 295.99 42.67 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 43.23 L 0.55 59.54 C 0.55 61.07 1.79 62.31 3.32 62.31 L 293.22 62.31 C 294.75 62.31 295.99 61.07 295.99 59.54 L 295.99 43.23 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 49.66)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p8.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p8.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p8.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 32-expert subset <span id="A2.SS5.p8.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:1.87em;--ltx-fo-depth:0em;" width="270.96" height="25.94" transform="matrix(1 0 0 -1 0 25.94)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p8.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p8.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p8.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">There were 4 cars. 4 * 15 = 60. So the answer is 60.</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p8.pic2" class="ltx_picture" height="135.23" overflow="visible" style="vertical-align:-67.62px" version="1.1" viewBox="0 0 296.54 135.23" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,135.23) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 131.91 C 0 133.74 1.49 135.23 3.32 135.23 L 293.22 135.23 C 295.05 135.23 296.54 133.74 296.54 131.91 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 77.61 L 295.99 77.61 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 78.17 L 0.55 131.91 C 0.55 133.44 1.79 134.68 3.32 134.68 L 293.22 134.68 C 294.75 134.68 295.99 133.44 295.99 131.91 L 295.99 78.17 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 84.52)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.17em;" width="270.96" height="48.64" transform="matrix(1 0 0 -1 0 46.22)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p8.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p8.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p8.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p8.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 32-expert subset <span id="A2.SS5.p8.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0.17em;" width="270.96" height="60.88" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p8.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p8.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p8.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Tom gets 4 car washes a month. Each car wash costs $15. So 4 x 15 = 60. So Tom pays 60 dollars a month. 60 x 12 = 720. So Tom pays 720 dollars a year.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p9" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p9.pic1" class="ltx_picture" height="97.66" overflow="visible" style="vertical-align:-48.83px" version="1.1" viewBox="0 0 296.54 97.66" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,97.66) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 94.34 C 0 96.18 1.49 97.66 3.32 97.66 L 293.22 97.66 C 295.05 97.66 296.54 96.18 296.54 94.34 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 77.61 L 295.99 77.61 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 78.17 L 0.55 94.34 C 0.55 95.87 1.79 97.11 3.32 97.11 L 293.22 97.11 C 294.75 97.11 295.99 95.87 295.99 94.34 L 295.99 78.17 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 84.52)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.62em;--ltx-fo-depth:0.17em;" width="270.96" height="11.07" transform="matrix(1 0 0 -1 0 8.65)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p9.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p9.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p9.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 128-expert subset <span id="A2.SS5.p9.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0.17em;" width="270.96" height="60.88" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p9.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p9.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p9.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Tom gets 4 car washes a month. 4 car washes a month will be 4 x 12 = 48. 48 car washes a year. 48 car washes a year will be 48 x 15 = 720. 720 dollars a year. So the answer is 720.</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p9.pic2" class="ltx_picture" height="135.23" overflow="visible" style="vertical-align:-67.62px" version="1.1" viewBox="0 0 296.54 135.23" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,135.23) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 131.91 C 0 133.74 1.49 135.23 3.32 135.23 L 293.22 135.23 C 295.05 135.23 296.54 133.74 296.54 131.91 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 77.61 L 295.99 77.61 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 78.17 L 0.55 131.91 C 0.55 133.44 1.79 134.68 3.32 134.68 L 293.22 134.68 C 294.75 134.68 295.99 133.44 295.99 131.91 L 295.99 78.17 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 84.52)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.17em;" width="270.96" height="48.64" transform="matrix(1 0 0 -1 0 46.22)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p9.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p9.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p9.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p9.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 128-expert subset <span id="A2.SS5.p9.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#2F8F3F;">✓</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0.17em;" width="270.96" height="60.88" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p9.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p9.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p9.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Tom gets 4 car washes a month. Each car wash costs $15. So 4 x 15 = 60 dollars. So Tom pays 60 dollars a month. 60 dollars a month for 12 months is 60 x 12 = 720 dollars a year.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p10" class="ltx_para ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p10.pic1" class="ltx_picture" height="107.23" overflow="visible" version="1.1" viewBox="0 0 600 107.23" width="600"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,107.23) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#7373FF;" fill="#7373FF" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.46 L 0 103.77 C 0 105.68 1.55 107.23 3.46 107.23 L 596.54 107.23 C 598.45 107.23 600 105.68 600 103.77 L 600 3.46 C 600 1.55 598.45 0 596.54 0 L 3.46 0 C 1.55 0 0 1.55 0 3.46 Z"></path></g><g style="--ltx-fill-color:#F7F7FF;" fill="#F7F7FF" fill-opacity="1.0"><path style="stroke:none" d="M 0.69 3.46 L 0.69 103.77 C 0.69 105.3 1.93 106.54 3.46 106.54 L 596.54 106.54 C 598.07 106.54 599.31 105.3 599.31 103.77 L 599.31 3.46 C 599.31 1.93 598.07 0.69 596.54 0.69 L 3.46 0.69 C 1.93 0.69 0.69 1.93 0.69 3.46 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 15.7 10.16)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:41.09em;--ltx-fo-height:6.28em;--ltx-fo-depth:0em;" width="568.6" height="86.9" transform="matrix(1 0 0 -1 0 86.9)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p10.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:41.09em;">
<span id="A2.SS5.p10.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p10.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold">Q.</span> Raymond and Samantha are cousins. Raymond was born 6 years before Samantha. Raymond had a son at the age of 23. If Samantha is now 31, how many years ago was Raymond’s son born?</span>
<span id="A2.SS5.p10.pic1.1.1.1.1.1.2" class="ltx_p"><span id="A2.SS5.p10.pic1.1.1.1.1.1.2.1" class="ltx_text ltx_font_bold">Gold:</span> <span id="A2.SS5.p10.pic1.1.1.1.1.1.2.2" class="ltx_text ltx_font_bold" style="--ltx-fg-color:#1F6E7C;">14</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p11" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p11.pic1" class="ltx_picture" height="65.28" overflow="visible" style="vertical-align:-32.64px" version="1.1" viewBox="0 0 296.54 65.28" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,65.28) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 61.96 C 0 63.8 1.49 65.28 3.32 65.28 L 293.22 65.28 C 295.05 65.28 296.54 63.8 296.54 61.96 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 45.1 L 295.99 45.1 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 45.65 L 0.55 61.96 C 0.55 63.49 1.79 64.73 3.32 64.73 L 293.22 64.73 C 294.75 64.73 295.99 63.49 295.99 61.96 L 295.99 45.65 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 52.08)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p11.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p11.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p11.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 8-expert subset <span id="A2.SS5.p11.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.75)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:1.82em;--ltx-fo-depth:0.22em;" width="270.96" height="28.37" transform="matrix(1 0 0 -1 0 25.25)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p11.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p11.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p11.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Olivia 3 5 = 4 9 4 Xe ? 5 4 3 Xe ? 5 5 4 = 9 so 20v 5 5 5 5 5 5 [+213 more]</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p11.pic2" class="ltx_picture" height="135.37" overflow="visible" style="vertical-align:-67.68px" version="1.1" viewBox="0 0 296.54 135.37" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,135.37) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 132.05 C 0 133.88 1.49 135.37 3.32 135.37 L 293.22 135.37 C 295.05 135.37 296.54 133.88 296.54 132.05 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 77.61 L 295.99 77.61 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 78.17 L 0.55 132.05 C 0.55 133.58 1.79 134.82 3.32 134.82 L 293.22 134.82 C 294.75 134.82 295.99 133.58 295.99 132.05 L 295.99 78.17 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 84.59)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.18em;" width="270.96" height="48.78" transform="matrix(1 0 0 -1 0 46.28)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p11.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p11.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p11.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p11.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 8-expert subset <span id="A2.SS5.p11.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0.17em;" width="270.96" height="60.88" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p11.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p11.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p11.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Raymond is 6 years older than Samantha. So Samantha is 31 - 6 = 25. Raymond is 6 years older than Samantha. So Raymond is 31 - 6 = 25 years old. So the answer is 25.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p12" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p12.pic1" class="ltx_picture" height="62.17" overflow="visible" style="vertical-align:-31.09px" version="1.1" viewBox="0 0 296.54 62.17" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,62.17) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 58.85 C 0 60.68 1.49 62.17 3.32 62.17 L 293.22 62.17 C 295.05 62.17 296.54 60.68 296.54 58.85 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 41.98 L 295.99 41.98 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 42.54 L 0.55 58.85 C 0.55 60.38 1.79 61.62 3.32 61.62 L 293.22 61.62 C 294.75 61.62 295.99 60.38 295.99 58.85 L 295.99 42.54 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 48.96)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p12.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p12.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p12.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 32-expert subset <span id="A2.SS5.p12.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:1.82em;--ltx-fo-depth:0em;" width="270.96" height="25.25" transform="matrix(1 0 0 -1 0 25.25)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p12.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p12.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p12.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Raymond was born 6 years before Samantha. So the answer is 6.</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p12.pic2" class="ltx_picture" height="149.55" overflow="visible" style="vertical-align:-74.78px" version="1.1" viewBox="0 0 296.54 149.55" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,149.55) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 146.23 C 0 148.06 1.49 149.55 3.32 149.55 L 293.22 149.55 C 295.05 149.55 296.54 148.06 296.54 146.23 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 91.8 L 295.99 91.8 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 92.35 L 0.55 146.23 C 0.55 147.76 1.79 149 3.32 149 L 293.22 149 C 294.75 149 295.99 147.76 295.99 146.23 L 295.99 92.35 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 98.78)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.18em;" width="270.96" height="48.78" transform="matrix(1 0 0 -1 0 46.28)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p12.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p12.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p12.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p12.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 32-expert subset <span id="A2.SS5.p12.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:5.42em;--ltx-fo-depth:0em;" width="270.96" height="75.07" transform="matrix(1 0 0 -1 0 75.07)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p12.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p12.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p12.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Raymond was born 6 years before Samantha. So Raymond was born 6 years before Samantha was born. 6 years before Samantha was born was 6 years before 31. So Raymond’s son was born 6 …</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
<div id="A2.SS5.p13" class="ltx_para ltx_align_left ltx_noindent">
<span class="ltx_inline-block"><svg id="A2.SS5.p13.pic1" class="ltx_picture" height="95.38" overflow="visible" style="vertical-align:-47.69px" version="1.1" viewBox="0 0 296.54 95.38" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,95.38) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#225C2E;" fill="#225C2E" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 92.06 C 0 93.89 1.49 95.38 3.32 95.38 L 293.22 95.38 C 295.05 95.38 296.54 93.89 296.54 92.06 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 75.19 L 295.99 75.19 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#EEF6E4;" fill="#EEF6E4" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 75.74 L 0.55 92.06 C 0.55 93.59 1.79 94.83 3.32 94.83 L 293.22 94.83 C 294.75 94.83 295.99 93.59 295.99 92.06 L 295.99 75.74 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 82.17)"><foreignObject style="--ltx-fg-color:#225C2E;--ltx-fo-width:19.58em;--ltx-fo-height:0.63em;--ltx-fo-depth:0.18em;" width="270.96" height="11.21" transform="matrix(1 0 0 -1 0 8.72)" overflow="visible" color="#225C2E"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p13.pic1.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p13.pic1.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p13.pic1.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Reg. MoE, 128-expert subset <span id="A2.SS5.p13.pic1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 8.64)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0em;" width="270.96" height="58.46" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p13.pic1.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p13.pic1.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p13.pic1.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Raymond was born 6 years before Samantha. So he was 6 years older than Samantha. So he was 6 + 31 = 37 years old when his son was born. So the answer is 37.</span></span>
</span></span></span></foreignObject></g></g></svg></span><span class="ltx_inline-block"><svg id="A2.SS5.p13.pic2" class="ltx_picture" height="135.37" overflow="visible" style="vertical-align:-67.68px" version="1.1" viewBox="0 0 296.54 135.37" width="296.54"><g style="--ltx-stroke-color:#000000;--ltx-fill-color:#000000;" transform="translate(0,135.37) matrix(1 0 0 -1 0 0)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><g style="--ltx-fill-color:#B8327C;" fill="#B8327C" fill-opacity="1.0"><path style="stroke:none" d="M 0 3.32 L 0 132.05 C 0 133.88 1.49 135.37 3.32 135.37 L 293.22 135.37 C 295.05 135.37 296.54 133.88 296.54 132.05 L 296.54 3.32 C 296.54 1.49 295.05 0 293.22 0 L 3.32 0 C 1.49 0 0 1.49 0 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 3.32 L 0.55 77.61 L 295.99 77.61 L 295.99 3.32 C 295.99 1.79 294.75 0.55 293.22 0.55 L 3.32 0.55 C 1.79 0.55 0.55 1.79 0.55 3.32 Z"></path></g><g style="--ltx-fill-color:#FBE7EF;" fill="#FBE7EF" fill-opacity="1.0"><path style="stroke:none" d="M 0.55 78.17 L 0.55 132.05 C 0.55 133.58 1.79 134.82 3.32 134.82 L 293.22 134.82 C 294.75 134.82 295.99 133.58 295.99 132.05 L 295.99 78.17 Z"></path></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 84.59)"><foreignObject style="--ltx-fg-color:#3F1052;--ltx-fo-width:19.58em;--ltx-fo-height:3.34em;--ltx-fo-depth:0.18em;" width="270.96" height="48.78" transform="matrix(1 0 0 -1 0 46.28)" overflow="visible" color="#3F1052"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p13.pic2.1.1.1.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:18.39em;">
<span id="A2.SS5.p13.pic2.1.1.1.1.1.1" class="ltx_p"><span id="A2.SS5.p13.pic2.1.1.1.1.1.1.1" class="ltx_text ltx_font_bold ltx_font_smallcaps" style="font-size:90%;">Emo<span id="A2.SS5.p13.pic2.1.1.1.1.1.1.1.1" class="ltx_text ltx_font_upright">, 128-expert subset <span id="A2.SS5.p13.pic2.1.1.1.1.1.1.1.1.1" class="ltx_text" style="--ltx-fg-color:#888888;">✗</span></span></span></span>
</span></span></span></foreignObject></g><g fill-opacity="1.0" transform="matrix(1.0 0.0 0.0 1.0 12.79 11.06)"><foreignObject style="--ltx-fg-color:#000000;--ltx-fo-width:19.58em;--ltx-fo-height:4.22em;--ltx-fo-depth:0.17em;" width="270.96" height="60.88" transform="matrix(1 0 0 -1 0 58.46)" overflow="visible" color="#000000"><span class="ltx_foreignobject_container"><span class="ltx_foreignobject_content">
<span id="A2.SS5.p13.pic2.2.2.2.1.1" class="ltx_inline-block ltx_minipage ltx_align_bottom" style="width:21.17em;">
<span id="A2.SS5.p13.pic2.2.2.2.1.1.1" class="ltx_p"><span id="A2.SS5.p13.pic2.2.2.2.1.1.1.1" class="ltx_text" style="font-size:90%;">Raymond was born 6 years before Samantha. So he was born 6 years before Samantha was 31. So he was born 6 years before 31 = 25 years ago. So the answer is 25 years ago.</span></span>
</span></span></span></foreignObject></g></g></svg></span>
</div>
</section>
</section>
<section id="A3" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix C </span>Evaluation Details</h2>

<figure id="A3.T4" class="ltx_table">
<table id="A3.T4.1" class="ltx_tabular ltx_centering ltx_align_middle">
<tbody class="ltx_tbody">
<tr id="A3.T4.1.1.1" class="ltx_tr">
<td id="A3.T4.1.1.1.1" class="ltx_td ltx_align_left ltx_border_tt" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.1.1.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Task</span></td>
<td id="A3.T4.1.1.1.2" class="ltx_td ltx_align_center ltx_border_tt" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.1.1.2.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Shots</span></td>
<td id="A3.T4.1.1.1.3" class="ltx_td ltx_align_right ltx_border_tt" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.1.1.3.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Train+Val</span></td>
<td id="A3.T4.1.1.1.4" class="ltx_td ltx_align_right ltx_border_tt" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.1.1.4.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Test</span></td>
<td id="A3.T4.1.1.1.5" class="ltx_td ltx_align_left ltx_border_tt" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.1.1.5.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Primary metric</span></td>
</tr>
<tr id="A3.T4.1.2.2" class="ltx_tr">
<td id="A3.T4.1.2.2.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;" colspan="5">
<span id="A3.T4.1.2.2.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">MMLU-Pro</span><span id="A3.T4.1.2.2.1.2" class="ltx_text" style="font-size:90%;"> (14 tasks)</span>
</td>
</tr>
<tr id="A3.T4.1.3.3" class="ltx_tr">
<td id="A3.T4.1.3.3.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.3.3.1.1" class="ltx_text" style="font-size:90%;">Math</span></td>
<td id="A3.T4.1.3.3.2" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.3.3.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.3.3.3" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.3.3.3.1" class="ltx_text" style="font-size:90%;">601</span></td>
<td id="A3.T4.1.3.3.4" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.3.3.4.1" class="ltx_text" style="font-size:90%;">750</span></td>
<td id="A3.T4.1.3.3.5" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.3.3.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.4.4" class="ltx_tr">
<td id="A3.T4.1.4.4.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.4.4.1.1" class="ltx_text" style="font-size:90%;">Health</span></td>
<td id="A3.T4.1.4.4.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.4.4.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.4.4.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.4.4.3.1" class="ltx_text" style="font-size:90%;">388</span></td>
<td id="A3.T4.1.4.4.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.4.4.4.1" class="ltx_text" style="font-size:90%;">430</span></td>
<td id="A3.T4.1.4.4.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.4.4.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.5.5" class="ltx_tr">
<td id="A3.T4.1.5.5.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.5.5.1.1" class="ltx_text" style="font-size:90%;">Physics</span></td>
<td id="A3.T4.1.5.5.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.5.5.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.5.5.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.5.5.3.1" class="ltx_text" style="font-size:90%;">580</span></td>
<td id="A3.T4.1.5.5.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.5.5.4.1" class="ltx_text" style="font-size:90%;">719</span></td>
<td id="A3.T4.1.5.5.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.5.5.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.6.6" class="ltx_tr">
<td id="A3.T4.1.6.6.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.6.6.1.1" class="ltx_text" style="font-size:90%;">Business</span></td>
<td id="A3.T4.1.6.6.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.6.6.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.6.6.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.6.6.3.1" class="ltx_text" style="font-size:90%;">376</span></td>
<td id="A3.T4.1.6.6.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.6.6.4.1" class="ltx_text" style="font-size:90%;">413</span></td>
<td id="A3.T4.1.6.6.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.6.6.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.7.7" class="ltx_tr">
<td id="A3.T4.1.7.7.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.7.7.1.1" class="ltx_text" style="font-size:90%;">Biology</span></td>
<td id="A3.T4.1.7.7.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.7.7.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.7.7.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.7.7.3.1" class="ltx_text" style="font-size:90%;">347</span></td>
<td id="A3.T4.1.7.7.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.7.7.4.1" class="ltx_text" style="font-size:90%;">370</span></td>
<td id="A3.T4.1.7.7.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.7.7.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.8.8" class="ltx_tr">
<td id="A3.T4.1.8.8.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.8.8.1.1" class="ltx_text" style="font-size:90%;">Chemistry</span></td>
<td id="A3.T4.1.8.8.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.8.8.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.8.8.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.8.8.3.1" class="ltx_text" style="font-size:90%;">513</span></td>
<td id="A3.T4.1.8.8.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.8.8.4.1" class="ltx_text" style="font-size:90%;">619</span></td>
<td id="A3.T4.1.8.8.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.8.8.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.9.9" class="ltx_tr">
<td id="A3.T4.1.9.9.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.9.9.1.1" class="ltx_text" style="font-size:90%;">Computer Science</span></td>
<td id="A3.T4.1.9.9.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.9.9.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.9.9.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.9.9.3.1" class="ltx_text" style="font-size:90%;">224</span></td>
<td id="A3.T4.1.9.9.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.9.9.4.1" class="ltx_text" style="font-size:90%;">186</span></td>
<td id="A3.T4.1.9.9.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.9.9.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.10.10" class="ltx_tr">
<td id="A3.T4.1.10.10.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.10.10.1.1" class="ltx_text" style="font-size:90%;">Economics</span></td>
<td id="A3.T4.1.10.10.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.10.10.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.10.10.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.10.10.3.1" class="ltx_text" style="font-size:90%;">398</span></td>
<td id="A3.T4.1.10.10.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.10.10.4.1" class="ltx_text" style="font-size:90%;">446</span></td>
<td id="A3.T4.1.10.10.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.10.10.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.11.11" class="ltx_tr">
<td id="A3.T4.1.11.11.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.11.11.1.1" class="ltx_text" style="font-size:90%;">Engineering</span></td>
<td id="A3.T4.1.11.11.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.11.11.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.11.11.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.11.11.3.1" class="ltx_text" style="font-size:90%;">448</span></td>
<td id="A3.T4.1.11.11.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.11.11.4.1" class="ltx_text" style="font-size:90%;">521</span></td>
<td id="A3.T4.1.11.11.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.11.11.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.12.12" class="ltx_tr">
<td id="A3.T4.1.12.12.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.12.12.1.1" class="ltx_text" style="font-size:90%;">Philosophy</span></td>
<td id="A3.T4.1.12.12.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.12.12.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.12.12.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.12.12.3.1" class="ltx_text" style="font-size:90%;">260</span></td>
<td id="A3.T4.1.12.12.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.12.12.4.1" class="ltx_text" style="font-size:90%;">239</span></td>
<td id="A3.T4.1.12.12.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.12.12.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.13.13" class="ltx_tr">
<td id="A3.T4.1.13.13.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.13.13.1.1" class="ltx_text" style="font-size:90%;">Other</span></td>
<td id="A3.T4.1.13.13.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.13.13.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.13.13.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.13.13.3.1" class="ltx_text" style="font-size:90%;">430</span></td>
<td id="A3.T4.1.13.13.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.13.13.4.1" class="ltx_text" style="font-size:90%;">494</span></td>
<td id="A3.T4.1.13.13.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.13.13.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.14.14" class="ltx_tr">
<td id="A3.T4.1.14.14.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.14.14.1.1" class="ltx_text" style="font-size:90%;">History</span></td>
<td id="A3.T4.1.14.14.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.14.14.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.14.14.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.14.14.3.1" class="ltx_text" style="font-size:90%;">213</span></td>
<td id="A3.T4.1.14.14.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.14.14.4.1" class="ltx_text" style="font-size:90%;">168</span></td>
<td id="A3.T4.1.14.14.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.14.14.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.15.15" class="ltx_tr">
<td id="A3.T4.1.15.15.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.15.15.1.1" class="ltx_text" style="font-size:90%;">Psychology</span></td>
<td id="A3.T4.1.15.15.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.15.15.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.15.15.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.15.15.3.1" class="ltx_text" style="font-size:90%;">380</span></td>
<td id="A3.T4.1.15.15.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.15.15.4.1" class="ltx_text" style="font-size:90%;">418</span></td>
<td id="A3.T4.1.15.15.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.15.15.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.16.16" class="ltx_tr">
<td id="A3.T4.1.16.16.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.16.16.1.1" class="ltx_text" style="font-size:90%;">Law</span></td>
<td id="A3.T4.1.16.16.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.16.16.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.16.16.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.16.16.3.1" class="ltx_text" style="font-size:90%;">501</span></td>
<td id="A3.T4.1.16.16.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.16.16.4.1" class="ltx_text" style="font-size:90%;">600</span></td>
<td id="A3.T4.1.16.16.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.16.16.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.17.17" class="ltx_tr">
<td id="A3.T4.1.17.17.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;" colspan="5">
<span id="A3.T4.1.17.17.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">MMLU</span><span id="A3.T4.1.17.17.1.2" class="ltx_text" style="font-size:90%;"> (17 tasks)</span>
</td>
</tr>
<tr id="A3.T4.1.18.18" class="ltx_tr">
<td id="A3.T4.1.18.18.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.18.18.1.1" class="ltx_text" style="font-size:90%;">Biology</span></td>
<td id="A3.T4.1.18.18.2" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.18.18.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.18.18.3" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.18.18.3.1" class="ltx_text" style="font-size:90%;">320</span></td>
<td id="A3.T4.1.18.18.4" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.18.18.4.1" class="ltx_text" style="font-size:90%;">182</span></td>
<td id="A3.T4.1.18.18.5" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.18.18.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.19.19" class="ltx_tr">
<td id="A3.T4.1.19.19.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.19.19.1.1" class="ltx_text" style="font-size:90%;">Business</span></td>
<td id="A3.T4.1.19.19.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.19.19.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.19.19.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.19.19.3.1" class="ltx_text" style="font-size:90%;">308</span></td>
<td id="A3.T4.1.19.19.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.19.19.4.1" class="ltx_text" style="font-size:90%;">176</span></td>
<td id="A3.T4.1.19.19.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.19.19.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.20.20" class="ltx_tr">
<td id="A3.T4.1.20.20.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.20.20.1.1" class="ltx_text" style="font-size:90%;">Chemistry</span></td>
<td id="A3.T4.1.20.20.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.20.20.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.20.20.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.20.20.3.1" class="ltx_text" style="font-size:90%;">211</span></td>
<td id="A3.T4.1.20.20.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.20.20.4.1" class="ltx_text" style="font-size:90%;">122</span></td>
<td id="A3.T4.1.20.20.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.20.20.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.21.21" class="ltx_tr">
<td id="A3.T4.1.21.21.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.21.21.1.1" class="ltx_text" style="font-size:90%;">Computer Science</span></td>
<td id="A3.T4.1.21.21.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.21.21.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.21.21.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.21.21.3.1" class="ltx_text" style="font-size:90%;">289</span></td>
<td id="A3.T4.1.21.21.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.21.21.4.1" class="ltx_text" style="font-size:90%;">165</span></td>
<td id="A3.T4.1.21.21.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.21.21.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.22.22" class="ltx_tr">
<td id="A3.T4.1.22.22.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.22.22.1.1" class="ltx_text" style="font-size:90%;">Culture</span></td>
<td id="A3.T4.1.22.22.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.22.22.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.22.22.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.22.22.3.1" class="ltx_text" style="font-size:90%;">232</span></td>
<td id="A3.T4.1.22.22.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.22.22.4.1" class="ltx_text" style="font-size:90%;">134</span></td>
<td id="A3.T4.1.22.22.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.22.22.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.23.23" class="ltx_tr">
<td id="A3.T4.1.23.23.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.23.23.1.1" class="ltx_text" style="font-size:90%;">Economics</span></td>
<td id="A3.T4.1.23.23.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.23.23.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.23.23.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.23.23.3.1" class="ltx_text" style="font-size:90%;">525</span></td>
<td id="A3.T4.1.23.23.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.23.23.4.1" class="ltx_text" style="font-size:90%;">298</span></td>
<td id="A3.T4.1.23.23.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.23.23.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.24.24" class="ltx_tr">
<td id="A3.T4.1.24.24.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.24.24.1.1" class="ltx_text" style="font-size:90%;">Engineering</span></td>
<td id="A3.T4.1.24.24.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.24.24.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.24.24.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.24.24.3.1" class="ltx_text" style="font-size:90%;">103</span></td>
<td id="A3.T4.1.24.24.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.24.24.4.1" class="ltx_text" style="font-size:90%;">58</span></td>
<td id="A3.T4.1.24.24.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.24.24.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.25.25" class="ltx_tr">
<td id="A3.T4.1.25.25.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.25.25.1.1" class="ltx_text" style="font-size:90%;">Geography</span></td>
<td id="A3.T4.1.25.25.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.25.25.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.25.25.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.25.25.3.1" class="ltx_text" style="font-size:90%;">140</span></td>
<td id="A3.T4.1.25.25.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.25.25.4.1" class="ltx_text" style="font-size:90%;">80</span></td>
<td id="A3.T4.1.25.25.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.25.25.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.26.26" class="ltx_tr">
<td id="A3.T4.1.26.26.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.26.26.1.1" class="ltx_text" style="font-size:90%;">Health</span></td>
<td id="A3.T4.1.26.26.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.26.26.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.26.26.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.26.26.3.1" class="ltx_text" style="font-size:90%;">1 162</span></td>
<td id="A3.T4.1.26.26.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.26.26.4.1" class="ltx_text" style="font-size:90%;">659</span></td>
<td id="A3.T4.1.26.26.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.26.26.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.27.27" class="ltx_tr">
<td id="A3.T4.1.27.27.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.27.27.1.1" class="ltx_text" style="font-size:90%;">History</span></td>
<td id="A3.T4.1.27.27.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.27.27.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.27.27.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.27.27.3.1" class="ltx_text" style="font-size:90%;">658</span></td>
<td id="A3.T4.1.27.27.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.27.27.4.1" class="ltx_text" style="font-size:90%;">373</span></td>
<td id="A3.T4.1.27.27.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.27.27.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.28.28" class="ltx_tr">
<td id="A3.T4.1.28.28.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.28.28.1.1" class="ltx_text" style="font-size:90%;">Law</span></td>
<td id="A3.T4.1.28.28.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.28.28.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.28.28.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.28.28.3.1" class="ltx_text" style="font-size:90%;">1 250</span></td>
<td id="A3.T4.1.28.28.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.28.28.4.1" class="ltx_text" style="font-size:90%;">707</span></td>
<td id="A3.T4.1.28.28.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.28.28.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.29.29" class="ltx_tr">
<td id="A3.T4.1.29.29.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.29.29.1.1" class="ltx_text" style="font-size:90%;">Math</span></td>
<td id="A3.T4.1.29.29.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.29.29.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.29.29.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.29.29.3.1" class="ltx_text" style="font-size:90%;">752</span></td>
<td id="A3.T4.1.29.29.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.29.29.4.1" class="ltx_text" style="font-size:90%;">427</span></td>
<td id="A3.T4.1.29.29.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.29.29.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.30.30" class="ltx_tr">
<td id="A3.T4.1.30.30.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.30.30.1.1" class="ltx_text" style="font-size:90%;">Other</span></td>
<td id="A3.T4.1.30.30.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.30.30.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.30.30.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.30.30.3.1" class="ltx_text" style="font-size:90%;">825</span></td>
<td id="A3.T4.1.30.30.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.30.30.4.1" class="ltx_text" style="font-size:90%;">467</span></td>
<td id="A3.T4.1.30.30.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.30.30.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.31.31" class="ltx_tr">
<td id="A3.T4.1.31.31.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.31.31.1.1" class="ltx_text" style="font-size:90%;">Philosophy</span></td>
<td id="A3.T4.1.31.31.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.31.31.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.31.31.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.31.31.3.1" class="ltx_text" style="font-size:90%;">1 427</span></td>
<td id="A3.T4.1.31.31.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.31.31.4.1" class="ltx_text" style="font-size:90%;">808</span></td>
<td id="A3.T4.1.31.31.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.31.31.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.32.32" class="ltx_tr">
<td id="A3.T4.1.32.32.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.32.32.1.1" class="ltx_text" style="font-size:90%;">Physics</span></td>
<td id="A3.T4.1.32.32.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.32.32.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.32.32.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.32.32.3.1" class="ltx_text" style="font-size:90%;">453</span></td>
<td id="A3.T4.1.32.32.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.32.32.4.1" class="ltx_text" style="font-size:90%;">257</span></td>
<td id="A3.T4.1.32.32.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.32.32.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.33.33" class="ltx_tr">
<td id="A3.T4.1.33.33.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.33.33.1.1" class="ltx_text" style="font-size:90%;">Politics</span></td>
<td id="A3.T4.1.33.33.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.33.33.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.33.33.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.33.33.3.1" class="ltx_text" style="font-size:90%;">459</span></td>
<td id="A3.T4.1.33.33.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.33.33.4.1" class="ltx_text" style="font-size:90%;">260</span></td>
<td id="A3.T4.1.33.33.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.33.33.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.34.34" class="ltx_tr">
<td id="A3.T4.1.34.34.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.34.34.1.1" class="ltx_text" style="font-size:90%;">Psychology</span></td>
<td id="A3.T4.1.34.34.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.34.34.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.34.34.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.34.34.3.1" class="ltx_text" style="font-size:90%;">823</span></td>
<td id="A3.T4.1.34.34.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.34.34.4.1" class="ltx_text" style="font-size:90%;">463</span></td>
<td id="A3.T4.1.34.34.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.34.34.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.35.35" class="ltx_tr">
<td id="A3.T4.1.35.35.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;" colspan="5">
<span id="A3.T4.1.35.35.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">MC9</span><span id="A3.T4.1.35.35.1.2" class="ltx_text" style="font-size:90%;"> (9 tasks)</span>
</td>
</tr>
<tr id="A3.T4.1.36.36" class="ltx_tr">
<td id="A3.T4.1.36.36.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.36.36.1.1" class="ltx_text" style="font-size:90%;">ARC-Easy</span></td>
<td id="A3.T4.1.36.36.2" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.36.36.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.36.36.3" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.36.36.3.1" class="ltx_text" style="font-size:90%;">2 821</span></td>
<td id="A3.T4.1.36.36.4" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.36.36.4.1" class="ltx_text" style="font-size:90%;">2 376</span></td>
<td id="A3.T4.1.36.36.5" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.36.36.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.37.37" class="ltx_tr">
<td id="A3.T4.1.37.37.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.37.37.1.1" class="ltx_text" style="font-size:90%;">ARC-Challenge</span></td>
<td id="A3.T4.1.37.37.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.37.37.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.37.37.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.37.37.3.1" class="ltx_text" style="font-size:90%;">1 418</span></td>
<td id="A3.T4.1.37.37.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.37.37.4.1" class="ltx_text" style="font-size:90%;">1 172</span></td>
<td id="A3.T4.1.37.37.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.37.37.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.38.38" class="ltx_tr">
<td id="A3.T4.1.38.38.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.38.38.1.1" class="ltx_text" style="font-size:90%;">BoolQ</span></td>
<td id="A3.T4.1.38.38.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.38.38.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.38.38.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.38.38.3.1" class="ltx_text" style="font-size:90%;">9 427</span></td>
<td id="A3.T4.1.38.38.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.38.38.4.1" class="ltx_text" style="font-size:90%;">3 270</span></td>
<td id="A3.T4.1.38.38.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.38.38.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.39.39" class="ltx_tr">
<td id="A3.T4.1.39.39.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.39.39.1.1" class="ltx_text" style="font-size:90%;">HellaSwag</span></td>
<td id="A3.T4.1.39.39.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.39.39.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.39.39.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.39.39.3.1" class="ltx_text" style="font-size:90%;">39 905</span></td>
<td id="A3.T4.1.39.39.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.39.39.4.1" class="ltx_text" style="font-size:90%;">10 042</span></td>
<td id="A3.T4.1.39.39.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.39.39.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.40.40" class="ltx_tr">
<td id="A3.T4.1.40.40.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.40.40.1.1" class="ltx_text" style="font-size:90%;">CSQA</span></td>
<td id="A3.T4.1.40.40.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.40.40.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.40.40.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.40.40.3.1" class="ltx_text" style="font-size:90%;">9 741</span></td>
<td id="A3.T4.1.40.40.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.40.40.4.1" class="ltx_text" style="font-size:90%;">1 221</span></td>
<td id="A3.T4.1.40.40.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.40.40.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.41.41" class="ltx_tr">
<td id="A3.T4.1.41.41.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.41.41.1.1" class="ltx_text" style="font-size:90%;">OpenBookQA</span></td>
<td id="A3.T4.1.41.41.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.41.41.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.41.41.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.41.41.3.1" class="ltx_text" style="font-size:90%;">5 457</span></td>
<td id="A3.T4.1.41.41.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.41.41.4.1" class="ltx_text" style="font-size:90%;">500</span></td>
<td id="A3.T4.1.41.41.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.41.41.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.42.42" class="ltx_tr">
<td id="A3.T4.1.42.42.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.42.42.1.1" class="ltx_text" style="font-size:90%;">PIQA</span></td>
<td id="A3.T4.1.42.42.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.42.42.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.42.42.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.42.42.3.1" class="ltx_text" style="font-size:90%;">16 113</span></td>
<td id="A3.T4.1.42.42.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.42.42.4.1" class="ltx_text" style="font-size:90%;">1 838</span></td>
<td id="A3.T4.1.42.42.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.42.42.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.43.43" class="ltx_tr">
<td id="A3.T4.1.43.43.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.43.43.1.1" class="ltx_text" style="font-size:90%;">SocialIQA</span></td>
<td id="A3.T4.1.43.43.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.43.43.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.43.43.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.43.43.3.1" class="ltx_text" style="font-size:90%;">33 410</span></td>
<td id="A3.T4.1.43.43.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.43.43.4.1" class="ltx_text" style="font-size:90%;">1 954</span></td>
<td id="A3.T4.1.43.43.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.43.43.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.44.44" class="ltx_tr">
<td id="A3.T4.1.44.44.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.44.44.1.1" class="ltx_text" style="font-size:90%;">WinoGrande</span></td>
<td id="A3.T4.1.44.44.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.44.44.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.44.44.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.44.44.3.1" class="ltx_text" style="font-size:90%;">40 398</span></td>
<td id="A3.T4.1.44.44.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.44.44.4.1" class="ltx_text" style="font-size:90%;">1 267</span></td>
<td id="A3.T4.1.44.44.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.44.44.5.1" class="ltx_text" style="font-size:90%;">acc/raw</span></td>
</tr>
<tr id="A3.T4.1.45.45" class="ltx_tr">
<td id="A3.T4.1.45.45.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;" colspan="5">
<span id="A3.T4.1.45.45.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">Gen5</span><span id="A3.T4.1.45.45.1.2" class="ltx_text" style="font-size:90%;"> (5 tasks)</span>
</td>
</tr>
<tr id="A3.T4.1.46.46" class="ltx_tr">
<td id="A3.T4.1.46.46.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.46.46.1.1" class="ltx_text" style="font-size:90%;">SQuAD</span></td>
<td id="A3.T4.1.46.46.2" class="ltx_td ltx_align_center ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.46.46.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.46.46.3" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.46.46.3.1" class="ltx_text" style="font-size:90%;">87 599</span></td>
<td id="A3.T4.1.46.46.4" class="ltx_td ltx_align_right ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.46.46.4.1" class="ltx_text" style="font-size:90%;">10 570</span></td>
<td id="A3.T4.1.46.46.5" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.46.46.5.1" class="ltx_text" style="font-size:90%;">F1</span></td>
</tr>
<tr id="A3.T4.1.47.47" class="ltx_tr">
<td id="A3.T4.1.47.47.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.47.47.1.1" class="ltx_text" style="font-size:90%;">CoQA</span></td>
<td id="A3.T4.1.47.47.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.47.47.2.1" class="ltx_text" style="font-size:90%;">0</span></td>
<td id="A3.T4.1.47.47.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.47.47.3.1" class="ltx_text" style="font-size:90%;">108 647</span></td>
<td id="A3.T4.1.47.47.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.47.47.4.1" class="ltx_text" style="font-size:90%;">7 983</span></td>
<td id="A3.T4.1.47.47.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.47.47.5.1" class="ltx_text" style="font-size:90%;">F1</span></td>
</tr>
<tr id="A3.T4.1.48.48" class="ltx_tr">
<td id="A3.T4.1.48.48.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.48.48.1.1" class="ltx_text" style="font-size:90%;">NaturalQS</span></td>
<td id="A3.T4.1.48.48.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.48.48.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.48.48.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.48.48.3.1" class="ltx_text" style="font-size:90%;">87 925</span></td>
<td id="A3.T4.1.48.48.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.48.48.4.1" class="ltx_text" style="font-size:90%;">3 610</span></td>
<td id="A3.T4.1.48.48.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.48.48.5.1" class="ltx_text" style="font-size:90%;">F1</span></td>
</tr>
<tr id="A3.T4.1.49.49" class="ltx_tr">
<td id="A3.T4.1.49.49.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.49.49.1.1" class="ltx_text" style="font-size:90%;">TriviaQA</span></td>
<td id="A3.T4.1.49.49.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.49.49.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.49.49.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.49.49.3.1" class="ltx_text" style="font-size:90%;">61 888</span></td>
<td id="A3.T4.1.49.49.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.49.49.4.1" class="ltx_text" style="font-size:90%;">7 993</span></td>
<td id="A3.T4.1.49.49.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.49.49.5.1" class="ltx_text" style="font-size:90%;">F1</span></td>
</tr>
<tr id="A3.T4.1.50.50" class="ltx_tr">
<td id="A3.T4.1.50.50.1" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.50.50.1.1" class="ltx_text" style="font-size:90%;">DROP</span></td>
<td id="A3.T4.1.50.50.2" class="ltx_td ltx_align_center" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.50.50.2.1" class="ltx_text" style="font-size:90%;">5</span></td>
<td id="A3.T4.1.50.50.3" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.50.50.3.1" class="ltx_text" style="font-size:90%;">77 409</span></td>
<td id="A3.T4.1.50.50.4" class="ltx_td ltx_align_right" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.50.50.4.1" class="ltx_text" style="font-size:90%;">9 536</span></td>
<td id="A3.T4.1.50.50.5" class="ltx_td ltx_align_left" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.50.50.5.1" class="ltx_text" style="font-size:90%;">F1</span></td>
</tr>
<tr id="A3.T4.1.51.51" class="ltx_tr">
<td id="A3.T4.1.51.51.1" class="ltx_td ltx_align_left ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;" colspan="5">
<span id="A3.T4.1.51.51.1.1" class="ltx_text ltx_font_bold" style="font-size:90%;">GSM8K</span><span id="A3.T4.1.51.51.1.2" class="ltx_text" style="font-size:90%;"> (1 task)</span>
</td>
</tr>
<tr id="A3.T4.1.52.52" class="ltx_tr">
<td id="A3.T4.1.52.52.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.52.52.1.1" class="ltx_text" style="font-size:90%;">GSM8K</span></td>
<td id="A3.T4.1.52.52.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.52.52.2.1" class="ltx_text" style="font-size:90%;">8</span></td>
<td id="A3.T4.1.52.52.3" class="ltx_td ltx_align_right ltx_border_bb ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.52.52.3.1" class="ltx_text" style="font-size:90%;">7 473</span></td>
<td id="A3.T4.1.52.52.4" class="ltx_td ltx_align_right ltx_border_bb ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.52.52.4.1" class="ltx_text" style="font-size:90%;">1 319</span></td>
<td id="A3.T4.1.52.52.5" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t" style="padding-left:4.0pt;padding-right:4.0pt;"><span id="A3.T4.1.52.52.5.1" class="ltx_text" style="font-size:90%;">EM</span></td>
</tr>
</tbody>
</table>
<figcaption class="ltx_caption ltx_centering" style="font-size:90%;"><span class="ltx_tag ltx_tag_table">Table 4: </span>Evaluation tasks grouped by <span id="A3.T4.16.1" class="ltx_text ltx_font_bold">MMLU-Pro</span> (14 categories), <span id="A3.T4.17.2" class="ltx_text ltx_font_bold">MMLU</span> (17 categories), <span id="A3.T4.18.3" class="ltx_text ltx_font_bold">MC9</span> (9 multiple-choice), <span id="A3.T4.19.4" class="ltx_text ltx_font_bold">Gen5</span> (5 generation), and <span id="A3.T4.20.5" class="ltx_text ltx_font_bold">GSM8K</span>. <span id="A3.T4.21.6" class="ltx_text ltx_font_italic">Train+Val</span> reports the size of the dataset used during expert selection and finetuning. </figcaption>
</figure>
<section id="A3.SS0.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">MMLU and MMLU-Pro Setup.</h4>

<div id="A3.SS0.SSS0.Px1.p1" class="ltx_para">
<p id="A3.SS0.SSS0.Px1.p1.1" class="ltx_p">For MMLU and MMLU-Pro, we randomly sample 40% of examples for the validation/training set and use the remaining 60% for evaluation. For MMLU, we group the original 57 subjects into 17 broader categories following <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib30" title="Measuring massive multitask language understanding" class="ltx_ref">15</a>]</cite> to ensure sufficient examples per category to train and evaluate with. Expert selection is performed at the category level: all subjects within the same category are processed by a single group of experts. When reporting aggregate MMLU and MMLU-Pro results, we exclude the “other” category, which is discussed in Appendix <a href="#A2.SS3" title='B.3 Selective Expert Use on "Other" Category in MMLU and MMLU Pro ‣ Appendix B Selective expert use Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity' class="ltx_ref"><span class="ltx_text ltx_ref_tag">B.3</span></a>.</p>
</div>
</section>
<section id="A3.SS0.SSS0.Px2" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">Evaluation and Finetuning Protocol.</h4>

<div id="A3.SS0.SSS0.Px2.p1" class="ltx_para">
<p id="A3.SS0.SSS0.Px2.p1.1" class="ltx_p">We provide the full list of evaluation categories and subjects in Table <a href="#A3.T4" title="Table 4 ‣ Appendix C Evaluation Details ‣ Emo: Pretraining Mixture of Experts for Emergent Modularity" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4</span></a>. For multiple-choice benchmarks, we score each answer choice by its log-likelihood and select the highest-scoring option, reporting raw accuracy (“acc-raw”). For generation benchmarks, we report recall on Gen5 and exact match on <span id="A3.SS0.SSS0.Px2.p1.1.1" class="ltx_text ltx_font_smallcaps">GSM8K</span>.</p>
</div>
<div id="A3.SS0.SSS0.Px2.p2" class="ltx_para">
<p id="A3.SS0.SSS0.Px2.p2.1" class="ltx_p">During selective expert use, we use the same set of examples as both the validation data for expert selection and training data for finetuning. For tasks other than MMLU and MMLU-Pro, we merge the original train and validation splits and use the combined set for both expert selection and finetuning. When finetuning is performed, we mask the input portion of the prompt and optimize only over output tokens. Unless otherwise specified, finetuning follows standard Hugging Face settings with one epoch, batch size 32, and learning rate <math id="A3.SS0.SSS0.Px2.p2.1.m1" class="ltx_Math" alttext="5\times 10^{-5}" display="inline" intent=":literal"><semantics><mrow><mn>5</mn><mo lspace="0.222em" rspace="0.222em">×</mo><msup><mn>10</mn><mrow><mo>−</mo><mn>5</mn></mrow></msup></mrow><annotation encoding="application/x-tex">5\times 10^{-5}</annotation></semantics></math>.</p>
</div>
</section>
</section>
<section id="A4" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix D </span>Token Clustering Details</h2>

<section id="A4.SS0.SSS0.Px1" class="ltx_paragraph">
<h4 class="ltx_title ltx_title_paragraph">
<span id="A4.SS0.SSS0.Px1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> forms a Distinct Cluster for the First Token of Each Document.</h4>

<div id="A4.SS0.SSS0.Px1.p1" class="ltx_para">
<p id="A4.SS0.SSS0.Px1.p1.1" class="ltx_p">We observe a dedicated expert cluster consistently activated for the first token in each document. This behavior is intuitive, as the model processes the first token without any context. Interestingly, this cluster does not extend to the second token: in most cases after observing just one token, the model transitions into a specific expert activation pattern that often remains stable throughout the rest of the document. This suggests that <span id="A4.SS0.SSS0.Px1.p1.1.1" class="ltx_text ltx_font_smallcaps">Emo</span> rapidly commits to a document-level routing pattern after minimal context, with the first token serving as a distinct initialization phase before more specialized processing begins.</p>
</div>
<div class="ltx_pagination ltx_role_newpage"></div>
</section>
</section>
</article>
</div>
</div>
<footer class="arxiv-html-footer">
  <div class="ltx_page_logo">
    Experimental support, please
    <a href="./2605.06663v1/__stdout.txt" class="ltx_ref"
    target="_blank" rel="nofollow">view the build logs</a>
    for errors. Generated by
    <a href="https://math.nist.gov/~BMiller/LaTeXML/" target="_blank" class="ltx_ref ltx_LaTeXML_logo">
      <span style="letter-spacing: -0.2em; margin-right: 0.1em;">
        L
        <span style="font-size: 70%; position: relative; bottom: 2.2pt;">A</span>
        T
        <span style="position: relative; bottom: -0.4ex;">E</span>
      </span>
      <span class="ltx_font_smallcaps">xml</span>
      <img alt="[LOGO]"
        src="data:image/png;base64, ">
    </a>.
  </div>
  <div class="keyboard-glossary">
    <h2>Instructions for reporting errors</h2>
    <p>We are continuing to improve HTML versions of papers, and your feedback helps enhance accessibility and mobile
      support. To report errors in the HTML that will help us improve conversion and rendering, choose any of the
      methods listed below:</p>
    <ul>
      <li>Click the "Report Issue" <span class="mobile-only">(<svg role="presentation"
            style="display: inline-block; vertical-align: middle; fill: var(--text-color);" aria-hidden="true"
            height="1em" viewBox="0 0 640 640">
            <path
              d="M224 160C224 107 267 64 320 64C373 64 416 107 416 160L416 163.6C416 179.3 403.3 192 387.6 192L252.5 192C236.8 192 224.1 179.3 224.1 163.6L224.1 160zM569.6 172.8C580.2 186.9 577.3 207 563.2 217.6L465.4 290.9C470.7 299.8 474.7 309.6 477.2 320L576 320C593.7 320 608 334.3 608 352C608 369.7 593.7 384 576 384L480 384L480 416C480 418.6 479.9 421.3 479.8 423.9L563.2 486.4C577.3 497 580.2 517.1 569.6 531.2C559 545.3 538.9 548.2 524.8 537.6L461.7 490.3C438.5 534.5 395.2 566.5 344 574.2L344 344C344 330.7 333.3 320 320 320C306.7 320 296 330.7 296 344L296 574.2C244.8 566.5 201.5 534.5 178.3 490.3L115.2 537.6C101.1 548.2 81 545.3 70.4 531.2C59.8 517.1 62.7 497 76.8 486.4L160.2 423.9C160.1 421.3 160 418.7 160 416L160 384L64 384C46.3 384 32 369.7 32 352C32 334.3 46.3 320 64 320L162.8 320C165.3 309.6 169.3 299.8 174.6 290.9L76.8 217.6C62.7 207 59.8 186.9 70.4 172.8C81 158.7 101.1 155.8 115.2 166.4L224 248C236.3 242.9 249.8 240 264 240L376 240C390.2 240 403.7 242.8 416 248L524.8 166.4C538.9 155.8 559 158.7 569.6 172.8z" />
          </svg>)</span> button, located in the page header.</li>
    </ul>
    <p><strong>Tip:</strong> You can select the relevant text first, to include it in your report.</p>
    <p>Our team has already identified <a class="ltx_ref" href="https://github.com/arXiv/html_feedback/issues"
        target="_blank">the following issues</a>. We appreciate your time reviewing and reporting rendering errors we
      may not have found yet. Your efforts will help us improve the HTML versions for all readers, because disability
      should not be a barrier to accessing research. Thank you for your continued support in championing open access for
      all.</p>
    <p>Have a free development cycle? Help support accessibility at arXiv! Our collaborators at LaTeXML maintain a <a
        class="ltx_ref" href="https://github.com/brucemiller/LaTeXML/wiki/Porting-LaTeX-packages-for-LaTeXML"
        target="_blank">list of packages that need conversion</a>, and welcome <a class="ltx_ref"
        href="https://github.com/brucemiller/LaTeXML/issues" target="_blank">developer contributions</a>.</p>
  </div>
</footer>
<div id="fixed-buttons-container">
  <div id="beta-badge" aria-hidden="true">BETA</div>
  <a id="disable-reading-mode-btn" class="header-button" href="javascript:toggleReadingMode();"
    title="Disable reading mode, show header and footer">
    <svg role="presentation" height="1.25rem"
      viewBox="0 0 448 512"><!--!Font Awesome Free v7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2026 Fonticons, Inc.-->
      <path
        d="M0 96C0 78.3 14.3 64 32 64l384 0c17.7 0 32 14.3 32 32s-14.3 32-32 32L32 128C14.3 128 0 113.7 0 96zM0 256c0-17.7 14.3-32 32-32l384 0c17.7 0 32 14.3 32 32s-14.3 32-32 32L32 288c-17.7 0-32-14.3-32-32zM448 416c0 17.7-14.3 32-32 32L32 448c-17.7 0-32-14.3-32-32s14.3-32 32-32l384 0c17.7 0 32 14.3 32 32z" />
    </svg>
  </a>
</div></body>
</html>
