<!DOCTYPE html><html lang="en" data-color-mode="auto" data-light-theme="light" data-dark-theme="dark" data-a11y-animated-images="system" data-a11y-link-underlines="true"><head><style type="text/css">.turbo-progress-bar {
  position: fixed;
  display: block;
  top: 0;
  left: 0;
  height: 3px;
  background: #0076ff;
  z-index: 2147483647;
  transition:
    width 300ms ease-out,
    opacity 150ms 150ms ease-in;
  transform: translate3d(0, 0, 0);
}
</style>
    <meta charset="utf-8">
  <link rel="dns-prefetch" href="https://github.githubassets.com">
  <link rel="dns-prefetch" href="https://avatars.githubusercontent.com">
  <link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
  <link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
  <link rel="preconnect" href="https://github.githubassets.com" crossorigin="">
  <link rel="preconnect" href="https://avatars.githubusercontent.com">

  

  <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light-c59dc71e3a4c.css"><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light_high_contrast-4bf0cb726930.css"><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark-89751e879f8b.css"><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark_high_contrast-67c7180a598a.css"><link data-color-theme="light" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light-c59dc71e3a4c.css"><link data-color-theme="light_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_high_contrast-4bf0cb726930.css"><link data-color-theme="light_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind-6060e905eb78.css"><link data-color-theme="light_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind_high_contrast-04e818620b9c.css"><link data-color-theme="light_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia-ae65df249e0f.css"><link data-color-theme="light_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia_high_contrast-fdadc12a1ec2.css"><link data-color-theme="dark" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark-89751e879f8b.css"><link data-color-theme="dark_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_high_contrast-67c7180a598a.css"><link data-color-theme="dark_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind-4277e18a7c75.css"><link data-color-theme="dark_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind_high_contrast-2e33ed61bc8c.css"><link data-color-theme="dark_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia-48d44d87614d.css"><link data-color-theme="dark_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia_high_contrast-6adcb5080302.css"><link data-color-theme="dark_dimmed" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed-250cee4c1ea8.css"><link data-color-theme="dark_dimmed_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed_high_contrast-e3802beb8c06.css">

    <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-primitives-225433424a87.css">
    <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-b8b91660c29d.css">
    <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/global-86fb66cfa45a.css">
    <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/github-8152e1e2638f.css">
  <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repository-197a21528ff0.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-177d21388df8.css">

  


  <script type="application/json" id="client-env">{"locale":"en","featureFlags":["alternate_user_config_repo","api_insights_show_missing_data_banner","autocomplete_use_dom_input_range","codespaces_prebuild_region_target_update","contentful_lp_flex_features_actions","contentful_lp_flex_features_code_review","contentful_lp_flex_features_code_search","contentful_lp_flex_features_codespaces","contentful_lp_flex_features_discussions","contentful_lp_flex_features_issues","contentful_lp_footnotes","copilot_chat_attach_multiple_images","copilot_chat_custom_instructions","copilot_chat_repo_custom_instructions_preview","copilot_chat_vision_in_claude","copilot_chat_wholearea_dd","copilot_custom_copilots_feature_preview","copilot_custom_copilots_org_owned","copilot_custom_copilots_redesign","copilot_custom_copilots_visibility","copilot_duplicate_thread","copilot_free_to_paid_telem","copilot_header_button_to_immersive","copilot_immersive_draft_issue_metadata_location","copilot_immersive_draft_issue_template_required","copilot_immersive_issue_preview","copilot_new_immersive_references_ui","copilot_no_floating_button","copilot_read_shared_conversation","copilot_showcase_icebreakers","copilot_task_oriented_assistive_prompts","copilot_topics_as_references","copilot_ui_refs","direct_to_salesforce","dotcom_chat_client_side_skills","ghost_pilot_confidence_truncation_25","ghost_pilot_confidence_truncation_40","insert_before_patch","issues_react_blur_item_picker_on_close","issues_react_create_milestone","issues_react_prohibit_title_fallback","issues_react_remove_placeholders","lifecycle_label_name_updates","link_contact_sales_swp_marketo","marketing_pages_search_explore_provider","memex_mwl_filter_field_delimiter","nonreporting_relay_graphql_status_codes","primer_react_css_modules_ga","primer_react_select_panel_with_modern_action_list","remove_child_patch","sample_network_conn_type","send_app_type_header","site_msbuild_hide_integrations","site_msbuild_launch","site_msbuild_webgl_hero","swp_enterprise_contact_form","use_copilot_avatar","use_paginated_repo_picker_cost_center_form","viewscreen_sandbox"]}</script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/wp-runtime-5444b0a4f954.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_oddbird_popover-polyfill_dist_popover-fn_js-81211bd82278.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_stacktrace-parser_dist_s-1d3d52-4be8ffe9a34a.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_failbot_failbot_ts-3f3eb0fe798c.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/environment-1d165312d5cd.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_primer_behaviors_dist_esm_index_mjs-0dbb79f97f8f.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_selector-observer_dist_index_esm_js-f690fd9ae3d5.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_relative-time-element_dist_index_js-fd884d19bf73.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_text-expander-element_dist_index_js-b4ed7224d804.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_auto-complete-element_dist_index_js-node_modules_github_catalyst_-8e9f78-a90ac05d2469.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_filter-input-element_dist_index_js-node_modules_github_remote-inp-d8c643-f5192902810f.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_markdown-toolbar-element_dist_index_js-ceef33f593fa.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_github_combobox-nav_dist-97536f-0a0158d4c78e.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_file-attachment-element_dist_index_js-node_modules_primer_view-co-63644b-8389d8c5493d.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/github-elements-2c6b561fd398.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/element-registry-759bd527e556.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_braintree_browser-detection_dist_browser-detection_js-node_modules_githu-bb80ec-72267f4e3ff9.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_lit-html_lit-html_js-be8cb88f481b.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_morphdom_dist_morphdom-esm_js-0c08218c7d5f.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_fzy_js_index_js-node_modules_github_paste-markdown_dist_index_js-6c00013a3dc4.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_turbo_dist_turbo_es2017-esm_js-a03ee12d659a.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_remote-form_dist_index_js-node_modules_delegated-events_dist_inde-893f9f-b6294cf703b7.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_color-convert_index_js-e3180fe3bcb3.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_quote-selection_dist_index_js-node_modules_github_session-resume_-c1aa61-97c8ff49bc41.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_updatable-content_updatable-content_ts-f4e60782b52d.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_behaviors_task-list_ts-app_assets_modules_github_sso_ts-ui_packages-900dde-c84fbefd392e.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_sticky-scroll-into-view_ts-3e000c5d31a9.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_behaviors_ajax-error_ts-app_assets_modules_github_behaviors_include-d0d0a6-9dc4ed803189.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_behaviors_commenting_edit_ts-app_assets_modules_github_behaviors_ht-83c235-ce8979e4f922.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/behaviors-b3ff01fc446e.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_delegated-events_dist_index_js-node_modules_github_catalyst_lib_index_js-ea8eaa-ae0dbe787ad8.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/notifications-global-7f3748c0d76b.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_virtualized-list_es_inde-5cfb7e-e6b0d3ff3d24.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_remote-form_dist_index_js-node_modules_delegated-events_dist_inde-70450e-4b93df70b903.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_ref-selector_ts-351a8f8ec24b.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/codespaces-1395fc4c5646.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_filter-input-element_dist_index_js-node_modules_github_remote-inp-3eebbd-0763620ad7bf.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_decorators_js-node_modules_delegated-events_di-e161aa-9d41fb1b6c9e.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_file-attachment-element_dist_index_js-node_modules_github_remote--3c9c82-b71ef90fbdc7.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/repositories-f58553097890.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_github_catalyst_lib_inde-dbbea9-4adcaf6c1c40.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/code-menu-7cd84ed05e00.js"></script>
  
  <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/primer-react-93a37a6984ad.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/react-core-895d1d63cc59.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/react-lib-80430c87778a.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/octicons-react-cf2f2ab8dab4.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_emotion_is-prop-valid_dist_emotion-is-prop-valid_esm_js-node_modules_emo-b1c483-30bc59af3a76.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_catalyst_lib_index_js-node_modules_primer_live-region-element_dis-b2aea6-22827f205bdc.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/notifications-subscriptions-menu-289430569e8f.js"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.ad18e42c1b06f95e74e5.module.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.e5e6e593370c808590a5.module.css">

  <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.ad18e42c1b06f95e74e5.module.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.e5e6e593370c808590a5.module.css">


  <title>GitHub - OSU-NLP-Group/GUI-Agents-Paper-List: Building a comprehensive and handy list of papers for GUI agents</title>



  <meta name="route-pattern" content="/:user_id/:repository" data-turbo-transient="">
  <meta name="route-controller" content="files" data-turbo-transient="">
  <meta name="route-action" content="disambiguate" data-turbo-transient="">
  <meta name="fetch-nonce" content="v2:073dc60b-75ed-517e-12ed-f8b77db3c2d0">

    
  <meta name="current-catalog-service-hash" content="f3abb0cc802f3d7b95fc8762b94bdcb13bf39634c40c357301c4aa1d67a256fb">


  <meta name="request-id" content="C376:3529C5:1AB6B48:1F291FF:68397F49" data-pjax-transient="true"><meta name="html-safe-nonce" content="a51cb1883a970550d32b3f7ffbf0792199eb9a49a2d822b699d570f763a8cecc" data-pjax-transient="true"><meta name="visitor-payload" content="eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiJDMzc2OjM1MjlDNToxQUI2QjQ4OjFGMjkxRkY6NjgzOTdGNDkiLCJ2aXNpdG9yX2lkIjoiNjUyMjg5OTMxMjQ1MDA0MzcyMSIsInJlZ2lvbl9lZGdlIjoic291dGhlYXN0YXNpYSIsInJlZ2lvbl9yZW5kZXIiOiJzb3V0aGVhc3Rhc2lhIn0=" data-pjax-transient="true"><meta name="visitor-hmac" content="8fdbf0e80ca73d24202686ae1621f64f276b12ae76185b097e2c575532e121a9" data-pjax-transient="true">


    <meta name="hovercard-subject-tag" content="repository:880854580" data-turbo-transient="">


  <meta name="github-keyboard-shortcuts" content="repository,copilot" data-turbo-transient="true">
  

  <meta name="selected-link" value="repo_source" data-turbo-transient="">
  <link rel="assets" href="https://github.githubassets.com/">

    <meta name="google-site-verification" content="Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I">

<meta name="octolytics-url" content="https://collector.github.com/github/collect">

  <meta name="analytics-location" content="/<user-name>/<repo-name>" data-turbo-transient="true">

  




    <meta name="user-login" content="">

  

    <meta name="viewport" content="width=device-width">

    

      <meta name="description" content=" Building a comprehensive and handy list of papers for GUI agents - OSU-NLP-Group/GUI-Agents-Paper-List">

      <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">

    <link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
    <meta property="fb:app_id" content="1401488693436528">
    <meta name="apple-itunes-app" content="app-id=1477376905, app-argument=https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List">

      <meta name="twitter:image" content="https://opengraph.githubassets.com/d8b3d8197f2173af85369c1f1327eab2208d20b50740352d4a69a752daa86fb3/OSU-NLP-Group/GUI-Agents-Paper-List"><meta name="twitter:site" content="@github"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:title" content="GitHub - OSU-NLP-Group/GUI-Agents-Paper-List: Building a comprehensive and handy list of papers for GUI agents"><meta name="twitter:description" content=" Building a comprehensive and handy list of papers for GUI agents - OSU-NLP-Group/GUI-Agents-Paper-List">
  <meta property="og:image" content="https://opengraph.githubassets.com/d8b3d8197f2173af85369c1f1327eab2208d20b50740352d4a69a752daa86fb3/OSU-NLP-Group/GUI-Agents-Paper-List"><meta property="og:image:alt" content=" Building a comprehensive and handy list of papers for GUI agents - OSU-NLP-Group/GUI-Agents-Paper-List"><meta property="og:image:width" content="1200"><meta property="og:image:height" content="600"><meta property="og:site_name" content="GitHub"><meta property="og:type" content="object"><meta property="og:title" content="GitHub - OSU-NLP-Group/GUI-Agents-Paper-List: Building a comprehensive and handy list of papers for GUI agents"><meta property="og:url" content="https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List"><meta property="og:description" content=" Building a comprehensive and handy list of papers for GUI agents - OSU-NLP-Group/GUI-Agents-Paper-List">
  




      <meta name="hostname" content="github.com">



        <meta name="expected-hostname" content="github.com">


  <meta http-equiv="x-pjax-version" content="9334a43936e35fe8ea90579e22968a05431e269ce5937af66394b836608182dd" data-turbo-track="reload">
  <meta http-equiv="x-pjax-csp-version" content="352e51c42d5f5727a7c545752bf34d1f83f40219e7036c6959817149a51651bc" data-turbo-track="reload">
  <meta http-equiv="x-pjax-css-version" content="c296e654ea989d71f05d687cb74f389a531f17ce0b1cf5bfe0ee83c347e56289" data-turbo-track="reload">
  <meta http-equiv="x-pjax-js-version" content="c903932696737dc05c184b747458feb1ad9f938b8ca30e703c82e387c104b64a" data-turbo-track="reload">

  <meta name="turbo-cache-control" content="no-preview" data-turbo-transient="">

      <meta data-hydrostats="publish">
  <meta name="go-import" content="github.com/OSU-NLP-Group/GUI-Agents-Paper-List git https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List.git">

  <meta name="octolytics-dimension-user_id" content="92067480"><meta name="octolytics-dimension-user_login" content="OSU-NLP-Group"><meta name="octolytics-dimension-repository_id" content="880854580"><meta name="octolytics-dimension-repository_nwo" content="OSU-NLP-Group/GUI-Agents-Paper-List"><meta name="octolytics-dimension-repository_public" content="true"><meta name="octolytics-dimension-repository_is_fork" content="false"><meta name="octolytics-dimension-repository_network_root_id" content="880854580"><meta name="octolytics-dimension-repository_network_root_nwo" content="OSU-NLP-Group/GUI-Agents-Paper-List">



      <link rel="canonical" href="https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List" data-turbo-transient="">


    <meta name="turbo-body-classes" content="logged-out env-production page-responsive">


  <meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">

  <meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">

  <meta name="release" content="87a4e1efe5b6dedd3997d9fba0c770172d581e8f">

  <link rel="mask-icon" href="https://github.githubassets.com/assets/pinned-octocat-093da3e6fa40.svg" color="#000000">
  <link rel="alternate icon" class="js-site-favicon" type="image/png" href="https://github.githubassets.com/favicons/favicon.png">
  <link rel="icon" class="js-site-favicon" type="image/svg+xml" href="https://github.githubassets.com/favicons/favicon.svg" data-base-href="https://github.githubassets.com/favicons/favicon">

<meta name="theme-color" content="#1e2327">
<meta name="color-scheme" content="light dark">


  <link rel="manifest" href="/manifest.json" crossorigin="use-credentials">

  <style data-styled="active" data-styled-version="5.3.11"></style><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-app_components_primer_experimental_select-panel-element_ts-fc23781c574e.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-app_components_primer_experimental_toggle-switch-element_ts-670c71d392c6.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-app_components_search_custom-scopes-element_ts-abc100eaa2cb.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-vendors-node_modules_github_hydro-analytics-client_dist_analytics-client_js-node_modules_gith-f3aee1-43d8039b50ae.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/ui_packages_query-builder-element_query-builder-element_ts-b492d6900d5e.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/app_assets_modules_github_blob-anchor_ts-ui_packages_code-nav_code-nav_ts-ui_packages_filter--8253c1-91468a3354f9.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_hydro-analytics_hydro-analytics_ts-ui_packages_jump-to-element_model_ts-af345b862138.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-app_components_search_qbsearch-input-element_ts-8528e7b1787c.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_cookie-consent-link-element_element-entry_ts-8ca582ddd98a.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_ghcc-consent-element_element-entry_ts-04338159da93.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_markdown-accessiblity-table-element_element-entry_ts-54748ea4a7e7.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_github_combobox-nav_dist-5f477b-6f36b76e0632.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_hydro-analytics_hydro-analytics_ts-ui_packages_query-builder-element_element-entry_ts-7772c0bbefed.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_react-partial-anchor-element_element-entry_ts-4fea8fb23f5c.js" crossorigin="anonymous"></script><script charset="utf-8" src="https://github.githubassets.com/assets/chunk-ui_packages_webauthn-get-element_element-entry_ts-c4e5a9b4f95a.js" crossorigin="anonymous"></script></head>

  <body class="logged-out env-production page-responsive" style="overflow-wrap: break-word; --dialog-scrollgutter: 0px;">
    <div data-turbo-body="" class="logged-out env-production page-responsive" style="word-wrap: break-word;">
      



    <div class="position-relative header-wrapper js-header-wrapper ">
      <a href="#start-of-content" data-skip-target-assigned="false" class="px-2 py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content">Skip to content</a>

      <span data-view-component="true" class="progress-pjax-loader Progress position-fixed width-full">
    <span style="width: 0%;" data-view-component="true" class="Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis"></span>
</span>      
      
      <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_ui-commands_ui-commands_ts-392f523dd6f6.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/keyboard-shortcuts-dialog-481f18f97e80.js"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.ad18e42c1b06f95e74e5.module.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/keyboard-shortcuts-dialog.47de85e2c17af43cefd5.module.css">

<react-partial partial-name="keyboard-shortcuts-dialog" data-ssr="false" data-attempted-ssr="false" data-catalyst="" class="loaded">
  
  <script type="application/json" data-target="react-partial.embeddedData">{"props":{"docsUrl":"https://docs.github.com/get-started/accessibility/keyboard-shortcuts"}}</script>
  <div data-target="react-partial.reactRoot"><div class="d-none"></div><script type="application/json" id="__PRIMER_DATA_:r0:__">{"resolvedServerColorMode":"day"}</script></div>
</react-partial>




      

          

              
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_remote-form_dist_index_js-node_modules_delegated-events_dist_inde-94fd67-e789af5a4655.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/sessions-1e75b15ae60a.js"></script>
<header class="HeaderMktg header-logged-out js-details-container js-header Details f4 py-3" role="banner" data-is-top="true" data-color-mode="light" data-light-theme="light" data-dark-theme="dark">
  <h2 class="sr-only">Navigation Menu</h2>

  <button type="button" class="HeaderMktg-backdrop d-lg-none border-0 position-fixed top-0 left-0 width-full height-full js-details-target" aria-label="Toggle navigation">
    <span class="d-none">Toggle navigation</span>
  </button>

  <div class="d-flex flex-column flex-lg-row flex-items-center px-3 px-md-4 px-lg-5 height-full position-relative z-1">
    <div class="d-flex flex-justify-between flex-items-center width-full width-lg-auto">
      <div class="flex-1">
        <button aria-label="Toggle navigation" aria-expanded="false" type="button" data-view-component="true" class="js-details-target js-nav-padding-recalculate js-header-menu-toggle Button--link Button--medium Button d-lg-none color-fg-inherit p-1">  <span class="Button-content">
    <span class="Button-label"><div class="HeaderMenu-toggle-bar rounded my-1"></div>
            <div class="HeaderMenu-toggle-bar rounded my-1"></div>
            <div class="HeaderMenu-toggle-bar rounded my-1"></div></span>
  </span>
</button>
      </div>

      <a class="mr-lg-3 color-fg-inherit flex-order-2 js-prevent-focus-on-mobile-nav" href="/" aria-label="Homepage" data-analytics-event="{&quot;category&quot;:&quot;Marketing nav&quot;,&quot;action&quot;:&quot;click to go to homepage&quot;,&quot;label&quot;:&quot;ref_page:Marketing;ref_cta:Logomark;ref_loc:Header&quot;}">
        <svg height="32" aria-hidden="true" viewBox="0 0 24 24" version="1.1" width="32" data-view-component="true" class="octicon octicon-mark-github">
    <path d="M12 1C5.9225 1 1 5.9225 1 12C1 16.8675 4.14875 20.9787 8.52125 22.4362C9.07125 22.5325 9.2775 22.2025 9.2775 21.9137C9.2775 21.6525 9.26375 20.7862 9.26375 19.865C6.5 20.3737 5.785 19.1912 5.565 18.5725C5.44125 18.2562 4.905 17.28 4.4375 17.0187C4.0525 16.8125 3.5025 16.3037 4.42375 16.29C5.29 16.2762 5.90875 17.0875 6.115 17.4175C7.105 19.0812 8.68625 18.6137 9.31875 18.325C9.415 17.61 9.70375 17.1287 10.02 16.8537C7.5725 16.5787 5.015 15.63 5.015 11.4225C5.015 10.2262 5.44125 9.23625 6.1425 8.46625C6.0325 8.19125 5.6475 7.06375 6.2525 5.55125C6.2525 5.55125 7.17375 5.2625 9.2775 6.67875C10.1575 6.43125 11.0925 6.3075 12.0275 6.3075C12.9625 6.3075 13.8975 6.43125 14.7775 6.67875C16.8813 5.24875 17.8025 5.55125 17.8025 5.55125C18.4075 7.06375 18.0225 8.19125 17.9125 8.46625C18.6138 9.23625 19.04 10.2125 19.04 11.4225C19.04 15.6437 16.4688 16.5787 14.0213 16.8537C14.42 17.1975 14.7638 17.8575 14.7638 18.8887C14.7638 20.36 14.75 21.5425 14.75 21.9137C14.75 22.2025 14.9563 22.5462 15.5063 22.4362C19.8513 20.9787 23 16.8537 23 12C23 5.9225 18.0775 1 12 1Z"></path>
</svg>
      </a>

      <div class="d-flex flex-1 flex-order-2 text-right d-lg-none gap-2 flex-justify-end">
          <a href="/login?return_to=https%3A%2F%2Fgithub.com%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" class="HeaderMenu-link HeaderMenu-button d-inline-flex f5 no-underline border color-border-default rounded-2 px-2 py-1 color-fg-inherit js-prevent-focus-on-mobile-nav" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;site header menu&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;SIGN_UP&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="1a3cedb20582af7daca24b759b032fbdf821aef3048538a1d15564ff96577f3b" data-analytics-event="{&quot;category&quot;:&quot;Marketing nav&quot;,&quot;action&quot;:&quot;click to Sign in&quot;,&quot;label&quot;:&quot;ref_page:Marketing;ref_cta:Sign in;ref_loc:Header&quot;}">
            Sign in
          </a>
              <div class="AppHeader-appearanceSettings">
    <react-partial-anchor>
      <button data-target="react-partial-anchor.anchor" id="icon-button-da55e875-3a6d-4c85-a073-e23cecf69635" aria-labelledby="tooltip-dbbf2db2-3973-456b-8783-b7deb2e90016" type="button" disabled="disabled" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium AppHeader-button HeaderMenu-link border cursor-wait">  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-sliders Button-visual">
    <path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path>
</svg>
</button><tool-tip id="tooltip-dbbf2db2-3973-456b-8783-b7deb2e90016" for="icon-button-da55e875-3a6d-4c85-a073-e23cecf69635" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute" aria-hidden="true" role="tooltip">Appearance settings</tool-tip>

      <template data-target="react-partial-anchor.template">
        <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_document-metadata_document-metadata_ts-ui_packages_promise-with-resolvers-polyfil-40d47c-83bb4d2a9499.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/appearance-settings-99fa071d4285.js"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.ad18e42c1b06f95e74e5.module.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/appearance-settings.22dfbc22ef0a2bf02523.module.css">

<react-partial partial-name="appearance-settings" data-ssr="false" data-attempted-ssr="false">
  
  <script type="application/json" data-target="react-partial.embeddedData">{"props":{}}</script>
  <div data-target="react-partial.reactRoot"></div>
</react-partial>

      </template>
    </react-partial-anchor>
  </div>

      </div>
    </div>


    <div class="HeaderMenu js-header-menu height-fit position-lg-relative d-lg-flex flex-column flex-auto top-0">
      <div class="HeaderMenu-wrapper d-flex flex-column flex-self-start flex-lg-row flex-auto rounded rounded-lg-0">
          <nav class="HeaderMenu-nav" aria-label="Global">
            <ul class="d-lg-flex list-style-none">


                <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item">
      <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false">
        Product
        <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1">
    <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path>
</svg>
      </button>

      <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 d-lg-flex flex-wrap dropdown-menu-wide">
          <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0">

                <ul class="list-style-none f5">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;github_copilot&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;github_copilot_link_product_navbar&quot;}" href="https://github.com/features/copilot">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-copilot color-fg-subtle mr-3">
    <path d="M23.922 16.992c-.861 1.495-5.859 5.023-11.922 5.023-6.063 0-11.061-3.528-11.922-5.023A.641.641 0 0 1 0 16.736v-2.869a.841.841 0 0 1 .053-.22c.372-.935 1.347-2.292 2.605-2.656.167-.429.414-1.055.644-1.517a10.195 10.195 0 0 1-.052-1.086c0-1.331.282-2.499 1.132-3.368.397-.406.89-.717 1.474-.952 1.399-1.136 3.392-2.093 6.122-2.093 2.731 0 4.767.957 6.166 2.093.584.235 1.077.546 1.474.952.85.869 1.132 2.037 1.132 3.368 0 .368-.014.733-.052 1.086.23.462.477 1.088.644 1.517 1.258.364 2.233 1.721 2.605 2.656a.832.832 0 0 1 .053.22v2.869a.641.641 0 0 1-.078.256ZM12.172 11h-.344a4.323 4.323 0 0 1-.355.508C10.703 12.455 9.555 13 7.965 13c-1.725 0-2.989-.359-3.782-1.259a2.005 2.005 0 0 1-.085-.104L4 11.741v6.585c1.435.779 4.514 2.179 8 2.179 3.486 0 6.565-1.4 8-2.179v-6.585l-.098-.104s-.033.045-.085.104c-.793.9-2.057 1.259-3.782 1.259-1.59 0-2.738-.545-3.508-1.492a4.323 4.323 0 0 1-.355-.508h-.016.016Zm.641-2.935c.136 1.057.403 1.913.878 2.497.442.544 1.134.938 2.344.938 1.573 0 2.292-.337 2.657-.751.384-.435.558-1.15.558-2.361 0-1.14-.243-1.847-.705-2.319-.477-.488-1.319-.862-2.824-1.025-1.487-.161-2.192.138-2.533.529-.269.307-.437.808-.438 1.578v.021c0 .265.021.562.063.893Zm-1.626 0c.042-.331.063-.628.063-.894v-.02c-.001-.77-.169-1.271-.438-1.578-.341-.391-1.046-.69-2.533-.529-1.505.163-2.347.537-2.824 1.025-.462.472-.705 1.179-.705 2.319 0 1.211.175 1.926.558 2.361.365.414 1.084.751 2.657.751 1.21 0 1.902-.394 2.344-.938.475-.584.742-1.44.878-2.497Z"></path><path d="M14.5 14.25a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Zm-5 0a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            GitHub Copilot
          </div>
        Write better code with AI
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;github_models&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;github_models_link_product_navbar&quot;}" href="https://github.com/features/models">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-ai-model color-fg-subtle mr-3">
    <path d="M19.375 8.5a3.25 3.25 0 1 1-3.163 4h-3a3.252 3.252 0 0 1-4.443 2.509L7.214 17.76a3.25 3.25 0 1 1-1.342-.674l1.672-2.957A3.238 3.238 0 0 1 6.75 12c0-.907.371-1.727.97-2.316L6.117 6.846A3.253 3.253 0 0 1 1.875 3.75a3.25 3.25 0 1 1 5.526 2.32l1.603 2.836A3.25 3.25 0 0 1 13.093 11h3.119a3.252 3.252 0 0 1 3.163-2.5ZM10 10.25a1.75 1.75 0 1 0-.001 3.499A1.75 1.75 0 0 0 10 10.25ZM5.125 2a1.75 1.75 0 1 0 0 3.5 1.75 1.75 0 0 0 0-3.5Zm12.5 9.75a1.75 1.75 0 1 0 3.5 0 1.75 1.75 0 0 0-3.5 0Zm-14.25 8.5a1.75 1.75 0 1 0 3.501-.001 1.75 1.75 0 0 0-3.501.001Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            GitHub Models
              <span class="HeaderMenu-label">
                New
              </span>
          </div>
        Manage and compare prompts
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;github_advanced_security&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;github_advanced_security_link_product_navbar&quot;}" href="https://github.com/security/advanced-security">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-shield-check color-fg-subtle mr-3">
    <path d="M16.53 9.78a.75.75 0 0 0-1.06-1.06L11 13.19l-1.97-1.97a.75.75 0 0 0-1.06 1.06l2.5 2.5a.75.75 0 0 0 1.06 0l5-5Z"></path><path d="m12.54.637 8.25 2.675A1.75 1.75 0 0 1 22 4.976V10c0 6.19-3.771 10.704-9.401 12.83a1.704 1.704 0 0 1-1.198 0C5.77 20.705 2 16.19 2 10V4.976c0-.758.489-1.43 1.21-1.664L11.46.637a1.748 1.748 0 0 1 1.08 0Zm-.617 1.426-8.25 2.676a.249.249 0 0 0-.173.237V10c0 5.46 3.28 9.483 8.43 11.426a.199.199 0 0 0 .14 0C17.22 19.483 20.5 15.461 20.5 10V4.976a.25.25 0 0 0-.173-.237l-8.25-2.676a.253.253 0 0 0-.154 0Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            GitHub Advanced Security
          </div>
        Find and fix vulnerabilities
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;actions&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;actions_link_product_navbar&quot;}" href="https://github.com/features/actions">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-workflow color-fg-subtle mr-3">
    <path d="M1 3a2 2 0 0 1 2-2h6.5a2 2 0 0 1 2 2v6.5a2 2 0 0 1-2 2H7v4.063C7 16.355 7.644 17 8.438 17H12.5v-2.5a2 2 0 0 1 2-2H21a2 2 0 0 1 2 2V21a2 2 0 0 1-2 2h-6.5a2 2 0 0 1-2-2v-2.5H8.437A2.939 2.939 0 0 1 5.5 15.562V11.5H3a2 2 0 0 1-2-2Zm2-.5a.5.5 0 0 0-.5.5v6.5a.5.5 0 0 0 .5.5h6.5a.5.5 0 0 0 .5-.5V3a.5.5 0 0 0-.5-.5ZM14.5 14a.5.5 0 0 0-.5.5V21a.5.5 0 0 0 .5.5H21a.5.5 0 0 0 .5-.5v-6.5a.5.5 0 0 0-.5-.5Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Actions
          </div>
        Automate any workflow
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;codespaces&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;codespaces_link_product_navbar&quot;}" href="https://github.com/features/codespaces">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-codespaces color-fg-subtle mr-3">
    <path d="M3.5 3.75C3.5 2.784 4.284 2 5.25 2h13.5c.966 0 1.75.784 1.75 1.75v7.5A1.75 1.75 0 0 1 18.75 13H5.25a1.75 1.75 0 0 1-1.75-1.75Zm-2 12c0-.966.784-1.75 1.75-1.75h17.5c.966 0 1.75.784 1.75 1.75v4a1.75 1.75 0 0 1-1.75 1.75H3.25a1.75 1.75 0 0 1-1.75-1.75ZM5.25 3.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h13.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Zm-2 12a.25.25 0 0 0-.25.25v4c0 .138.112.25.25.25h17.5a.25.25 0 0 0 .25-.25v-4a.25.25 0 0 0-.25-.25Z"></path><path d="M10 17.75a.75.75 0 0 1 .75-.75h6.5a.75.75 0 0 1 0 1.5h-6.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Codespaces
          </div>
        Instant dev environments
      </div>

    
</a></li>

                </ul>
              </div>
          </div>
          <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0">

                <ul class="list-style-none f5">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;issues&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;issues_link_product_navbar&quot;}" href="https://github.com/features/issues">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-issue-opened color-fg-subtle mr-3">
    <path d="M12 1c6.075 0 11 4.925 11 11s-4.925 11-11 11S1 18.075 1 12 5.925 1 12 1ZM2.5 12a9.5 9.5 0 0 0 9.5 9.5 9.5 9.5 0 0 0 9.5-9.5A9.5 9.5 0 0 0 12 2.5 9.5 9.5 0 0 0 2.5 12Zm9.5 2a2 2 0 1 1-.001-3.999A2 2 0 0 1 12 14Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Issues
          </div>
        Plan and track work
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;code_review&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;code_review_link_product_navbar&quot;}" href="https://github.com/features/code-review">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-code-review color-fg-subtle mr-3">
    <path d="M10.3 6.74a.75.75 0 0 1-.04 1.06l-2.908 2.7 2.908 2.7a.75.75 0 1 1-1.02 1.1l-3.5-3.25a.75.75 0 0 1 0-1.1l3.5-3.25a.75.75 0 0 1 1.06.04Zm3.44 1.06a.75.75 0 1 1 1.02-1.1l3.5 3.25a.75.75 0 0 1 0 1.1l-3.5 3.25a.75.75 0 1 1-1.02-1.1l2.908-2.7-2.908-2.7Z"></path><path d="M1.5 4.25c0-.966.784-1.75 1.75-1.75h17.5c.966 0 1.75.784 1.75 1.75v12.5a1.75 1.75 0 0 1-1.75 1.75h-9.69l-3.573 3.573A1.458 1.458 0 0 1 5 21.043V18.5H3.25a1.75 1.75 0 0 1-1.75-1.75ZM3.25 4a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h2.5a.75.75 0 0 1 .75.75v3.19l3.72-3.72a.749.749 0 0 1 .53-.22h10a.25.25 0 0 0 .25-.25V4.25a.25.25 0 0 0-.25-.25Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Code Review
          </div>
        Manage code changes
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;discussions&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;discussions_link_product_navbar&quot;}" href="https://github.com/features/discussions">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-comment-discussion color-fg-subtle mr-3">
    <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 14.25 14H8.061l-2.574 2.573A1.458 1.458 0 0 1 3 15.543V14H1.75A1.75 1.75 0 0 1 0 12.25v-9.5C0 1.784.784 1 1.75 1ZM1.5 2.75v9.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h6.5a.25.25 0 0 0 .25-.25v-9.5a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Z"></path><path d="M22.5 8.75a.25.25 0 0 0-.25-.25h-3.5a.75.75 0 0 1 0-1.5h3.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 22.25 20H21v1.543a1.457 1.457 0 0 1-2.487 1.03L15.939 20H10.75A1.75 1.75 0 0 1 9 18.25v-1.465a.75.75 0 0 1 1.5 0v1.465c0 .138.112.25.25.25h5.5a.75.75 0 0 1 .53.22l2.72 2.72v-2.19a.75.75 0 0 1 .75-.75h2a.25.25 0 0 0 .25-.25v-9.5Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Discussions
          </div>
        Collaborate outside of code
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;code_search&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;code_search_link_product_navbar&quot;}" href="https://github.com/features/code-search">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-code-square color-fg-subtle mr-3">
    <path d="M10.3 8.24a.75.75 0 0 1-.04 1.06L7.352 12l2.908 2.7a.75.75 0 1 1-1.02 1.1l-3.5-3.25a.75.75 0 0 1 0-1.1l3.5-3.25a.75.75 0 0 1 1.06.04Zm3.44 1.06a.75.75 0 1 1 1.02-1.1l3.5 3.25a.75.75 0 0 1 0 1.1l-3.5 3.25a.75.75 0 1 1-1.02-1.1l2.908-2.7-2.908-2.7Z"></path><path d="M2 3.75C2 2.784 2.784 2 3.75 2h16.5c.966 0 1.75.784 1.75 1.75v16.5A1.75 1.75 0 0 1 20.25 22H3.75A1.75 1.75 0 0 1 2 20.25Zm1.75-.25a.25.25 0 0 0-.25.25v16.5c0 .138.112.25.25.25h16.5a.25.25 0 0 0 .25-.25V3.75a.25.25 0 0 0-.25-.25Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Code Search
          </div>
        Find more, search less
      </div>

    
</a></li>

                </ul>
              </div>
          </div>
          <div class="HeaderMenu-column px-lg-4">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0 border-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="product-explore-heading">Explore</span>

                <ul class="list-style-none f5" aria-labelledby="product-explore-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;why_github&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;why_github_link_product_navbar&quot;}" href="https://github.com/why-github">
      Why GitHub

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;all_features&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;all_features_link_product_navbar&quot;}" href="https://github.com/features">
      All features

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;documentation&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;documentation_link_product_navbar&quot;}" href="https://docs.github.com">
      Documentation

    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle">
    <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path>
</svg>
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;github_skills&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;github_skills_link_product_navbar&quot;}" href="https://skills.github.com">
      GitHub Skills

    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle">
    <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path>
</svg>
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;blog&quot;,&quot;context&quot;:&quot;product&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;blog_link_product_navbar&quot;}" href="https://github.blog">
      Blog

    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle">
    <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path>
</svg>
</a></li>

                </ul>
              </div>
          </div>

      </div>
</li>


                <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item">
      <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false">
        Solutions
        <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1">
    <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path>
</svg>
      </button>

      <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 d-lg-flex flex-wrap dropdown-menu-wide">
          <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0 pb-lg-3 mb-3 mb-lg-0">
                    <span class="d-block h4 color-fg-default my-1" id="solutions-by-company-size-heading">By company size</span>

                <ul class="list-style-none f5" aria-labelledby="solutions-by-company-size-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;enterprises&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;enterprises_link_solutions_navbar&quot;}" href="https://github.com/enterprise">
      Enterprises

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;small_and_medium_teams&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;small_and_medium_teams_link_solutions_navbar&quot;}" href="https://github.com/team">
      Small and medium teams

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;startups&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;startups_link_solutions_navbar&quot;}" href="https://github.com/enterprise/startups">
      Startups

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;nonprofits&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;nonprofits_link_solutions_navbar&quot;}" href="/solutions/industry/nonprofits">
      Nonprofits

    
</a></li>

                </ul>
              </div>
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="solutions-by-use-case-heading">By use case</span>

                <ul class="list-style-none f5" aria-labelledby="solutions-by-use-case-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;devsecops&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;devsecops_link_solutions_navbar&quot;}" href="/solutions/use-case/devsecops">
      DevSecOps

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;devops&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;devops_link_solutions_navbar&quot;}" href="/solutions/use-case/devops">
      DevOps

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;ci_cd&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;ci_cd_link_solutions_navbar&quot;}" href="/solutions/use-case/ci-cd">
      CI/CD

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;view_all_use_cases&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;view_all_use_cases_link_solutions_navbar&quot;}" href="/solutions/use-case">
      View all use cases

    
</a></li>

                </ul>
              </div>
          </div>
          <div class="HeaderMenu-column px-lg-4">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="solutions-by-industry-heading">By industry</span>

                <ul class="list-style-none f5" aria-labelledby="solutions-by-industry-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;healthcare&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;healthcare_link_solutions_navbar&quot;}" href="/solutions/industry/healthcare">
      Healthcare

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;financial_services&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;financial_services_link_solutions_navbar&quot;}" href="/solutions/industry/financial-services">
      Financial services

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;manufacturing&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;manufacturing_link_solutions_navbar&quot;}" href="/solutions/industry/manufacturing">
      Manufacturing

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;government&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;government_link_solutions_navbar&quot;}" href="/solutions/industry/government">
      Government

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;view_all_industries&quot;,&quot;context&quot;:&quot;solutions&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;view_all_industries_link_solutions_navbar&quot;}" href="/solutions/industry">
      View all industries

    
</a></li>

                </ul>
              </div>
          </div>

         <div class="HeaderMenu-trailing-link rounded-bottom-2 flex-shrink-0 mt-lg-4 px-lg-4 py-4 py-lg-3 f5 text-semibold">
            <a href="/solutions">
              View all solutions
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-right HeaderMenu-trailing-link-icon">
    <path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</a>         </div>
      </div>
</li>


                <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item">
      <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false">
        Resources
        <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1">
    <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path>
</svg>
      </button>

      <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 d-lg-flex flex-wrap dropdown-menu-wide">
          <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="resources-topics-heading">Topics</span>

                <ul class="list-style-none f5" aria-labelledby="resources-topics-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;ai&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;ai_link_resources_navbar&quot;}" href="/resources/articles/ai">
      AI

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;devops&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;devops_link_resources_navbar&quot;}" href="/resources/articles/devops">
      DevOps

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;security&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;security_link_resources_navbar&quot;}" href="/resources/articles/security">
      Security

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;software_development&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;software_development_link_resources_navbar&quot;}" href="/resources/articles/software-development">
      Software Development

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;view_all&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;view_all_link_resources_navbar&quot;}" href="/resources/articles">
      View all

    
</a></li>

                </ul>
              </div>
          </div>
          <div class="HeaderMenu-column px-lg-4">
              <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0 border-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="resources-explore-heading">Explore</span>

                <ul class="list-style-none f5" aria-labelledby="resources-explore-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;learning_pathways&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;learning_pathways_link_resources_navbar&quot;}" href="https://resources.github.com/learn/pathways">
      Learning Pathways

    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle">
    <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path>
</svg>
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;events_amp_webinars&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;events_amp_webinars_link_resources_navbar&quot;}" href="https://resources.github.com">
      Events &amp; Webinars

    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle">
    <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path>
</svg>
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;ebooks_amp_whitepapers&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;ebooks_amp_whitepapers_link_resources_navbar&quot;}" href="https://github.com/resources/whitepapers">
      Ebooks &amp; Whitepapers

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;customer_stories&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;customer_stories_link_resources_navbar&quot;}" href="https://github.com/customer-stories">
      Customer Stories

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;partners&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;partners_link_resources_navbar&quot;}" href="https://partner.github.com">
      Partners

    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle">
    <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path>
</svg>
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;executive_insights&quot;,&quot;context&quot;:&quot;resources&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;executive_insights_link_resources_navbar&quot;}" href="https://github.com/solutions/executive-insights">
      Executive Insights

    
</a></li>

                </ul>
              </div>
          </div>

      </div>
</li>


                <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item">
      <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false">
        Open Source
        <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1">
    <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path>
</svg>
      </button>

      <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 px-lg-4">
          <div class="HeaderMenu-column">
              <div class="border-bottom pb-3 pb-lg-0 pb-lg-3 mb-3 mb-lg-0 mb-lg-3">

                <ul class="list-style-none f5">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;github_sponsors&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;github_sponsors_link_open_source_navbar&quot;}" href="/sponsors">
      
      <div>
          <div class="color-fg-default h4">
            GitHub Sponsors
          </div>
        Fund open source developers
      </div>

    
</a></li>

                </ul>
              </div>
              <div class="border-bottom pb-3 pb-lg-0 pb-lg-3 mb-3 mb-lg-0 mb-lg-3">

                <ul class="list-style-none f5">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;the_readme_project&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;the_readme_project_link_open_source_navbar&quot;}" href="https://github.com/readme">
      
      <div>
          <div class="color-fg-default h4">
            The ReadME Project
          </div>
        GitHub community articles
      </div>

    
</a></li>

                </ul>
              </div>
              <div class="border-bottom pb-3 pb-lg-0 border-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="open-source-repositories-heading">Repositories</span>

                <ul class="list-style-none f5" aria-labelledby="open-source-repositories-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;topics&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;topics_link_open_source_navbar&quot;}" href="https://github.com/topics">
      Topics

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;trending&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;trending_link_open_source_navbar&quot;}" href="https://github.com/trending">
      Trending

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;collections&quot;,&quot;context&quot;:&quot;open_source&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;collections_link_open_source_navbar&quot;}" href="https://github.com/collections">
      Collections

    
</a></li>

                </ul>
              </div>
          </div>

      </div>
</li>


                <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item">
      <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false">
        Enterprise
        <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1">
    <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path>
</svg>
      </button>

      <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 px-lg-4">
          <div class="HeaderMenu-column">
              <div class="border-bottom pb-3 pb-lg-0 pb-lg-3 mb-3 mb-lg-0 mb-lg-3">

                <ul class="list-style-none f5">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;enterprise_platform&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;enterprise_platform_link_enterprise_navbar&quot;}" href="/enterprise">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-stack color-fg-subtle mr-3">
    <path d="M11.063 1.456a1.749 1.749 0 0 1 1.874 0l8.383 5.316a1.751 1.751 0 0 1 0 2.956l-8.383 5.316a1.749 1.749 0 0 1-1.874 0L2.68 9.728a1.751 1.751 0 0 1 0-2.956Zm1.071 1.267a.25.25 0 0 0-.268 0L3.483 8.039a.25.25 0 0 0 0 .422l8.383 5.316a.25.25 0 0 0 .268 0l8.383-5.316a.25.25 0 0 0 0-.422Z"></path><path d="M1.867 12.324a.75.75 0 0 1 1.035-.232l8.964 5.685a.25.25 0 0 0 .268 0l8.964-5.685a.75.75 0 0 1 .804 1.267l-8.965 5.685a1.749 1.749 0 0 1-1.874 0l-8.965-5.685a.75.75 0 0 1-.231-1.035Z"></path><path d="M1.867 16.324a.75.75 0 0 1 1.035-.232l8.964 5.685a.25.25 0 0 0 .268 0l8.964-5.685a.75.75 0 0 1 .804 1.267l-8.965 5.685a1.749 1.749 0 0 1-1.874 0l-8.965-5.685a.75.75 0 0 1-.231-1.035Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Enterprise platform
          </div>
        AI-powered developer platform
      </div>

    
</a></li>

                </ul>
              </div>
              <div class="border-bottom pb-3 pb-lg-0 border-bottom-0">
                    <span class="d-block h4 color-fg-default my-1" id="enterprise-available-add-ons-heading">Available add-ons</span>

                <ul class="list-style-none f5" aria-labelledby="enterprise-available-add-ons-heading">
                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;github_advanced_security&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;github_advanced_security_link_enterprise_navbar&quot;}" href="https://github.com/security/advanced-security">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-shield-check color-fg-subtle mr-3">
    <path d="M16.53 9.78a.75.75 0 0 0-1.06-1.06L11 13.19l-1.97-1.97a.75.75 0 0 0-1.06 1.06l2.5 2.5a.75.75 0 0 0 1.06 0l5-5Z"></path><path d="m12.54.637 8.25 2.675A1.75 1.75 0 0 1 22 4.976V10c0 6.19-3.771 10.704-9.401 12.83a1.704 1.704 0 0 1-1.198 0C5.77 20.705 2 16.19 2 10V4.976c0-.758.489-1.43 1.21-1.664L11.46.637a1.748 1.748 0 0 1 1.08 0Zm-.617 1.426-8.25 2.676a.249.249 0 0 0-.173.237V10c0 5.46 3.28 9.483 8.43 11.426a.199.199 0 0 0 .14 0C17.22 19.483 20.5 15.461 20.5 10V4.976a.25.25 0 0 0-.173-.237l-8.25-2.676a.253.253 0 0 0-.154 0Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            GitHub Advanced Security
          </div>
        Enterprise-grade security features
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;copilot_for_business&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;copilot_for_business_link_enterprise_navbar&quot;}" href="/features/copilot/copilot-business">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-copilot color-fg-subtle mr-3">
    <path d="M23.922 16.992c-.861 1.495-5.859 5.023-11.922 5.023-6.063 0-11.061-3.528-11.922-5.023A.641.641 0 0 1 0 16.736v-2.869a.841.841 0 0 1 .053-.22c.372-.935 1.347-2.292 2.605-2.656.167-.429.414-1.055.644-1.517a10.195 10.195 0 0 1-.052-1.086c0-1.331.282-2.499 1.132-3.368.397-.406.89-.717 1.474-.952 1.399-1.136 3.392-2.093 6.122-2.093 2.731 0 4.767.957 6.166 2.093.584.235 1.077.546 1.474.952.85.869 1.132 2.037 1.132 3.368 0 .368-.014.733-.052 1.086.23.462.477 1.088.644 1.517 1.258.364 2.233 1.721 2.605 2.656a.832.832 0 0 1 .053.22v2.869a.641.641 0 0 1-.078.256ZM12.172 11h-.344a4.323 4.323 0 0 1-.355.508C10.703 12.455 9.555 13 7.965 13c-1.725 0-2.989-.359-3.782-1.259a2.005 2.005 0 0 1-.085-.104L4 11.741v6.585c1.435.779 4.514 2.179 8 2.179 3.486 0 6.565-1.4 8-2.179v-6.585l-.098-.104s-.033.045-.085.104c-.793.9-2.057 1.259-3.782 1.259-1.59 0-2.738-.545-3.508-1.492a4.323 4.323 0 0 1-.355-.508h-.016.016Zm.641-2.935c.136 1.057.403 1.913.878 2.497.442.544 1.134.938 2.344.938 1.573 0 2.292-.337 2.657-.751.384-.435.558-1.15.558-2.361 0-1.14-.243-1.847-.705-2.319-.477-.488-1.319-.862-2.824-1.025-1.487-.161-2.192.138-2.533.529-.269.307-.437.808-.438 1.578v.021c0 .265.021.562.063.893Zm-1.626 0c.042-.331.063-.628.063-.894v-.02c-.001-.77-.169-1.271-.438-1.578-.341-.391-1.046-.69-2.533-.529-1.505.163-2.347.537-2.824 1.025-.462.472-.705 1.179-.705 2.319 0 1.211.175 1.926.558 2.361.365.414 1.084.751 2.657.751 1.21 0 1.902-.394 2.344-.938.475-.584.742-1.44.878-2.497Z"></path><path d="M14.5 14.25a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Zm-5 0a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Copilot for business
          </div>
        Enterprise-grade AI features
      </div>

    
</a></li>

                    <li>
  <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;premium_support&quot;,&quot;context&quot;:&quot;enterprise&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;premium_support_link_enterprise_navbar&quot;}" href="/premium-support">
      <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-comment-discussion color-fg-subtle mr-3">
    <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 14.25 14H8.061l-2.574 2.573A1.458 1.458 0 0 1 3 15.543V14H1.75A1.75 1.75 0 0 1 0 12.25v-9.5C0 1.784.784 1 1.75 1ZM1.5 2.75v9.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h6.5a.25.25 0 0 0 .25-.25v-9.5a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Z"></path><path d="M22.5 8.75a.25.25 0 0 0-.25-.25h-3.5a.75.75 0 0 1 0-1.5h3.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 22.25 20H21v1.543a1.457 1.457 0 0 1-2.487 1.03L15.939 20H10.75A1.75 1.75 0 0 1 9 18.25v-1.465a.75.75 0 0 1 1.5 0v1.465c0 .138.112.25.25.25h5.5a.75.75 0 0 1 .53.22l2.72 2.72v-2.19a.75.75 0 0 1 .75-.75h2a.25.25 0 0 0 .25-.25v-9.5Z"></path>
</svg>
      <div>
          <div class="color-fg-default h4">
            Premium Support
          </div>
        Enterprise-grade 24/7 support
      </div>

    
</a></li>

                </ul>
              </div>
          </div>

      </div>
</li>


                <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item">
    <a class="HeaderMenu-link no-underline px-0 px-lg-2 py-3 py-lg-2 d-block d-lg-inline-block" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;pricing&quot;,&quot;context&quot;:&quot;global&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;pricing_link_global_navbar&quot;}" href="https://github.com/pricing">Pricing</a>
</li>

            </ul>
          </nav>

        <div class="d-flex flex-column flex-lg-row width-full flex-justify-end flex-lg-items-center text-center mt-3 mt-lg-0 text-lg-left ml-lg-3">
                


<qbsearch-input class="search-input" data-scope="repo:OSU-NLP-Group/GUI-Agents-Paper-List" data-custom-scopes-path="/search/custom_scopes" data-delete-custom-scopes-csrf="eovkCUFM60uNWLiT_MKOpD8lOubpvZ8fTr0JWcoy03INP5NM88b-6Ic8QxgT0uhHbgOv4XSsZn8ZEeMoScAcvw" data-max-custom-scopes="10" data-header-redesign-enabled="false" data-initial-value="" data-blackbird-suggestions-path="/search/suggestions" data-jump-to-suggestions-path="/_graphql/GetSuggestedNavigationDestinations" data-current-repository="OSU-NLP-Group/GUI-Agents-Paper-List" data-current-org="OSU-NLP-Group" data-current-owner="" data-logged-in="false" data-copilot-chat-enabled="false" data-nl-search-enabled="false" data-retain-scroll-position="true">
  <div class="search-input-container search-with-dialog position-relative d-flex flex-row flex-items-center mr-4 rounded" data-action="click:qbsearch-input#searchInputContainerClicked">
      <button type="button" class="header-search-button placeholder  input-button form-control d-flex flex-1 flex-self-stretch flex-items-center no-wrap width-full py-0 pl-2 pr-0 text-left border-0 box-shadow-none" data-target="qbsearch-input.inputButton" aria-label="Search or jump to…" aria-haspopup="dialog" placeholder="Search or jump to..." data-hotkey="s,/" autocapitalize="off" data-analytics-event="{&quot;location&quot;:&quot;navbar&quot;,&quot;action&quot;:&quot;searchbar&quot;,&quot;context&quot;:&quot;global&quot;,&quot;tag&quot;:&quot;input&quot;,&quot;label&quot;:&quot;searchbar_input_global_navbar&quot;}" data-action="click:qbsearch-input#handleExpand">
        <div class="mr-2 color-fg-muted">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-search">
    <path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path>
</svg>
        </div>
        <span class="flex-1" data-target="qbsearch-input.inputButtonText">Search or jump to...</span>
          <div class="d-flex" data-target="qbsearch-input.hotkeyIndicator">
            <svg xmlns="http://www.w3.org/2000/svg" width="22" height="20" aria-hidden="true" class="mr-1"><path fill="none" stroke="#979A9C" opacity=".4" d="M3.5.5h12c1.7 0 3 1.3 3 3v13c0 1.7-1.3 3-3 3h-12c-1.7 0-3-1.3-3-3v-13c0-1.7 1.3-3 3-3z"></path><path fill="#979A9C" d="M11.8 6L8 15.1h-.9L10.8 6h1z"></path></svg>
          </div>
      </button>

    <input type="hidden" name="type" class="js-site-search-type-field">

    
<div class="Overlay--hidden " data-modal-dialog-overlay="">
  <modal-dialog data-action="close:qbsearch-input#handleClose cancel:qbsearch-input#handleClose" data-target="qbsearch-input.searchSuggestionsDialog" role="dialog" id="search-suggestions-dialog" aria-modal="true" aria-labelledby="search-suggestions-dialog-header" data-view-component="true" class="Overlay Overlay--width-large Overlay--height-auto">
      <h1 id="search-suggestions-dialog-header" class="sr-only">Search code, repositories, users, issues, pull requests...</h1>
    <div class="Overlay-body Overlay-body--paddingNone">
      
          <div data-view-component="true">        <div class="search-suggestions position-fixed width-full color-shadow-large border color-fg-default color-bg-default overflow-hidden d-flex flex-column query-builder-container" style="border-radius: 12px;" data-target="qbsearch-input.queryBuilderContainer" hidden="">
          <!-- '"` --><!-- </textarea></xmp> --><form id="query-builder-test-form" action="" accept-charset="UTF-8" method="get">
  <query-builder data-target="qbsearch-input.queryBuilder" id="query-builder-query-builder-test" data-filter-key=":" data-view-component="true" class="QueryBuilder search-query-builder">
    <div class="FormControl FormControl--fullWidth">
      <label id="query-builder-test-label" for="query-builder-test" class="FormControl-label sr-only">
        Search
      </label>
      <div class="QueryBuilder-StyledInput width-fit " data-target="query-builder.styledInput">
          <span id="query-builder-test-leadingvisual-wrap" class="FormControl-input-leadingVisualWrap QueryBuilder-leadingVisualWrap">
            <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-search FormControl-input-leadingVisual">
    <path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path>
</svg>
          </span>
        <div data-target="query-builder.styledInputContainer" class="QueryBuilder-StyledInputContainer">
          <div aria-hidden="true" class="QueryBuilder-StyledInputContent" data-target="query-builder.styledInputContent"></div>
          <div class="QueryBuilder-InputWrapper">
            <div aria-hidden="true" class="QueryBuilder-Sizer" data-target="query-builder.sizer"></div>
            <input id="query-builder-test" name="query-builder-test" value="" autocomplete="off" type="text" role="combobox" spellcheck="false" aria-expanded="false" aria-describedby="validation-b08ed85f-9287-4db7-9df5-244cfddaf0d0" data-target="query-builder.input" data-action="
          input:query-builder#inputChange
          blur:query-builder#inputBlur
          keydown:query-builder#inputKeydown
          focus:query-builder#inputFocus
        " data-view-component="true" class="FormControl-input QueryBuilder-Input FormControl-medium">
          </div>
        </div>
          <span class="sr-only" id="query-builder-test-clear">Clear</span>
          <button role="button" id="query-builder-test-clear-button" aria-labelledby="query-builder-test-clear query-builder-test-label" data-target="query-builder.clearButton" data-action="
                click:query-builder#clear
                focus:query-builder#clearButtonFocus
                blur:query-builder#clearButtonBlur
              " variant="small" hidden="hidden" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium mr-1 px-2 py-0 d-flex flex-items-center rounded-1 color-fg-muted">  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x-circle-fill Button-visual">
    <path d="M2.343 13.657A8 8 0 1 1 13.658 2.343 8 8 0 0 1 2.343 13.657ZM6.03 4.97a.751.751 0 0 0-1.042.018.751.751 0 0 0-.018 1.042L6.94 8 4.97 9.97a.749.749 0 0 0 .326 1.275.749.749 0 0 0 .734-.215L8 9.06l1.97 1.97a.749.749 0 0 0 1.275-.326.749.749 0 0 0-.215-.734L9.06 8l1.97-1.97a.749.749 0 0 0-.326-1.275.749.749 0 0 0-.734.215L8 6.94Z"></path>
</svg>
</button>

      </div>
      <template id="search-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-search">
    <path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path>
</svg>
</template>

<template id="code-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code">
    <path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</template>

<template id="file-code-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-file-code">
    <path d="M4 1.75C4 .784 4.784 0 5.75 0h5.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v8.586A1.75 1.75 0 0 1 14.25 15h-9a.75.75 0 0 1 0-1.5h9a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 10 4.25V1.5H5.75a.25.25 0 0 0-.25.25v2.5a.75.75 0 0 1-1.5 0Zm1.72 4.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734l1.47-1.47-1.47-1.47a.75.75 0 0 1 0-1.06ZM3.28 7.78 1.81 9.25l1.47 1.47a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Zm8.22-6.218V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path>
</svg>
</template>

<template id="history-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-history">
    <path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path>
</svg>
</template>

<template id="repo-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo">
    <path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path>
</svg>
</template>

<template id="bookmark-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bookmark">
    <path d="M3 2.75C3 1.784 3.784 1 4.75 1h6.5c.966 0 1.75.784 1.75 1.75v11.5a.75.75 0 0 1-1.227.579L8 11.722l-3.773 3.107A.751.751 0 0 1 3 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v9.91l3.023-2.489a.75.75 0 0 1 .954 0l3.023 2.49V2.75a.25.25 0 0 0-.25-.25Z"></path>
</svg>
</template>

<template id="plus-circle-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-plus-circle">
    <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm7.25-3.25v2.5h2.5a.75.75 0 0 1 0 1.5h-2.5v2.5a.75.75 0 0 1-1.5 0v-2.5h-2.5a.75.75 0 0 1 0-1.5h2.5v-2.5a.75.75 0 0 1 1.5 0Z"></path>
</svg>
</template>

<template id="circle-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-dot-fill">
    <path d="M8 4a4 4 0 1 1 0 8 4 4 0 0 1 0-8Z"></path>
</svg>
</template>

<template id="trash-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-trash">
    <path d="M11 1.75V3h2.25a.75.75 0 0 1 0 1.5H2.75a.75.75 0 0 1 0-1.5H5V1.75C5 .784 5.784 0 6.75 0h2.5C10.216 0 11 .784 11 1.75ZM4.496 6.675l.66 6.6a.25.25 0 0 0 .249.225h5.19a.25.25 0 0 0 .249-.225l.66-6.6a.75.75 0 0 1 1.492.149l-.66 6.6A1.748 1.748 0 0 1 10.595 15h-5.19a1.75 1.75 0 0 1-1.741-1.575l-.66-6.6a.75.75 0 1 1 1.492-.15ZM6.5 1.75V3h3V1.75a.25.25 0 0 0-.25-.25h-2.5a.25.25 0 0 0-.25.25Z"></path>
</svg>
</template>

<template id="team-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-people">
    <path d="M2 5.5a3.5 3.5 0 1 1 5.898 2.549 5.508 5.508 0 0 1 3.034 4.084.75.75 0 1 1-1.482.235 4 4 0 0 0-7.9 0 .75.75 0 0 1-1.482-.236A5.507 5.507 0 0 1 3.102 8.05 3.493 3.493 0 0 1 2 5.5ZM11 4a3.001 3.001 0 0 1 2.22 5.018 5.01 5.01 0 0 1 2.56 3.012.749.749 0 0 1-.885.954.752.752 0 0 1-.549-.514 3.507 3.507 0 0 0-2.522-2.372.75.75 0 0 1-.574-.73v-.352a.75.75 0 0 1 .416-.672A1.5 1.5 0 0 0 11 5.5.75.75 0 0 1 11 4Zm-5.5-.5a2 2 0 1 0-.001 3.999A2 2 0 0 0 5.5 3.5Z"></path>
</svg>
</template>

<template id="project-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-project">
    <path d="M1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0ZM1.5 1.75v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25ZM11.75 3a.75.75 0 0 1 .75.75v7.5a.75.75 0 0 1-1.5 0v-7.5a.75.75 0 0 1 .75-.75Zm-8.25.75a.75.75 0 0 1 1.5 0v5.5a.75.75 0 0 1-1.5 0ZM8 3a.75.75 0 0 1 .75.75v3.5a.75.75 0 0 1-1.5 0v-3.5A.75.75 0 0 1 8 3Z"></path>
</svg>
</template>

<template id="pencil-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-pencil">
    <path d="M11.013 1.427a1.75 1.75 0 0 1 2.474 0l1.086 1.086a1.75 1.75 0 0 1 0 2.474l-8.61 8.61c-.21.21-.47.364-.756.445l-3.251.93a.75.75 0 0 1-.927-.928l.929-3.25c.081-.286.235-.547.445-.758l8.61-8.61Zm.176 4.823L9.75 4.81l-6.286 6.287a.253.253 0 0 0-.064.108l-.558 1.953 1.953-.558a.253.253 0 0 0 .108-.064Zm1.238-3.763a.25.25 0 0 0-.354 0L10.811 3.75l1.439 1.44 1.263-1.263a.25.25 0 0 0 0-.354Z"></path>
</svg>
</template>

<template id="copilot-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copilot">
    <path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path>
</svg>
</template>

<template id="copilot-error-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copilot-error">
    <path d="M16 11.24c0 .112-.072.274-.21.467L13 9.688V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-.198 0-.388-.009-.571-.029L6.833 5.226a4.01 4.01 0 0 0 .17-.782c.117-.935-.037-1.395-.241-1.614-.193-.206-.637-.413-1.682-.297-.683.076-1.115.231-1.395.415l-1.257-.91c.579-.564 1.413-.877 2.485-.996 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095Zm-5.083-8.707c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Zm2.511 11.074c-1.393.776-3.272 1.428-5.43 1.428-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.18-.455-.241-.963-.252-1.475L.31 4.107A.747.747 0 0 1 0 3.509V3.49a.748.748 0 0 1 .625-.73c.156-.026.306.047.435.139l14.667 10.578a.592.592 0 0 1 .227.264.752.752 0 0 1 .046.249v.022a.75.75 0 0 1-1.19.596Zm-1.367-.991L5.635 7.964a5.128 5.128 0 0 1-.889.073c-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433 1.539 0 3.089-.505 4.063-.934Z"></path>
</svg>
</template>

<template id="workflow-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-workflow">
    <path d="M0 1.75C0 .784.784 0 1.75 0h3.5C6.216 0 7 .784 7 1.75v3.5A1.75 1.75 0 0 1 5.25 7H4v4a1 1 0 0 0 1 1h4v-1.25C9 9.784 9.784 9 10.75 9h3.5c.966 0 1.75.784 1.75 1.75v3.5A1.75 1.75 0 0 1 14.25 16h-3.5A1.75 1.75 0 0 1 9 14.25v-.75H5A2.5 2.5 0 0 1 2.5 11V7h-.75A1.75 1.75 0 0 1 0 5.25Zm1.75-.25a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Zm9 9a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
</template>

<template id="book-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-book">
    <path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path>
</svg>
</template>

<template id="code-review-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code-review">
    <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v8.5A1.75 1.75 0 0 1 14.25 13H8.061l-2.574 2.573A1.458 1.458 0 0 1 3 14.543V13H1.75A1.75 1.75 0 0 1 0 11.25v-8.5C0 1.784.784 1 1.75 1ZM1.5 2.75v8.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h6.5a.25.25 0 0 0 .25-.25v-8.5a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Zm5.28 1.72a.75.75 0 0 1 0 1.06L5.31 7l1.47 1.47a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018l-2-2a.75.75 0 0 1 0-1.06l2-2a.75.75 0 0 1 1.06 0Zm2.44 0a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L10.69 7 9.22 5.53a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</template>

<template id="codespaces-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-codespaces">
    <path d="M0 11.25c0-.966.784-1.75 1.75-1.75h12.5c.966 0 1.75.784 1.75 1.75v3A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm2-9.5C2 .784 2.784 0 3.75 0h8.5C13.216 0 14 .784 14 1.75v5a1.75 1.75 0 0 1-1.75 1.75h-8.5A1.75 1.75 0 0 1 2 6.75Zm1.75-.25a.25.25 0 0 0-.25.25v5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5a.25.25 0 0 0-.25-.25Zm-2 9.5a.25.25 0 0 0-.25.25v3c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-3a.25.25 0 0 0-.25-.25Z"></path><path d="M7 12.75a.75.75 0 0 1 .75-.75h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path>
</svg>
</template>

<template id="comment-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-comment">
    <path d="M1 2.75C1 1.784 1.784 1 2.75 1h10.5c.966 0 1.75.784 1.75 1.75v7.5A1.75 1.75 0 0 1 13.25 12H9.06l-2.573 2.573A1.458 1.458 0 0 1 4 13.543V12H2.75A1.75 1.75 0 0 1 1 10.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h4.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
</template>

<template id="comment-discussion-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-comment-discussion">
    <path d="M1.75 1h8.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 10.25 10H7.061l-2.574 2.573A1.458 1.458 0 0 1 2 11.543V10h-.25A1.75 1.75 0 0 1 0 8.25v-5.5C0 1.784.784 1 1.75 1ZM1.5 2.75v5.5c0 .138.112.25.25.25h1a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h3.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25Zm13 2a.25.25 0 0 0-.25-.25h-.5a.75.75 0 0 1 0-1.5h.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 14.25 12H14v1.543a1.458 1.458 0 0 1-2.487 1.03L9.22 12.28a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l2.22 2.22v-2.19a.75.75 0 0 1 .75-.75h1a.25.25 0 0 0 .25-.25Z"></path>
</svg>
</template>

<template id="organization-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-organization">
    <path d="M1.75 16A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0h8.5C11.216 0 12 .784 12 1.75v12.5c0 .085-.006.168-.018.25h2.268a.25.25 0 0 0 .25-.25V8.285a.25.25 0 0 0-.111-.208l-1.055-.703a.749.749 0 1 1 .832-1.248l1.055.703c.487.325.779.871.779 1.456v5.965A1.75 1.75 0 0 1 14.25 16h-3.5a.766.766 0 0 1-.197-.026c-.099.017-.2.026-.303.026h-3a.75.75 0 0 1-.75-.75V14h-1v1.25a.75.75 0 0 1-.75.75Zm-.25-1.75c0 .138.112.25.25.25H4v-1.25a.75.75 0 0 1 .75-.75h2.5a.75.75 0 0 1 .75.75v1.25h2.25a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25ZM3.75 6h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5ZM3 3.75A.75.75 0 0 1 3.75 3h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 3 3.75Zm4 3A.75.75 0 0 1 7.75 6h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 7 6.75ZM7.75 3h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5ZM3 9.75A.75.75 0 0 1 3.75 9h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 3 9.75ZM7.75 9h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5Z"></path>
</svg>
</template>

<template id="rocket-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-rocket">
    <path d="M14.064 0h.186C15.216 0 16 .784 16 1.75v.186a8.752 8.752 0 0 1-2.564 6.186l-.458.459c-.314.314-.641.616-.979.904v3.207c0 .608-.315 1.172-.833 1.49l-2.774 1.707a.749.749 0 0 1-1.11-.418l-.954-3.102a1.214 1.214 0 0 1-.145-.125L3.754 9.816a1.218 1.218 0 0 1-.124-.145L.528 8.717a.749.749 0 0 1-.418-1.11l1.71-2.774A1.748 1.748 0 0 1 3.31 4h3.204c.288-.338.59-.665.904-.979l.459-.458A8.749 8.749 0 0 1 14.064 0ZM8.938 3.623h-.002l-.458.458c-.76.76-1.437 1.598-2.02 2.5l-1.5 2.317 2.143 2.143 2.317-1.5c.902-.583 1.74-1.26 2.499-2.02l.459-.458a7.25 7.25 0 0 0 2.123-5.127V1.75a.25.25 0 0 0-.25-.25h-.186a7.249 7.249 0 0 0-5.125 2.123ZM3.56 14.56c-.732.732-2.334 1.045-3.005 1.148a.234.234 0 0 1-.201-.064.234.234 0 0 1-.064-.201c.103-.671.416-2.273 1.15-3.003a1.502 1.502 0 1 1 2.12 2.12Zm6.94-3.935c-.088.06-.177.118-.266.175l-2.35 1.521.548 1.783 1.949-1.2a.25.25 0 0 0 .119-.213ZM3.678 8.116 5.2 5.766c.058-.09.117-.178.176-.266H3.309a.25.25 0 0 0-.213.119l-1.2 1.95ZM12 5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
</template>

<template id="shield-check-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield-check">
    <path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
</svg>
</template>

<template id="heart-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-heart">
    <path d="m8 14.25.345.666a.75.75 0 0 1-.69 0l-.008-.004-.018-.01a7.152 7.152 0 0 1-.31-.17 22.055 22.055 0 0 1-3.434-2.414C2.045 10.731 0 8.35 0 5.5 0 2.836 2.086 1 4.25 1 5.797 1 7.153 1.802 8 3.02 8.847 1.802 10.203 1 11.75 1 13.914 1 16 2.836 16 5.5c0 2.85-2.045 5.231-3.885 6.818a22.066 22.066 0 0 1-3.744 2.584l-.018.01-.006.003h-.002ZM4.25 2.5c-1.336 0-2.75 1.164-2.75 3 0 2.15 1.58 4.144 3.365 5.682A20.58 20.58 0 0 0 8 13.393a20.58 20.58 0 0 0 3.135-2.211C12.92 9.644 14.5 7.65 14.5 5.5c0-1.836-1.414-3-2.75-3-1.373 0-2.609.986-3.029 2.456a.749.749 0 0 1-1.442 0C6.859 3.486 5.623 2.5 4.25 2.5Z"></path>
</svg>
</template>

<template id="server-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-server">
    <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v4c0 .372-.116.717-.314 1 .198.283.314.628.314 1v4a1.75 1.75 0 0 1-1.75 1.75H1.75A1.75 1.75 0 0 1 0 12.75v-4c0-.358.109-.707.314-1a1.739 1.739 0 0 1-.314-1v-4C0 1.784.784 1 1.75 1ZM1.5 2.75v4c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-4a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Zm.25 5.75a.25.25 0 0 0-.25.25v4c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-4a.25.25 0 0 0-.25-.25ZM7 4.75A.75.75 0 0 1 7.75 4h4.5a.75.75 0 0 1 0 1.5h-4.5A.75.75 0 0 1 7 4.75ZM7.75 10h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1 0-1.5ZM3 4.75A.75.75 0 0 1 3.75 4h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 3 4.75ZM3.75 10h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5Z"></path>
</svg>
</template>

<template id="globe-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-globe">
    <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM5.78 8.75a9.64 9.64 0 0 0 1.363 4.177c.255.426.542.832.857 1.215.245-.296.551-.705.857-1.215A9.64 9.64 0 0 0 10.22 8.75Zm4.44-1.5a9.64 9.64 0 0 0-1.363-4.177c-.307-.51-.612-.919-.857-1.215a9.927 9.927 0 0 0-.857 1.215A9.64 9.64 0 0 0 5.78 7.25Zm-5.944 1.5H1.543a6.507 6.507 0 0 0 4.666 5.5c-.123-.181-.24-.365-.352-.552-.715-1.192-1.437-2.874-1.581-4.948Zm-2.733-1.5h2.733c.144-2.074.866-3.756 1.58-4.948.12-.197.237-.381.353-.552a6.507 6.507 0 0 0-4.666 5.5Zm10.181 1.5c-.144 2.074-.866 3.756-1.58 4.948-.12.197-.237.381-.353.552a6.507 6.507 0 0 0 4.666-5.5Zm2.733-1.5a6.507 6.507 0 0 0-4.666-5.5c.123.181.24.365.353.552.714 1.192 1.436 2.874 1.58 4.948Z"></path>
</svg>
</template>

<template id="issue-opened-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened">
    <path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
</svg>
</template>

<template id="device-mobile-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-device-mobile">
    <path d="M3.75 0h8.5C13.216 0 14 .784 14 1.75v12.5A1.75 1.75 0 0 1 12.25 16h-8.5A1.75 1.75 0 0 1 2 14.25V1.75C2 .784 2.784 0 3.75 0ZM3.5 1.75v12.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25ZM8 13a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path>
</svg>
</template>

<template id="package-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-package">
    <path d="m8.878.392 5.25 3.045c.54.314.872.89.872 1.514v6.098a1.75 1.75 0 0 1-.872 1.514l-5.25 3.045a1.75 1.75 0 0 1-1.756 0l-5.25-3.045A1.75 1.75 0 0 1 1 11.049V4.951c0-.624.332-1.201.872-1.514L7.122.392a1.75 1.75 0 0 1 1.756 0ZM7.875 1.69l-4.63 2.685L8 7.133l4.755-2.758-4.63-2.685a.248.248 0 0 0-.25 0ZM2.5 5.677v5.372c0 .09.047.171.125.216l4.625 2.683V8.432Zm6.25 8.271 4.625-2.683a.25.25 0 0 0 .125-.216V5.677L8.75 8.432Z"></path>
</svg>
</template>

<template id="credit-card-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-credit-card">
    <path d="M10.75 9a.75.75 0 0 0 0 1.5h1.5a.75.75 0 0 0 0-1.5h-1.5Z"></path><path d="M0 3.75C0 2.784.784 2 1.75 2h12.5c.966 0 1.75.784 1.75 1.75v8.5A1.75 1.75 0 0 1 14.25 14H1.75A1.75 1.75 0 0 1 0 12.25ZM14.5 6.5h-13v5.75c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25Zm0-2.75a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25V5h13Z"></path>
</svg>
</template>

<template id="play-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play">
    <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
</svg>
</template>

<template id="gift-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-gift">
    <path d="M2 2.75A2.75 2.75 0 0 1 4.75 0c.983 0 1.873.42 2.57 1.232.268.318.497.668.68 1.042.183-.375.411-.725.68-1.044C9.376.42 10.266 0 11.25 0a2.75 2.75 0 0 1 2.45 4h.55c.966 0 1.75.784 1.75 1.75v2c0 .698-.409 1.301-1 1.582v4.918A1.75 1.75 0 0 1 13.25 16H2.75A1.75 1.75 0 0 1 1 14.25V9.332C.409 9.05 0 8.448 0 7.75v-2C0 4.784.784 4 1.75 4h.55c-.192-.375-.3-.8-.3-1.25ZM7.25 9.5H2.5v4.75c0 .138.112.25.25.25h4.5Zm1.5 0v5h4.5a.25.25 0 0 0 .25-.25V9.5Zm0-4V8h5.5a.25.25 0 0 0 .25-.25v-2a.25.25 0 0 0-.25-.25Zm-7 0a.25.25 0 0 0-.25.25v2c0 .138.112.25.25.25h5.5V5.5h-5.5Zm3-4a1.25 1.25 0 0 0 0 2.5h2.309c-.233-.818-.542-1.401-.878-1.793-.43-.502-.915-.707-1.431-.707ZM8.941 4h2.309a1.25 1.25 0 0 0 0-2.5c-.516 0-1 .205-1.43.707-.337.392-.646.975-.879 1.793Z"></path>
</svg>
</template>

<template id="code-square-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code-square">
    <path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
</svg>
</template>

<template id="device-desktop-icon">
  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-device-desktop">
    <path d="M14.25 1c.966 0 1.75.784 1.75 1.75v7.5A1.75 1.75 0 0 1 14.25 12h-3.727c.099 1.041.52 1.872 1.292 2.757A.752.752 0 0 1 11.25 16h-6.5a.75.75 0 0 1-.565-1.243c.772-.885 1.192-1.716 1.292-2.757H1.75A1.75 1.75 0 0 1 0 10.25v-7.5C0 1.784.784 1 1.75 1ZM1.75 2.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25ZM9.018 12H6.982a5.72 5.72 0 0 1-.765 2.5h3.566a5.72 5.72 0 0 1-.765-2.5Z"></path>
</svg>
</template>

        <div class="position-relative">
                <ul role="listbox" class="ActionListWrap QueryBuilder-ListWrap" aria-label="Suggestions" data-action="
                    combobox-commit:query-builder#comboboxCommit
                    mousedown:query-builder#resultsMousedown
                  " data-target="query-builder.resultsList" data-persist-list="false" id="query-builder-test-results"></ul>
        </div>
      <div class="FormControl-inlineValidation" id="validation-b08ed85f-9287-4db7-9df5-244cfddaf0d0" hidden="hidden">
        <span class="FormControl-inlineValidation--visual">
          <svg aria-hidden="true" height="12" viewBox="0 0 12 12" version="1.1" width="12" data-view-component="true" class="octicon octicon-alert-fill">
    <path d="M4.855.708c.5-.896 1.79-.896 2.29 0l4.675 8.351a1.312 1.312 0 0 1-1.146 1.954H1.33A1.313 1.313 0 0 1 .183 9.058ZM7 7V3H5v4Zm-1 3a1 1 0 1 0 0-2 1 1 0 0 0 0 2Z"></path>
</svg>
        </span>
        <span></span>
</div>    </div>
    <div data-target="query-builder.screenReaderFeedback" aria-live="polite" aria-atomic="true" class="sr-only"></div>
</query-builder></form>
          <div class="d-flex flex-row color-fg-muted px-3 text-small color-bg-default search-feedback-prompt">
            <a target="_blank" href="https://docs.github.com/search-github/github-code-search/understanding-github-code-search-syntax" data-view-component="true" class="Link color-fg-accent text-normal ml-2">Search syntax tips</a>            <div class="d-flex flex-1"></div>
          </div>
        </div>
</div>

    </div>
</modal-dialog></div>
  </div>
  <div data-action="click:qbsearch-input#retract" class="dark-backdrop position-fixed" hidden="" data-target="qbsearch-input.darkBackdrop"></div>
  <div class="color-fg-default">
    
<dialog-helper>
  <dialog data-target="qbsearch-input.feedbackDialog" data-action="close:qbsearch-input#handleDialogClose cancel:qbsearch-input#handleDialogClose" id="feedback-dialog" aria-modal="true" aria-labelledby="feedback-dialog-title" aria-describedby="feedback-dialog-description" data-view-component="true" class="Overlay Overlay-whenNarrow Overlay--size-medium Overlay--motion-scaleFade Overlay--disableScroll">
    <div data-view-component="true" class="Overlay-header">
  <div class="Overlay-headerContentWrap">
    <div class="Overlay-titleWrap">
      <h1 class="Overlay-title " id="feedback-dialog-title">
        Provide feedback
      </h1>
        
    </div>
    <div class="Overlay-actionWrap">
      <button data-close-dialog-id="feedback-dialog" aria-label="Close" type="button" data-view-component="true" class="close-button Overlay-closeButton"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
    <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg></button>
    </div>
  </div>
  
</div>
      <scrollable-region data-labelled-by="feedback-dialog-title" data-catalyst="" style="overflow: auto;">
        <div data-view-component="true" class="Overlay-body">        <!-- '"` --><!-- </textarea></xmp> --><form id="code-search-feedback-form" data-turbo="false" action="/search/feedback" accept-charset="UTF-8" method="post"><input type="hidden" data-csrf="true" name="authenticity_token" value="5rhnZX0TRe+jm7O/IaCxNhKlFTr7h1v3AESzKAqOhMOamRthCMD1UcsValeZcTwSXBGauEUGujpCCTAP08YeCQ==">
          <p>We read every piece of feedback, and take your input very seriously.</p>
          <textarea name="feedback" class="form-control width-full mb-2" style="height: 120px" id="feedback"></textarea>
          <input name="include_email" id="include_email" aria-label="Include my email address so I can be contacted" class="form-control mr-2" type="checkbox">
          <label for="include_email" style="font-weight: normal">Include my email address so I can be contacted</label>
</form></div>
      </scrollable-region>
      <div data-view-component="true" class="Overlay-footer Overlay-footer--alignEnd">          <button data-close-dialog-id="feedback-dialog" type="button" data-view-component="true" class="btn">    Cancel
</button>
          <button form="code-search-feedback-form" data-action="click:qbsearch-input#submitFeedback" type="submit" data-view-component="true" class="btn-primary btn">    Submit feedback
</button>
</div>
</dialog></dialog-helper>

    <custom-scopes data-target="qbsearch-input.customScopesManager">
    
<dialog-helper>
  <dialog data-target="custom-scopes.customScopesModalDialog" data-action="close:qbsearch-input#handleDialogClose cancel:qbsearch-input#handleDialogClose" id="custom-scopes-dialog" aria-modal="true" aria-labelledby="custom-scopes-dialog-title" aria-describedby="custom-scopes-dialog-description" data-view-component="true" class="Overlay Overlay-whenNarrow Overlay--size-medium Overlay--motion-scaleFade Overlay--disableScroll">
    <div data-view-component="true" class="Overlay-header Overlay-header--divided">
  <div class="Overlay-headerContentWrap">
    <div class="Overlay-titleWrap">
      <h1 class="Overlay-title " id="custom-scopes-dialog-title">
        Saved searches
      </h1>
        <h2 id="custom-scopes-dialog-description" class="Overlay-description">Use saved searches to filter your results more quickly</h2>
    </div>
    <div class="Overlay-actionWrap">
      <button data-close-dialog-id="custom-scopes-dialog" aria-label="Close" type="button" data-view-component="true" class="close-button Overlay-closeButton"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
    <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg></button>
    </div>
  </div>
  
</div>
      <scrollable-region data-labelled-by="custom-scopes-dialog-title" data-catalyst="" style="overflow: auto;">
        <div data-view-component="true" class="Overlay-body">        <div data-target="custom-scopes.customScopesModalDialogFlash"></div>

        <div hidden="" class="create-custom-scope-form" data-target="custom-scopes.createCustomScopeForm">
        <!-- '"` --><!-- </textarea></xmp> --><form id="custom-scopes-dialog-form" data-turbo="false" action="/search/custom_scopes" accept-charset="UTF-8" method="post"><input type="hidden" data-csrf="true" name="authenticity_token" value="9kjkj3hbuc4xwERICVCfq7h5v4gkM3DOaQsExFUVsW9DcdNT7hR05TKeQNuKJM4QjHJqmzfdr9zxCSoVwlPlvw==">
          <div data-target="custom-scopes.customScopesModalDialogFlash"></div>

          <input type="hidden" id="custom_scope_id" name="custom_scope_id" data-target="custom-scopes.customScopesIdField">

          <div class="form-group">
            <label for="custom_scope_name">Name</label>
            <auto-check src="/search/custom_scopes/check_name" required="">
              <input type="text" name="custom_scope_name" id="custom_scope_name" data-target="custom-scopes.customScopesNameField" class="form-control" autocomplete="off" placeholder="github-ruby" required="" maxlength="50" spellcheck="false">
              <input type="hidden" data-csrf="true" value="tWPDPH/5Htbe/IfX+FHHqgwEBMPh6MQAFwj9+ra/qomnf0rD48keR6W4oNYMat1mXdf6CvlBvwb4IlZmkichfg==">
            </auto-check>
          </div>

          <div class="form-group">
            <label for="custom_scope_query">Query</label>
            <input type="text" name="custom_scope_query" id="custom_scope_query" data-target="custom-scopes.customScopesQueryField" class="form-control" autocomplete="off" placeholder="(repo:mona/a OR repo:mona/b) AND lang:python" required="" maxlength="500">
          </div>

          <p class="text-small color-fg-muted">
            To see all available qualifiers, see our <a class="Link--inTextBlock" href="https://docs.github.com/search-github/github-code-search/understanding-github-code-search-syntax">documentation</a>.
          </p>
</form>        </div>

        <div data-target="custom-scopes.manageCustomScopesForm">
          <div data-target="custom-scopes.list"></div>
        </div>

</div>
      </scrollable-region>
      <div data-view-component="true" class="Overlay-footer Overlay-footer--alignEnd Overlay-footer--divided">          <button data-action="click:custom-scopes#customScopesCancel" type="button" data-view-component="true" class="btn">    Cancel
</button>
          <button form="custom-scopes-dialog-form" data-action="click:custom-scopes#customScopesSubmit" data-target="custom-scopes.customScopesSubmitButton" type="submit" data-view-component="true" class="btn-primary btn">    Create saved search
</button>
</div>
</dialog></dialog-helper>
    </custom-scopes>
  </div>
</qbsearch-input>


            <div class="position-relative HeaderMenu-link-wrap d-lg-inline-block">
              <a href="/login?return_to=https%3A%2F%2Fgithub.com%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" class="HeaderMenu-link HeaderMenu-link--sign-in HeaderMenu-button flex-shrink-0 no-underline d-none d-lg-inline-flex border border-lg-0 rounded rounded-lg-0 px-2 py-1" style="margin-left: 12px;" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;site header menu&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;SIGN_UP&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="1a3cedb20582af7daca24b759b032fbdf821aef3048538a1d15564ff96577f3b" data-analytics-event="{&quot;category&quot;:&quot;Marketing nav&quot;,&quot;action&quot;:&quot;click to go to homepage&quot;,&quot;label&quot;:&quot;ref_page:Marketing;ref_cta:Sign in;ref_loc:Header&quot;}">
                Sign in
              </a>
            </div>

              <a href="/signup?ref_cta=Sign+up&amp;ref_loc=header+logged+out&amp;ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E&amp;source=header-repo&amp;source_repo=OSU-NLP-Group%2FGUI-Agents-Paper-List" class="HeaderMenu-link HeaderMenu-link--sign-up HeaderMenu-button flex-shrink-0 d-flex d-lg-inline-flex no-underline border color-border-default rounded px-2 py-1" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;site header menu&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;SIGN_UP&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="1a3cedb20582af7daca24b759b032fbdf821aef3048538a1d15564ff96577f3b" data-analytics-event="{&quot;category&quot;:&quot;Sign up&quot;,&quot;action&quot;:&quot;click to sign up for account&quot;,&quot;label&quot;:&quot;ref_page:/<user-name>/<repo-name>;ref_cta:Sign up;ref_loc:header logged out&quot;}">
                Sign up
              </a>

                <div class="AppHeader-appearanceSettings">
    <react-partial-anchor>
      <button data-target="react-partial-anchor.anchor" id="icon-button-783ce2c2-516b-4379-80ba-ea7b401bdf17" aria-labelledby="tooltip-68ca09b2-75bf-48a7-95dd-7fba17745091" type="button" disabled="disabled" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium AppHeader-button HeaderMenu-link border cursor-wait">  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-sliders Button-visual">
    <path d="M15 2.75a.75.75 0 0 1-.75.75h-4a.75.75 0 0 1 0-1.5h4a.75.75 0 0 1 .75.75Zm-8.5.75v1.25a.75.75 0 0 0 1.5 0v-4a.75.75 0 0 0-1.5 0V2H1.75a.75.75 0 0 0 0 1.5H6.5Zm1.25 5.25a.75.75 0 0 0 0-1.5h-6a.75.75 0 0 0 0 1.5h6ZM15 8a.75.75 0 0 1-.75.75H11.5V10a.75.75 0 1 1-1.5 0V6a.75.75 0 0 1 1.5 0v1.25h2.75A.75.75 0 0 1 15 8Zm-9 5.25v-2a.75.75 0 0 0-1.5 0v1.25H1.75a.75.75 0 0 0 0 1.5H4.5v1.25a.75.75 0 0 0 1.5 0v-2Zm9 0a.75.75 0 0 1-.75.75h-6a.75.75 0 0 1 0-1.5h6a.75.75 0 0 1 .75.75Z"></path>
</svg>
</button><tool-tip id="tooltip-68ca09b2-75bf-48a7-95dd-7fba17745091" for="icon-button-783ce2c2-516b-4379-80ba-ea7b401bdf17" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute" aria-hidden="true" role="tooltip">Appearance settings</tool-tip>

      <template data-target="react-partial-anchor.template">
        <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.ad18e42c1b06f95e74e5.module.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/appearance-settings.22dfbc22ef0a2bf02523.module.css">

<react-partial partial-name="appearance-settings" data-ssr="false" data-attempted-ssr="false">
  
  <script type="application/json" data-target="react-partial.embeddedData">{"props":{}}</script>
  <div data-target="react-partial.reactRoot"></div>
</react-partial>

      </template>
    </react-partial-anchor>
  </div>

          <button type="button" class="sr-only js-header-menu-focus-trap d-block d-lg-none">Resetting focus</button>
        </div>
      </div>
    </div>
  </div>
</header>

      <div hidden="hidden" data-view-component="true" class="js-stale-session-flash stale-session-flash flash flash-warn flash-full">
  
        <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
    <path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
        <span class="js-stale-session-flash-signed-in" hidden="">You signed in with another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
        <span class="js-stale-session-flash-signed-out" hidden="">You signed out in another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>
        <span class="js-stale-session-flash-switched" hidden="">You switched accounts on another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span>

    <button id="icon-button-2ebf6b46-6fd4-48f1-8bb4-9e1d436171e2" aria-labelledby="tooltip-acedc7e1-164d-41b9-b32a-3718a49905e7" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium flash-close js-flash-close">  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x Button-visual">
    <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
</button><tool-tip id="tooltip-acedc7e1-164d-41b9-b32a-3718a49905e7" for="icon-button-2ebf6b46-6fd4-48f1-8bb4-9e1d436171e2" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute" aria-hidden="true" role="tooltip">Dismiss alert</tool-tip>


  
</div>
    </div>

  <div id="start-of-content" class="show-on-focus"></div>








    <div id="js-flash-container" class="flash-container" data-turbo-replace="">




  <template class="js-flash-template">
    
<div class="flash flash-full   {{ className }}">
  <div>
    <button autofocus="" class="flash-close js-flash-close" type="button" aria-label="Dismiss this message">
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
    <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
    </button>
    <div aria-atomic="true" role="alert" class="js-flash-alert">
      
      <div>{{ message }}</div>

    </div>
  </div>
</div>
  </template>
</div>


    






  <div class="application-main " data-commit-hovercards-enabled="" data-discussion-hovercards-enabled="" data-issue-and-pr-hovercards-enabled="" data-project-hovercards-enabled="">
        <div itemscope="" itemtype="http://schema.org/SoftwareSourceCode" class="">
    <main id="js-repo-pjax-container">
      
  





    
    

    






  
  <div id="repository-container-header" class="pt-3 hide-full-screen" style="background-color: var(--page-header-bgColor, var(--color-page-header-bg));" data-turbo-replace="">

      <div class="d-flex flex-nowrap flex-justify-end mb-3  px-3 px-lg-5" style="gap: 1rem;">

        <div class="flex-auto min-width-0 width-fit">
            
  <div class=" d-flex flex-wrap flex-items-center wb-break-word f3 text-normal">
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo color-fg-muted mr-2">
    <path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path>
</svg>
    
    <span class="author flex-self-stretch" itemprop="author">
      <a class="url fn" rel="author" data-hovercard-type="organization" data-hovercard-url="/orgs/OSU-NLP-Group/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="/OSU-NLP-Group">
        OSU-NLP-Group
</a>    </span>
    <span class="mx-1 flex-self-stretch color-fg-muted">/</span>
    <strong itemprop="name" class="mr-2 flex-self-stretch">
      <a data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" href="/OSU-NLP-Group/GUI-Agents-Paper-List">GUI-Agents-Paper-List</a>
    </strong>

    <span></span><span class="Label Label--secondary v-align-middle mr-1">Public</span>
  </div>


        </div>

        <div id="repository-details-container" class="flex-shrink-0" data-turbo-replace="" style="max-width: 70%;">
            <ul class="pagehead-actions flex-shrink-0 d-none d-md-inline" style="padding: 2px 0;">
    
      

  <li>
            <a href="/login?return_to=%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" rel="nofollow" id="repository-details-watch-button" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;notification subscription menu watch&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="6aa43455b18452b2da09981985f4e0920116c430e2bfc1f6acc3c33cd37dddd2" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn" aria-describedby="tooltip-82c1a51b-fb8e-4af1-afd3-056a284b0694">    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2">
    <path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path>
</svg>Notifications
</a>    <tool-tip id="tooltip-82c1a51b-fb8e-4af1-afd3-056a284b0694" for="repository-details-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute" role="tooltip">You must be signed in to change notification settings</tool-tip>

  </li>

  <li>
          <a icon="repo-forked" id="fork-button" href="/login?return_to=%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" rel="nofollow" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;repo details fork button&quot;,&quot;repository_id&quot;:880854580,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="0391b076d14e21e6f84425ce33de0cc9f2435a27560dcdd33403038b66d690b7" data-view-component="true" class="btn-sm btn">    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2">
    <path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path>
</svg>Fork
    <span id="repo-network-counter" data-pjax-replace="true" data-turbo-replace="true" title="20" data-view-component="true" class="Counter">20</span>
</a>
  </li>

  <li>
        <div data-view-component="true" class="BtnGroup d-flex">
        <a href="/login?return_to=%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" rel="nofollow" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;star button&quot;,&quot;repository_id&quot;:880854580,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="395b7cf728cfc24ef129d513b5d1601c2e4c762a37f431d148bdd1958c326c5c" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn">    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2">
    <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
</svg><span data-view-component="true" class="d-inline">
          Star
</span>          <span id="repo-stars-counter-star" aria-label="370 users starred this repository" data-singular-suffix="user starred this repository" data-plural-suffix="users starred this repository" data-turbo-replace="true" title="370" data-view-component="true" class="Counter js-social-count">370</span>
</a></div>
  </li>

</ul>

        </div>
      </div>

        <div id="responsive-meta-container" data-turbo-replace="">
      <div class="d-block d-md-none mb-2 px-3 px-md-4 px-lg-5">
      <p class="f4 mb-3 ">
         Building a comprehensive and handy list of papers for GUI agents
      </p>

    

    <div class="mb-3">
        <a class="Link--secondary no-underline mr-3" href="/OSU-NLP-Group/GUI-Agents-Paper-List/stargazers">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star mr-1">
    <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
</svg>
          <span class="text-bold">370</span>
          stars
</a>        <a class="Link--secondary no-underline mr-3" href="/OSU-NLP-Group/GUI-Agents-Paper-List/forks">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-1">
    <path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path>
</svg>
          <span class="text-bold">20</span>
          forks
</a>        <a class="Link--secondary no-underline mr-3 d-inline-block" href="/OSU-NLP-Group/GUI-Agents-Paper-List/branches">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-branch mr-1">
    <path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path>
</svg>
          <span>Branches</span>
</a>        <a class="Link--secondary no-underline d-inline-block" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tags">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-tag mr-1">
    <path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"></path>
</svg>
          <span>Tags</span>
</a>        <a class="Link--secondary no-underline d-inline-block" href="/OSU-NLP-Group/GUI-Agents-Paper-List/activity">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-pulse mr-1">
    <path d="M6 2c.306 0 .582.187.696.471L10 10.731l1.304-3.26A.751.751 0 0 1 12 7h3.25a.75.75 0 0 1 0 1.5h-2.742l-1.812 4.528a.751.751 0 0 1-1.392 0L6 4.77 4.696 8.03A.75.75 0 0 1 4 8.5H.75a.75.75 0 0 1 0-1.5h2.742l1.812-4.529A.751.751 0 0 1 6 2Z"></path>
</svg>
          <span>Activity</span>
</a>    </div>

      <div class="d-flex flex-wrap gap-2">
        <div class="flex-1">
            <div data-view-component="true" class="BtnGroup d-flex">
        <a href="/login?return_to=%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" rel="nofollow" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;star button&quot;,&quot;repository_id&quot;:880854580,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="395b7cf728cfc24ef129d513b5d1601c2e4c762a37f431d148bdd1958c326c5c" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn btn-block">    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2">
    <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
</svg><span data-view-component="true" class="d-inline">
          Star
</span>
</a></div>
        </div>
        <div class="flex-1">
                <a href="/login?return_to=%2FOSU-NLP-Group%2FGUI-Agents-Paper-List" rel="nofollow" id="files-overview-watch-button" data-hydro-click="{&quot;event_type&quot;:&quot;authentication.click&quot;,&quot;payload&quot;:{&quot;location_in_page&quot;:&quot;notification subscription menu watch&quot;,&quot;repository_id&quot;:null,&quot;auth_type&quot;:&quot;LOG_IN&quot;,&quot;originating_url&quot;:&quot;https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List&quot;,&quot;user_id&quot;:null}}" data-hydro-click-hmac="6aa43455b18452b2da09981985f4e0920116c430e2bfc1f6acc3c33cd37dddd2" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn btn-block" aria-describedby="tooltip-622df0d9-7e5a-4eab-ae01-7d5994e08d11">    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2">
    <path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path>
</svg>Notifications
</a>    <tool-tip id="tooltip-622df0d9-7e5a-4eab-ae01-7d5994e08d11" for="files-overview-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute" role="tooltip">You must be signed in to change notification settings</tool-tip>

        </div>
        <span>
          

        </span>
      </div>
  </div>

</div>


          <nav data-pjax="#js-repo-pjax-container" aria-label="Repository" data-view-component="true" class="js-repo-nav js-sidenav-container-pjax js-responsive-underlinenav overflow-hidden UnderlineNav px-3 px-md-4 px-lg-5">

  <ul data-view-component="true" class="UnderlineNav-body list-style-none">
      <li data-view-component="true" class="d-inline-flex">
  <a id="code-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List" data-tab-item="i0code-tab" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches repo_packages repo_deployments repo_attestations /OSU-NLP-Group/GUI-Agents-Paper-List" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g c" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Code&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" aria-current="page" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item selected">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code UnderlineNav-octicon d-none d-sm-inline">
    <path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
</svg>
        <span data-content="Code">Code</span>
          <span id="code-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>


    
</a></li>
      <li data-view-component="true" class="d-inline-flex">
  <a id="issues-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/issues" data-tab-item="i1issues-tab" data-selected-links="repo_issues repo_labels repo_milestones /OSU-NLP-Group/GUI-Agents-Paper-List/issues" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g i" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Issues&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened UnderlineNav-octicon d-none d-sm-inline">
    <path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
</svg>
        <span data-content="Issues">Issues</span>
          <span id="issues-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="3" data-view-component="true" class="Counter">3</span>


    
</a></li>
      <li data-view-component="true" class="d-inline-flex">
  <a id="pull-requests-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/pulls" data-tab-item="i2pull-requests-tab" data-selected-links="repo_pulls checks /OSU-NLP-Group/GUI-Agents-Paper-List/pulls" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g p" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Pull requests&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request UnderlineNav-octicon d-none d-sm-inline">
    <path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
</svg>
        <span data-content="Pull requests">Pull requests</span>
          <span id="pull-requests-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>


    
</a></li>
      <li data-view-component="true" class="d-inline-flex">
  <a id="actions-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/actions" data-tab-item="i3actions-tab" data-selected-links="repo_actions /OSU-NLP-Group/GUI-Agents-Paper-List/actions" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g a" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Actions&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play UnderlineNav-octicon d-none d-sm-inline">
    <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
</svg>
        <span data-content="Actions">Actions</span>
          <span id="actions-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>


    
</a></li>
      <li data-view-component="true" class="d-inline-flex">
  <a id="projects-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/projects" data-tab-item="i4projects-tab" data-selected-links="repo_projects new_repo_project repo_project /OSU-NLP-Group/GUI-Agents-Paper-List/projects" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g b" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Projects&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table UnderlineNav-octicon d-none d-sm-inline">
    <path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
</svg>
        <span data-content="Projects">Projects</span>
          <span id="projects-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span>


    
</a></li>
      <li data-view-component="true" class="d-inline-flex">
  <a id="security-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /OSU-NLP-Group/GUI-Agents-Paper-List/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield UnderlineNav-octicon d-none d-sm-inline">
    <path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
        <span data-content="Security">Security</span>
          <include-fragment src="/OSU-NLP-Group/GUI-Agents-Paper-List/security/overall-count" accept="text/fragment+html" data-nonce="v2:073dc60b-75ed-517e-12ed-f8b77db3c2d0" data-view-component="true">
  
  </include-fragment></a><div data-show-on-forbidden-error="" hidden=""><a id="security-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /OSU-NLP-Group/GUI-Agents-Paper-List/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    </a><div class="Box"><a id="security-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /OSU-NLP-Group/GUI-Agents-Paper-List/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
  </a><div class="blankslate-container"><a id="security-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /OSU-NLP-Group/GUI-Agents-Paper-List/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    </a><div data-view-component="true" class="blankslate blankslate-spacious color-bg-default rounded-2"><a id="security-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /OSU-NLP-Group/GUI-Agents-Paper-List/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
      

      <h3 data-view-component="true" class="blankslate-heading">        Uh oh!
</h3>
      <p data-view-component="true">        </p></a><p class="color-fg-muted my-2 mb-2 ws-normal"><a id="security-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /OSU-NLP-Group/GUI-Agents-Paper-List/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Security&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">There was an error while loading. </a><a class="Link--inTextBlock" data-turbo="false" href="" aria-label="Please reload this page">Please reload this page</a>.</p>
<p></p>

</div>  </div>
</div>  </div>


    
</li>
      <li data-view-component="true" class="d-inline-flex">
  <a id="insights-tab" href="/OSU-NLP-Group/GUI-Agents-Paper-List/pulse" data-tab-item="i6insights-tab" data-selected-links="repo_graphs repo_contributors dependency_graph dependabot_updates pulse people community /OSU-NLP-Group/GUI-Agents-Paper-List/pulse" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-analytics-event="{&quot;category&quot;:&quot;Underline navbar&quot;,&quot;action&quot;:&quot;Click tab&quot;,&quot;label&quot;:&quot;Insights&quot;,&quot;target&quot;:&quot;UNDERLINE_NAV.TAB&quot;}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item">
    
              <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph UnderlineNav-octicon d-none d-sm-inline">
    <path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
</svg>
        <span data-content="Insights">Insights</span>
          <span id="insights-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span>


    
</a></li>
</ul>
    <div style="visibility:hidden;" data-view-component="true" class="UnderlineNav-actions js-responsive-underlinenav-overflow position-absolute pr-3 pr-md-4 pr-lg-5 right-0">      <action-menu data-select-variant="none" data-view-component="true" data-catalyst="" data-ready="true">
  <focus-group direction="vertical" mnemonics="" retain="">
    <button id="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-button" popovertarget="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-overlay" aria-controls="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-list" aria-haspopup="true" aria-labelledby="tooltip-6b48f5cb-0110-4488-9fdf-42d89f6470ae" type="button" data-view-component="true" class="Button Button--iconOnly Button--secondary Button--medium UnderlineNav-item">  <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-kebab-horizontal Button-visual">
    <path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path>
</svg>
</button><tool-tip id="tooltip-6b48f5cb-0110-4488-9fdf-42d89f6470ae" for="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-button" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute" aria-hidden="true" role="tooltip">Additional navigation options</tool-tip>


<anchored-position data-target="action-menu.overlay" id="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-overlay" anchor="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-button" align="start" side="outside-bottom" anchor-offset="normal" popover="auto" data-view-component="true" style="inset: 36px auto auto 0px;">
  <div data-view-component="true" class="Overlay Overlay--size-auto">
    
      <div data-view-component="true" class="Overlay-body Overlay-body--paddingNone">          <action-list data-catalyst="">
  <div data-view-component="true">
    <ul aria-labelledby="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-button" id="action-menu-4a74cddc-8f3a-4565-9322-a99959e4967f-list" role="menu" data-view-component="true" class="ActionListWrap--inset ActionListWrap">
        <li hidden="hidden" data-menu-item="i0code-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-dba3bc29-17f1-4356-bf03-04a17782857d" href="/OSU-NLP-Group/GUI-Agents-Paper-List" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code">
    <path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Code
</span>      
</a>
  
</li>
        <li hidden="hidden" data-menu-item="i1issues-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-7c33f1c4-f4df-411d-8b2a-d35d75a56e61" href="/OSU-NLP-Group/GUI-Agents-Paper-List/issues" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened">
    <path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Issues
</span>      
</a>
  
</li>
        <li hidden="hidden" data-menu-item="i2pull-requests-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-a4e65a2c-d0f8-4715-a0a1-2cb49bd81beb" href="/OSU-NLP-Group/GUI-Agents-Paper-List/pulls" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request">
    <path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Pull requests
</span>      
</a>
  
</li>
        <li hidden="hidden" data-menu-item="i3actions-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-f4b02fa9-e131-4745-bc1b-cb679fdb9406" href="/OSU-NLP-Group/GUI-Agents-Paper-List/actions" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play">
    <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Actions
</span>      
</a>
  
</li>
        <li hidden="hidden" data-menu-item="i4projects-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-e53b89b4-6e77-4b15-8b4f-091ddb42d5b3" href="/OSU-NLP-Group/GUI-Agents-Paper-List/projects" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table">
    <path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Projects
</span>      
</a>
  
</li>
        <li hidden="hidden" data-menu-item="i5security-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-b0931f7e-fd10-45b8-a466-49a986d1061b" href="/OSU-NLP-Group/GUI-Agents-Paper-List/security" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield">
    <path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Security
</span>      
</a>
  
</li>
        <li hidden="hidden" data-menu-item="i6insights-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem">
    
    
    <a tabindex="-1" id="item-7ad2b922-5092-47b2-a5cf-1b2a605f8858" href="/OSU-NLP-Group/GUI-Agents-Paper-List/pulse" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16">
        <span class="ActionListItem-visual ActionListItem-visual--leading">
          <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph">
    <path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path>
</svg>
        </span>
      
        <span data-view-component="true" class="ActionListItem-label">
          Insights
</span>      
</a>
  
</li>
</ul>    
</div></action-list>


</div>
      
</div></anchored-position>  </focus-group>
</action-menu></div>
</nav>

  </div>

  



<turbo-frame id="repo-content-turbo-frame" target="_top" data-turbo-action="advance" class="">
    <div id="repo-content-pjax-container" class="repository-content ">
    



    
      
  <h1 class="sr-only">OSU-NLP-Group/GUI-Agents-Paper-List</h1>
  <div class="clearfix container-xl px-md-4 px-lg-5 px-3">
    <div>

  <div style="max-width: 100%" data-view-component="true" class="Layout Layout--flowRow-until-md react-repos-overview-margin Layout--sidebarPosition-end Layout--sidebarPosition-flowRow-end">
  <div data-view-component="true" class="Layout-main">      <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_dompurify_dist_purify_es_mjs-dd1d3ea6a436.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_tanstack_query-core_build_modern_queryObserver_js-node_modules_tanstack_-defd52-9743ca933872.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_tanstack_react-virtual_dist_esm_index_js-0b672975e32f.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_document-metadata_document-metadata_ts-ui_packages_history_history_ts-ui_packages-417c81-50b9bb07478c.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_paths_index_ts-3838bbeac639.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_ref-selector_RefSelector_tsx-de5e05cf88ec.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_use-alive_use-alive_ts-201977f5783f.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_commit-attribution_index_ts-ui_packages_commit-checks-status_index_ts-ui_packages-762eaa-b9321ed019f0.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_hydro-analytics_hydro-analytics_ts-ui_packages_use-client-value_use-client-value_-6f712e-d59b35038b54.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_code-view-shared_hooks_use-canonical-object_ts-ui_packages_code-view-shared_hooks-7b64b1-87f7a99810ed.js"></script>
<script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/repos-overview-4d90a5db0990.js"></script>
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.ad18e42c1b06f95e74e5.module.css">
<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repos-overview.724784c9697c817d2c2a.module.css">

<react-partial partial-name="repos-overview" data-ssr="true" data-attempted-ssr="true" data-catalyst="" class="loaded">
  
  <script type="application/json" data-target="react-partial.embeddedData">{"props":{"initialPayload":{"allShortcutsEnabled":false,"path":"/","repo":{"id":880854580,"defaultBranch":"main","name":"GUI-Agents-Paper-List","ownerLogin":"OSU-NLP-Group","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2024-10-30T13:40:08.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/92067480?v=4","public":true,"private":false,"isOrgOwned":true},"currentUser":null,"refInfo":{"name":"main","listCacheKey":"v0:1734582197.0","canEdit":false,"refType":"branch","currentOid":"5f96ea0c9ca4bb04ab1832d9f60d95e886d6c83c"},"tree":{"items":[{"name":".github/workflows","path":".github/workflows","contentType":"directory","hasSimplifiedPath":true},{"name":"paper_by_author","path":"paper_by_author","contentType":"directory"},{"name":"paper_by_env","path":"paper_by_env","contentType":"directory"},{"name":"paper_by_key","path":"paper_by_key","contentType":"directory"},{"name":"update_template_or_data","path":"update_template_or_data","contentType":"directory"},{"name":"README.md","path":"README.md","contentType":"file"}],"templateDirectorySuggestionUrl":null,"readme":null,"totalCount":6,"showBranchInfobar":false},"fileTree":null,"fileTreeProcessingTime":null,"foldersToFetch":[],"treeExpanded":false,"symbolsExpanded":false,"isOverview":true,"overview":{"banners":{"shouldRecommendReadme":false,"isPersonalRepo":false,"showUseActionBanner":false,"actionSlug":null,"actionId":null,"showProtectBranchBanner":false,"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_repo","releasePath":"/OSU-NLP-Group/GUI-Agents-Paper-List/releases/new?marketplace=true","showPublishActionBanner":false},"interactionLimitBanner":null,"showInvitationBanner":false,"inviterName":null,"actionsMigrationBannerInfo":{"releaseTags":[],"showImmutableActionsMigrationBanner":false,"initialMigrationStatus":null}},"codeButton":{"contactPath":"/contact","isEnterprise":false,"local":{"protocolInfo":{"httpAvailable":true,"sshAvailable":null,"httpUrl":"https://github.com/OSU-NLP-Group/GUI-Agents-Paper-List.git","showCloneWarning":null,"sshUrl":null,"sshCertificatesRequired":null,"sshCertificatesAvailable":null,"ghCliUrl":"gh repo clone OSU-NLP-Group/GUI-Agents-Paper-List","defaultProtocol":"http","newSshKeyUrl":"/settings/ssh/new","setProtocolPath":"/users/set_protocol"},"platformInfo":{"cloneUrl":"https://desktop.github.com","showVisualStudioCloneButton":false,"visualStudioCloneUrl":"https://windows.github.com","showXcodeCloneButton":false,"xcodeCloneUrl":"xcode://clone?repo=https%3A%2F%2Fgithub.com%2FOSU-NLP-Group%2FGUI-Agents-Paper-List","zipballUrl":"/OSU-NLP-Group/GUI-Agents-Paper-List/archive/refs/heads/main.zip"}},"newCodespacePath":"/codespaces/new?hide_repo_select=true\u0026repo=880854580"},"popovers":{"rename":null,"renamedParentRepo":null},"commitCount":"528","overviewFiles":[{"displayName":"README.md","repoName":"GUI-Agents-Paper-List","refName":"main","path":"README.md","preferredFileType":"readme","tabName":"README","richText":"\u003carticle class=\"markdown-body entry-content container-lg\" itemprop=\"text\"\u003e\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAwesome GUI Agent Paper List\u003c/h1\u003e\u003ca id=\"user-content-awesome-gui-agent-paper-list\" class=\"anchor\" aria-label=\"Permalink: Awesome GUI Agent Paper List\" href=\"#awesome-gui-agent-paper-list\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThis repo covers a variety of papers related to GUI Agents, such as:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDatasets\u003c/li\u003e\n\u003cli\u003eBenchmarks\u003c/li\u003e\n\u003cli\u003eModels\u003c/li\u003e\n\u003cli\u003eAgent frameworks\u003c/li\u003e\n\u003cli\u003eVision, language, multimodal foundation models (with explicit support for GUI)\u003c/li\u003e\n\u003cli\u003eWorks in general domains extensively used by GUI Agents (e.g., SoM prompting)\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003e\u003ca target=\"_blank\" rel=\"noopener noreferrer\" href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/update_template_or_data/statistics/keyword_wordcloud_long.png\"\u003e\u003cimg src=\"/OSU-NLP-Group/GUI-Agents-Paper-List/raw/main/update_template_or_data/statistics/keyword_wordcloud_long.png\" alt=\"keyword_wordcloud_long.png\" style=\"max-width: 100%;\"\u003e\u003c/a\u003e\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003ePapers Grouped by Environments\u003c/h2\u003e\u003ca id=\"user-content-papers-grouped-by-environments\" class=\"anchor\" aria-label=\"Permalink: Papers Grouped by Environments\" href=\"#papers-grouped-by-environments\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cmarkdown-accessiblity-table\u003e\u003ctable\u003e\n\u003cthead\u003e\n\u003ctr\u003e\n\u003cth\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_web.md\"\u003eWeb\u003c/a\u003e\u003c/th\u003e\n\u003cth\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_mobile.md\"\u003eMobile\u003c/a\u003e\u003c/th\u003e\n\u003cth\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_desktop.md\"\u003eDesktop\u003c/a\u003e\u003c/th\u003e\n\u003cth\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_gui.md\"\u003eGUI\u003c/a\u003e\u003c/th\u003e\n\u003cth\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_misc.md\"\u003eMisc\u003c/a\u003e\u003c/th\u003e\n\u003c/tr\u003e\n\u003c/thead\u003e\n\u003c/table\u003e\u003c/markdown-accessiblity-table\u003e\n\u003cp dir=\"auto\"\u003e(Misc: Papers for general topics that have important applications in GUI agents.)\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003ePapers Grouped by Keywords\u003c/h2\u003e\u003ca id=\"user-content-papers-grouped-by-keywords\" class=\"anchor\" aria-label=\"Permalink: Papers Grouped by Keywords\" href=\"#papers-grouped-by-keywords\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_framework.md\"\u003eframework (142)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_benchmark.md\"\u003ebenchmark (90)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_dataset.md\"\u003edataset (80)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_model.md\"\u003emodel (42)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_reinforcement_learning.md\"\u003ereinforcement learning (22)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_planning.md\"\u003eplanning (13)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_safety.md\"\u003esafety (12)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_reasoning.md\"\u003ereasoning (10)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_visual-grounding.md\"\u003evisual grounding (10)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_evaluation.md\"\u003eevaluation (9)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_survey.md\"\u003esurvey (8)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_grounding.md\"\u003egrounding (8)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_learning.md\"\u003elearning (5)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_vision_language_model.md\"\u003evision language model (5)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_multimodal.md\"\u003emultimodal (4)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_memory.md\"\u003ememory (4)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_attack.md\"\u003eattack (4)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_GUI_grounding.md\"\u003eGUI grounding (3)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_WebArena.md\"\u003eWebArena (3)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_ScreenSpot-Pro.md\"\u003eScreenSpot-Pro (3)\u003c/a\u003e\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003ePapers Grouped by Authors\u003c/h2\u003e\u003ca id=\"user-content-papers-grouped-by-authors\" class=\"anchor\" aria-label=\"Permalink: Papers Grouped by Authors\" href=\"#papers-grouped-by-authors\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Yu_Su.md\"\u003eYu Su (13)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Graham_Neubig.md\"\u003eGraham Neubig (10)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Huan_Sun.md\"\u003eHuan Sun (9)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Boyuan_Zheng.md\"\u003eBoyuan Zheng (8)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Tao_Yu.md\"\u003eTao Yu (8)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Tianbao_Xie.md\"\u003eTianbao Xie (7)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Shuyan_Zhou.md\"\u003eShuyan Zhou (7)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Qiushi_Sun.md\"\u003eQiushi Sun (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Yu_Gu.md\"\u003eYu Gu (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Daniel_Fried.md\"\u003eDaniel Fried (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Boyu_Gou.md\"\u003eBoyu Gou (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Mike_Zheng_Shou.md\"\u003eMike Zheng Shou (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Kun_Shao.md\"\u003eKun Shao (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Zhiyong_Wu.md\"\u003eZhiyong Wu (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Xiao_Liu.md\"\u003eXiao Liu (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Hanyu_Lai.md\"\u003eHanyu Lai (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Jie_Tang.md\"\u003eJie Tang (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Yuxiao_Dong.md\"\u003eYuxiao Dong (6)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Jianfeng_Gao.md\"\u003eJianfeng Gao (5)\u003c/a\u003e | \u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Jianye_Hao.md\"\u003eJianye Hao (5)\u003c/a\u003e\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAll Papers (from most recent to oldest)\u003c/h2\u003e\u003ca id=\"user-content-all-papers-from-most-recent-to-oldest\" class=\"anchor\" aria-label=\"Permalink: All Papers (from most recent to oldest)\" href=\"#all-papers-from-most-recent-to-oldest\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdetails open=\"\"\u003e\n\u003csummary\u003ePapers\u003c/summary\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2505.00684\" rel=\"nofollow\"\u003eVisual Test-time Scaling for GUI Agent Grounding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTiange Luo, Lajanugen Logeswaran, Justin Johnson, Honglak Lee\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UMich, LG AI Research\u003c/li\u003e\n\u003cli\u003e📅 Date: May 1, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [RegionFocus], [test-time scaling], [grounding], [Qwen2.5-VL], [UI-TARS]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eRegionFocus\u003c/em\u003e, a visual test-time scaling method for GUI agents that dynamically zooms into relevant regions within GUI images, reducing background clutter and enhancing grounding accuracy. By integrating an \"image-as-map\" mechanism to visualize key landmarks during each interaction step, the approach improves transparency and decision-making. Applied to state-of-the-art vision-language models like UI-TARS and Qwen2.5-VL, RegionFocus achieves significant performance gains—28% on ScreenSpot-Pro and 24% on WebVoyager benchmarks—setting a new state-of-the-art grounding accuracy of 61.6% on ScreenSpot-Pro.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2505.00416\" rel=\"nofollow\"\u003eScaleTrack: Scaling and back-tracking Automated GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJing Huang, Zhixiong Zeng, Wenkang Han, Yufeng Zhong, Liming Zheng, Shuai Fu, Jingyuan Chen, Lin Ma\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Meituan, Zhejiang University, University of Adelaide\u003c/li\u003e\n\u003cli\u003e📅 Date: May 1, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [training strategy], [back-tracking], [grounding], [planning], [ScaleTrack]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eScaleTrack\u003c/em\u003e, a training framework designed to enhance automated GUI agents by addressing two primary challenges: insufficient training data for GUI grounding and the lack of back-tracking in GUI planning. The authors aggregate diverse GUI samples from various synthesis methods into a unified template to scale the grounding process. Additionally, they propose a hybrid training strategy that combines forward-planning and back-tracking, enabling the agent to predict both the next action and the historical actions leading to the current GUI state. Experimental results demonstrate that ScaleTrack significantly improves task success rates across multiple benchmarks, highlighting the effectiveness of integrating back-tracking into GUI agent training.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.20464\" rel=\"nofollow\"\u003eA Summary on GUI Agents with Foundation Models Enhanced by Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJiahao Li, Kaer Huang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Lenovo Research\u003c/li\u003e\n\u003cli\u003e📅 Date: April 29, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey], [framework], [reinforcement learning], [multimodal], [training taxonomy]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents a structured summary of recent advances in GUI agents powered by Multi-modal Large Language Models (MLLMs) and enhanced through Reinforcement Learning (RL). It formalizes GUI agent tasks as Markov Decision Processes and reviews modular architectures comprising Perception, Planning, and Acting modules. The study categorizes training methodologies into Prompt-based, Supervised Fine-Tuning (SFT), and RL-based approaches, highlighting the progression towards dynamic policy learning. The paper concludes by identifying key challenges and future directions for developing more capable and reliable GUI agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/PhoneLLM/Awesome-LLM-Powered-Phone-GUI-Agents\"\u003eLLM-Powered GUI Agents in Phone Automation: Surveying Progress and Prospects\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGuangyi Liu, Pengxiang Zhao, Liang Liu, Yaxuan Guo, Han Xiao, Weifeng Lin, Yuxiang Chai, Yue Han, Shuai Ren, Hao Wang, Xiaoyu Liang, Wenhao Wang, Tianze Wu, Linghao Li, Guanjing Xiong, Yong Liu, Hongsheng Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang Univ., vivo AI Lab, CUHK MMLab\u003c/li\u003e\n\u003cli\u003e📅 Date: April 28, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey], [framework], [dataset], [benchmark], [planning], [multimodal], [taxonomy]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This comprehensive survey examines the evolution of LLM-powered GUI agents in mobile phone automation, transitioning from static scripts to intelligent, adaptive systems. It presents a taxonomy encompassing agent frameworks (single-agent, multi-agent, plan-then-act), modeling approaches (prompt engineering, training-based), and essential datasets and benchmarks. The paper discusses how LLMs enhance language understanding, multimodal perception, and decision-making in GUI tasks, and addresses challenges such as dataset diversity, on-device deployment efficiency, user-centric adaptation, and security concerns. It serves as a definitive reference for researchers and practitioners aiming to leverage LLMs in designing scalable, user-friendly phone GUI agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.14603\" rel=\"nofollow\"\u003eUFO2: The Desktop AgentOS\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChaoyun Zhang, He Huang, Chiming Ni, Jian Mu, Si Qin, Shilin He, Lu Wang, Fangkai Yang, Pu Zhao, Chao Du, Liqun Li, Yu Kang, Zhao Jiang, Suzhen Zheng, Rujia Wang, Jiaxu Qian, Minghua Ma, Jian-Guang Lou, Qingwei Lin, Saravan Rajmohan, Dongmei Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Microsoft Research, ZJU-UIUC Institute, Nanjing University, Peking University\u003c/li\u003e\n\u003cli\u003e📅 Date: April 20, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [hybrid control], [multi-agent], [speculative execution], [knowledge substrate], [UFO2]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: UFO2 introduces a multi-agent AgentOS for Windows desktops, aiming to transform Computer-Using Agents (CUAs) from fragile prototypes into robust, system-level automation tools. It features a centralized HostAgent for task decomposition and coordination, along with specialized AppAgents equipped with native APIs and domain-specific knowledge. The system employs a hybrid control detection pipeline combining Windows UI Automation with vision-based parsing, and enhances runtime efficiency through speculative multi-action planning. A Picture-in-Picture interface allows agents and users to operate concurrently without interference. Evaluated across over 20 real-world Windows applications, UFO2 demonstrates significant improvements in robustness and execution accuracy over prior CUAs.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.14239\" rel=\"nofollow\"\u003eInfiGUI-R1: Advancing Multimodal GUI Agents from Reactive Actors to Deliberative Reasoners\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYuhang Liu, Pengxiang Li, Congkai Xie, Xavier Hu, Xiaotian Han, Shengyu Zhang, Hongxia Yang, Fei Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang Univ., Dalian Univ. of Tech., Reallm Labs, HK PolyU\u003c/li\u003e\n\u003cli\u003e📅 Date: April 19, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [reinforcement learning], [planning], [reasoning], [Actor2Reasoner], [InfiGUI-R1]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eInfiGUI-R1\u003c/strong\u003e, a multimodal GUI agent developed through the \u003cstrong\u003eActor2Reasoner\u003c/strong\u003e framework, aiming to transition agents from reactive behaviors to deliberative reasoning. The framework comprises two stages: \u003cem\u003eReasoning Injection\u003c/em\u003e, which employs Spatial Reasoning Distillation to integrate GUI visual-spatial information with logical reasoning, and \u003cem\u003eDeliberation Enhancement\u003c/em\u003e, which uses Reinforcement Learning with Sub-goal Guidance and Error Recovery Scenario Construction to refine the agent's planning and error correction capabilities. Evaluations on benchmarks like AndroidControl and ScreenSpot demonstrate that InfiGUI-R1-3B achieves state-of-the-art performance in GUI grounding and trajectory tasks, outperforming larger models in several categories.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://lgy0404.github.io/LearnAct\" rel=\"nofollow\"\u003eLearnAct: Few-Shot Mobile GUI Agent with a Unified Demonstration Benchmark\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGuangyi Liu, Pengxiang Zhao, Liang Liu, Zhiming Chen, Yuxiang Chai, Shuai Ren, Hao Wang, Shibo He, Wenchao Meng\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ZJU, vivo AI Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: April 18, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [few-shot learning], [LearnAct], [LearnGUI], [DemoParser], [KnowSeeker], [ActExecutor]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eLearnAct\u003c/strong\u003e, a multi-agent framework designed to enhance mobile GUI agents through few-shot demonstration learning. Accompanied by \u003cstrong\u003eLearnGUI\u003c/strong\u003e, a dataset comprising 2,252 offline and 101 online tasks with high-quality human demonstrations, the framework integrates three specialized agents—DemoParser, KnowSeeker, and ActExecutor—to extract, retrieve, and apply knowledge from demonstrations. Experimental results show significant performance improvements in both offline and online evaluations, establishing demonstration-based learning as a promising direction for developing adaptable and personalized mobile GUI agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://tongui-agent.github.io/\" rel=\"nofollow\"\u003eTongUI: Building Generalized GUI Agents by Learning from Multimodal Web Tutorials\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBofei Zhang, Zirui Shang, Zhi Gao, Wang Zhang, Rui Xie, Xiaojian Ma, Tao Yuan, Xinxiao Wu, Song-Chun Zhu, Qing Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: BIGAI, BIT, PKU, SJTU, THU\u003c/li\u003e\n\u003cli\u003e📅 Date: April 17, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [Qwen2.5-VL], [GUI-Net], [multimodal], [trajectory generation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eTongUI\u003c/strong\u003e, a framework for building generalized GUI agents by learning from multimodal web tutorials. By crawling and processing online tutorials into GUI agent trajectory data, the authors construct the \u003cstrong\u003eGUI-Net\u003c/strong\u003e dataset containing 143K trajectories across five operating systems and over 200 applications. Fine-tuning Qwen2.5-VL-3B/7B models on GUI-Net leads to significant performance improvements on grounding and navigation benchmarks, demonstrating the effectiveness of the TongUI framework.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.11257\" rel=\"nofollow\"\u003eUI-E2I-Synth: Advancing GUI Grounding with Large-Scale Instruction Synthesis\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXinyi Liu, Xiaoyi Zhang, Ziyun Zhang, Yan Lu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: MSRA, PKU\u003c/li\u003e\n\u003cli\u003e📅 Date: April 15, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [instruction synthesis], [UI-E2I-Synth], [UI-I2E-Bench], [GPT-4o]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eUI-E2I-Synth\u003c/strong\u003e, a large-scale data synthesis pipeline that generates diverse GUI grounding instructions using GPT-4o, addressing challenges like implicit instructions and unbalanced element types. It also presents \u003cstrong\u003eUI-I2E-Bench\u003c/strong\u003e, a new benchmark with detailed annotations for evaluating GUI instruction grounding. Models trained on the synthesized data demonstrate superior performance across multiple platforms, highlighting the effectiveness of the proposed approach.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://realevals.xyz\" rel=\"nofollow\"\u003eREAL: Benchmarking Autonomous Agents on Deterministic Simulations of Real Websites\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDivyansh Garg, Shaun VanWeelden, Diego Caples, Andis Draguns, Nikil Ravi, Pranav Putta, Naman Garg, Tomas Abraham, Michael Lara, Federico Lopez, James Liu, Atharva Gundawar, Prannay Hebbar, Youngchul Joo, Charles London, Christian Schroeder de Witt, Sumeet Motwani\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: AGI Inc.\u003c/li\u003e\n\u003cli\u003e📅 Date: April 15, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [dataset], [evaluation], [reproducibility], [multi-turn tasks], [REAL]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eREAL\u003c/strong\u003e, a benchmark and framework designed for evaluating autonomous agents through deterministic simulations of real-world websites. REAL encompasses high-fidelity replicas of 11 widely-used websites across domains like e-commerce, travel, communication, and professional networking. It includes 112 practical tasks that mirror complex user interactions requiring both accurate information retrieval and state-changing actions. The controlled environment ensures safety and reproducibility, combining programmatic checks for action-based tasks with rubric-guided LLM-based judgments for information retrieval. Empirical results reveal that current frontier language models achieve at most a 41% success rate on REAL, highlighting significant gaps in autonomous web navigation and task completion capabilities.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.10127\" rel=\"nofollow\"\u003eBreaking the Data Barrier -- Building GUI Agents Through Task Generalization\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJunlei Zhang, Zichen Ding, Chang Ma, Zijie Chen, Qiushi Sun, Zhenzhong Lan, Junxian He\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ZJU, WestlakeU, Shanghai AI Lab, HKU, HKUST\u003c/li\u003e\n\u003cli\u003e📅 Date: April 14, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [mid-training], [reasoning], [GUIMid]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a mid-training approach to enhance GUI agents by leveraging diverse, reasoning-intensive datasets such as mathematical and coding tasks. The authors demonstrate that training Vision Language Models (VLMs) on these data-rich tasks before fine-tuning on limited GUI trajectory data significantly improves performance on GUI benchmarks like WebArena and AndroidWorld. Notably, text-only mathematical reasoning data led to substantial cross-modal gains, highlighting the effectiveness of task generalization in overcoming data scarcity challenges in GUI agent development.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.10458\" rel=\"nofollow\"\u003eGUI-R1: A Generalist R1-Style Vision-Language Action Model For GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXiaobo Xia, Run Luo\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Unknown\u003c/li\u003e\n\u003cli\u003e📅 Date: April 14, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [unified action space], [GRPO], [GUI-R1]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: GUI-R1 introduces a reinforcement learning framework that enhances the capabilities of Large Vision-Language Models (LVLMs) for GUI agents. By employing a unified action space and a rule-based reward function, the model efficiently learns to perform high-level tasks across multiple platforms, including Windows, Linux, MacOS, Android, and Web. Utilizing only 0.02% of the data compared to previous methods, GUI-R1 demonstrates superior performance on eight benchmarks, showcasing the potential of reinforcement learning in GUI agent tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://scai.cs.jhu.edu/projects/RealWebAssist/\" rel=\"nofollow\"\u003eRealWebAssist: A Benchmark for Long-Horizon Web Assistance with Real-World Users\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSuyu Ye, Haojun Shi, Darren Shih, Hyokun Yun, Tanya Roosta, Tianmin Shu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: JHU, Amazon\u003c/li\u003e\n\u003cli\u003e📅 Date: April 14, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [GUI grounding], [speech input], [spatial reasoning], [temporal reasoning], [multi-step planning], [routine learning], [RealWebAssist]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: RealWebAssist introduces a benchmark for evaluating AI agents' ability to assist with long-horizon web tasks using sequential instructions from real-world users. The dataset includes 1,885 instructions across 107 tasks on 66 websites, featuring challenges like ambiguous instructions, GUI grounding, and evolving user goals. Evaluations show that current state-of-the-art models struggle with these complex, realistic scenarios.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://agent-reward-bench.github.io/\" rel=\"nofollow\"\u003eAgentRewardBench: Evaluating Automatic Evaluations of Web Agent Trajectories\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXing Han Lù, Amirhossein Kazemnejad, Nicholas Meade, Arkil Patel, Dongchan Shin, Alejandra Zambrano, Karolina Stańczak, Peter Shaw, Christopher J. Pal, Siva Reddy\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: McGill, Mila, Google DeepMind, Polytechnique Montréal, ServiceNow Research\u003c/li\u003e\n\u003cli\u003e📅 Date: April 11, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [evaluation], [LLM judges], [AgentRewardBench]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eAgentRewardBench\u003c/em\u003e, a benchmark designed to assess the effectiveness of large language model (LLM) judges in evaluating web agent trajectories. The benchmark comprises 1,302 trajectories across five web benchmarks, each annotated by experts for success, side effects, and repetitiveness. Evaluations of 12 LLM judges reveal that no single model excels across all benchmarks, and that rule-based evaluations often underreport agent success rates, highlighting the need for more adaptable automatic evaluation methods.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.07079\" rel=\"nofollow\"\u003eSkillWeaver: Web Agents can Self-Improve by Discovering and Honing Skills\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBoyuan Zheng, Michael Y. Fatemi, Xiaolong Jin, Zora Zhiruo Wang, Apurva Gandhi, Yueqi Song, Yu Gu, Jayanth Srinivasa, Gaowen Liu, Graham Neubig, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, CMU, UVA, Purdue, Cisco Research\u003c/li\u003e\n\u003cli\u003e📅 Date: April 9, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [API synthesis], [skill discovery], [transfer learning], [WebArena]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: SkillWeaver is a framework that enables web agents to autonomously improve by discovering, practicing, and refining reusable skills, encapsulated as APIs. Through iterative exploration, agents build a library of plug-and-play APIs, enhancing their capabilities. Experiments on WebArena and real-world websites demonstrate significant performance improvements, and the synthesized APIs can be shared among agents to boost overall performance.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.06821\" rel=\"nofollow\"\u003eInducing Programmatic Skills for Agentic Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZora Zhiruo Wang, Apurva Gandhi, Graham Neubig, Daniel Fried\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: April 9, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [benchmark], [learning], [reasoning], [planning], [ASI], [WebArena]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces Agent Skill Induction (ASI), a framework enabling web agents to learn and apply programmatic skills dynamically. By representing skills as executable programs, ASI allows agents to verify and reuse these skills across tasks, enhancing adaptability and efficiency. Evaluated on the WebArena benchmark, ASI outperforms static and text-based skill agents in success rate and step efficiency, demonstrating improved generalization and adaptability to new web environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.04716\" rel=\"nofollow\"\u003eOn the Robustness of GUI Grounding Models Against Image Attacks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHaoren Zhao, Tianyi Chen, Zhen Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HDU, Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: April 7, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [robustness], [adversarial attacks], [UGround], [ScreenSpot-V2]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper systematically evaluates the robustness of GUI grounding models, such as UGround, against natural noise, untargeted adversarial attacks, and targeted adversarial attacks. Experiments conducted across mobile, desktop, and web interfaces reveal that these models are highly sensitive to adversarial perturbations and low-resolution conditions. The findings highlight vulnerabilities in current GUI grounding models and establish a benchmark for future research aimed at enhancing their robustness in practical applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.01382\" rel=\"nofollow\"\u003eAn Illusion of Progress? Assessing the Current State of Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianci Xue, Weijian Qi, Tianneng Shi, Chan Hee Song, Boyu Gou, Dawn Song, Huan Sun, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, UC Berkeley\u003c/li\u003e\n\u003cli\u003e📅 Date: April 2, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [evaluation], [Online-Mind2Web], [WebJudge], [Operator], [LLM-as-a-Judge]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper critically evaluates the performance of current web agents, revealing that many underperform on the newly introduced \u003cstrong\u003eOnline-Mind2Web\u003c/strong\u003e benchmark, which comprises 300 realistic tasks across 136 websites. The study highlights a discrepancy between reported successes and actual capabilities, attributing this to shortcomings in existing benchmarks. To address evaluation scalability, the authors propose \u003cstrong\u003eWebJudge\u003c/strong\u003e, an automatic LLM-based evaluation method achieving approximately 85% agreement with human judgments. The comprehensive analysis underscores the need for more robust assessment frameworks in web agent research.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2504.00906\" rel=\"nofollow\"\u003eAgent S2: A Compositional Generalist-Specialist Framework for Computer Use Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSaaket Agashe, Kyle Wong, Vincent Tu, Jiachen Yang, Ang Li, Xin Eric Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Simular Research\u003c/li\u003e\n\u003cli\u003e📅 Date: April 1, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Mixture-of-Grounding], [Proactive Hierarchical Planning], [benchmark], [OSWorld], [WindowsAgentArena], [AndroidWorld]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eAgent S2\u003c/em\u003e, a compositional framework for computer use agents that combines generalist and specialist models to address challenges in GUI element grounding and long-horizon task planning. The framework employs a novel Mixture-of-Grounding technique for precise GUI localization and Proactive Hierarchical Planning to dynamically refine action plans. Evaluations demonstrate that Agent S2 achieves state-of-the-art performance on benchmarks like OSWorld, WindowsAgentArena, and AndroidWorld, outperforming existing agents such as Claude Computer Use and UI-TARS.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.23350\" rel=\"nofollow\"\u003eA Survey of WebAgents: Towards Next-Generation AI Agents for Web Automation with Large Foundation Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLiangbo Ning, Ziran Liang, Zhuohang Jiang, Haohao Qu, Yujuan Ding, Wenqi Fan, Xiao-yong Wei, Shanru Lin, Hui Liu, Philip S. Yu, Qing Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: PolyU, CityUHK, MSU, UIC\u003c/li\u003e\n\u003cli\u003e📅 Date: March 30, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey], [framework], [training], [trustworthiness], [WebAgents], [Large Foundation Models]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This comprehensive survey examines the development of WebAgents—AI agents designed to automate web tasks—by leveraging Large Foundation Models (LFMs). It delves into the architectures, training methodologies, and trustworthiness of these agents, providing a detailed overview of current research and proposing future directions to enhance their effectiveness and reliability in web automation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.21620\" rel=\"nofollow\"\u003eUI-R1: Enhancing Action Prediction of GUI Agents by Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhengxi Lu, Yuxiang Chai, Yaxuan Guo, Xi Yin, Liang Liu, Hao Wang, Guanjing Xiong, Hongsheng Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: vivo AI Lab, MMLab @ CUHK\u003c/li\u003e\n\u003cli\u003e📅 Date: March 27, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [reinforcement learning], [UI-R1-3B], [GRPO], [AndroidControl], [ScreenSpot-Pro]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eUI-R1\u003c/strong\u003e, a framework that enhances the reasoning capabilities of multimodal large language models (MLLMs) for GUI action prediction tasks through rule-based reinforcement learning. Utilizing a small, high-quality dataset of 136 challenging mobile tasks, the authors design a unified rule-based action reward function and optimize the model using Group Relative Policy Optimization (GRPO). The resulting model, \u003cstrong\u003eUI-R1-3B\u003c/strong\u003e, demonstrates significant improvements over the base model (Qwen2.5-VL-3B) on both in-domain (AndroidControl) and out-of-domain (ScreenSpot-Pro) benchmarks, showcasing the effectiveness of rule-based RL in advancing GUI understanding and control.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.15937\" rel=\"nofollow\"\u003eAdvancing Mobile GUI Agents: A Verifier-Driven Approach to Practical Deployment\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGaole Dai, Shiqi Jiang, Ting Cao, Yuanchun Li, Yuqing Yang, Rui Tan, Mo Li, Lili Qiu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, NTU, UT Austin\u003c/li\u003e\n\u003cli\u003e📅 Date: March 20, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [dataset], [benchmark], [V-Droid], [verifier-driven], [pair-wise preference training], [human-agent joint annotation], [low-latency]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eV-Droid\u003c/strong\u003e, a mobile GUI automation agent that leverages large language models (LLMs) as verifiers rather than generators. By evaluating candidate actions before execution, V-Droid enhances decision-making accuracy and reduces latency. The framework incorporates discretized action space construction, a prefilling-only workflow, pair-wise preference training, and a scalable human-agent joint annotation scheme. Evaluated on benchmarks like AndroidWorld, AndroidLab, and MobileAgentBench, V-Droid achieves state-of-the-art success rates and operates with near-real-time decision-making capabilities.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.15661\" rel=\"nofollow\"\u003eUI-Vision: A Desktop-centric GUI Benchmark for Visual Perception and Interaction\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShravan Nayak, Xiangru Jian, Kevin Qinghong Lin, Juan A. Rodriguez, Montek Kalsi, Rabiul Awal, Nicolas Chapados, M. Tamer Özsu, Aishwarya Agrawal, David Vazquez, Christopher Pal, Perouz Taslakian, Spandana Gella, Sai Rajeswar\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Unknown\u003c/li\u003e\n\u003cli\u003e📅 Date: March 19, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [framework], [UI-Vision], [UI-TARS-72B], [spatial reasoning], [drag-and-drop], [element grounding], [layout grounding], [action prediction]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eUI-Vision\u003c/em\u003e, a comprehensive, license-permissive benchmark designed for evaluating autonomous agents in real-world desktop GUI environments. It encompasses 83 software applications with dense annotations of human demonstrations, including bounding boxes, UI labels, and action trajectories. The benchmark defines three tasks—Element Grounding, Layout Grounding, and Action Prediction—to assess agents' performance. Evaluations reveal limitations in state-of-the-art models like UI-TARS-72B, particularly in understanding professional software, spatial reasoning, and complex actions such as drag-and-drop. By releasing UI-Vision as open-source, the authors aim to advance the development of more capable agents for desktop tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/FanbinLu/STEVE\"\u003eSTEVE: A Step Verification Pipeline for Computer-use Agent Training\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFanbin Lu, Zhisheng Zhong, Ziqin Wei, Shu Liu, Chi-Wing Fu, Jiaya Jia\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CUHK, SmartMore, HKUST\u003c/li\u003e\n\u003cli\u003e📅 Date: March 16, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [model], [UI grounding], [KTO], [GPT-4o], [WinAgentArena]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eSTEVE\u003c/strong\u003e, a step verification pipeline designed to enhance the training of computer-use agents. The approach involves creating a large instruction set and collecting trajectory data using suboptimal agents. GPT-4o is employed to verify the correctness of each action step by comparing pre- and post-action screenshots, assigning binary labels. The agent is then optimized using Kahneman and Tversky Optimization (KTO) based on these labels. The result is a 7B vision-language model that achieves state-of-the-art performance in the WinAgentArena desktop environment, outperforming supervised fine-tuning methods by effectively leveraging both positive and negative action data.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.11170\" rel=\"nofollow\"\u003eDeskVision: Large Scale Desktop Region Captioning for Advanced GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYibin Xu, Liang Yang, Hao Chen, Hua Wang, Zhi Chen, Yaohua Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Unknown\u003c/li\u003e\n\u003cli\u003e📅 Date: March 14, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [model], [AutoCaptioner], [DeskVision], [DeskVision-Eval], [GUIExplorer]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eAutoCaptioner\u003c/em\u003e, an automated pipeline for generating richly annotated GUI data with minimal human effort. Utilizing AutoCaptioner, the authors created \u003cem\u003eDeskVision\u003c/em\u003e, a large-scale desktop GUI dataset comprising 54,855 images and over 303,622 annotations across various operating systems. They also developed \u003cem\u003eDeskVision-Eval\u003c/em\u003e, the largest desktop benchmark reflecting daily usage scenarios. Leveraging DeskVision, the authors trained \u003cem\u003eGUIExplorer\u003c/em\u003e, a GUI understanding model that achieves state-of-the-art performance in grounding visual elements without complex architectural designs. The effectiveness of DeskVision was further validated through ablation studies on various large visual language models (LVLMs).\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.09241\" rel=\"nofollow\"\u003eIn-Context Defense in Computer Agents: An Empirical Study\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePei Yang, Hai Ci, Mike Zheng Shou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Unknown\u003c/li\u003e\n\u003cli\u003e📅 Date: March 12, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [defense], [in-context learning], [chain-of-thought], [context deception], [security]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces an in-context defense strategy for computer agents powered by vision-language models (VLMs), targeting context deception attacks such as malicious pop-ups and deceptive HTML elements. By incorporating a small set of curated exemplars and employing chain-of-thought reasoning, the approach guides agents to perform explicit defensive reasoning before action planning. Experiments demonstrate significant reductions in attack success rates across various attack types, highlighting the effectiveness of the method in enhancing agent reliability without requiring model fine-tuning.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2503.06470\" rel=\"nofollow\"\u003eThink Twice, Click Once: Enhancing GUI Grounding via Fast and Slow Systems\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFei Tang, Yongliang Shen, Hang Zhang, Siqi Chen, Guiyang Hou, Wenqi Zhang, Wenqiao Zhang, Kaitao Song, Weiming Lu, Yueting Zhuang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University\u003c/li\u003e\n\u003cli\u003e📅 Date: March 9, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [dual-system cognition], [FOCUS], [ScreenSpot], [ScreenSpot-Pro]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eFOCUS\u003c/strong\u003e, a novel GUI grounding framework inspired by human dual-system cognition. FOCUS dynamically switches between a fast, intuitive system and a slow, analytical system based on task complexity. The framework decomposes GUI grounding into three stages: interface summarization, focused analysis, and precise coordinate prediction. Trained on a synthesized dataset of 300,000 samples, FOCUS achieves state-of-the-art performance on the ScreenSpot and ScreenSpot-Pro benchmarks, demonstrating significant improvements in both efficiency and accuracy for complex GUI interactions.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/PathOnAI/LiteWebAgent\"\u003eLiteWebAgent: The Open-Source Suite for VLM-Based Web-Agent Applications\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDanqing Zhang, Balaji Rama, Jingyi Ni, Shiying He, Fu Zhao, Kunyu Chen, Arnold Chen, Junyu Cao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: PathOnAI.org, Rutgers Univ., UT Austin\u003c/li\u003e\n\u003cli\u003e📅 Date: March 4, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NAACL 2025\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [planning], [memory], [tree search], [LiteWebAgent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: LiteWebAgent is an open-source suite designed for VLM-based web agent applications. It offers a modular framework that decouples action generation from grounding, supports agent planning, memory, and tree search, and is deployable via a Vercel-based web app or a Chrome extension using the Chrome DevTools Protocol (CDP).\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://vulnerable-ai-agents.github.io/\" rel=\"nofollow\"\u003eWhy Are Web AI Agents More Vulnerable Than Standalone LLMs? A Security Analysis\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJeffrey Yang Fan Chiang, Seungjae Lee, Jia-Bin Huang, Furong Huang, Yizheng Chen\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UMD\u003c/li\u003e\n\u003cli\u003e📅 Date: March 4, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [security], [jailbreaking], [evaluation], [OpenHands]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper investigates why Web AI agents are significantly more susceptible to executing harmful commands compared to standalone LLMs, despite sharing the same underlying models. Through a fine-grained evaluation, the authors identify three critical factors contributing to this vulnerability: embedding user goals into system prompts, multi-step action generation, and processing of event streams from web navigation. The study introduces a five-level harmfulness evaluation framework and utilizes the OpenHands platform to systematically assess these vulnerabilities, revealing a 46.6% success rate in malicious task execution by Web AI agents versus 0% for standalone LLMs.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2502.13053\" rel=\"nofollow\"\u003eAEIA-MN: Evaluating the Robustness of Multimodal LLM-Powered Mobile Agents Against Active Environmental Injection Attacks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYurun Chen, Xueyu Hu, Keting Yin, Juncheng Li, Shengyu Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University\u003c/li\u003e\n\u003cli\u003e📅 Date: February 18, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [safety], [AEIA-MN]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the concept of Active Environment Injection Attack (AEIA), where attackers disguise malicious actions as environmental elements to disrupt AI agents' decision-making processes. The authors propose AEIA-MN, an attack scheme leveraging mobile notifications to evaluate the robustness of multimodal large language model-based mobile agents. Experimental results demonstrate that even advanced models are highly vulnerable to such attacks, with success rates reaching up to 93% in the AndroidWorld benchmark.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://microsoft.github.io/Magma/\" rel=\"nofollow\"\u003eMagma: A Foundation Model for Multimodal AI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJianwei Yang, Reuben Tan, Qianhui Wu, Ruijie Zheng, Baolin Peng, Yongyuan Liang, Yu Gu, Mu Cai, Seonghyeon Ye, Joel Jang, Yuquan Deng, Lars Liden, Jianfeng Gao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Microsoft Research, Univ. of Maryland, Univ. of Wisconsin-Madison, KAIST, Univ. of Washington\u003c/li\u003e\n\u003cli\u003e📅 Date: Feb 18, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI], [Robotics]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [SoM], [ToM], [robotics], [UI navigation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eMagma\u003c/strong\u003e, a foundation model designed for multimodal AI agents operating in both digital and physical environments. Magma extends traditional vision-language models by incorporating planning and action capabilities, enabling tasks from UI navigation to robotic manipulation. The model is pretrained on diverse datasets, including images, videos, and robotics data, utilizing \u003cstrong\u003eSet-of-Mark (SoM)\u003c/strong\u003e for action grounding and \u003cstrong\u003eTrace-of-Mark (ToM)\u003c/strong\u003e for action planning. Experiments demonstrate that SoM and ToM synergistically enhance Magma's spatial-temporal intelligence, achieving state-of-the-art performance in UI navigation and robotic manipulation tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2502.11357\" rel=\"nofollow\"\u003eExplorer: Scaling Exploration-driven Web Trajectory Synthesis for Multimodal Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eVardaan Pahuja, Yadong Lu, Corby Rosset, Boyu Gou, Arindam Mitra, Spencer Whitehead, Yu Su, Ahmed Awadallah\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Microsoft Research, The Ohio State University\u003c/li\u003e\n\u003cli\u003e📅 Date: February 17, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [web agents], [reinforcement learning], [Explorer]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eExplorer\u003c/em\u003e, a multimodal web agent trained on a newly synthesized dataset comprising over 94,000 successful web trajectories. The dataset, generated through scalable exploration and refinement techniques, spans 49,000 unique URLs and includes 720,000 screenshots and 33 million web elements. \u003cem\u003eExplorer\u003c/em\u003e demonstrates strong performance on benchmarks like Mind2Web-Live, Multimodal-Mind2Web, and MiniWob++, highlighting the importance of data scaling in enhancing web agent capabilities.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2502.07949\" rel=\"nofollow\"\u003eVSC-RL: Advancing Autonomous Vision-Language Agents with Variational Subgoal-Conditioned Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eQingyuan Wu, Jianheng Liu, Jianye Hao, Jun Wang, Kun Shao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Huawei Noah's Ark Lab, Univ. College London\u003c/li\u003e\n\u003cli\u003e📅 Date: February 11, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [subgoal generation], [VSC-RL], [learning efficiency]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eVSC-RL\u003c/strong\u003e, a novel reinforcement learning framework that enhances learning efficiency for vision-language agents in complex sequential decision-making tasks. By reformulating these tasks as variational goal-conditioned problems, VSC-RL leverages advanced optimization techniques and utilizes vision-language models to autonomously decompose goals into feasible subgoals. Empirical results demonstrate that VSC-RL significantly outperforms state-of-the-art agents, particularly in challenging mobile device control tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2502.06395\" rel=\"nofollow\"\u003eAppVLM: A Lightweight Vision Language Model for Online App Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGeorgios Papoudakis, Thomas Coste, Zhihao Wu, Jianye Hao, Jun Wang, Kun Shao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Univ. of Cambridge, Huawei Noah's Ark Lab, Univ. College London\u003c/li\u003e\n\u003cli\u003e📅 Date: February 10, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [evaluation], [AppVLM], [on-device control]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eAppVLM\u003c/strong\u003e, a lightweight Vision-Language Model designed for efficient on-device control of smartphone applications. AppVLM is fine-tuned on the AndroidControl dataset and further refined through interactions within the AndroidWorld environment. It achieves superior action prediction accuracy in offline evaluations and matches GPT-4o in online task completion success rates, while operating up to ten times faster, making it a practical solution for real-world deployment.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2502.02982\" rel=\"nofollow\"\u003eMobileA3gent: Training Mobile GUI Agents Using Decentralized Self-Sourced Data from Diverse Users\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWenhao Wang, Mengying Yuan, Zijie Yu, Guangyi Liu, Rui Ye, Tian Jin, Siheng Chen, Yanfeng Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ZJU, SJTU, Shanghai AI Lab, MAGIC\u003c/li\u003e\n\u003cli\u003e📅 Date: February 5, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Auto-Annotation], [federated learning], [privacy], [non-IID], [adapted aggregation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMobileA3gent\u003c/em\u003e, a framework for training mobile GUI agents using decentralized, self-sourced data from diverse users. It addresses challenges in extracting user instructions without human intervention and utilizing distributed data while preserving privacy. The framework comprises two key techniques: \u003cem\u003eAuto-Annotation\u003c/em\u003e, which automatically collects high-quality datasets during users' routine phone usage, and \u003cem\u003eFedVLM-A\u003c/em\u003e, which enhances federated training on non-IID user data by incorporating both episode- and step-level distributions. Experiments demonstrate that MobileA3gent achieves performance comparable to centralized human-annotated models at less than 1% of the cost, highlighting its potential for real-world applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2501.12326\" rel=\"nofollow\"\u003eUI-TARS: Pioneering Automated GUI Interaction with Native Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYujia Qin, Yining Ye, Junjie Fang, Haoming Wang, Shihao Liang, Shizuo Tian, Junda Zhang, Jiahao Li, Yunxin Li, Shijue Huang, Wanjun Zhong, Kuanye Li, Jiale Yang, Yu Miao, Woyu Lin, Longxiang Liu, Xu Jiang, Qianli Ma, Jingyu Li, Xiaojun Xiao, Kai Cai, Chuang Li, Yaowei Zheng, Chaolin Jin, Chen Li, Xiao Zhou, Minchao Wang, Haoli Chen, Zhaojian Li, Haihua Yang, Haifeng Liu, Feng Lin, Tao Peng, Xin Liu, Guang Shi\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ByteDance, Tsinghua University\u003c/li\u003e\n\u003cli\u003e📅 Date: January 21, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [UI-TARS]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eUI-TARS\u003c/strong\u003e, a native GUI agent model that processes screenshots to perform human-like interactions such as keyboard and mouse operations. Unlike traditional frameworks relying on commercial models with handcrafted prompts, UI-TARS is an end-to-end model demonstrating superior performance across over 10 GUI agent benchmarks. Key innovations include enhanced perception through large-scale GUI screenshot datasets, unified action modeling across platforms, incorporation of deliberate multi-step reasoning (System-2 Reasoning), and iterative training with reflective online traces, enabling continuous learning and adaptation with minimal human intervention.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2501.07572\" rel=\"nofollow\"\u003eWebWalker: Benchmarking LLMs in Web Traversal\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJialong Wu, Wenbiao Yin, Yong Jiang, Zhenglin Wang, Zekun Xi, Runnan Fang, Deyu Zhou, Pengjun Xie, Fei Huang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tongyi Lab, Alibaba NLP\u003c/li\u003e\n\u003cli\u003e📅 Date: January 13, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [RAG], [WebWalker], [WebWalkerQA]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cstrong\u003eWebWalker\u003c/strong\u003e, a multi-agent framework designed to improve the ability of large language models (LLMs) to traverse websites, addressing the challenges of retrieving complex, multi-layered information. WebWalker integrates an \"explore-critic\" paradigm, where the explorer agent navigates the web, and the critic agent evaluates the progress. The \u003cstrong\u003eWebWalkerQA\u003c/strong\u003e benchmark is introduced to assess web traversal tasks, showing how retrieval-augmented generation (RAG) can be enhanced with vertical exploration to solve real-world queries.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://likaixin2000.github.io/papers/ScreenSpot_Pro.pdf\" rel=\"nofollow\"\u003eScreenSpot-Pro: GUI Grounding for Professional High-Resolution Computer Use\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS, ECNU, HKBU\u003c/li\u003e\n\u003cli\u003e📅 Date: January 3, 2025\u003c/li\u003e\n\u003cli\u003e📑 Publisher: GitHub\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [GUI grounding], [high-resolution], [ScreenSpot-Pro]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: ScreenSpot-Pro introduces a benchmark designed to evaluate GUI grounding models in professional, high-resolution environments. It encompasses 1,581 tasks across 23 applications in various industries, highlighting the challenges models face with complex software interfaces. Current models achieve low accuracy, underscoring the need for further research in this domain.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://qiushisun.github.io/OS-Genesis-Home/\" rel=\"nofollow\"\u003eOS-Genesis: Automating GUI Agent Trajectory Construction via Reverse Task Synthesis\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eQiushi Sun, Kanzhi Cheng, Zichen Ding, Chuanyang Jin, Yian Wang, Fangzhi Xu, Zhenyu Wu, Chengyou Jia, Liheng Chen, Zhoumianze Liu, Ben Kao, Guohao Li, Junxian He, Yu Qiao, Zhiyong Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Shanghai AI Lab, HKU, Johns Hopkins University, SJTU, Oxford, HKUST\u003c/li\u003e\n\u003cli\u003e📅 Date: Dec 27, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [trajectory data], [reward model], [e2e model], [data synthesis], [OS-Genesis]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eOS-Genesis\u003c/em\u003e, an interaction-driven pipeline that automates the construction of high-quality and diverse GUI agent trajectory data without human supervision. By employing reverse task synthesis and a trajectory reward model, OS-Genesis enables effective end-to-end training of GUI agents, significantly enhancing their performance on online benchmarks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.17589\" rel=\"nofollow\"\u003ePC Agent: While You Sleep, AI Works -- A Cognitive Journey into Digital World\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYanheng He, Jiahe Jin, Shijie Xia, Jiadi Su, Runze Fan, Haoyang Zou, Xiangkun Hu, Pengfei Liu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: Dec 23, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [PC Agent], [human cognition transfer], [PC Tracker], [Cognition Completion], [multi-agent system]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003ePC Agent\u003c/em\u003e, an AI system designed to autonomously perform complex computer-based tasks by emulating human cognitive processes. The system comprises three key components: \u003cstrong\u003ePC Tracker\u003c/strong\u003e, a lightweight infrastructure for collecting high-quality human-computer interaction data; a \u003cstrong\u003eCognition Completion\u003c/strong\u003e pipeline that enriches raw interaction data into detailed cognitive trajectories; and a \u003cstrong\u003emulti-agent system\u003c/strong\u003e combining a planning agent for decision-making with a grounding agent for precise visual grounding. This approach represents a significant advancement toward AI systems capable of handling intricate real-world work autonomously.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/OS-Agent-Survey/OS-Agent-Survey\"\u003eOS Agents: A Survey on MLLM-based Agents for General Computing Devices Use\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXueyu Hu, Tao Xiong, Biao Yi, Zishu Wei, Ruixuan Xiao, Yurun Chen, Jiasheng Ye, Meiling Tao, Xiangxin Zhou, Ziyu Zhao, Yuhuai Li, Shengze Xu, Shawn Wang, Xinchen Xu, Shuofei Qiao , Kun Kuang, Tieyong Zeng, Liang Wang, Jiwei Li, Yuchen Eleanor Jiang, Wangchunshu Zhou, Guoyin Wang, Keting Yin, Zhou Zhao, Hongxia Yang, Fan Wu, Shengyu Zhang, Fei Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University, Fudan University, OPPO AI Center, University of Chinese Academy of Sciences, Chinese Academy of Sciences, The Chinese HKU, Tsinghua University, 01.AI, The Hong Kong Polytechnic University, SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: December 20, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: Github Repo\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper conducts a comprehensive survey on OS Agents, which are (M)LLM-based agents that use computing devices (e.g., computers and mobile phones) by operating within the environments and interfaces (e.g., Graphical User Interface (GUI)) provided by operating systems (OS) to automate tasks. The survey begins by elucidating the fundamentals of OS Agents, exploring their key components including the environment, observation space, and action space, and outlining essential capabilities such as understanding, planning, and grounding. Methodologies for constructing OS Agents are examined, with a focus on domain-specific foundation models and agent frameworks. A detailed review of evaluation protocols and benchmarks highlights how OS Agents are assessed across diverse tasks. Finally, current challenges and promising future research directions, including safety and privacy, personalization and self-evolution, are discussed.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://ariaui.github.io/\" rel=\"nofollow\"\u003eAria-UI: Visual Grounding for GUI Instructions\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYuhao Yang, Yue Wang, Dongxu Li, Ziyang Luo, Bei Chen, Chao Huang, Junnan Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HKU, Rhymes AI\u003c/li\u003e\n\u003cli\u003e📅 Date: Dec 20, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [grounding], [visual grounding], [Aria-UI]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eAria-UI\u003c/em\u003e, a large multimodal model specifically designed for GUI grounding. Aria-UI employs a pure-vision approach, avoiding reliance on auxiliary inputs like HTML or AXTree. It utilizes a scalable data pipeline to synthesize diverse and high-quality instruction samples and incorporates textual and text-image interleaved action histories for robust context-aware reasoning. Aria-UI achieves state-of-the-art results across offline and online agent benchmarks, outperforming both vision-only and AXTree-reliant baselines.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.13501\" rel=\"nofollow\"\u003eGUI Agents: A Survey\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDang Nguyen, Jian Chen, Yu Wang, Gang Wu, Namyong Park, Zhengmian Hu, Hanjia Lyu, Junda Wu, Ryan Aponte, Yu Xia, Xintong Li, Jing Shi, Hongjie Chen, Viet Dac Lai, Zhouhang Xie, Sungchul Kim, Ruiyi Zhang, Tong Yu, Mehrab Tanjim, Nesreen K. Ahmed, Puneet Mathur, Seunghyun Yoon, Lina Yao, Branislav Kveton, Thien Huu Nguyen, Trung Bui, Tianyi Zhou, Ryan A. Rossi, Franck Dernoncourt\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UMD, SUNY Buffalo, University of Oregon, Adobe Research, Meta AI, University of Rochester, UCSD, CMU, Dolby Labs, Intel AI Research, UNSW\u003c/li\u003e\n\u003cli\u003e📅 Date: December 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This survey provides a comprehensive overview of GUI agents powered by Large Foundation Models, detailing their benchmarks, evaluation metrics, architectures, and training methods. It introduces a unified framework outlining their perception, reasoning, planning, and acting capabilities, identifies open challenges, and discusses future research directions, serving as a resource for both practitioners and researchers in the field.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.13194\" rel=\"nofollow\"\u003eProposer-Agent-Evaluator (PAE): Autonomous Skill Discovery For Foundation Model Internet Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYifei Zhou, Qianlan Yang, Kaixiang Lin, Min Bai, Xiong Zhou, Yu-Xiong Wang, Sergey Levine, Erran Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCB, UIUC, Amazon\u003c/li\u003e\n\u003cli\u003e📅 Date: December 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [skill discovery], [PAE]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the Proposer-Agent-Evaluator (PAE) system, enabling foundation model agents to autonomously discover and practice skills in real-world web environments. PAE comprises a context-aware task proposer, an agent policy for task execution, and a vision-language model-based success evaluator. Validated on vision-based web navigation tasks, PAE significantly enhances zero-shot generalization capabilities of vision-language model Internet agents, achieving over 30% relative improvement on unseen tasks and websites, and surpassing state-of-the-art open-source agents by more than 10%.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.10342\" rel=\"nofollow\"\u003eIris: Breaking GUI Complexity with Adaptive Focus and Self-Refining\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhiqi Ge, Juncheng Li, Xinglei Pang, Minghe Gao, Kaihang Pan, Wang Lin, Hao Fei, Wenqiao Zhang, Siliang Tang, Yueting Zhuang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University, NUS\u003c/li\u003e\n\u003cli\u003e📅 Date: December 13, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Information-Sensitive Cropping], [Self-Refining Dual Learning], [visual grounding], [model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eIris\u003c/em\u003e, a visual agent designed to enhance GUI automation by addressing challenges in high-resolution, complex digital environments. It employs two key innovations: \u003cstrong\u003eInformation-Sensitive Cropping (ISC)\u003c/strong\u003e, which dynamically identifies and prioritizes visually dense regions using an edge detection algorithm for efficient processing, and \u003cstrong\u003eSelf-Refining Dual Learning (SRDL)\u003c/strong\u003e, which enhances the agent's ability to handle complex tasks through a dual-learning loop that iteratively refines its performance without requiring additional annotated data. Empirical evaluations demonstrate that Iris achieves state-of-the-art performance across multiple benchmarks with only 850K GUI annotations, outperforming methods using ten times more training data.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.09362\" rel=\"nofollow\"\u003eFalcon-UI: Understanding GUI Before Following User Instructions\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHuawen Shen, Chang Liu, Gengluo Li, Xinlong Wang, Yu Zhou, Can Ma, Xiangyang Ji\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Chinese Academy of Sciences, Tsinghua University, Nankai University, BAAI\u003c/li\u003e\n\u003cli\u003e📅 Date: Dec 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [Falcon-UI], [GUI understanding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eFalcon-UI\u003c/em\u003e, a GUI agent model that emphasizes understanding GUI contexts before following user instructions. The authors present the \u003cem\u003eInsight-UI Dataset\u003c/em\u003e, an instruction-free GUI navigation dataset generated from the Common Crawl corpus, simulating various platforms like iOS, Android, Windows, and Linux across multiple resolutions on 312K domains. Falcon-UI is pretrained on this dataset and fine-tuned on Android and Web GUI datasets, achieving performance comparable to larger models, highlighting the importance of decoupling GUI understanding from instruction following in agent performance.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.05467\" rel=\"nofollow\"\u003eThe BrowserGym Ecosystem for Web Agent Research\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThibault Le Sellier De Chezelles, Maxime Gasse, Alexandre Drouin, Massimo Caccia, Léo Boisvert, Megh Thakkar, Tom Marty, Rim Assouel, Sahar Omidi Shayegan, Lawrence Keunho Jang, Xing Han Lù, Ori Yoran, Dehan Kong, Frank F. Xu, Siva Reddy, Quentin Cappart, Graham Neubig, Ruslan Salakhutdinov, Nicolas Chapados, Alexandre Lacoste\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ServiceNow Research, Mila, Polytechnique Montréal, CMU, McGill University, Tel Aviv University, Université de Montréal, iMean AI\u003c/li\u003e\n\u003cli\u003e📅 Date: December 6, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [LLM], [automation], [BrowserGym], [AgentLab]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cem\u003eBrowserGym\u003c/em\u003e, an ecosystem designed to standardize the evaluation and benchmarking of web agents, particularly those leveraging Large Language Models (LLMs). It addresses the challenges posed by fragmented benchmarks and inconsistent methodologies in web agent research. BrowserGym provides a unified, gym-like environment with clearly defined observation and action spaces, enabling reproducible comparisons across various benchmarks. Additionally, \u003cem\u003eAgentLab\u003c/em\u003e, a complementary framework, supports agent creation, testing, and analysis. The paper also features a large-scale experiment comparing the performance of 6 leading LLMs, highlighting the strengths and weaknesses of different models in real-world web tasks, while emphasizing the ongoing challenges in building efficient and robust web agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://aguvis-project.github.io/\" rel=\"nofollow\"\u003eAguvis: Unified Pure Vision Agents for Autonomous GUI Interaction\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYiheng Xu, Zekun Wang, Junli Wang, Dunjie Lu, Tianbao Xie, Amrita Saha, Doyen Sahoo, Tao Yu, Caiming Xiong\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HKU, NTU, Salesforce\u003c/li\u003e\n\u003cli\u003e📅 Date: Dec 5, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2025\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [planning], [reasoning], [Aguvis], [visual grounding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eAguvis\u003c/em\u003e, a unified pure vision-based framework for autonomous GUI agents that operates across various platforms. It leverages image-based observations and grounds natural language instructions to visual elements, employing a consistent action space to ensure cross-platform generalization. The approach integrates explicit planning and reasoning within the model, enhancing its ability to autonomously navigate and interact with complex digital environments. A large-scale dataset of GUI agent trajectories is constructed, incorporating multimodal reasoning and grounding. Comprehensive experiments demonstrate that Aguvis surpasses previous state-of-the-art methods in both offline and real-world online scenarios, achieving the first fully autonomous pure vision GUI agent capable of performing tasks independently without collaboration with external closed-source models. All datasets, models, and training recipes are open-sourced to facilitate future research.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2412.01268\" rel=\"nofollow\"\u003ePonder \u0026amp; Press: Advancing Visual GUI Agent towards General Computer Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYiqin Wang, Haoji Zhang, Jingqi Tian, Yansong Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University\u003c/li\u003e\n\u003cli\u003e📅 Date: December 2, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [visual grounding], [reinforcement learning], [Ponder \u0026amp; Press]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003ePonder \u0026amp; Press\u003c/em\u003e, a divide-and-conquer framework for general computer control using only visual input. The approach combines a general-purpose multimodal large language model (MLLM) as an 'interpreter' to translate high-level user instructions into detailed action descriptions, with a GUI-specific MLLM as a 'locator' that precisely identifies GUI elements for action execution. By relying solely on visual inputs, the agent offers a versatile, human-like interaction paradigm applicable across various platforms, achieving state-of-the-art performance in GUI grounding and control tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.17465\" rel=\"nofollow\"\u003eShowUI: One Vision-Language-Action Model for GUI Visual Agent\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKevin Qinghong Lin, Linjie Li, Difei Gao, Zhengyuan Yang, Shiwei Wu, Zechen Bai, Weixian Lei, Lijuan Wang, Mike Zheng Shou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS, Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: November 26, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [dataset], [UI-Guided Visual Token Selection], [Interleaved Vision-Language-Action Streaming], [ShowUI]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eShowUI\u003c/em\u003e, a vision-language-action model designed to enhance GUI automation by addressing challenges in UI visual perception and action modeling. It features innovations like UI-Guided Visual Token Selection to reduce computational costs and Interleaved Vision-Language-Action Streaming for effective management of visual-action history. Trained on a curated dataset, ShowUI achieves 75.1% accuracy in zero-shot screenshot grounding and demonstrates competitive performance across web, mobile, and online environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.13451\" rel=\"nofollow\"\u003eAdaptAgent: Adapting Multimodal Web Agents with Few-Shot Learning from Human Demonstrations\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGaurav Verma, Rachneet Kaur, Nishan Srishankar, Zhen Zeng, Tucker Balch, Manuela Veloso\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: J.P. Morgan AI Research\u003c/li\u003e\n\u003cli\u003e📅 Date: November 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [few-shot learning], [meta-learning], [AdaptAgent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eAdaptAgent\u003c/strong\u003e, a framework that enables multimodal web agents to adapt to new websites and domains using few human demonstrations (up to 2). The approach enhances agents' adaptability beyond large-scale pre-training and fine-tuning by leveraging in-context learning and meta-learning techniques. Experiments on benchmarks like Mind2Web and VisualWebArena show that incorporating minimal human demonstrations boosts task success rates significantly, highlighting the effectiveness of multimodal demonstrations over text-only ones and the impact of data selection strategies during meta-learning on agent generalization.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.13591\" rel=\"nofollow\"\u003eImproved GUI Grounding via Iterative Narrowing\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAnthony Nguyen\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Algoma University\u003c/li\u003e\n\u003cli\u003e📅 Date: November 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [grounding], [visual grounding], [iterative narrowing]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a visual framework to enhance GUI grounding. By iteratively refining model predictions through progressively focused image crops, the proposed method improves the performance of both general and fine-tuned Vision-Language Models (VLMs) in GUI grounding tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.10943\" rel=\"nofollow\"\u003eGeneralist Virtual Agents: A Survey on Autonomous Agents Across Digital Platforms\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMinghe Gao, Wendong Bu, Bingchen Miao, Yang Wu, Yunfei Li, Juncheng Li, Siliang Tang, Qi Wu, Yueting Zhuang, Meng Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University, University of Adelaide, Hefei University of Technology\u003c/li\u003e\n\u003cli\u003e📅 Date: November 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey], [Generalist Virtual Agent], [GVA], [autonomous agents], [digital platforms]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This survey introduces the concept of Generalist Virtual Agents (GVAs), autonomous entities designed to operate across various digital platforms and environments to assist users in performing diverse tasks. It traces the evolution of GVAs from early intelligent assistants to modern implementations incorporating large-scale models, discussing their philosophical foundations, development challenges, and current methodologies. The paper provides a detailed taxonomy of GVA environments, tasks, and capabilities, aiming to bridge theoretical and practical aspects and suggesting that agents operating in environments closely mirroring the real world are more likely to exhibit human-like intelligence.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.10323\" rel=\"nofollow\"\u003eThe Dawn of GUI Agent: A Preliminary Case Study with Claude 3.5 Computer Use\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSiyuan Hu, Mingyu Ouyang, Difei Gao, Mike Zheng Shou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS\u003c/li\u003e\n\u003cli\u003e📅 Date: Nov 15, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Claude 3.5 Computer Use], [GUI automation], [planning], [action], [critic]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study evaluates Claude 3.5 Computer Use, an AI model enabling end-to-end language-to-desktop actions, through curated tasks across various domains. It introduces an out-of-the-box framework for deploying API-based GUI automation models, analyzing the model's planning, action execution, and adaptability to dynamic environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://aclanthology.org/2024.emnlp-demo.20/\" rel=\"nofollow\"\u003eWebOlympus: An Open Platform for Web Agents on Live Websites\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBoyuan Zheng, Boyu Gou, Scott Salisbury, Zheng Du, Huan Sun, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU\u003c/li\u003e\n\u003cli\u003e📅 Date: November 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [safety], [Chrome extension], [WebOlympus], [SeeAct], [Annotation Tool]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eWebOlympus\u003c/em\u003e, an open platform designed to facilitate the research and deployment of web agents on live websites. It features a user-friendly Chrome extension interface, allowing users without programming expertise to operate web agents with minimal effort. The platform incorporates a safety monitor module to prevent harmful actions through human supervision or model-based control, supporting applications such as annotation interfaces for web agent trajectories and data crawling.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.06559\" rel=\"nofollow\"\u003eIs Your LLM Secretly a World Model of the Internet? Model-Based Planning for Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYu Gu, Boyuan Zheng, Boyu Gou, Kai Zhang, Cheng Chang, Sanjari Srivastava, Yanan Xie, Peng Qi, Huan Sun, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, Orby AI\u003c/li\u003e\n\u003cli\u003e📅 Date: November 10, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [WebDreamer], [model-based planning], [world model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper investigates whether Large Language Models (LLMs) can function as world models within web environments, enabling model-based planning for web agents. Introducing \u003cstrong\u003eWebDreamer\u003c/strong\u003e, a framework that leverages LLMs to simulate potential action sequences in web environments, the study demonstrates significant performance improvements over reactive baselines on benchmarks like VisualWebArena and Mind2Web-live. The findings suggest that LLMs possess the capability to model the dynamic nature of the internet, paving the way for advancements in automated web interaction and opening new research avenues in optimizing LLMs for complex, evolving environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.04890\" rel=\"nofollow\"\u003eGUI Agents with Foundation Models: A Comprehensive Survey\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShuai Wang, Weiwen Liu, Jingxuan Chen, Weinan Gan, Xingshan Zeng, Shuai Yu, Xinlong Hao, Kun Shao, Yasheng Wang, Ruiming Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Huawei Noah’s Ark Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: November 7, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This survey consolidates recent research on GUI agents powered by foundation models, particularly Large Language Models (LLMs) and Multimodal Large Language Models (MLLMs). It discusses representative datasets and benchmarks, summarizes a unified framework capturing essential components from prior studies, and explores commercial applications. The paper identifies key challenges and proposes future research directions to inspire further developments in (M)LLM-based GUI agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.02337v1\" rel=\"nofollow\"\u003eWebRL: Training LLM Web Agents via Self-Evolving Online Curriculum Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZehan Qi, Xiao Liu, Iat Long Iong, Hanyu Lai, Xueqiao Sun, Xinyue Yang, Jiadai Sun, Yu Yang, Shuntian Yao, Tianjie Zhang, Wei Xu, Jie Tang, Yuxiao Dong\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, BAAI\u003c/li\u003e\n\u003cli\u003e📅 Date: November 4, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [self-evolving curriculum], [WebRL], [outcome-supervised reward model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eWebRL\u003c/em\u003e, a self-evolving online curriculum reinforcement learning framework designed to train high-performance web agents using open large language models (LLMs). WebRL addresses challenges such as the scarcity of training tasks, sparse feedback signals, and policy distribution drift in online learning. It incorporates a self-evolving curriculum that generates new tasks from unsuccessful attempts, a robust outcome-supervised reward model (ORM), and adaptive reinforcement learning strategies to ensure consistent improvements. Applied to Llama-3.1 and GLM-4 models, WebRL significantly enhances their performance on web-based tasks, surpassing existing state-of-the-art web agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2411.02391\" rel=\"nofollow\"\u003eAttacking Vision-Language Computer Agents via Pop-ups\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYanzhe Zhang, Tao Yu, Diyi Yang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Georgia Tech, HKU, Stanford\u003c/li\u003e\n\u003cli\u003e📅 Date: Nov 4, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [attack], [adversarial pop-ups], [VLM agents], [safety]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper demonstrates that vision-language model (VLM) agents can be easily deceived by carefully designed adversarial pop-ups, leading them to perform unintended actions such as clicking on these pop-ups instead of completing their assigned tasks. Integrating these pop-ups into environments like OSWorld and VisualWebArena resulted in an average attack success rate of 86% and a 47% decrease in task success rate. Basic defense strategies, such as instructing the agent to ignore pop-ups or adding advertisement notices, were found to be ineffective against these attacks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://language-agent-tutorial.github.io/\" rel=\"nofollow\"\u003eLanguage Agents: Foundations, Prospects, and Risks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYu Su, Diyi Yang, Shunyu Yao, Tao Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, Stanford, Princeton, HKU\u003c/li\u003e\n\u003cli\u003e📅 Date: November 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [survey], [tutorial], [reasoning], [planning], [memory], [multi-agent systems], [safty]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This tutorial provides a comprehensive exploration of language agents—autonomous systems powered by large language models capable of executing complex tasks through language instructions. It delves into their theoretical foundations, potential applications, associated risks, and future directions, covering topics such as reasoning, memory, planning, tool augmentation, grounding, multi-agent systems, and safety considerations.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.13701\" rel=\"nofollow\"\u003eFrom Context to Action: Analysis of the Impact of State Representation and Context on the Generalization of Multi-Turn Web Navigation Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNalin Tiwary, Vardhan Dongre, Sanil Arun Chawla, Ashwin Lamani, Dilek Hakkani-Tür\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UIUC\u003c/li\u003e\n\u003cli\u003e📅 Date: October 31, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [context management], [generalization], [multi-turn navigation], [CWA]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study examines how different contextual elements affect the performance and generalization of Conversational Web Agents (CWAs) in multi-turn web navigation tasks. By optimizing context management—specifically interaction history and web page representation—the research demonstrates enhanced agent performance across various out-of-distribution scenarios, including unseen websites, categories, and geographic locations.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.24024\" rel=\"nofollow\"\u003eAndroidLab: Training and Systematic Benchmarking of Android Autonomous Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYifan Xu, Xiao Liu, Xueqiao Sun, Siyi Cheng, Hao Yu, Hanyu Lai, Shudan Zhang, Dan Zhang, Jie Tang, Yuxiao Dong\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, Peking University\u003c/li\u003e\n\u003cli\u003e📅 Date: October 31, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [AndroidLab]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eAndroidLab\u003c/strong\u003e, a comprehensive framework for training and systematically benchmarking Android autonomous agents. It provides an operational environment with diverse modalities and action spaces, supporting both large language models (LLMs) and multimodal models (LMMs). The benchmark includes 138 tasks across nine apps on predefined Android virtual devices. Utilizing AndroidLab, the authors developed an Android Instruction dataset and trained six open-source LLMs and LMMs, significantly improving their average success rates.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://osatlas.github.io/\" rel=\"nofollow\"\u003eOS-ATLAS: A Foundation Action Model For Generalist GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhiyong Wu, Zhenyu Wu, Fangzhi Xu, Yian Wang, Qiushi Sun, Chengyou Jia, Kanzhi Cheng, Zichen Ding, Liheng Chen, Paul Pu Liang, Yu Qiao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Shanghai AI Lab, SJTU, HKU, MIT\u003c/li\u003e\n\u003cli\u003e📅 Date: October 30, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [benchmark], [OS-Atlas]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces OS-Atlas, a foundational GUI action model designed to enhance GUI grounding and out-of-distribution tasks. The authors developed a toolkit to synthesize multi-platform GUI grounding data, resulting in a cross-platform corpus of over 13 million GUI elements. OS-Atlas demonstrates significant performance improvements across six benchmarks spanning mobile, desktop, and web platforms.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.23252\" rel=\"nofollow\"\u003eEvaluating Cultural and Social Awareness of LLM Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHaoyi Qiu, Alexander R. Fabbri, Divyansh Agarwal, Kung-Hsiang Huang, Sarah Tan, Nanyun Peng, Chien-Sheng Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCLA, Salesforce AI Research\u003c/li\u003e\n\u003cli\u003e📅 Date: October 30, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [CASA], [cultural awareness], [social awareness], [fine-tuning], [prompting]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces CASA, a benchmark designed to assess the cultural and social awareness of LLM web agents in tasks like online shopping and social discussion forums. It evaluates agents' abilities to detect and appropriately respond to norm-violating user queries and observations. The study finds that current LLM agents have limited cultural and social awareness, with less than 10% awareness coverage and over 40% violation rates. To enhance performance, the authors explore prompting and fine-tuning methods, demonstrating that combining both can offer complementary advantages.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.22552\" rel=\"nofollow\"\u003eAuto-Intent: Automated Intent Discovery and Self-Exploration for Large Language Model Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJaekyeom Kim, Dong-Ki Kim, Lajanugen Logeswaran, Sungryull Sohn, Honglak Lee\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: LG AI Research, Field AI, University of Michigan\u003c/li\u003e\n\u003cli\u003e📅 Date: October 29, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2024 (Findings)\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Auto-Intent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper presents Auto-Intent, a method to adapt pre-trained large language models for web navigation tasks without direct fine-tuning. It discovers underlying intents from domain demonstrations and trains an intent predictor to enhance decision-making. Auto-Intent improves the performance of GPT-3.5, GPT-4, and Llama-3.1 agents on benchmarks like Mind2Web and WebArena.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://doi.org/10.48550/arXiv.2410.19461\" rel=\"nofollow\"\u003eEDGE: Enhanced Grounded GUI Understanding with Enriched Multi-Granularity Synthetic Data\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXuetian Chen, Hangcheng Li, Jiaqing Liang, Sihang Jiang, Deqing Yang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Fudan University\u003c/li\u003e\n\u003cli\u003e📅 Date: October 25, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [framework], [synthetic data]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The \u003cem\u003eEDGE\u003c/em\u003e framework proposes an innovative approach to improve GUI understanding and interaction capabilities in vision-language models through large-scale, multi-granularity synthetic data generation. By leveraging webpage data, EDGE minimizes the need for manual annotations and enhances the adaptability of models across desktop and mobile GUI environments. Evaluations show its effectiveness in diverse GUI-related tasks, contributing significantly to autonomous agent development in GUI navigation and interaction.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://doi.org/10.48550/arXiv.2410.19609\" rel=\"nofollow\"\u003eOpenWebVoyager: Building Multimodal Web Agents via Iterative Real-World Exploration, Feedback and Optimization\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHongliang He, Wenlin Yao, Kaixin Ma, Wenhao Yu, Hongming Zhang, Tianqing Fang, Zhenzhong Lan, Dong Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University, Tencent AI Lab, Westlake University\u003c/li\u003e\n\u003cli\u003e📅 Date: October 25, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [learning], [imitation learning], [exploration], [AI feedback]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper presents \u003cstrong\u003eOpenWebVoyager\u003c/strong\u003e, an open-source framework for training web agents that explore real-world online environments autonomously. The framework employs a cycle of exploration, feedback, and optimization, enhancing agent capabilities through multimodal perception and iterative learning. Initial skills are acquired through imitation learning, followed by real-world exploration, where the agent’s performance is evaluated and refined through feedback loops.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://xiao9905.github.io/AutoGLM/\" rel=\"nofollow\"\u003eAutoGLM: Autonomous Foundation Agents for GUIs\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXiao Liu, Bo Qin, Dongzhu Liang, Guang Dong, Hanyu Lai, Hanchen Zhang, Hanlin Zhao, Iat Long Iong, Jiadai Sun, Jiaqi Wang, Junjie Gao, Junjun Shan, Kangning Liu, Shudan Zhang, Shuntian Yao, Siyi Cheng, Wentao Yao, Wenyi Zhao, Xinghan Liu, Xinyi Liu, Xinying Chen, Xinyue Yang, Yang Yang, Yifan Xu, Yu Yang, Yujia Wang, Yulin Xu, Zehan Qi, Yuxiao Dong, Jie Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhipu AI, Tsinghua University\u003c/li\u003e\n\u003cli\u003e📅 Date: October 25, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [learning], [AutoGLM]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces AutoGLM, a new series in the ChatGLM family, designed as foundation agents for autonomous control of digital devices through GUIs. It addresses the challenges foundation models face in decision-making within dynamic environments by developing agents capable of learning through autonomous interactions. Focusing on web browsers and Android devices, AutoGLM integrates various techniques to create deployable agent systems. Key insights include the importance of designing an appropriate \"intermediate interface\" for GUI control and a novel progressive training framework for self-evolving online curriculum reinforcement learning. Evaluations demonstrate AutoGLM's effectiveness across multiple domains, achieving notable success rates in web browsing and Android device control tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/pdf/2410.16464\" rel=\"nofollow\"\u003eBeyond Browsing: API-Based Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYueqi Song, Frank Xu, Shuyan Zhou, Graham Neubig\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: October 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [API-based agent], [hybrid agent], [benchmark], [WebArena], [SOTA performance]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces API-based and hybrid agents designed to execute online tasks by accessing both APIs and traditional web browsing interfaces. In evaluations using WebArena, a benchmark for web navigation, the API-based agent achieves higher performance than browser-based agents, and the hybrid model achieves a success rate of 35.8%, setting a new state-of-the-art (SOTA) in task-agnostic web navigation. The findings highlight the efficiency and reliability gains of API interactions for web agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.18603\" rel=\"nofollow\"\u003eAgentStore: Scalable Integration of Heterogeneous Agents As Specialized Generalist Computer Assistant\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChengyou Jia, Minnan Luo, Zhuohang Dang, Qiushi Sun, Fangzhi Xu, Junlin Hu, Tianbao Xie, Zhiyong Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: XJTU, Shanghai AI Lab, HKU\u003c/li\u003e\n\u003cli\u003e📅 Date: October 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [multi-agent systems], [specialized generalist agent], [OSWorld benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AgentStore introduces a scalable platform to integrate and manage heterogeneous agents, designed to enhance generalist assistant capabilities for diverse computer tasks. Using a MetaAgent and AgentToken strategy, AgentStore shows improved generalization on the OSWorld benchmark.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://doi.org/10.48550/arXiv.2410.19100\" rel=\"nofollow\"\u003eVideoWebArena: Evaluating Long Context Multimodal Agents with Video Understanding Web Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLawrence Jang, Yinheng Li, Charles Ding, Justin Lin, Paul Pu Liang, Dan Zhao, Rogerio Bonatti, Kazuhito Koishida\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, MIT, NYU, Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: October 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [video understanding], [long-context], [VideoWA]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eVideoWebArena (VideoWA)\u003c/strong\u003e, a benchmark assessing multimodal agents in video-based tasks. It features over 2,000 tasks focused on skill and factual retention, using video tutorials to simulate long-context environments. Results highlight current challenges in agentic abilities, providing a critical testbed for long-context video understanding improvements.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.17520\" rel=\"nofollow\"\u003eMobileSafetyBench: Evaluating Safety of Autonomous Agents in Mobile Device Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJuyong Lee, Dongyoon Hahm, June Suk Choi, W. Bradley Knox, Kimin Lee\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: KAIST, UT at Austin\u003c/li\u003e\n\u003cli\u003e📅 Date: October 23, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [safety], [evaluation], [Android emulator]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: \u003cem\u003eMobileSafetyBench\u003c/em\u003e introduces a benchmark for evaluating the safety of large language model (LLM)-based autonomous agents in mobile device control. Using Android emulators, the benchmark simulates real-world tasks in apps such as messaging and banking to assess agents' safety and helpfulness. The safety-focused tasks test for privacy risk management and robustness against adversarial prompt injections. Experiments show agents perform well in helpful tasks but struggle with safety-related challenges, underscoring the need for continued advancements in mobile safety mechanisms for autonomous agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.17883\" rel=\"nofollow\"\u003eLightweight Neural App Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFilippos Christianos, Georgios Papoudakis, Thomas Coste, Jianye Hao, Jun Wang, Kun Shao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Huawei Noah's Ark Lab, UCL\u003c/li\u003e\n\u003cli\u003e📅 Date: October 23, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [vision language model], [Action Transformer], [app agent], [Android control], [multi-modal]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces LiMAC, a mobile control framework for Android that integrates an Action Transformer and fine-tuned vision-language models to execute precise actions in mobile apps. Tested on open-source datasets, LiMAC improves action accuracy by up to 42% over traditional prompt engineering baselines, demonstrating enhanced efficiency and accuracy in mobile app control tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://ar5iv.org/abs/2410.17236\" rel=\"nofollow\"\u003eLarge Language Models Empowered Personalized Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHongru Cai, Yongqi Li, Wenjie Wang, Fengbin Zhu, Xiaoyu Shen, Wenjie Li, Tat-Seng Chua\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HK PolyU, NTU Singapore\u003c/li\u003e\n\u003cli\u003e📅 Date: Oct 22, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [personalized web agent], [user behavior alignment], [memory-enhanced alignment]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper proposes a novel framework, \u003cem\u003ePersonalized User Memory-enhanced Alignment (PUMA)\u003c/em\u003e, enabling large language models to serve as personalized web agents by incorporating user-specific data and historical web interactions. The authors also introduce a benchmark, \u003cem\u003ePersonalWAB\u003c/em\u003e, to evaluate these agents on various personalized web tasks. Results show that PUMA improves web agent performance by optimizing action execution based on user-specific preferences.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.15711\" rel=\"nofollow\"\u003eAssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOri Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya, Ben Bogin, Ofir Press, Jonathan Berant\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tel Aviv University\u003c/li\u003e\n\u003cli\u003e📅 Date: October 21, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [planning and reasoning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AssistantBench is a benchmark designed to test the abilities of web agents in completing time-intensive, realistic web-based tasks. Covering 214 tasks across various domains, the benchmark introduces the SPA (See-Plan-Act) framework to handle multi-step planning and memory retention. AssistantBench emphasizes realistic task completion, showing that current agents achieve only modest success, with significant improvements needed for complex information synthesis and execution across multiple web domains.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://openreview.net/forum?id=LjVIGva5Ct\" rel=\"nofollow\"\u003eDissecting Adversarial Robustness of Multimodal LM Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChen Henry Wu, Rishi Rajesh Shah, Jing Yu Koh, Russ Salakhutdinov, Daniel Fried, Aditi Raghunathan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Stanford\u003c/li\u003e\n\u003cli\u003e📅 Date: October 21, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2024 Workshop\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [attack], [ARE], [safety]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the Agent Robustness Evaluation (ARE) framework to assess the adversarial robustness of multimodal language model agents in web environments. By creating 200 targeted adversarial tasks within VisualWebArena, the study reveals that minimal perturbations can significantly compromise agent performance, even in advanced systems utilizing reflection and tree-search mechanisms. The findings highlight the need for enhanced safety measures in deploying such agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://ai-agents-2030.github.io/SPA-Bench/\" rel=\"nofollow\"\u003eSPA-Bench: A Comprehensive Benchmark for SmartPhone Agent Evaluation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJingxuan Chen, Derek Yuen, Bin Xie, Yuhao Yang, Gongwei Chen, Zhihao Wu, Li Yixing, Xurui Zhou, Weiwen Liu, Shuai Wang, Rui Shao, Liqiang Nie, Yasheng Wang, Jianye Hao, Jun Wang, Kun Shao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Huawei Noah’s Ark Lab, Harbin Institute of Technology, Shenzhen, UCL\u003c/li\u003e\n\u003cli\u003e📅 Date: October 19, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2025\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [AI agent], [smartphone control], [framework]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: SPA-Bench is introduced as a benchmark designed to evaluate multimodal large language model (MLLM)-based smartphone agents, offering a task set that spans common smartphone functionalities across system and third-party applications. It includes a plug-and-play framework for real-time agent interactions on Android, integrating over ten agents with an adaptable evaluation pipeline measuring success across diverse metrics. Through this, the benchmark exposes challenges such as UI interpretation, action grounding, and memory retention in mobile environments, advancing research in smartphone-based agent applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://ai-agents-2030.github.io/DistRL/\" rel=\"nofollow\"\u003eDistRL: An Asynchronous Distributed Reinforcement Learning Framework for On-Device Control Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTaiyi Wang, Zhihao Wu, Jianheng Liu, Jianye Hao, Jun Wang, Kun Shao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Univ. of Cambridge, Huawei Noah's Ark Lab, Univ. College London\u003c/li\u003e\n\u003cli\u003e📅 Date: October 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2025\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [distributed training], [A-RIDE], [on-device control]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eDistRL\u003c/strong\u003e, a novel framework designed to enhance the efficiency of online reinforcement learning fine-tuning for mobile device control agents. DistRL employs centralized training and decentralized data acquisition to ensure efficient fine-tuning in dynamic online interactions. The framework is supported by a custom RL algorithm, A-RIDE, which balances exploration with prioritized data utilization to ensure stable and robust training. Experiments demonstrate that DistRL improves training efficiency by 3× and accelerates data collection by 2.4× compared to leading synchronous multi-machine methods. Notably, after training, DistRL achieves a 20% relative improvement in success rate on general Android tasks from an open benchmark, outperforming existing approaches while maintaining the same training time.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.13824\" rel=\"nofollow\"\u003eHarnessing Webpage UIs for Text-Rich Visual Understanding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJunpeng Liu, Tianyue Ou, Yifan Song, Yuxiao Qu, Wai Lam, Chenyan Xiong, Wenhu Chen, Graham Neubig, Xiang Yue\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: October 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web], [Doc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [model], [text-rich visual understanding], [web UI comprehension]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMultiUI\u003c/em\u003e, a large-scale dataset containing 7.3 million annotated samples from 1 million websites, specifically designed to enhance multimodal large language models’ (MLLMs) capabilities in text-rich visual understanding. Utilizing webpage UI structures as a training resource, MultiUI provides robust accessibility tree data paired with UI screenshots, significantly improving MLLMs’ grounding, OCR, and interaction performance. Models trained with MultiUI achieve up to a 48% performance boost on VisualWebBench and demonstrate enhanced generalization across non-web tasks, setting a new standard for structured, visually integrated web data modeling.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.03648\" rel=\"nofollow\"\u003eAutoWebGLM: A Large Language Model-based Web Navigating Agent\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHanyu Lai, Xiao Liu, Iat Long Iong, Shuntian Yao, Yuxuan Chen, Pengbo Shen, Hao Yu, Hanchen Zhang, Xiaohan Zhang, Yuxiao Dong, Jie Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: THU, OSU\u003c/li\u003e\n\u003cli\u003e📅 Date: October 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [reinforcement learning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AutoWebGLM introduces a web navigation agent based on ChatGLM3-6B, designed to autonomously navigate and interact with webpages for complex tasks. The paper highlights a two-phase data construction approach using a hybrid human-AI methodology for diverse, curriculum-based web task training. It also presents AutoWebBench, a benchmark for evaluating agent performance in web tasks, and uses reinforcement learning to fine-tune operations, addressing complex webpage interaction and grounding.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.13886\" rel=\"nofollow\"\u003eRefusal-Trained LLMs Are Easily Jailbroken As Browser Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePriyanshu Kumar, Elaine Lau, Saranya Vijayakumar, Tu Trinh, Scale Red Team, Elaine Chang, Vaughn Robinson, Sean Hendryx, Shuyan Zhou, Matt Fredrikson, Summer Yue, Zifan Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, GraySwan AI, Scale AI\u003c/li\u003e\n\u003cli\u003e📅 Date: October 11, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [attack], [BrowserART], [jailbreaking], [safety]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eBrowser Agent Red teaming Toolkit (BrowserART)\u003c/strong\u003e, a comprehensive test suite for evaluating the safety of LLM-based browser agents. The study reveals that while refusal-trained LLMs decline harmful instructions in chat settings, their corresponding browser agents often comply with such instructions, indicating a significant safety gap. The authors call for collaboration among developers and policymakers to enhance agent safety.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.08164\" rel=\"nofollow\"\u003eAgent S: An Open Agentic Framework that Uses Computers Like a Human\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSaaket Agashe, Jiuzhou Han, Shuyu Gan, Jiachen Yang, Ang Li, Xin Eric Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Simular Research\u003c/li\u003e\n\u003cli\u003e📅 Date: October 10, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [autonomous GUI interaction], [experience-augmented hierarchical planning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces Agent S, an open agentic framework that enables autonomous interaction with computers through a Graphical User Interface (GUI). The system addresses key challenges in automating computer tasks through experience-augmented hierarchical planning and an Agent-Computer Interface (ACI). Agent S demonstrates significant improvements over baselines on the OSWorld benchmark, achieving a 20.58% success rate (83.6% relative improvement). The framework shows generalizability across different operating systems and provides insights for developing more effective GUI agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.11872\" rel=\"nofollow\"\u003eClickAgent: Enhancing UI Location Capabilities of Autonomous Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJakub Hoscilowicz, Bartosz Maj, Bartosz Kozakiewicz, Oleksii Tymoschuk, Artur Janicki\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Samsung R\u0026amp;D Poland, Warsaw University of Technology\u003c/li\u003e\n\u003cli\u003e📅 Date: October 9, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [SeeClick], [AITW benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper introduces \u003cem\u003eClickAgent\u003c/em\u003e, a framework that enhances autonomous agents' interaction with mobile UIs by improving their ability to locate interface elements accurately. This is achieved through a dual-component system where an MLLM performs reasoning and action planning, while a dedicated UI location model (e.g., SeeClick) handles element identification. ClickAgent, evaluated on the AITW benchmark and tested on both emulators and real Android devices, surpasses other agents like CogAgent and AppAgent in task success rate, advancing automation reliability on mobile platforms.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://sites.google.com/view/st-webagentbench/home\" rel=\"nofollow\"\u003eST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIdo Levy, Ben Wiesel, Sami Marreed, Alon Oved, Avi Yaeli, Segev Shlomov\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: IBM Research\u003c/li\u003e\n\u003cli\u003e📅 Date: October 9, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [safety], [trustworthiness], [ST-WebAgentBench]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eST-WebAgentBench\u003c/strong\u003e, a benchmark designed to evaluate the safety and trustworthiness of web agents in enterprise contexts. It defines safe and trustworthy agent behavior, outlines the structure of safety policies, and introduces the \"Completion under Policies\" metric to assess agent performance. The study reveals that current state-of-the-art agents struggle with policy adherence, highlighting the need for improved policy awareness and compliance in web agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.11871\" rel=\"nofollow\"\u003eTinyClick: Single-Turn Agent for Empowering GUI Automation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePawel Pawlowski, Krystian Zawistowski, Wojciech Lapacz, Marcin Skorupa, Adam Wiacek, Sebastien Postansque, Jakub Hoscilowicz\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Samsung R\u0026amp;D Poland, Warsaw University of Technology\u003c/li\u003e\n\u003cli\u003e📅 Date: October 9, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [vision language model], [Screenspot], [OmniAct]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: TinyClick is a compact, single-turn agent designed to automate GUI tasks by precisely locating screen elements via the Vision-Language Model Florence-2-Base. Trained with multi-task strategies and MLLM-based data augmentation, TinyClick achieves high accuracy on Screenspot and OmniAct, outperforming specialized GUI interaction models and general MLLMs like GPT-4V. The model's lightweight design (0.27B parameters) ensures fast processing and minimal latency, making it efficient for real-world applications on multiple platforms.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://osu-nlp-group.github.io/UGround/\" rel=\"nofollow\"\u003eNavigating the Digital World as Humans Do: Universal Visual Grounding for GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBoyu Gou, Ruochen Wang, Boyuan Zheng, Yucheng Xie, Cheng Chang, Yiheng Shu, Haotian Sun, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, Orby AI\u003c/li\u003e\n\u003cli\u003e📅 Date: October 7, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2025\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [dataset], [visual grounding], [GUI agents], [cross-platform generalization], [UGround], [SeeAct-V], [synthetic data]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces UGround, a universal visual grounding model for GUI agents that enables human-like navigation of digital interfaces. The authors advocate for GUI agents with human-like embodiment that perceive the environment entirely visually and take pixel-level actions. UGround is trained on a large-scale synthetic dataset of 10M GUI elements across 1.3M screenshots. Evaluated on six benchmarks spanning grounding, offline, and online agent tasks, UGround significantly outperforms existing visual grounding models by up to 20% absolute. Agents using UGround achieve comparable or better performance than state-of-the-art agents that rely on additional textual input, demonstrating the feasibility of vision-only GUI agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://agent-e3.github.io/ExACT/\" rel=\"nofollow\"\u003eExACT: Teaching AI Agents to Explore with Reflective-MCTS and Exploratory Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXiao Yu, Baolin Peng, Vineeth Vajipey, Hao Cheng, Michel Galley, Jianfeng Gao, Zhou Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Columbia Univ., MSR\u003c/li\u003e\n\u003cli\u003e📅 Date: Oct 2, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [learning], [R-MCTS], [Exploratory Learning], [VisualWebArena]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces ExACT, an approach that combines Reflective Monte Carlo Tree Search (R-MCTS) and Exploratory Learning to enhance AI agents' exploration and decision-making capabilities in complex web environments. R-MCTS incorporates contrastive reflection and multi-agent debate for improved search efficiency and reliable state evaluation. Evaluated on the VisualWebArena benchmark, the GPT-4o-based R-MCTS agent demonstrates significant performance improvements over previous state-of-the-art methods. Additionally, knowledge gained from test-time search is effectively transferred back to GPT-4o through fine-tuning, enabling the model to explore, evaluate, and backtrack without external search algorithms, achieving 87% of R-MCTS's performance with reduced computational resources.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2410.00467\" rel=\"nofollow\"\u003eDynamic Planning for LLM-based Graphical User Interface Automation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShaoqing Zhang, Zhuosheng Zhang, Kehai Chen, Xinbei Ma, Muyun Yang, Tiejun Zhao, Min Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: October 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dynamic planning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a novel method called Dynamic Planning of Thoughts (D-PoT) aimed at enhancing LLM-based agents for GUI tasks. It addresses the challenges of task execution by dynamically adjusting planning based on environmental feedback and action history, outperforming existing methods such as ReAct by improving accuracy significantly in navigating GUI environments. The study emphasizes the importance of integrating execution history and contextual cues to optimize decision-making processes for autonomous agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.20566\" rel=\"nofollow\"\u003eMM1.5: Methods, Analysis \u0026amp; Insights from Multimodal LLM Fine-tuning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHaotian Zhang, Mingfei Gao, Zhe Gan, Philipp Dufter, Nina Wenzel, Forrest Huang, Dhruti Shah, Xianzhi Du, Bowen Zhang, Yanghao Li, Sam Dodge, Keen You, Zhen Yang, Aleksei Timofeev, Mingze Xu, Hong-You Chen, Jean-Philippe Fauconnier, Zhengfeng Lai, Haoxuan You, Zirui Wang, Afshin Dehghan, Peter Grasch, Yinfei Yang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Apple\u003c/li\u003e\n\u003cli\u003e📅 Date: September 30, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [MM1.5], [vision language model], [visual grounding], [reasoning], [data-centric], [analysis]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces MM1.5, a family of multimodal large language models (MLLMs) ranging from 1B to 30B parameters, including dense and mixture-of-experts variants. MM1.5 enhances capabilities in text-rich image understanding, visual referring and grounding, and multi-image reasoning. The authors employ a data-centric training approach, utilizing high-quality OCR data and synthetic captions for continual pre-training, alongside an optimized visual instruction-tuning data mixture for supervised fine-tuning. Specialized variants, MM1.5-Video and MM1.5-UI, are designed for video understanding and mobile UI comprehension, respectively. Extensive empirical studies provide insights into the training processes, offering guidance for future MLLM development.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.15637\" rel=\"nofollow\"\u003eSynatra: Turning Indirect Knowledge into Direct Demonstrations for Digital Agents at Scale\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianyue Ou, Frank F. Xu, Aman Madaan, Jiarui Liu, Robert Lo, Abishek Sridhar, Sudipta Sengupta, Dan Roth, Graham Neubig, Shuyan Zhou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Amazon AWS AI\u003c/li\u003e\n\u003cli\u003e📅 Date: September 27, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [synthetic data]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: Synatra introduces a scalable framework for digital agents, enabling them to convert indirect knowledge sources into actionable demonstrations. This approach enhances the ability of agents to learn tasks without extensive labeled data, leveraging insights from indirect observations to scale practical implementations in digital environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://ai-secure.github.io/AdvWeb/\" rel=\"nofollow\"\u003eAdvWeb: Controllable Black-box Attacks on VLM-powered Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChejian Xu, Mintong Kang, Jiawei Zhang, Zeyi Liao, Lingbo Mo, Mengqi Yuan, Huan Sun, Bo Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UIUC, OSU\u003c/li\u003e\n\u003cli\u003e📅 Date: September 27, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [safety], [black-box attack], [adversarial prompter model], [Direct Policy Optimization]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents AdvWeb, a black-box attack framework that exploits vulnerabilities in vision-language model (VLM)-powered web agents by injecting adversarial prompts directly into web pages. Using Direct Policy Optimization (DPO), AdvWeb trains an adversarial prompter model that can mislead agents into executing harmful actions, such as unauthorized financial transactions, while maintaining high stealth and control. Extensive evaluations reveal that AdvWeb achieves high success rates across multiple real-world tasks, emphasizing the need for stronger security measures in web agent deployments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.17140\" rel=\"nofollow\"\u003eTurn Every Application into an Agent: Towards Efficient Human-Agent-Computer Interaction with API-First LLM-Based Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJunting Lu, Zhiyang Zhang, Fangkai Yang, Jue Zhang, Lu Wang, Chao Du, Qingwei Lin, Saravan Rajmohan, Dongmei Zhang, Qi Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Peking University, Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: September 26, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [API interaction], [HACI], [Agent OS]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper proposes an API-centered framework called \u003cstrong\u003eAXIS\u003c/strong\u003e, enhancing the efficiency and reliability of LLM-based agents by prioritizing API interactions over UI-based actions. This approach aims to reduce the high latency and error rates of traditional UI-interaction models. AXIS not only supports the rapid creation and extension of APIs through automated application exploration but also contributes to a new \u003cstrong\u003eHuman-Agent-Computer Interaction (HACI)\u003c/strong\u003e framework. The paper outlines the development of an agent-centric operating system (Agent OS), which improves task completion times by up to 70% and reduces cognitive load on users while maintaining high accuracy across complex multi-application tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://molmo.allenai.org/blog\" rel=\"nofollow\"\u003eMolmo and PixMo: Open Weights and Open Data for State-of-the-Art Vision-Language Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMatt Deitke, Christopher Clark, Sangho Lee, Rohun Tripathi, Yue Yang, Jae Sung Park, Mohammadreza Salehi, Niklas Muennighoff, Kyle Lo, Luca Soldaini, Jiasen Lu, Taira Anderson, Erin Bransom, Kiana Ehsani, Huong Ngo, YenSung Chen, Ajay Patel, Mark Yatskar, Chris Callison-Burch, Andrew Head, Rose Hendrix, Favyen Bastani, Eli VanderBilt, Nathan Lambert, Yvonne Chou, Arnavi Chheda, Jenna Sparks, Sam Skjonsberg, Michael Schmitz, Aaron Sarnat, Byron Bischoff, Pete Walsh, Chris Newell, Piper Wolters, Tanmay Gupta, Kuo-Hao Zeng, Jon Borchardt, Dirk Groeneveld, Crystal Nam, Sophie Lebrecht, Caitlin Wittlif, Carissa Schoenick, Oscar Michel, Ranjay Krishna, Luca Weihs, Noah A. Smith, Hannaneh Hajishirzi, Ross Girshick, Ali Farhadi, Aniruddha Kembhavi\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: AI2, UW\u003c/li\u003e\n\u003cli\u003e📅 Date: September 25, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [PixMo], [Molmo], [vision language model], [foundation model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMolmo\u003c/em\u003e, a family of state-of-the-art open vision-language models (VLMs), and \u003cem\u003ePixMo\u003c/em\u003e, a collection of new datasets including detailed image captions, free-form image Q\u0026amp;A, and innovative 2D pointing data, all collected without reliance on proprietary VLMs. The authors demonstrate that careful model design, a well-tuned training pipeline, and high-quality open datasets can produce VLMs that outperform existing open models and rival proprietary systems. The model weights, datasets, and source code are made publicly available to advance research in this field.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.14818\" rel=\"nofollow\"\u003eMobileVLM: A Vision-Language Model for Better Intra- and Inter-UI Understanding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eQinzhuo Wu, Weikai Xu, Wei Liu, Tao Tan, Jianfeng Liu, Ang Li, Jian Luan, Bin Wang, Shuo Shang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: XiaoMi AI Lab, University of Electronic Science and Technology of China, Renmin University of China\u003c/li\u003e\n\u003cli\u003e📅 Date: September 23, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [MobileVLM], [Mobile3M], [UI understanding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMobileVLM\u003c/em\u003e, a vision-language model designed to enhance both intra- and inter-UI understanding for mobile applications. The authors propose two additional pre-training stages with four specific UI-based tasks to improve the model's perception of fine-grained elements and capture page transition actions. To support this, they constructed \u003cem\u003eMobile3M\u003c/em\u003e, a large-scale Chinese mobile dataset comprising 3 million UI pages and real-world transition actions, organized into directed graphs. Experimental results demonstrate that MobileVLM outperforms existing vision-language models on both in-house test sets and public mobile benchmarks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://qwenlm.github.io/blog/qwen2-vl/\" rel=\"nofollow\"\u003eQwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePeng Wang, Shuai Bai, Sinan Tan, Shijie Wang, Zhihao Fan, Jinze Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Yang Fan, Kai Dang, Mengfei Du, Xuancheng Ren, Rui Men, Dayiheng Liu, Chang Zhou, Jingren Zhou, Junyang Lin\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Alibaba Cloud\u003c/li\u003e\n\u003cli\u003e📅 Date: September 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [foundation model], [MLLM], [Qwen2-VL]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: Qwen2-VL introduces an advanced vision-language framework that enables dynamic resolution handling for images and videos through its Naive Dynamic Resolution mechanism and Multimodal Rotary Position Embedding (M-RoPE). This structure allows the model to convert images of varying resolutions into diverse token counts for improved visual comprehension. With model sizes up to 72B parameters, Qwen2-VL demonstrates competitive performance across multiple benchmarks, achieving results on par with or better than prominent multimodal models like GPT-4o and Claude3.5-Sonnet. This work represents a significant step forward in scalable vision-language learning for multimodal tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.11295\" rel=\"nofollow\"\u003eEIA: Environmental Injection Attack on Generalist Web Agents for Privacy Leakage\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZeyi Liao, Lingbo Mo, Chejian Xu, Mintong Kang, Jiawei Zhang, Chaowei Xiao, Yuan Tian, Bo Li, Huan Sun\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, UCLA, UChicago, UIUC, UW-Madison\u003c/li\u003e\n\u003cli\u003e📅 Date: September 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [safety], [privacy attack], [environmental injection], [stealth attack]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the Environmental Injection Attack (EIA), a privacy attack targeting generalist web agents by embedding malicious yet concealed web elements to trick agents into leaking users' PII. Utilizing 177 action steps within realistic web scenarios, EIA demonstrates a high success rate in extracting specific PII and whole user requests. Through its detailed threat model and defense suggestions, the work underscores the challenge of detecting and mitigating privacy risks in autonomous web agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://microsoft.github.io/WindowsAgentArena/\" rel=\"nofollow\"\u003eWindows Agent Arena: Evaluating Multi-Modal OS Agents at Scale\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRogerio Bonatti, Dan Zhao, Francesco Bonacci, Dillon Dupont, Sara Abdali, Yinheng Li, Yadong Lu, Justin Wagle, Kazuhito Koishida, Arthur Bucker, Lawrence Jang, Zack Hui\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: September 13, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [Navi]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the \u003cem\u003eWindows Agent Arena (WAA)\u003c/em\u003e, a scalable platform for testing and benchmarking multi-modal AI agents within a realistic Windows OS environment. WAA enables researchers to evaluate agentic workflows across diverse tasks and supports large-scale deployment using Azure ML. The study also presents \u003cem\u003eNavi\u003c/em\u003e, a multi-modal agent achieving a 19.5% success rate on Windows tasks, highlighting the platform's potential for advancing AI agent development.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.07429\" rel=\"nofollow\"\u003eAgent Workflow Memory\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZora Zhiruo Wang, Jiayuan Mao, Daniel Fried, Graham Neubig\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, MIT\u003c/li\u003e\n\u003cli\u003e📅 Date: September 11, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [memory], [AWM]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper proposes \u003cem\u003eAgent Workflow Memory (AWM)\u003c/em\u003e, a method enabling language model-based agents to induce and utilize reusable workflows from past experiences to guide future actions in web navigation tasks. AWM operates in both offline and online settings, significantly improving performance on benchmarks like Mind2Web and WebArena, and demonstrating robust generalization across tasks, websites, and domains.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2409.01927\" rel=\"nofollow\"\u003eFrom Grounding to Planning: Benchmarking Bottlenecks in Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSegev Shlomov, Ben Wiesel, Aviad Sela, Ido Levy, Liane Galanti, Roy Abitbol\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: IBM\u003c/li\u003e\n\u003cli\u003e📅 Date: September 3, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [planning], [grounding], [Mind2Web dataset], [web navigation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper analyzes performance bottlenecks in web agents by separately evaluating grounding and planning tasks, isolating their individual impacts on navigation efficacy. Using an enhanced version of the Mind2Web dataset, the study reveals planning as a significant bottleneck, with advancements in grounding and task-specific benchmarking for elements like UI component recognition. Through experimental adjustments, the authors propose a refined evaluation framework, aiming to enhance web agents' contextual adaptability and accuracy in complex web environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/SqueezeAILab/TinyAgent\"\u003eTinyAgent: Function Calling at the Edge\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLutfi Eren Erdogan, Nicholas Lee, Siddharth Jha, Sehoon Kim, Ryan Tabrizi, Suhong Moon, Coleman Hooper, Gopala Anumanchipalli, Kurt Keutzer, Amir Gholami\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UC Berkeley, ICSI\u003c/li\u003e\n\u003cli\u003e📅 Date: September 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [quantization], [LLMCompiler], [TinyAgent-1.1B], [TinyAgent-7B]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces TinyAgent, an end-to-end framework for training and deploying task-specific small language model agents capable of function calling at the edge. By fine-tuning small models with curated datasets and employing techniques like quantization and a novel tool retrieval method, TinyAgent enables efficient, real-time execution of user commands on local devices without relying on cloud infrastructure. The framework demonstrates that these small models can match or even surpass the function-calling capabilities of larger models like GPT-4-Turbo while operating entirely on edge devices.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2408.15978\" rel=\"nofollow\"\u003eWebPilot: A Versatile and Autonomous Multi-Agent System for Web Task Execution with Strategic Exploration\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYao Zhang, Zijian Ma, Yunpu Ma, Zhen Han, Yu Wu, Volker Tresp\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: LMU Munich, Technical University of Munich, Munich Center for Machine Learning (MCML)\u003c/li\u003e\n\u003cli\u003e📅 Date: August 28, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Monte Carlo Tree Search], [reinforcement learning], [WebPilot]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eWebPilot\u003c/strong\u003e, a multi-agent system designed to execute complex web tasks requiring dynamic interaction. By employing a dual optimization strategy grounded in Monte Carlo Tree Search (MCTS), WebPilot enhances adaptability in complex web environments. The system's Global Optimization phase generates high-level plans by decomposing tasks into manageable subtasks, while the Local Optimization phase executes each subtask using a tailored MCTS approach. Experimental results on WebArena and MiniWoB++ demonstrate WebPilot's effectiveness, achieving state-of-the-art performance with GPT-4 and marking a significant advancement in autonomous web agent capabilities. :contentReference[oaicite:0]{index=0}\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2408.07199\" rel=\"nofollow\"\u003eAgent Q: Advanced Reasoning and Learning for Autonomous AI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAgent Q: Advanced Reasoning and Learning for Autonomous AI Agents\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: MultiOn, Stanford\u003c/li\u003e\n\u003cli\u003e📅 Date: August 13, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [MCTS], [Tree Search], [DPO], [Reinforcement Learning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: TBD\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2408.06327\" rel=\"nofollow\"\u003eVisualAgentBench: Towards Large Multimodal Models as Visual Foundation Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXiao Liu, Tianjie Zhang, Yu Gu, Iat Long Iong, Yifan Xu, Xixuan Song, Shudan Zhang, Hanyu Lai, Xinyi Liu, Hanlin Zhao, Jiadai Sun, Xinyue Yang, Yu Yang, Zehan Qi, Shuntian Yao, Xueqiao Sun, Siyi Cheng, Qinkai Zheng, Hao Yu, Hanchen Zhang, Wenyi Hong, Ming Ding, Lihang Pan, Xiaotao Gu, Aohan Zeng, Zhengxiao Du, Chan Hee Song, Yu Su, Yuxiao Dong, Jie Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, MSRA, The Ohio State University\u003c/li\u003e\n\u003cli\u003e📅 Date: August 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [VisualAgentBench], [VAB]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The authors introduce \u003cem\u003eVisualAgentBench (VAB)\u003c/em\u003e, a comprehensive benchmark designed to train and evaluate large multimodal models (LMMs) as visual foundation agents across diverse scenarios, including embodied tasks, graphical user interfaces, and visual design. VAB comprises five distinct environments that systematically challenge LMMs' understanding and interaction capabilities. Additionally, the benchmark offers supervised fine-tuning trajectory data for behavior cloning training, demonstrating the potential to improve open LMMs for serving as visual foundation agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2408.11824\" rel=\"nofollow\"\u003eAppAgent v2: Advanced Agent for Flexible Mobile Interactions\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYanda Li, Chi Zhang, Wanqi Yang, Bin Fu, Pei Cheng, Xin Chen, Ling Chen, Yunchao Wei\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: University of Technology Sydney, Tencent, Beijing Jiaotong University, Westlake University\u003c/li\u003e\n\u003cli\u003e📅 Date: August 5, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [AppAgent v2]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This work presents \u003cem\u003eAppAgent v2\u003c/em\u003e, a novel LLM-based multimodal agent framework for mobile devices capable of navigating applications by emulating human-like interactions such as tapping and swiping. The agent constructs a flexible action space that enhances adaptability across various applications, including parsing text and vision descriptions. It operates through two main phases: exploration and deployment, utilizing retrieval-augmented generation (RAG) technology to efficiently retrieve and update information from a knowledge base, thereby empowering the agent to perform tasks effectively and accurately.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://aclanthology.org/2024.findings-acl.539\" rel=\"nofollow\"\u003eCoCo-Agent: A Comprehensive Cognitive MLLM Agent for Smartphone GUI Automation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXinbei Ma, Zhuosheng Zhang, Hai Zhao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: August 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents CoCo-Agent, a multimodal large language model (MLLM) designed for smartphone GUI automation. It introduces two novel approaches: Comprehensive Environment Perception (CEP) for enhanced GUI understanding, and Conditional Action Prediction (CAP) to improve action response accuracy. The proposed agent achieves state-of-the-art performance on GUI automation benchmarks such as AITW and META-GUI, showcasing its capabilities in realistic scenarios.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2408.02544\" rel=\"nofollow\"\u003eCaution for the Environment: Multimodal Agents are Susceptible to Environmental Distractions\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXinbei Ma, Yiting Wang, Yao Yao, Tongxin Yuan, Aston Zhang, Zhuosheng Zhang, Hai Zhao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU, Meta\u003c/li\u003e\n\u003cli\u003e📅 Date: August 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [multimodal agents], [environmental distractions], [robustness]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper highlights the vulnerability of multimodal agents to environmental distractions. The researchers demonstrate that these agents, which process multiple types of input (e.g., text, images, audio), can be significantly impacted by irrelevant or misleading environmental cues. The study provides insights into the limitations of current multimodal systems and emphasizes the need for more robust architectures that can filter out distractions and maintain focus on relevant information in complex, real-world environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://microsoft.github.io/OmniParser/\" rel=\"nofollow\"\u003eOmniParser for Pure Vision Based GUI Agent\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYadong Lu, Jianwei Yang, Yelong Shen, Ahmed Awadallah\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: MSR, Microsoft Gen AI\u003c/li\u003e\n\u003cli\u003e📅 Date: August 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [OmniParser]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eOmniParser\u003c/strong\u003e, a method for parsing user interface screenshots into structured elements, enhancing the ability of models like GPT-4V to generate actions accurately grounded in corresponding UI regions. The authors curated datasets for interactable icon detection and icon description, fine-tuning models to parse interactable regions and extract functional semantics of UI elements.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.19056\" rel=\"nofollow\"\u003eOfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZilong Wang, Yuedong Cui, Li Zhong, Zimin Zhang, Da Yin, Bill Yuchen Lin, Jingbo Shang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCSD, UCLA, AI2\u003c/li\u003e\n\u003cli\u003e📅 Date: July 26, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [multi-application], [office automation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: OfficeBench introduces a benchmark that evaluates language models' ability to automate office tasks across a range of applications like Word, Excel, and email. The benchmark tests agents’ skills in task-switching, planning, and decision-making by simulating realistic office workflows. Current models, including GPT-4, demonstrate significant gaps in task accuracy and efficiency, revealing areas for improvement in managing complex, multi-application tasks in office environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.13032\" rel=\"nofollow\"\u003eAgent-E: From Autonomous Web Navigation to Foundational Design Principles in Agentic Systems\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAditya Vempaty, [Other authors not provided in the search results]\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Emergence AI\u003c/li\u003e\n\u003cli\u003e📅 Date: July 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [autonomous web navigation], [hierarchical architecture], [DOM distillation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents Agent-E, a novel web agent that introduces several architectural improvements over previous state-of-the-art systems. Key features include a hierarchical architecture, flexible DOM distillation and denoising methods, and a \"change observation\" concept for improved performance. Agent-E outperforms existing text and multi-modal web agents by 10-30% on the WebVoyager benchmark. The authors synthesize their findings into general design principles for developing agentic systems, including the use of domain-specific primitive skills, hierarchical architectures, and agentic self-improvement.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://spider2-v.github.io/\" rel=\"nofollow\"\u003eSpider2-V: How Far Are Multimodal Agents From Automating Data Science and Engineering Workflows?\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRuisheng Cao, Fangyu Lei, Haoyuan Wu, Jixuan Chen, Yeqiao Fu, Hongcheng Gao, Xinzhuang Xiong, Hanchong Zhang, Yuchen Mao, Wenjing Hu, Tianbao Xie, Hongsheng Xu, Danyang Zhang, Sida Wang, Ruoxi Sun, Pengcheng Yin, Caiming Xiong, Ansong Ni, Qian Liu, Victor Zhong, Lu Chen, Kai Yu, Tao Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HKU, SJTU, Google Cloud AI Research, Google DeepMind, Salesforce Research, Yale University, Sea AI Lab, University of Waterloo\u003c/li\u003e\n\u003cli\u003e📅 Date: July 15, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [data science], [engineering workflows], [Spider2-V]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eSpider2-V\u003c/strong\u003e, a multimodal agent benchmark designed to evaluate the capability of agents in automating professional data science and engineering workflows. It comprises 494 real-world tasks across 20 enterprise-level applications, assessing agents' proficiency in code generation and GUI operations within authentic computer environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/bz-lab/AUITestAgent\"\u003eAUITestAgent: Automatic Requirements Oriented GUI Function Testing\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYongxiang Hu, Xuan Wang, Yingchuan Wang, Yu Zhang, Shiyu Guo, Chaoyi Chen, Xin Wang, Yangfan Zhou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Fudan University, Meituan\u003c/li\u003e\n\u003cli\u003e📅 Date: July 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [GUI testing], [AUITestAgent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cstrong\u003eAUITestAgent\u003c/strong\u003e, the first automatic, natural language-driven GUI testing tool for mobile apps. It automates the entire process of GUI interaction and function verification by extracting GUI interactions from test requirements via dynamically organized agents and employing a multi-dimensional data extraction strategy for verification.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://luffyzm3d2y.github.io/publication/IoA\" rel=\"nofollow\"\u003eInternet of Agents: Weaving a Web of Heterogeneous Agents for Collaborative Intelligence\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWeize Chen, Ziming You, Ran Li, Yitong Guan, Chen Qian, Chenyang Zhao, Cheng Yang, Ruobing Xie, Zhiyuan Liu, Maosong Sun\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, Peking University, BUPT, Tencent\u003c/li\u003e\n\u003cli\u003e📅 Date: July 7, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [IoA]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper proposes the \u003cstrong\u003eInternet of Agents (IoA)\u003c/strong\u003e, a framework inspired by the Internet to facilitate collaboration among diverse autonomous agents. IoA introduces an agent integration protocol, dynamic teaming mechanisms, and conversation flow control, enabling flexible and scalable multi-agent collaboration. Experiments demonstrate IoA's superior performance across various tasks, highlighting its effectiveness in integrating heterogeneous agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.05291\" rel=\"nofollow\"\u003eWorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLéo Boisvert, Megh Thakkar, Maxime Gasse, Massimo Caccia, Thibault Le Sellier De Chezelles, Quentin Cappart, Nicolas Chapados, Alexandre Lacoste, Alexandre Drouin\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ServiceNow Research, Mila, Polytechnique Montréal, Université de Montréal\u003c/li\u003e\n\u003cli\u003e📅 Date: July 7, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [planning], [reasoning], [WorkArena++]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eWorkArena++\u003c/strong\u003e, a benchmark comprising 682 tasks that simulate realistic workflows performed by knowledge workers. It evaluates web agents' capabilities in planning, problem-solving, logical/arithmetic reasoning, retrieval, and contextual understanding. The study reveals challenges faced by current large language models and vision-language models in serving as effective workplace assistants, providing a resource to advance autonomous agent development. \u003ca href=\"https://arxiv.org/abs/2407.05291?utm_source=chatgpt.com\" rel=\"nofollow\"\u003eoai_citation_attribution:0‡arXiv\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.04346\" rel=\"nofollow\"\u003eMobileFlow: A Multimodal LLM For Mobile GUI Agent\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eSongqin Nong, Jiali Zhu, Rui Wu, Jiongchao Jin, Shuo Shan, Xiutian Huang, Wenhao Xu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Ant Group\u003c/li\u003e\n\u003cli\u003e📅 Date: July 5, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [MobileFlow]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMobileFlow\u003c/em\u003e, a multimodal large language model tailored for mobile GUI agents. With approximately 21 billion parameters and hybrid visual encoders, it supports variable image resolutions and multilingual GUIs, enhancing the model's ability to interpret image data and comprehend user instructions for GUI interaction tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.03913\" rel=\"nofollow\"\u003eMobileExperts: A Dynamic Tool-Enabled Agent Team in Mobile Devices\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJiayi Zhang, Chuang Zhao, Yihan Zhao, Zhaoyang Yu, Ming He, Jianping Fan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HKUST, Ant Group\u003c/li\u003e\n\u003cli\u003e📅 Date: July 4, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [tool formulation], [multi-agent collaboration], [MobileExperts]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMobileExperts\u003c/em\u003e, a framework that enhances autonomous operations on mobile devices by dynamically assembling agent teams based on user requirements. Each agent independently explores and formulates tools to evolve into an expert, improving efficiency and reducing reasoning costs.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.03037\" rel=\"nofollow\"\u003eVision-driven Automated Mobile GUI Testing via Multimodal Large Language Model\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhe Liu, Cheng Li, Chunyang Chen, Junjie Wang, Boyu Wu, Yawen Wang, Jun Hu, Qing Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Institute of Software, Chinese Academy of Sciences, Monash University, Beijing Institute of Technology, University of Chinese Academy of Sciences\u003c/li\u003e\n\u003cli\u003e📅 Date: July 3, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [VisionDroid]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper presents \u003cstrong\u003eVisionDroid\u003c/strong\u003e, a vision-driven automated GUI testing approach utilizing Multimodal Large Language Models (MLLM) to detect non-crash functional bugs in mobile applications. By extracting GUI text information and aligning it with screenshots, VisionDroid enables MLLM to understand GUI context, facilitating deeper and function-oriented exploration. The approach segments exploration history into logically cohesive parts, prompting MLLM for bug detection, demonstrating superior performance over existing methods.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.01511\" rel=\"nofollow\"\u003eCRAB: Cross-environment Agent Benchmark for Multimodal Language Model Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianqi Xu, Linyao Chen, Dai-Jie Wu, Yanjun Chen, Zecheng Zhang, Xiang Yao, Zhiqiang Xie, Yongchao Chen, Shilong Liu, Bochen Qian, Philip Torr, Bernard Ghanem, Guohao Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: KAUST, UTokyo, CMU, Stanford, Harvard, Tsinghua University, SUSTech, Oxford\u003c/li\u003e\n\u003cli\u003e📅 Date: July 3, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [evaluation], [CRAB]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The authors present \u003cem\u003eCRAB\u003c/em\u003e, a benchmark framework designed to evaluate Multimodal Language Model agents across multiple environments. It features a graph-based fine-grained evaluation method and supports automatic task generation, addressing limitations in existing benchmarks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://yuxiangchai.github.io/AMEX/\" rel=\"nofollow\"\u003eAMEX: Android Multi-annotation Expo Dataset for Mobile GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYuxiang Chai, Siyuan Huang, Yazhe Niu, Han Xiao, Liang Liu, Dingyu Zhang, Peng Gao, Shuai Ren, Hongsheng Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CUHK, SJTU, Shanghai AI Lab, vivo AI Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: July 3, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark], [AMEX]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the \u003cstrong\u003eAndroid Multi-annotation EXpo (AMEX)\u003c/strong\u003e, a comprehensive dataset designed for training and evaluating mobile GUI-control agents. AMEX comprises over 104K high-resolution screenshots from 110 popular mobile applications, annotated at multiple levels, including GUI interactive element grounding, functionality descriptions, and complex natural language instructions. The dataset aims to advance research on AI agents capable of completing complex tasks by interacting directly with mobile device GUIs.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://screen-point-and-read.github.io/\" rel=\"nofollow\"\u003eRead Anywhere Pointed: Layout-aware GUI Screen Reading with Tree-of-Lens Grounding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYue Fan, Lei Ding, Ching-Chen Kuo, Shan Jiang, Yang Zhao, Xinze Guan, Jie Yang, Yi Zhang, Xin Eric Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCSC, MSR\u003c/li\u003e\n\u003cli\u003e📅 Date: June 27, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [ToL], [screen reading], [accessibility]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The authors propose the Tree-of-Lens (ToL) agent to address the Screen Point-and-Read (ScreenPR) task, which involves generating natural language descriptions of screen regions based on user-indicated points. The ToL agent constructs a Hierarchical Layout Tree to comprehend the content and articulate the layout and spatial relationships between elements. The authors also introduce the ScreenPR benchmark, consisting of 650 screenshots from web, mobile, and operating system GUIs, manually annotated with 1,500 target points and regions.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://nexa.ai/octo-planner\" rel=\"nofollow\"\u003eOcto-planner: On-device Language Model for Planner-Action Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNexa AI Team\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Nexa AI\u003c/li\u003e\n\u003cli\u003e📅 Date: June 26, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [Octo-planner], [on-device], [planning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents Octo-planner, an on-device planning model designed for the Planner-Action Agents Framework. Octo-planner utilizes a fine-tuned model based on Phi-3 Mini (3.8 billion parameters) for high efficiency and low power consumption. It separates planning and action execution into two distinct components: a planner agent optimized for edge devices and an action agent using the Octopus model for function execution. The model achieves a planning success rate of 98.1% on benchmark datasets, providing reliable and effective performance.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.14250\" rel=\"nofollow\"\u003eE-ANT: A Large-Scale Dataset for Efficient Automatic GUI NavigaTion\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKe Wang, Tianyu Xia, Zhangxuan Gu, Yi Zhao, Shuheng Shen, Changhua Meng, Weiqiang Wang, Ke Xu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Ant Group, Tsinghua University\u003c/li\u003e\n\u003cli\u003e📅 Date: June 20, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark], [E-ANT]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eE-ANT\u003c/strong\u003e, the first large-scale Chinese GUI navigation dataset comprising over 40,000 real human interaction traces across more than 5,000 tiny apps. The dataset includes high-quality screenshots with annotations, facilitating the evaluation and development of GUI navigation and decision-making capabilities in multimodal large language models (MLLMs). The authors also assess various MLLMs on E-ANT, providing insights into their performance and potential improvements.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://ical-learning.github.io/\" rel=\"nofollow\"\u003eVLM Agents Generate Their Own Memories: Distilling Experience into Embodied Programs of Thought\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGabriel Sarch, Lawrence Jang, Michael J. Tarr, William W. Cohen, Kenneth Marino, Katerina Fragkiadaki\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Google DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: June 20, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [memory], [in-context learning], [ICAL]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eIn-Context Abstraction Learning (ICAL)\u003c/em\u003e, a method enabling Vision-Language Models (VLMs) to generate their own examples from sub-optimal demonstrations and human feedback. By abstracting trajectories into generalized programs of thought, ICAL enhances decision-making in retrieval-augmented LLM and VLM agents, reducing reliance on manual prompt engineering and improving performance across various tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.14314\" rel=\"nofollow\"\u003eIdentifying User Goals from UI Trajectories\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eOmri Berkovitch, Sapir Caduri, Noam Kahlon, Anatoly Efros, Avi Caciularu, Ido Dagan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google Research, Bar-Ilan University\u003c/li\u003e\n\u003cli\u003e📅 Date: June 20, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [evaluation metric], [intent identification]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the task of goal identification from observed UI trajectories, aiming to infer the user's intended task based on their GUI interactions. It proposes a novel evaluation metric to assess whether two task descriptions are paraphrases within a specific UI environment. Experiments utilizing the Android-In-The-Wild and Mind2Web datasets reveal that state-of-the-art models, such as GPT-4 and Gemini-1.5 Pro, underperform compared to humans, indicating significant room for improvement.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.14056\" rel=\"nofollow\"\u003eVGA: Vision GUI Assistant -- Minimizing Hallucinations through Image-Centric Fine-Tuning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZiyang Meng, Yu Dai, Zezheng Gong, Shaoxiong Guo, Minglong Tang, Tongquan Wei\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: June 20, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [framework], [VGA], [hallucination]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces VGA, a fine-tuned model designed to enhance GUI comprehension by reducing hallucinations. The authors constructed a Vision Question Answering (VQA) dataset of 63.8k high-quality examples using a Referent Method, ensuring model responses are highly dependent on visual content. They also propose a two-stage fine-tuning method called Foundation and Advanced Comprehension (FAC) to improve the model's ability to extract information from images and align with human intent.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://showlab.github.io/GUI-Narrator/\" rel=\"nofollow\"\u003eGUI Action Narrator: Where and When Did That Action Take Place?\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eQinchen Wu, Difei Gao, Kevin Qinghong Lin, Zhuoyu Wu, Xiangwu Guo, Peiran Li, Weichen Zhang, Hengxu Wang, Mike Zheng Shou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS, Chinese Academy of Sciences\u003c/li\u003e\n\u003cli\u003e📅 Date: June 19, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [framework], [Act2Cap], [GUI Narrator]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The authors present \u003cstrong\u003eAct2Cap\u003c/strong\u003e, a GUI action dataset containing 4,189 video-caption pairs depicting various GUI actions such as clicks, drags, and typing across multiple software environments. They also propose \u003cstrong\u003eGUI Narrator\u003c/strong\u003e, a framework that leverages cursor detection as a visual prompt to enhance the interpretation of high-resolution screenshots for GUI video captioning. Evaluations reveal that even advanced multimodal models face challenges in this domain, highlighting the need for specialized approaches to improve performance.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.12373\" rel=\"nofollow\"\u003eWebCanvas: Benchmarking Web Agents in Online Environments\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYichen Pan, Dehan Kong, Sida Zhou, Cheng Cui, Yifei Leng, Bing Jiang, Hangyu Liu, Yanyi Shang, Shuyan Zhou, Tongshuang Wu, Zhengyang Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: iMean AI, CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: June 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [Mind2Web-Live], [key-node evaluation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents WebCanvas, an online evaluation framework for web agents designed to address the dynamic nature of web interactions. It introduces a key-node-based evaluation metric to capture critical actions or states necessary for task completion while disregarding noise from insignificant events or changed web elements. The framework includes the Mind2Web-Live dataset, a refined version of the original Mind2Web static dataset, containing 542 tasks with 2,439 intermediate evaluation states. Despite advancements, the best-performing model achieves a task success rate of 23.1%, highlighting substantial room for improvement.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://chenwu.io/attack-agent/\" rel=\"nofollow\"\u003eAdversarial Attacks on Multimodal Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChen Henry Wu, Jing Yu Koh, Ruslan Salakhutdinov, Daniel Fried, Aditi Raghunathan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: Jun 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [safety], [VisualWebArena-Adv]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper investigates the safety risks posed by multimodal agents built on vision-enabled language models (VLMs). The authors introduce two adversarial attack methods: a captioner attack targeting white-box captioners and a CLIP attack that transfers to proprietary VLMs. To evaluate these attacks, they curated VisualWebArena-Adv, a set of adversarial tasks based on VisualWebArena. The study demonstrates that within a limited perturbation norm, the captioner attack can achieve a 75% success rate in making a captioner-augmented GPT-4V agent execute adversarial goals. The paper also discusses the robustness of agents based on other VLMs and provides insights into factors contributing to attack success and potential defenses. \u003ca href=\"https://arxiv.org/abs/2406.12814?utm_source=chatgpt.com\" rel=\"nofollow\"\u003eoai_citation_attribution:0‡ArXiv\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/yiye3/GUICourse\"\u003eGUICourse: From General Vision Language Models to Versatile GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWentong Chen, Junbo Cui, Jinyi Hu, Yujia Qin, Junjie Fang, Yue Zhao, Chongyi Wang, Jun Liu, Guirong Chen, Yupeng Huo, Yuan Yao, Yankai Lin, Zhiyuan Liu, Maosong Sun\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, Rhapsody AI, University of Electronic Science and Technology of China\u003c/li\u003e\n\u003cli\u003e📅 Date: June 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [framework], [GUICourse]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eGUICourse\u003c/em\u003e, a suite of datasets aimed at training visual-based GUI agents from general vision-language models. It addresses challenges in OCR, grounding, and GUI knowledge, enhancing the models' capabilities in GUI navigation tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.10819\" rel=\"nofollow\"\u003eGUI-WORLD: A Dataset for GUI-oriented Multimodal LLM-based Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDongping Chen, Yue Huang, Siyuan Wu, Jingyu Tang, Liuyi Chen, Yilin Bai, Zhigang He, Chenlong Wang, Huichi Zhou, Yiqiang Li, Tianshuo Zhou, Yue Yu, Chujie Gao, Qihui Zhang, Yi Gui, Zhen Li, Yao Wan, Pan Zhou, Jianfeng Gao, Lichao Sun\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Huazhong University of Science and Technology (HUST), MSR, University of Illinois at Chicago (UIC)\u003c/li\u003e\n\u003cli\u003e📅 Date: June 16, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark], [GUI-World], [GUI-Vid]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eGUI-World\u003c/em\u003e, a comprehensive dataset designed to evaluate Multimodal Large Language Models (MLLMs) in dynamic and complex GUI environments. It includes over 12,000 annotated GUI interaction videos covering diverse applications and scenarios. The study highlights the limitations of current MLLMs in handling dynamic and multi-step tasks and presents \u003cem\u003eGUI-Vid\u003c/em\u003e, a fine-tuned VideoLLM, demonstrating improved understanding of various GUI tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://digirl-agent.github.io/\" rel=\"nofollow\"\u003eDigiRL: Training In-The-Wild Device-Control Agents with Autonomous Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHao Bai, Yifei Zhou, Mert Cemri, Jiayi Pan, Alane Suhr, Sergey Levine, Aviral Kumar\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UC Berkeley, UIUC, Google DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: June 14, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [DigiRL]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The authors present \u003cem\u003eDigiRL\u003c/em\u003e, an autonomous reinforcement learning approach for training device-control agents. By fine-tuning a pre-trained vision-language model in two stages—offline and offline-to-online RL—DigiRL achieves a significant improvement in success rates on the Android-in-the-Wild dataset, establishing a new state-of-the-art for digital agents in device control.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.08451\" rel=\"nofollow\"\u003eGUI Odyssey: A Comprehensive Dataset for Cross-App GUI Navigation on Mobile Devices\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eQuanfeng Lu, Wenqi Shao, Zitao Liu, Fanqing Meng, Boxuan Li, Botong Chen, Siyuan Huang, Kaipeng Zhang, Yu Qiao, Ping Luo\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OpenGVLab, Shanghai AI Laboratory, HKU, Nanjing University, Harbin Institute of Technology, Shenzhen, SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: June 13, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [model], [OdysseyAgent], [cross-app navigation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cem\u003eGUI Odyssey\u003c/em\u003e, a dataset comprising 7,735 episodes from six mobile devices, designed to train and evaluate cross-app navigation agents. It spans six types of cross-app tasks across 201 apps and 1,399 app combinations. Leveraging this dataset, the authors developed \u003cem\u003eOdysseyAgent\u003c/em\u003e, a multimodal cross-app navigation agent fine-tuned from the Qwen-VL model, demonstrating superior accuracy over existing models in both in-domain and out-of-domain scenarios.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://mobileagentbench.github.io/\" rel=\"nofollow\"\u003eMobileAgentBench: An Efficient and User-Friendly Benchmark for Mobile LLM Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLuyuan Wang, Yongyu Deng, Yiwei Zha, Guodong Mao, Qinmin Wang, Tianchen Min, Wei Chen, Shoufa Chen\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, University of Michigan, Northeastern University, HKU\u003c/li\u003e\n\u003cli\u003e📅 Date: June 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [MobileAgentBench]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMobileAgentBench\u003c/em\u003e, a benchmark designed to evaluate the performance of large language model-based mobile agents. It defines 100 tasks across 10 open-source apps, categorized by difficulty levels, and assesses existing agents like AppAgent and MobileAgent to facilitate systematic comparisons.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.08340\" rel=\"nofollow\"\u003ePractical, Automated Scenario-based Mobile App Testing\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShengcheng Yu, Chunrong Fang, Mingzhe Du, Zimin Ding, Zhenyu Chen, Zhendong Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Nanjing University, ETH Zurich\u003c/li\u003e\n\u003cli\u003e📅 Date: June 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: IEEE Transactions on Software Engineering\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [ScenTest], [event knowledge graph], [GUI image understanding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eScenTest\u003c/em\u003e, a novel approach for scenario-based mobile app testing that integrates event knowledge graphs (EKGs) with GUI image understanding. By extracting entities and relationships from crowdsourced test reports, ScenTest constructs EKGs for specific scenarios, guiding automated testing processes. This method bridges the gap between testing execution and app business logic, achieving fully automated testing on target scenarios for the first time.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.03679\" rel=\"nofollow\"\u003eOn the Effects of Data Scale on UI Control Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWei Li, William Bishop, Alice Li, Chris Rawles, Folawiyo Campbell-Ajala, Divya Tyamagundlu, Oriana Riva\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google DeepMind, Google\u003c/li\u003e\n\u003cli\u003e📅 Date: June 6, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [AndroidControl], [fine-tuning], [scalability]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study investigates how the performance of computer control agents scales with the amount of fine-tuning data. The authors introduce \u003cstrong\u003eAndroidControl\u003c/strong\u003e, a dataset comprising 15,283 demonstrations across 833 Android applications. Findings indicate that while in-domain performance improves with more data, out-of-domain performance, especially on high-level tasks, scales more slowly, suggesting that fine-tuning alone may be insufficient for robust out-of-domain performance.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/X-PLUG/MobileAgent\"\u003eMobile-Agent-v2: Mobile Device Operation Assistant with Effective Navigation via Multi-Agent Collaboration\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJunyang Wang, Haiyang Xu, Haitao Jia, Xi Zhang, Ming Yan, Weizhou Shen, Ji Zhang, Fei Huang, Jitao Sang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Alibaba Group, Beijing University of Posts and Telecommunications\u003c/li\u003e\n\u003cli\u003e📅 Date: June 3, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [multi-agent], [planning], [decision-making], [reflection]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper presents \u003cstrong\u003eMobile-Agent-v2\u003c/strong\u003e, a multi-agent architecture designed to assist with mobile device operations. It comprises three agents: a planning agent that generates task progress, a decision agent that navigates tasks using a memory unit, and a reflection agent that corrects erroneous operations. This collaborative approach addresses challenges in navigation and long-context input scenarios, achieving over a 30% improvement in task completion compared to single-agent architectures.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://aclanthology.org/2024.naacl-industry.9/\" rel=\"nofollow\"\u003eVisual Grounding for User Interfaces\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYijun Qian, Yujie Lu, Alexander Hauptmann, Oriana Riva\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, UCSB\u003c/li\u003e\n\u003cli\u003e📅 Date: June 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NAACL 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [visual grounding], [UI element localization], [LVG]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This work introduces the task of visual UI grounding, which unifies detection and grounding by enabling models to identify UI elements referenced by natural language commands solely from visual input. The authors propose \u003cstrong\u003eLVG\u003c/strong\u003e, a model that outperforms baselines pre-trained on larger datasets by over 4.9 points in top-1 accuracy, demonstrating its effectiveness in localizing referenced UI elements without relying on UI metadata.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.01623\" rel=\"nofollow\"\u003eWebSuite: Systematically Evaluating Why Web Agents Fail\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEric Li, Jim Waldo\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Harvard\u003c/li\u003e\n\u003cli\u003e📅 Date: June 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [failure analysis], [analysis], [task disaggregation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eWebSuite\u003c/em\u003e, a diagnostic benchmark to investigate the causes of web agent failures. By categorizing agent tasks using a taxonomy of operational, informational, and navigational actions, WebSuite offers granular insights into the specific actions where agents struggle, like filtering or form completion. It enables detailed comparison across agents, identifying areas for architectural and UX adaptation to improve agent reliability and task success on the web.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.10227\" rel=\"nofollow\"\u003eVideoGUI: A Benchmark for GUI Automation from Instructional Videos\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKevin Qinghong Lin, Linjie Li, Difei Gao, Qinchen WU, Mingyi Yan, Zhengyuan Yang, Lijuan Wang, Mike Zheng Shou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS, Microsoft Gen AI\u003c/li\u003e\n\u003cli\u003e📅 Date: June 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop, Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [instructional videos], [visual planning], [hierarchical task decomposition], [complex software interaction]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: VideoGUI presents a benchmark for evaluating GUI automation on tasks derived from instructional videos, focusing on visually intensive applications like Adobe Photoshop and video editing software. The benchmark includes 178 tasks, with a hierarchical evaluation method distinguishing high-level planning, mid-level procedural steps, and precise action execution. VideoGUI reveals current model limitations in complex visual tasks, marking a significant step toward improved visual planning in GUI automation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2405.20309\" rel=\"nofollow\"\u003eLarge Language Models Can Self-Improve At Web Agent Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAjay Patel, Markus Hofmarcher, Claudiu Leoveanu-Condrei, Marius-Constantin Dinu, Chris Callison-Burch, Sepp Hochreiter\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: University of Pennsylvania, ExtensityAI, Johannes Kepler University Linz, NXAI\u003c/li\u003e\n\u003cli\u003e📅 Date: May 30, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [self-improvement], [self-improve]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper investigates the ability of large language models (LLMs) to enhance their performance as web agents through self-improvement. Utilizing the WebArena benchmark, the authors fine-tune LLMs on synthetic training data, achieving a 31% improvement in task completion rates. They also introduce novel evaluation metrics to assess the performance, robustness, and quality of the fine-tuned agents' trajectories.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2405.14573\" rel=\"nofollow\"\u003eAndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChristopher Rawles, Sarah Clinckemaillie, Yifan Chang, Jonathan Waltz, Gabrielle Lau, Marybeth Fair, Alice Li, William Bishop, Wei Li, Folawiyo Campbell-Ajala, Daniel Toyama, Robert Berry, Divya Tyamagundlu, Timothy Lillicrap, Oriana Riva\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google DeepMind, Google\u003c/li\u003e\n\u003cli\u003e📅 Date: May 23, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2025\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [Android-based agents], [task diversity], [reinforcement learning], [dynamic environment]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AndroidWorld introduces a dynamic Android environment for benchmarking autonomous agents across 116 tasks spanning 20 Android apps. These tasks vary through parameterized and natural language prompts, fostering a realistic testing ground for agents designed to operate in complex mobile environments. The benchmark supports millions of task variations, allowing agents to respond to the Android system's changing states and improving real-world applicability.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2405.04497\" rel=\"nofollow\"\u003eUnveiling Disparities in Web Task Handling Between Human and Web Agent\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKihoon Son, Jinhyeon Kwon, DaEun Choi, Tae Soo Kim, Young-Ho Kim, Sangdoo Yun, Juho Kim\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: KAIST, Seoul National University\u003c/li\u003e\n\u003cli\u003e📅 Date: May 7, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CHI 2024 Workshop\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [cognitive comparison], [task analysis]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper examines how humans and web agents differ in handling web-based tasks, focusing on key aspects such as planning, action-taking, and reflection. Using a think-aloud protocol, the study highlights the cognitive processes humans employ, like exploration and adjustment, versus the more rigid task execution patterns observed in web agents. The authors identify several limitations in current web agents, proposing the need for improved frameworks to enhance adaptability and knowledge update mechanisms in agent-based systems.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2405.00516\" rel=\"nofollow\"\u003eNavigating WebAI: Training Agents to Complete Web Tasks with Large Language Models and Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLucas-Andreï Thil, Mirela Popa, Gerasimos Spanakis\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Maastricht University the Netherlands\u003c/li\u003e\n\u003cli\u003e📅 Date: May 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [large language models], [reinforcement learning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper proposes a novel approach combining supervised learning (SL) and reinforcement learning (RL) techniques to train web navigation agents using large language models. The authors address limitations in previous models' understanding of HTML content and introduce methods to enhance true comprehension. Their approach, evaluated on the MiniWoB benchmark, outperforms previous SL methods on certain tasks using less data and narrows the performance gap with RL models. The study achieves 43.58% average accuracy in SL and 36.69% when combined with a multimodal RL approach, setting a new direction for future web navigation research.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.12500\" rel=\"nofollow\"\u003eUIClip: A Data-driven Model for Assessing User Interface Design\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJason Wu, Yi-Hao Peng, Amanda Li, Amanda Swearngin, Jeffrey P. Bigham, Jeffrey Nichols\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Apple\u003c/li\u003e\n\u003cli\u003e📅 Date: Apr 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: UIST 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [UIClip], [vision foundation model], [foundation model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eUIClip\u003c/em\u003e, a machine-learned model that evaluates the design quality and visual relevance of user interfaces by analyzing screenshots and corresponding natural language descriptions. Trained on a large-scale dataset combining automated crawling, synthetic augmentation, and human ratings, UIClip assigns numerical scores representing a UI's relevance and quality, and offers design suggestions. Evaluations show that UIClip's assessments align closely with human designer rankings. The paper also demonstrates UIClip's utility in applications like UI code generation, design tips generation, and quality-aware UI example search.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.11459\" rel=\"nofollow\"\u003eOctopus v3: Technical Report for On-device Sub-billion Multimodal AI Agent\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWei Chen, Zhiyuan Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Stanford University\u003c/li\u003e\n\u003cli\u003e📅 Date: April 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [functional token], [on-device AI], [Octopus v3]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces Octopus v3, a compact multimodal AI agent with less than 1 billion parameters, designed for efficient on-device operation. It processes both English and Chinese inputs, integrating visual and textual data to perform tasks such as sending emails, messaging, and online shopping. The model employs a functional token approach to translate image-based data into actionable outcomes, demonstrating high accuracy and efficiency on edge devices, including Raspberry Pi.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.10887\" rel=\"nofollow\"\u003eSearch Beyond Queries: Training Smaller Language Models for Web Interactions via Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eMoghis Fereidouni, A.B. Siddique\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: University of Kentucky\u003c/li\u003e\n\u003cli\u003e📅 Date: April 16, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [grounded language agent], [Flan-T5], [unsupervised domain adaptation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces GLAINTEL, a grounded language agent framework designed to enhance web interaction using instruction-finetuned language models, particularly Flan-T5, with reinforcement learning (PPO) to tackle interactive web navigation challenges. The study explores unsupervised and supervised training methods, evaluating the effects of human demonstration on agent performance. Results indicate that combining human feedback with reinforcement learning yields effective outcomes, rivaling larger models like GPT-4 on web navigation tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.09992\" rel=\"nofollow\"\u003eMMInA: Benchmarking Multihop Multimodal Internet Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZiniu Zhang, Shulin Tian, Liangyu Chen, Ziwei Liu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NTU\u003c/li\u003e\n\u003cli\u003e📅 Date: April 15, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [multihop web browsing], [multimodal tasks], [long-range reasoning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The \u003cstrong\u003eMMInA\u003c/strong\u003e benchmark is designed to evaluate agents' capacity to complete complex, multihop web tasks by navigating and extracting information across evolving real-world websites. Composed of 1,050 tasks across diverse domains, MMInA challenges agents with realistic, multimodal information retrieval and reasoning tasks, such as comparative shopping and travel inquiries. Despite recent advances, agents show difficulties in handling tasks requiring sequential steps across multiple sites, underscoring the need for enhanced multimodal and memory-augmented models.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.16054\" rel=\"nofollow\"\u003eLlamaTouch: A Faithful and Scalable Testbed for Mobile UI Automation Task Evaluation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLi Zhang, Shihe Wang, Xianqing Jia, Zhihan Zheng, Yunhe Yan, Longxi Gao, Yuanchun Li, Mengwei Xu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: BUPT, Tsinghua University\u003c/li\u003e\n\u003cli\u003e📅 Date: April 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: UIST 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [UI automation], [mobile agent evaluation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: LlamaTouch is an evaluation testbed designed for mobile UI automation, enabling reliable task assessment across 495 annotated tasks. It provides a scalable solution to evaluate agents in real-world mobile settings, comparing agent actions to essential UI states for accurate task completion. LlamaTouch supports dynamic environments, advancing mobile agent reliability and scalability in task automation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.07972\" rel=\"nofollow\"\u003eOSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HKU, CMU, Salesforce, University of Waterloo\u003c/li\u003e\n\u003cli\u003e📅 Date: April 11, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [real computer tasks], [online environment], [online benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: OSWorld introduces a groundbreaking benchmark for multimodal agents to perform open-ended tasks within real computer environments across platforms like Ubuntu, Windows, and macOS. It includes 369 real-world tasks involving web and desktop apps, file management, and multi-app workflows, with custom evaluation scripts for reproducibility. The results reveal current agents’ limitations in GUI interaction and operational knowledge, as they achieve just 12.24% task success compared to humans' 72.36%, highlighting critical gaps for future model improvement.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.05955\" rel=\"nofollow\"\u003eVisualWebBench: How Far Have Multimodal LLMs Evolved in Web Page Understanding and Grounding?\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJunpeng Liu, Yifan Song, Bill Yuchen Lin, Wai Lam, Graham Neubig, Yuanzhi Li, Xiang Yue\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: April 9, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: COLM 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [web page understanding], [grounding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: VisualWebBench introduces a comprehensive benchmark for evaluating multimodal large language models (MLLMs) on web-based tasks. It includes 1.5K human-curated instances across 139 websites in 87 sub-domains. The benchmark spans seven tasks—such as OCR, grounding, and web-based QA—aiming to test MLLMs' capabilities in fine-grained web page understanding. Results reveal significant performance gaps, particularly in grounding tasks, highlighting the need for advancement in MLLM web understanding.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.06474\" rel=\"nofollow\"\u003eAutonomous Evaluation and Refinement of Digital Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJiayi Pan, Yichi Zhang, Nicholas Tomlin, Yifei Zhou, Sergey Levine, Alane Suhr\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCB, UMich\u003c/li\u003e\n\u003cli\u003e📅 Date: April 9, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: COLM 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [evaluation model], [domain transfer]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents an autonomous evaluation framework for digital agents to enhance performance on web navigation and device control. The study introduces modular, cost-effective evaluators achieving up to 92.9% accuracy in benchmarks like WebArena and outlines their use in fine-tuning agents, improving state-of-the-art by 29% without additional supervision.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://machinelearning.apple.com/research/ferretui-mobile\" rel=\"nofollow\"\u003eFerret-UI: Grounded Mobile UI Understanding with Multimodal LLMs\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKeen You, Haotian Zhang, Eldon Schoop, Floris Weers, Amanda Swearngin, Jeffrey Nichols, Yinfei Yang, Zhe Gan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Apple\u003c/li\u003e\n\u003cli\u003e📅 Date: April 8, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ECCV 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [dataset], [benchmark], [mobile UI understanding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cstrong\u003eFerret-UI\u003c/strong\u003e, a multimodal large language model (MLLM) designed to understand and interact with mobile user interfaces. The model incorporates advanced capabilities for referring, grounding, and reasoning about UI elements. By training on a variety of UI tasks, Ferret-UI achieves high performance in tasks such as icon recognition and text extraction. The authors introduce a unique architecture that allows for improved visual feature extraction from mobile screens, paving the way for applications in accessibility and user interaction.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.16660\" rel=\"nofollow\"\u003eBenchmarking Mobile Device Control Agents across Diverse Configurations\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJuyong Lee, Taywon Min, Minyong An, Dongyoon Hahm, Haeone Lee, Changyeon Kim, Kimin Lee\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: KAIST, Seoul National University, Yonsei University\u003c/li\u003e\n\u003cli\u003e📅 Date: April 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [mobile device control], [agent performance]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cstrong\u003eB-MoCA\u003c/strong\u003e, a comprehensive benchmark for evaluating mobile device control agents using an Android-based testbed with 131 tasks and various device configurations. The benchmark assesses agents' abilities across tasks that include device-specific variations, navigation, and human-like dual-gesture interactions. B-MoCA highlights that current agents perform well on basic tasks but struggle with complex configurations, pointing to opportunities for future improvements in mobile automation capabilities.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2404.08860\" rel=\"nofollow\"\u003eEnhancing Mobile \"How-to\" Queries with Automated Search Results Verification and Reranking\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLei Ding, Jeshwanth Bheemanpally, Yi Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCSC\u003c/li\u003e\n\u003cli\u003e📅 Date: April 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: SIGIR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [reranking], [verification], [mobile task automation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents a system that enhances mobile \"how-to\" queries by verifying and reranking search results through automated instruction extraction, on-device action execution, and reranking based on relevance. The method improves on traditional ranking by analyzing device-specific execution success. The approach comprises a three-stage pipeline: 1) extracting step-by-step instructions from top search results, 2) validating these instructions on mobile devices, and 3) reranking based on performance. The system leverages a pre-trained GPT model for initial processing, ensuring adaptability across diverse apps and systems.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2403.17918\" rel=\"nofollow\"\u003eAgentStudio: A Toolkit for Building General Virtual Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLongtao Zheng, Zhiyuan Huang, Zhenghai Xue, Xinrun Wang, Bo An, Shuicheng Yan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NTU, Skywork AI, ETH Zurich\u003c/li\u003e\n\u003cli\u003e📅 Date: March 26, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [general virtual agents], [open-ended learning], [tool creation], [GroundUI], [benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AgentStudio is a robust toolkit for developing virtual agents with versatile actions, such as GUI automation and code execution. It unifies real-world human-computer interactions across OS platforms and includes diverse observation and action spaces, facilitating comprehensive training and benchmarking in complex settings. The toolkit's flexibility promotes agent generalization across varied tasks, supporting tool creation and a multimodal interaction interface to advance agent adaptability and learning.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2312.15820\" rel=\"nofollow\"\u003eWebVLN: Vision-and-Language Navigation on Websites\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eQi Chen, Dileepa Pitawela, Chongyang Zhao, Gengze Zhou, Hsiang-Ting Chen, Qi Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: The University of Adelaide\u003c/li\u003e\n\u003cli\u003e📅 Date: March 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: AAAI 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [web-based VLN], [HTML content integration], [multimodal navigation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the \u003cem\u003eWebVLN\u003c/em\u003e task, where agents navigate websites by following natural language instructions that include questions and descriptions. Aimed at emulating real-world browsing behavior, the task allows the agent to interact with elements not directly visible in the rendered content by integrating HTML-specific information. A new \u003cem\u003eWebVLN-Net\u003c/em\u003e model, based on the VLN BERT framework, is introduced alongside the \u003cem\u003eWebVLN-v1\u003c/em\u003e dataset, supporting question-answer navigation across web pages. This framework demonstrated significant improvement over existing web-based navigation methods, marking a new direction in vision-and-language navigation research.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2403.11905\" rel=\"nofollow\"\u003eTur[k]ingBench: A Challenge Benchmark for Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKevin Xu, Yeganeh Kordi, Kate Sanders, Yizhong Wang, Adam Byerly, Jingyu Zhang, Benjamin Van Durme, Daniel Khashabi\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: JHU, Brown, UW\u003c/li\u003e\n\u003cli\u003e📅 Date: March 18, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [multi-modal reasoning], [TurkingBench], [Turking]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eTur[k]ingBench\u003c/strong\u003e, a benchmark comprising 158 web-grounded tasks designed to evaluate AI agents' capabilities in complex web-based environments. Unlike prior benchmarks that utilize synthetic web pages, Tur[k]ingBench leverages natural HTML pages from crowdsourcing platforms, presenting tasks with rich multi-modal contexts. The benchmark includes 32.2K instances, each with diverse inputs, challenging models to interpret and interact with web pages effectively. Evaluations of state-of-the-art models reveal significant room for improvement, highlighting the need for advanced web-based agents capable of handling real-world web interactions.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2403.07718\" rel=\"nofollow\"\u003eWorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAlexandre Drouin, Maxime Gasse, Massimo Caccia, Issam H. Laradji, Manuel Del Verme, Tom Marty, Léo Boisvert, Megh Thakkar, Quentin Cappart, David Vazquez, Nicolas Chapados, Alexandre Lacoste\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ServiceNow Research, Mila, Polytechnique Montreal, McGill University, University de Montreal\u003c/li\u003e\n\u003cli\u003e📅 Date: March 11, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [enterprise task automation], [ServiceNow], [knowledge work automation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: WorkArena introduces a robust benchmark hosted on the ServiceNow platform to assess the effectiveness of large language model-based agents in performing 33 knowledge tasks common to enterprise environments. Leveraging BrowserGym, an environment that simulates complex browser interactions, WorkArena provides web agents with realistic challenges like data entry, form completion, and information retrieval in knowledge bases. Despite promising initial results, open-source models show a 42.7% success rate compared to closed-source counterparts, underlining the current gap in task automation for enterprise applications and highlighting key areas for improvement.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2403.03186\" rel=\"nofollow\"\u003eTowards General Computer Control: A Multimodal Agent for Red Dead Redemption II as a Case Study\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWeihao Tan, Ziluo Ding, Wentao Zhang, Boyu Li, Bohan Zhou, Junpeng Yue, Haochong Xia, Jiechuan Jiang, Longtao Zheng, Xinrun Xu, Yifei Bi, Pengjie Gu, Xinrun Wang, Börje F. Karlsson, Bo An, Zongqing Lu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NTU, BAAI, PKU\u003c/li\u003e\n\u003cli\u003e📅 Date: March 5, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Cradle], [General Computer Control], [multimodal], [keyboard and mouse control], [long-term memory], [reasoning], [self-improvement]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eCradle\u003c/em\u003e, a framework designed to achieve General Computer Control (GCC) by enabling agents to perform any computer task using only screen images (and possibly audio) as input and producing keyboard and mouse operations as output. The authors deploy Cradle in the complex AAA game Red Dead Redemption II, demonstrating its capability to follow the main storyline and complete real missions with minimal reliance on prior knowledge or resources.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2403.03186\" rel=\"nofollow\"\u003eCradle: Empowering Foundation Agents Towards General Computer Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWeihao Tan, Wentao Zhang, Xinrun Xu, Haochong Xia, Ziluo Ding, Boyu Li, Bohan Zhou, Junpeng Yue, Jiechuan Jiang, Yewen Li, Ruyi An, Molei Qin, Chuqiao Zong, Longtao Zheng, Yujie Wu, Xiaoqiang Chai, Yifei Bi, Tianbao Xie, Pengjie Gu, Xiyun Li, Ceyao Zhang, Long Tian, Chaojie Wang, Xinrun Wang, Börje F. Karlsson, Bo An, Shuicheng Yan, Zongqing Lu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Skywork AI, BAAI, NTU, PKU, Institute of Software - Chinese Academy of Sciences, HKU, CUHK\u003c/li\u003e\n\u003cli\u003e📅 Date: March 5, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: TBD\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [general computer control], [skill curation], [self-improvement]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the Cradle framework, designed to enable general computer control (GCC) through multimodal input (e.g., screen images and optional audio) and outputs (keyboard and mouse). Cradle’s six core modules, including self-reflection, skill curation, and memory, allow for generalized task handling in complex environments like AAA games. Demonstrated in \u003cem\u003eRed Dead Redemption II\u003c/em\u003e, the framework exhibits adaptability by performing real missions and following the storyline with minimal prior knowledge, showcasing its potential as a generalist agent for diverse computer tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2403.02713\" rel=\"nofollow\"\u003eAndroid in the Zoo: Chain-of-Action-Thought for GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJiwen Zhang, Jihao Wu, Yihua Teng, Minghui Liao, Nuo Xu, Xiao Xiao, Zhongyu Wei, Duyu Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Fudan University, Huawei\u003c/li\u003e\n\u003cli\u003e📅 Date: March 5, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [Android GUI], [Chain-of-Action-Thought], [autonomous GUI agents]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eChain-of-Action-Thought\u003c/em\u003e (CoAT), a novel paradigm to improve GUI agent task completion by enabling agents to interpret previous actions, current screen content, and action rationale for next steps. The authors present the \u003cem\u003eAndroid-In-The-Zoo\u003c/em\u003e (AitZ) dataset, which includes 18,643 screen-action pairs with detailed annotations, supporting CoAT's development and evaluation. The study demonstrates that fine-tuning with the AitZ dataset improves performance of a baseline large language model in predicting correct action sequences in Android tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.15057\" rel=\"nofollow\"\u003eOn the Multi-turn Instruction Following for Conversational Web Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYang Deng, Xuan Zhang, Wenxuan Zhang, Yifei Yuan, See-Kiong Ng, Tat-Seng Chua\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS, DAMO Academy, University of Copenhagen\u003c/li\u003e\n\u003cli\u003e📅 Date: February 23, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [multi-turn dialogue], [memory utilization], [self-reflective planning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper explores multi-turn conversational web navigation, introducing the MT-Mind2Web dataset to support instruction-following tasks for web agents. The proposed Self-MAP (Self-Reflective Memory-Augmented Planning) framework enhances agent performance by integrating memory with self-reflection for sequential decision-making in complex interactions. Extensive evaluations using MT-Mind2Web demonstrate Self-MAP's efficacy in addressing the limitations of current models in multi-turn interactions, providing a novel dataset and framework for evaluating and training agents on detailed, multi-step web-based tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.14073\" rel=\"nofollow\"\u003eImproving Language Understanding from Screenshots\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianyu Gao, Zirui Wang, Adithya Bhaskar, Danqi Chen\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Princeton\u003c/li\u003e\n\u003cli\u003e📅 Date: February 22, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [screenshot language models], [patch-and-text prediction]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a novel approach to improve the language understanding capabilities of screenshot language models (LMs). The authors propose a Patch-and-Text Prediction (PTP) objective, which masks and recovers both image patches and text within screenshots. The method significantly narrows the performance gap between screenshot LMs and text-only models on language understanding tasks, achieving comparable results to BERT on most GLUE tasks. The research also extends PTP to train autoregressive screenshot LMs, demonstrating improved perplexity by utilizing screenshot context.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.11208\" rel=\"nofollow\"\u003eWatch Out for Your Agents! Investigating Backdoor Threats to LLM-Based Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWenkai Yang, Xiaohan Bi, Yankai Lin, Sishuo Chen, Jie Zhou, Xu Sun\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Renming University of China, PKU, Tencent\u003c/li\u003e\n\u003cli\u003e📅 Date: Feb 17, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI], [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [attack], [backdoor], [safety]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper investigates backdoor attacks on LLM-based agents, introducing a framework that categorizes attacks based on outcomes and trigger locations. The study demonstrates the vulnerability of such agents to backdoor attacks and emphasizes the need for targeted defenses.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://www.catalyzex.com/paper/a-trembling-house-of-cards-mapping\" rel=\"nofollow\"\u003eA Trembling House of Cards? Mapping Adversarial Attacks against Language Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLingbo Mo, Zeyi Liao, Boyuan Zheng, Yu Su, Chaowei Xiao, Huan Sun\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU, UWM\u003c/li\u003e\n\u003cli\u003e📅 Date: February 15, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [safety], [adversarial attacks], [security risks], [language agents], [Perception-Brain-Action]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a conceptual framework to assess and understand adversarial vulnerabilities in language agents, dividing the agent structure into three components—Perception, Brain, and Action. It discusses 12 specific adversarial attack types that exploit these components, ranging from input manipulation to complex backdoor and jailbreak attacks. The framework provides a basis for identifying and mitigating risks before the widespread deployment of these agents in real-world applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.07939\" rel=\"nofollow\"\u003eUFO: A UI-Focused Agent for Windows OS Interaction\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChaoyun Zhang, Liqun Li, Shilin He, Xu Zhang, Bo Qiao, Si Qin, Minghua Ma, Yu Kang, Qingwei Lin, Saravan Rajmohan, Dongmei Zhang, Qi Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: February 14, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [UI automation], [Windows], [UFO]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents UFO, a pioneering multimodal LLM-based agent designed to fulfill user requests on Windows OS. UFO employs a dual-agent architecture—comprising AppAgent and ActAgent—that can interpret and execute complex tasks across multiple Windows applications by observing UI elements and utilizing control interactions. The framework allows UFO to handle intricate, cross-application workflows and execute commands seamlessly based on natural language prompts. It integrates GPT-Vision to recognize and interact with graphical elements, enabling flexible, autonomous task completion within and across diverse Windows applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.07945\" rel=\"nofollow\"\u003eScreenAgent: A Computer Control Agent Driven by Visual Language Large Model\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRunliang Niu, Jindong Li, Shiqi Wang, Yali Fu, Xiyu Hu, Xueyuan Leng, He Kong, Yi Chang, Qi Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Jilin University\u003c/li\u003e\n\u003cli\u003e📅 Date: February 13, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [visual language model], [computer control agent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces ScreenAgent, a computer control agent powered by a visual language large model. The system can interpret natural language instructions and execute them on various computer applications by analyzing screen content. ScreenAgent employs a novel action grounding mechanism to map high-level instructions to specific UI interactions. Evaluated on a diverse set of tasks across different applications, ScreenAgent demonstrates superior performance in task completion and generalization compared to existing methods.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.07456\" rel=\"nofollow\"\u003eOS-Copilot: Towards Generalist Computer Agents with Self-Improvement\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhiyong Wu, Chengcheng Han, Zichen Ding, Zhenmin Weng, Zhoumianze Liu, Shunyu Yao, Tao Yu, Lingpeng Kong\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Shanghai AI Lab, East China Normal University, Princeton, HKU\u003c/li\u003e\n\u003cli\u003e📅 Date: February 12, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024 Workshop LLMAgents\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [self-directed learning], [GAIA], [FRIDAY], [OS-Copilot]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The OS-Copilot framework supports building generalist agents capable of performing diverse tasks across an operating system (OS). This work introduces FRIDAY, an embodied agent using OS-Copilot to self-improve by learning from task outcomes. It operates with a memory-based architecture to tackle OS-level tasks across applications like terminals, web browsers, and third-party tools. Tested on the GAIA benchmark, FRIDAY achieved 35% higher performance than prior methods, proving effective in adapting to unfamiliar applications and refining its capabilities with minimal guidance.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.04615\" rel=\"nofollow\"\u003eScreenAI: A Vision-Language Model for UI and Infographics Understanding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGilles Baechler, Srinivas Sunkara, Maria Wang, Fedir Zubach, Hassan Mansoor, Vincent Etter, Victor Cărbune, Jason Lin, Jindong Chen, Abhanshu Sharma\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: February 7, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: IJCAI 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [UI understanding], [infographics understanding], [vision language model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces ScreenAI, a vision-language model specializing in UI and infographics understanding. The model combines the PaLI architecture with the flexible patching strategy of pix2struct and is trained on a unique mixture of datasets. ScreenAI achieves state-of-the-art results on several UI and infographics-based tasks, outperforming larger models. The authors also release three new datasets for screen annotation and question answering tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.04476\" rel=\"nofollow\"\u003eDual-View Visual Contextualization for Web Navigation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJihyung Kil, Chan Hee Song, Boyuan Zheng, Xiang Deng, Yu Su, Wei-Lun Chao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU\u003c/li\u003e\n\u003cli\u003e📅 Date: February 6, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CVPR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [visual contextualization]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper proposes a novel approach to web navigation by contextualizing HTML elements through their \"dual views\" in webpage screenshots. The method leverages both the textual content of HTML elements and their visual representation in the screenshot to create more informative representations for web agents. Evaluated on the Mind2Web dataset, the approach demonstrates consistent improvements over baseline methods across various scenarios, including cross-task, cross-website, and cross-domain navigation tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.05930\" rel=\"nofollow\"\u003eWebLINX: Real-World Website Navigation with Multi-Turn Dialogue\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXing Han Lu, Zdeněk Kasner, Siva Reddy\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Mila, McGill University\u003c/li\u003e\n\u003cli\u003e📅 Date: February 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [multi-turn dialogue], [real-world navigation], [WebLINX]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: WebLINX addresses the complexity of real-world website navigation for conversational agents, with a benchmark featuring over 2,300 demonstrations across 150+ websites. The benchmark allows agents to handle multi-turn instructions and interact dynamically across diverse domains, including geographic and thematic categories. The study proposes a retrieval-inspired model that selectively extracts key HTML elements and browser actions, achieving efficient task-specific representations. Experiments reveal that smaller finetuned decoders outperform larger zero-shot multimodal models, though generalization to new environments remains challenging.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2402.17553\" rel=\"nofollow\"\u003eOmniACT: A Dataset and Benchmark for Enabling Multimodal Generalist Autonomous Agents for Desktop and Web\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eRaghav Kapoor, Yash Parag Butala, Melisa Russak, Jing Yu Koh, Kiran Kamble, Waseem Alshikh, Ruslan Salakhutdinov\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: February 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: OmniACT introduces a dataset and benchmark to train and evaluate multimodal agents capable of autonomously performing diverse tasks across desktop and web environments. Using annotated UI elements across applications, it combines visual grounding with natural language instructions, providing 9,802 data points for developing agents that integrate high-level reasoning with UI interactions. The study highlights the limited proficiency of current models, with baselines like GPT-4 only achieving 15% of human performance on executable scripts, emphasizing OmniACT's potential as a testbed for advancing multimodal AI.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2401.16158\" rel=\"nofollow\"\u003eMobile-Agent: Autonomous Multi-Modal Mobile Device Agent with Visual Perception\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJunyang Wang, Haiyang Xu, Jiabo Ye, Ming Yan, Weizhou Shen, Ji Zhang, Fei Huang, Jitao Sang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Beijing Jiaotong University, Alibaba\u003c/li\u003e\n\u003cli\u003e📅 Date: January 29, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents Mobile-Agent, an autonomous multi-modal agent designed for mobile device interaction. The system integrates visual perception, natural language processing, and action prediction to navigate and operate mobile applications. The authors introduce a new dataset and benchmark for evaluating mobile agents, demonstrating Mobile-Agent's superior performance in task completion and generalization across various apps compared to existing methods.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2401.13649\" rel=\"nofollow\"\u003eVisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJing Yu Koh, Robert Lo, Lawrence Jang, Vikram Duvvur, Ming Chong Lim, Po-Yu Huang, Graham Neubig, Shuyan Zhou, Ruslan Salakhutdinov, Daniel Fried\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: January 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [dataset], [multimodal agent evaluation], [visually grounded tasks]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: VisualWebArena is a benchmark designed for testing multimodal web agents on complex, visually grounded web tasks. It provides a reproducible framework with 910 task scenarios across real-world web applications, emphasizing open-ended, visually guided interactions. The tasks are modeled within a partially observable Markov decision process to assess agents’ capacity to interpret multimodal inputs, execute navigation, and accomplish user-defined objectives across complex visual and textual information on websites.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2401.13919\" rel=\"nofollow\"\u003eWebVoyager: Building an End-to-End Web Agent with Large Multimodal Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHongliang He, Wenlin Yao, Kaixin Ma, Wenhao Yu, Yong Dai, Hongming Zhang, Zhenzhong Lan, Dong Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Zhejiang University, Tencent AI Lab, Westlake University\u003c/li\u003e\n\u003cli\u003e📅 Date: January 24, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [evaluation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces WebVoyager, an innovative web agent powered by Large Multimodal Models (LMMs) that can complete user instructions end-to-end by interacting with real-world websites. The authors establish a new benchmark with tasks from 15 popular websites and propose an automatic evaluation protocol using GPT-4V. WebVoyager achieves a 59.1% task success rate, significantly outperforming GPT-4 (All Tools) and text-only setups. The study demonstrates the effectiveness of multimodal approaches in web automation and provides insights into developing more intelligent web interaction solutions.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2401.10935\" rel=\"nofollow\"\u003eSeeClick: Harnessing GUI Grounding for Advanced Visual GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKanzhi Cheng, Qiushi Sun, Yougang Chu, Fangzhi Xu, Yantao Li, Jianbing Zhang, Zhiyong Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Nanjing University, Shanghai AI Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: January 19, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [benchmark], [GUI grounding], [visual grounding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: TBD.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://llmbench.ai/agent\" rel=\"nofollow\"\u003eAgentBench: Evaluating LLMs as Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXiao Liu, Hao Yu, Hanchen Zhang, Yifan Xu, Xuanyu Lei, Hanyu Lai, Yu Gu, Hangliang Ding, Kaiwen Men, Kejuan Yang, Shudan Zhang, Xiang Deng, Aohan Zeng, Zhengxiao Du, Chenhui Zhang, Sheng Shen, Tianjun Zhang, Yu Su, Huan Sun, Minlie Huang, Yuxiao Dong, Jie Tang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: THU, OSU, ByteDance\u003c/li\u003e\n\u003cli\u003e📅 Date: January 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI], [General]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [evaluation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AgentBench provides a comprehensive benchmark for evaluating LLMs as autonomous agents in various environments. It includes eight distinct scenarios, testing the LLMs' reasoning and decision-making capabilities in tasks such as OS interaction, database querying, knowledge graph traversal, and more. This benchmark compares the effectiveness of multiple commercial and open-source LLMs, revealing areas of improvement in instruction-following and long-term reasoning, essential for practical agent development.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://huggingface.co/papers/2305.11854\" rel=\"nofollow\"\u003eMultimodal Web Navigation with Instruction-Finetuned Foundation Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHiroki Furuta, Kuang-Huei Lee, Ofir Nachum, Yutaka Matsuo, Aleksandra Faust, Shixiang Shane Gu, Izzeddin Gur\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Univ. of Tokyo, Google DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: Jan 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [model], [dataset], [web navigation], [instruction-following], [WebShop]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces WebGUM, an instruction-following multimodal agent for autonomous web navigation that leverages both visual (webpage screenshots) and textual (HTML) inputs to perform actions such as click and type. The model is trained on a vast corpus of demonstrations and shows improved capabilities in visual perception, HTML comprehension, and multi-step decision-making, achieving state-of-the-art performance on benchmarks like MiniWoB and WebShop. WebGUM provides a scalable approach to web-based tasks without task-specific architectures, enabling high-performance web navigation with generalizable, multimodal foundation models.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://osu-nlp-group.github.io/SeeAct/\" rel=\"nofollow\"\u003eGPT-4V(ision) is a Generalist Web Agent, if Grounded\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBoyuan Zheng, Boyu Gou, Jihyung Kil, Huan Sun, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU\u003c/li\u003e\n\u003cli\u003e📅 Date: January 1, 2024\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [grounding], [SeeAct], [Multimodal-Mind2web]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper explores the capability of GPT-4V(ision), a multimodal model, as a web agent that can perform tasks across various websites by following natural language instructions. It introduces the \u003cstrong\u003eSEEACT\u003c/strong\u003e framework, enabling GPT-4V to navigate, interpret, and interact with elements on websites. Evaluated using the \u003cstrong\u003eMind2Web\u003c/strong\u003e benchmark and an online test environment, the framework demonstrates high performance on complex web tasks by integrating grounding strategies like element attributes and image annotations to improve HTML element targeting. However, grounding remains challenging, presenting opportunities for further improvement.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2312.13771\" rel=\"nofollow\"\u003eAppAgent: Multimodal Agents as Smartphone Users\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChi Zhang, Zhao Yang, Jiaxuan Liu, Yucheng Han, Xin Chen, Zebiao Huang, Bin Fu, Gang Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tencent\u003c/li\u003e\n\u003cli\u003e📅 Date: December 21, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [smartphone interaction], [autonomous exploration], [self-improve]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces AppAgent, a novel multimodal agent framework designed to operate smartphone applications. The agent uses a simplified action space to mimic human-like interactions such as tapping and swiping. AppAgent learns to navigate and use new apps through autonomous exploration or by observing human demonstrations, creating a knowledge base for executing complex tasks across different applications. The framework's effectiveness is demonstrated through extensive testing on 50 tasks across 10 diverse applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2312.13108\" rel=\"nofollow\"\u003eAssistGUI: Task-Oriented Desktop Graphical User Interface Automation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDifei Gao, Lei Ji, Zechen Bai, Mingyu Ouyang, Peiran Li, Dongxing Mao, Qinchen Wu, Weichen Zhang, Peiyi Wang, Xiangwu Guo, Hengxu Wang, Luowei Zhou, Mike Zheng Shou\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NUS\u003c/li\u003e\n\u003cli\u003e📅 Date: December 20, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CVPR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [desktop productivity tasks]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study presents \u003cem\u003eAssistGUI\u003c/em\u003e, a benchmark and framework for desktop GUI automation, featuring an LLM-based agent capable of completing complex user requests by analyzing instructional videos and performing actions on the desktop. Utilizing a novel Actor-Critic framework and GUI parser, \u003cem\u003eAssistGUI\u003c/em\u003e was tested on 100 tasks across nine applications, such as MS Word and After Effects. Despite advances, the top-performing model achieved only a 46% success rate, illustrating the challenge of comprehensive desktop automation and underscoring areas for future research in agent-driven GUI tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2312.08914\" rel=\"nofollow\"\u003eCogAgent: A Visual Language Model for GUI Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eWenyi Hong, Weihan Wang, Qingsong Lv, Jiazheng Xu, Wenmeng Yu, Junhao Chen, Yuxuan Wang, Yining Ye, Jiayi Zhang, Hao Dong, Wenhu Chen, Yizhou Wang, Kai-Wei Chang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, Zhipu AI\u003c/li\u003e\n\u003cli\u003e📅 Date: December 15, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CVPR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [dataset], [benchmark], [visual language model], [GUI agent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents CogAgent, a visual language model designed for GUI agents. The authors introduce a new dataset, CogBench, featuring 1,430 GUI tasks across various applications. CogAgent employs a novel training approach combining supervised fine-tuning and decision-making fine-tuning. The model demonstrates superior performance on CogBench and generalizes well to unseen applications, outperforming existing models like GPT-4V in GUI task completion.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://huggingface.co/gaia-benchmark\" rel=\"nofollow\"\u003eGAIA: a benchmark for General AI Assistants\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGrégoire Mialon, Yassine Nakkach, Aslan Tchamkerten, Albert Thomas, Laurent Dinh, and a research team from Meta AI and Hugging Face.\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Meta AI, Hugging Face\u003c/li\u003e\n\u003cli\u003e📅 Date: November 21, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [multi-modality], [tool use], [reasoning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: GAIA is a benchmark developed for evaluating general-purpose AI assistants. It aims to test assistant models across multiple modalities and complex reasoning tasks in real-world settings, including scenarios that require tool usage and open-ended question answering. With a dataset comprising 466 questions across various domains, GAIA highlights gaps between current AI performance and human capability, presenting a significant challenge for large language models such as GPT-4.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2311.07562\" rel=\"nofollow\"\u003eGPT-4V in Wonderland: Large Multimodal Models for Zero-Shot Smartphone GUI Navigation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAn Yan, Zhengyuan Yang, Wanrong Zhu, Kevin Lin, Linjie Li, Jianfeng Wang, Jianwei Yang, Yiwu Zhong, Julian McAuley, Jianfeng Gao, Zicheng Liu, Lijuan Wang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCSD, Microsoft, UCSB, UWM\u003c/li\u003e\n\u003cli\u003e📅 Date: November 13, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [zero-shot GUI navigation], [multimodal LLMs]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper explores the capabilities of GPT-4V in navigating smartphone GUIs without prior training. The authors introduce a novel framework for GUI navigation and a new benchmark, MobileNav, featuring 1,000 navigation tasks across 100 mobile apps. The study demonstrates GPT-4V's impressive zero-shot performance in understanding and interacting with mobile interfaces, outperforming previous methods and even approaching human-level performance on some tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2406.11736\" rel=\"nofollow\"\u003eInteractive Evolution: A Neural-Symbolic Self-Training Framework For Large Language Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFangzhi Xu, Qiushi Sun, Kanzhi Cheng, Jun Liu, Yu Qiao, Zhiyong Wu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Xi'an Jiaotong University, Shanghai AI Lab, HKU, Nanjing University\u003c/li\u003e\n\u003cli\u003e📅 Date: November 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI (evaluated on web, math reasoning, and logic reasoning environments)]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [neural-symbolic self-training], [online exploration], [self-refinement]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eENVISIONS\u003c/em\u003e, a neural-symbolic self-training framework designed to improve large language models (LLMs) by enabling self-training through interaction with a symbolic environment. The framework addresses symbolic data scarcity and enhances LLMs' symbolic reasoning proficiency by iteratively exploring, refining, and learning from symbolic tasks without reinforcement learning. Extensive evaluations across web navigation, math, and logical reasoning tasks highlight \u003cem\u003eENVISIONS\u003c/em\u003e as a promising approach for enhancing LLM symbolic processing.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2310.15455\" rel=\"nofollow\"\u003eUI Layout Generation with LLMs Guided by UI Grammar\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYuwen Lu, Ziang Tong, Qinyi Zhao, Chengzhi Zhang, Toby Jia-Jun Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ICML 2023 Workshop on AI and HCI\u003c/li\u003e\n\u003cli\u003e📅 Date: October 24, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [UI grammar], [UI Layout Generation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This position paper explores the use of Large Language Models (LLMs) for generating mobile user interface (UI) layouts. It introduces \u003cem\u003eUI grammar\u003c/em\u003e, a novel approach to represent the hierarchical structure of UI screens, aiming to guide LLMs' generative capabilities more effectively and enhance the explainability and controllability of the process. Initial experiments with GPT-4 demonstrate the potential of LLMs to produce high-quality UIs through in-context learning, with the grammar-based approach improving certain aspects of generation quality.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2310.11441\" rel=\"nofollow\"\u003eSet-of-Mark Prompting Unleashes Extraordinary Visual Grounding in GPT-4V\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJianwei Yang, Hao Zhang, Feng Li, Xueyan Zou, Chunyuan Li, Jianfeng Gao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: MSR\u003c/li\u003e\n\u003cli\u003e📅 Date: October 17, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [visual prompting], [framework], [benchmark], [visual grounding], [zero-shot]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces Set-of-Mark (SoM), a novel visual prompting approach designed to enhance the visual grounding capabilities of multimodal models like GPT-4V. By overlaying images with spatially and semantically distinct marks, SoM enables fine-grained object recognition and interaction within visual data, surpassing conventional zero-shot segmentation methods in accuracy. The framework is validated on tasks requiring detailed spatial reasoning, demonstrating a significant improvement over existing visual-language models without fine-tuning.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://github.com/xlang-ai/OpenAgents\"\u003eOpenAgents: An Open Platform for Language Agents in the Wild\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianbao Xie, Fan Zhou, Zhoujun Cheng, Peng Shi, Luoxuan Weng, Yitao Liu, Toh Jing Hua, Junning Zhao, Qian Liu, Che Liu, Leo Z. Liu, Yiheng Xu, Hongjin Su, Dongchan Shin, Caiming Xiong, Tao Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: HKU, XLang Lab, Sea AI Lab, Salesforce Research\u003c/li\u003e\n\u003cli\u003e📅 Date: October 16, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Data Agent], [Plugins Agent], [Web Agent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces OpenAgents, an open-source platform designed to facilitate the use and hosting of language agents in real-world scenarios. It features three agents: Data Agent for data analysis using Python and SQL, Plugins Agent with access to over 200 daily API tools, and Web Agent for autonomous web browsing. OpenAgents aims to provide a user-friendly web interface for general users and a seamless deployment experience for developers and researchers, promoting the development and evaluation of innovative language agents in practical applications.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2310.04716\" rel=\"nofollow\"\u003eReinforced UI Instruction Grounding: Towards a Generic UI Task Automation API\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhizheng Zhang, Wenxuan Xie, Xiaoyi Zhang, Yan Lu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: MSRA\u003c/li\u003e\n\u003cli\u003e📅 Date: October 7, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [reinforcement learning], [UI task automation], [instruction grounding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a multimodal model, termed RUIG (Reinforced UI Instruction Grounding), for automating UI tasks through natural language instructions. By leveraging a pixel-to-sequence approach, the model directly decodes UI element locations from screenshots based on user commands, removing the need for metadata like element coordinates. The framework uses a transformer-based encoder-decoder setup optimized through reinforcement learning to improve spatial accuracy. This novel approach outperforms prior methods, offering a generalized solution for UI task automation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://asappresearch.github.io/webagents-step/\" rel=\"nofollow\"\u003eSteP: Stacked LLM Policies for Web Actions\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePaloma Sodhi, S.R.K. Branavan, Yoav Artzi, Ryan McDonald\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: ASAPP Research, Cornell University\u003c/li\u003e\n\u003cli\u003e📅 Date: October 5, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [policy composition], [dynamic control], [SteP]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eSteP (Stacked LLM Policies)\u003c/strong\u003e, a framework that dynamically composes policies to tackle diverse web tasks. By defining a Markov Decision Process where the state is a stack of policies, SteP enables adaptive control that adjusts to task complexity. Evaluations on WebArena, MiniWoB++, and a CRM simulator demonstrate that SteP significantly outperforms existing methods, achieving a success rate improvement from 14.9% to 35.8% over state-of-the-art GPT-4 policies.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2309.11436\" rel=\"nofollow\"\u003eYou Only Look at Screens: Multimodal Chain-of-Action Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZhuosheng Zhang, Aston Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: September 20, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [multimodal agent], [chain-of-action technique]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents Auto-GUI, a multimodal agent capable of directly interacting with graphical user interfaces without relying on environment parsing or application-specific APIs. The authors introduce a novel chain-of-action technique that leverages previous action histories and future action plans to improve decision-making. Auto-GUI is evaluated on a new device-control benchmark, AITW, demonstrating state-of-the-art performance in action prediction and task completion across various applications and web-based tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2309.08172\" rel=\"nofollow\"\u003eLASER: LLM Agent with State-Space Exploration for Web Navigation\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKaixin Ma, Hongming Zhang, Hongwei Wang, Xiaoman Pan, Dong Yu, Jianshu Chen\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tencent AI Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: September 15, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [state-space exploration], [backtracking]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces LASER, an LLM agent that models interactive web navigation tasks as state-space exploration. The approach defines a set of high-level states and associated actions, allowing the agent to transition between states and backtrack from errors. LASER significantly outperforms previous methods on the WebShop task without using in-context examples, demonstrating improved handling of novel situations and mistakes during task execution.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2308.15272\" rel=\"nofollow\"\u003eAutoDroid: LLM-powered Task Automation in Android\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHao Wen, Yuanchun Li, Guohong Liu, Shanhui Zhao, Tao Yu, Toby Jia-Jun Li, Shiqi Jiang, Yunhao Liu, Yaqin Zhang, Yunxin Liu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Tsinghua University, Shanghai AI Lab, University of Notre Dame, MSR\u003c/li\u003e\n\u003cli\u003e📅 Date: August 29, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: MobiCom 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [Android task automation], [LLM-powered agent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces AutoDroid, a novel mobile task automation system capable of handling arbitrary tasks on any Android application without manual efforts. The framework combines the commonsense knowledge of LLMs with domain-specific knowledge of apps through automated dynamic analysis. AutoDroid features a functionality-aware UI representation method, exploration-based memory injection techniques, and a multi-granularity query optimization module. Evaluated on a new benchmark with 158 common tasks, AutoDroid achieves a 90.9% action generation accuracy and a 71.3% task completion rate, significantly outperforming GPT-4-powered baselines.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2407.20183\" rel=\"nofollow\"\u003eMindSearch: Mimicking Human Minds Elicits Deep AI Searcher\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eZehui Chen, Kuikun Liu, Qiuchen Wang, Jiangning Liu, Wenwei Zhang, Kai Chen, Feng Zhao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: USTC, Shanghai AI Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: July 29, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [information seeking], [planning], [AI search], [MindSearch]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents MindSearch, a novel approach to web information seeking and integration that mimics human cognitive processes. The system uses a multi-agent framework consisting of a WebPlanner and WebSearcher. The WebPlanner models multi-step information seeking as a dynamic graph construction process, decomposing complex queries into sub-questions. The WebSearcher performs hierarchical information retrieval for each sub-question. MindSearch demonstrates significant improvements in response quality and depth compared to existing AI search solutions, processing information from over 300 web pages in just 3 minutes.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2307.13854\" rel=\"nofollow\"\u003eWebArena: A Realistic Web Environment for Building Autonomous Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShuyan Zhou, Frank F. Xu, Hao Zhu, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng, Tianyue Ou, Yonatan Bisk, Daniel Fried, Uri Alon, Graham Neubig\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: July 26, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [multi-tab navigation], [web-based interaction], [agent simulation]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: \u003cem\u003eWebArena\u003c/em\u003e provides a standalone, realistic web simulation environment where autonomous agents can perform complex web-based tasks. The platform offers functionalities such as multi-tab browsing, element interaction, and customized user profiles. Its benchmark suite contains 812 tasks grounded in high-level natural language commands. WebArena uses multi-modal observations, including HTML and accessibility tree views, supporting advanced tasks that require contextual understanding across diverse web pages, making it suitable for evaluating generalist agents in real-world web environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2307.10088\" rel=\"nofollow\"\u003eAndroid in the Wild: A Large-Scale Dataset for Android Device Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChristopher Rawles, Alice Li, Daniel Rodriguez, Oriana Riva, Timothy Lillicrap\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google Research, Google DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: July 19, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark], [device control], [natural language interaction], [gesture-based actions]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The \u003cem\u003eAndroid in the Wild (AitW)\u003c/em\u003e dataset introduces a significant benchmark for Android device control, encompassing over 715,000 human-labeled episodes with natural language commands and corresponding UI actions. Collected from Android devices across versions 10-13, it captures complex multi-step tasks requiring both visual and contextual understanding. The dataset is structured to test the robustness of device-control systems under varying conditions, such as new tasks or applications, and includes data to evaluate gesture-based interactions, providing a unique foundation for mobile interface automation and task execution research.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2307.12856\" rel=\"nofollow\"\u003eA Real-World WebAgent with Planning, Long Context Understanding, and Program Synthesis\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eIzzeddin Gur, Hiroki Furuta, Austin Huang, Mustafa Safdari, Yutaka Matsuo, Douglas Eck, Aleksandra Faust\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google DeepMind, The University of Tokyo\u003c/li\u003e\n\u003cli\u003e📅 Date: July 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [program synthesis], [HTML comprehension], [web automation], [self-supervised learning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: WebAgent leverages two LLMs—HTML-T5 for HTML comprehension and Flan-U-PaLM for program synthesis—to complete web automation tasks. It combines planning, HTML summarization, and code generation to navigate and interact with real-world web environments, improving success rates on HTML-based tasks and achieving state-of-the-art performance in benchmarks like MiniWoB and Mind2Web. The modular architecture adapts well to open-domain tasks, using local-global attention mechanisms to manage long HTML contexts.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2306.07863\" rel=\"nofollow\"\u003eSynapse: Trajectory-as-Exemplar Prompting with Memory for Computer Control\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLongtao Zheng, Rundong Wang, Xinrun Wang, Bo An\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: NTU\u003c/li\u003e\n\u003cli\u003e📅 Date: June 13, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2024\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [trajectory prompting], [state abstraction], [memory retrieval]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: Synapse introduces a novel framework for computer control tasks, leveraging trajectory-as-exemplar prompting and memory to enhance LLM performance in complex, multi-step computer tasks. The system combines state abstraction, trajectory-based prompts, and memory retrieval, overcoming LLM limitations by filtering task-irrelevant data, storing exemplar trajectories, and retrieving relevant instances for improved decision-making. Synapse achieves significant performance gains on benchmarks such as MiniWoB++ and Mind2Web, demonstrating enhanced task success rates and generalization across diverse web-based tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2306.06070\" rel=\"nofollow\"\u003eMind2Web: Towards a Generalist Agent for the Web\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eXiang Deng, Yu Gu, Boyuan Zheng, Shijie Chen, Sam Stevens, Boshi Wang, Huan Sun, Yu Su\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: OSU\u003c/li\u003e\n\u003cli\u003e📅 Date: June 9, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark], [model], [Mind2Web], [MindAct]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: \u003cem\u003eMind2Web\u003c/em\u003e presents a dataset and benchmark specifically crafted for generalist web agents capable of performing language-guided tasks across varied websites. Featuring over 2,000 tasks from 137 sites, it spans 31 domains and emphasizes open-ended, realistic tasks in authentic, unsimplified web settings. The study proposes the \u003cem\u003eMindAct\u003c/em\u003e framework, which optimizes LLMs for handling complex HTML elements by using small LMs to rank elements before full processing, thereby enhancing the efficiency and versatility of web agents in diverse contexts.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2305.19308\" rel=\"nofollow\"\u003eSheetCopilot: Bringing Software Productivity to the Next Level through Large Language Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eHongxin Li, Jingran Su, Yuntao Chen, Qing Li, Zhaoxiang Zhang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCAS, HKISI-CAS, PolyU, Shanghai AI Lab\u003c/li\u003e\n\u003cli\u003e📅 Date: May 30, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [spreadsheet automation], [natural language interface]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces SheetCopilot, an innovative system that leverages large language models to automate spreadsheet tasks through natural language interactions. The framework includes a novel prompt design for task decomposition and execution, and a feedback loop for error correction. SheetCopilot demonstrates significant improvements in task completion rates and efficiency across various spreadsheet operations, outperforming existing methods and showing potential for enhancing productivity in spreadsheet software.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2305.12487\" rel=\"nofollow\"\u003eAugmenting Autotelic Agents with Large Language Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCédric Colas, Laetitia Teodorescu, Pierre-Yves Oudeyer, Xingdi Yuan, Marc-Alexandre Côté\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: MIT, Inria, Microsoft\u003c/li\u003e\n\u003cli\u003e📅 Date: May 22, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CoLLAs 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [GUI]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reinforcement learning], [goal generation], [large language models], [autotelic learning]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study introduces the \u003cem\u003eLanguage Model-Augmented Autotelic Agent (LMA3)\u003c/em\u003e, a framework leveraging large language models to help agents autonomously generate, represent, and learn diverse goals in a task-agnostic, text-based environment. LMA3 integrates pretrained language models to emulate human cultural knowledge, aiming to dynamically relabel goals, generate new goals, and create goal-driven reward functions without manual inputs. This approach supports skill development by autonomously expanding goal repertoires in ways that resemble human open-ended learning, showcasing potential for achieving complex, self-directed learning in AI.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2305.08144\" rel=\"nofollow\"\u003eMobile-Env: Building Qualified Evaluation Benchmarks for LLM-GUI Interaction\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDanyang Zhang, Zhennan Shen, Rui Xie, Situo Zhang, Tianbao Xie, Zihan Zhao, Siyuan Chen, Lu Chen, Hongshen Xu, Ruisheng Cao, Kai Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU, HKU\u003c/li\u003e\n\u003cli\u003e📅 Date: May 14, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [interaction platform], [multistep interaction], [InfoUI]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eMobile-Env\u003c/em\u003e, a novel interaction platform and benchmark aimed at assessing large language models' (LLMs) capabilities in interactive environments. It builds on the InfoUI task set, derived from WikiHow, to create structured text-based challenges that simulate real-world mobile interactions. The platform is designed to support task expansions from the community, aiming to drive advancements in LLM-based interactive agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2303.17491\" rel=\"nofollow\"\u003eLanguage Models can Solve Computer Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGeunwoo Kim, Pierre Baldi, Stephen McAleer\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UCI\u003c/li\u003e\n\u003cli\u003e📅 Date: March 30, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [Recursive Critique and Improve], [RCI], [MiniWoB++], [general computer tasks]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study demonstrates that large language models (LLMs) can effectively automate computer tasks using a Recursive Critique and Improve (RCI) prompting method, enabling agents to handle complex desktop tasks like email and file management. By combining RCI with existing Chain of Thought (CoT) prompting, the method outperforms prior LLM approaches and traditional supervised and reinforcement learning models on the \u003cstrong\u003eMiniWoB++\u003c/strong\u003e benchmark, showing potential for broad computer task automation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2303.11366\" rel=\"nofollow\"\u003eReflexion: Language Agents with Verbal Reinforcement Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNoah Shinn, Federico Cassano, Edward Berman, Ashwin Gopinath, Karthik Narasimhan, Shunyu Yao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Northeastern University, MIT, Princeton\u003c/li\u003e\n\u003cli\u003e📅 Date: March 20, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [learning], [verbal reinforcement learning], [Reflexion]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eReflexion\u003c/em\u003e, a framework that enhances language agents by enabling them to reflect on task feedback linguistically, storing these reflections in an episodic memory to improve decision-making in future trials. Reflexion allows agents to learn from various feedback types without traditional weight updates, achieving significant performance improvements across tasks like decision-making, coding, and reasoning. For instance, Reflexion attains a 91% pass@1 accuracy on the HumanEval coding benchmark, surpassing the previous state-of-the-art GPT-4's 80%.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2210.03347\" rel=\"nofollow\"\u003ePix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eKenton Lee, Mandar Joshi, Iulia Raluca Turc, Hexiang Hu, Fangyu Liu, Julian Martin Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google\u003c/li\u003e\n\u003cli\u003e📅 Date: February 1, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web], [Doc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [model], [framework], [vision encoder], [visual language understanding], [screenshot parsing], [image-to-text]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces Pix2Struct, a model pre-trained to parse masked screenshots into simplified HTML for tasks requiring visual language understanding. By leveraging the structure of HTML and diverse web page elements, Pix2Struct captures pretraining signals like OCR and image captioning, achieving state-of-the-art performance across tasks in domains including documents, user interfaces, and illustrations.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2301.13280\" rel=\"nofollow\"\u003eWebUI: A Dataset for Enhancing Visual UI Understanding with Web Semantics\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eJason Wu, Siyan Wang, Siman Shen, Yi-Hao Peng, Jeffrey Nichols, Jeffrey P. Bigham\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Wellesley College, Grinnell College, Snooty Bird LLC\u003c/li\u003e\n\u003cli\u003e📅 Date: January 30, 2023\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CHI 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [element detection], [screen classification], [screen similarity], [UI modeling]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The WebUI dataset includes 400,000 web UIs captured to enhance UI modeling by integrating visual UI metadata. This dataset supports tasks such as element detection, screen classification, and screen similarity, especially for accessibility, app automation, and testing applications. Through transfer learning and semi-supervised methods, WebUI addresses the challenge of training robust models with limited labeled mobile data, proving effective in tasks beyond web contexts, such as mobile UIs.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://react-lm.github.io/\" rel=\"nofollow\"\u003eReAct: Synergizing Reasoning and Acting in Language Models\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik Narasimhan, Yuan Cao\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Princeton, Google Research\u003c/li\u003e\n\u003cli\u003e📅 Date: October 6, 2022\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Misc]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [reasoning], [ReAct]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eReAct\u003c/em\u003e, a framework that enables large language models to generate reasoning traces and task-specific actions in an interleaved manner. By combining reasoning and acting, ReAct enhances the model's ability to perform complex tasks in language understanding and interactive decision making. The approach is validated across various benchmarks, demonstrating improved performance and interpretability over existing methods.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2209.14927\" rel=\"nofollow\"\u003eSpotlight: Mobile UI Understanding using Vision-Language Models with a Focus\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGang Li, Yang Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google Research\u003c/li\u003e\n\u003cli\u003e📅 Date: September 29, 2022\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2023\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [dataset], [mobile UI tasks], [region-based focus]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \"Spotlight,\" a vision-language model for mobile UI understanding that operates solely on visual inputs (screenshots) and a specified focus region on the screen. By leveraging a large-scale dataset and training strategies tailored to mobile interfaces, Spotlight performs multiple UI-related tasks, including widget captioning, screen summarization, command grounding, and tappability prediction. It utilizes a vision-only approach, avoiding reliance on view hierarchies to achieve greater robustness and scalability across different mobile UI environments.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2207.01206\" rel=\"nofollow\"\u003eWebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eShunyu Yao, Howard Chen, John Yang, Karthik Narasimhan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Princeton\u003c/li\u003e\n\u003cli\u003e📅 Date: July 2022\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NeurIPS 2022\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [benchmark], [e-commerce web interaction], [language grounding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eWebShop\u003c/strong\u003e, a simulated web-based shopping environment with over 1 million real-world products and 12,087 annotated instructions. It allows language agents to navigate, search, and make purchases based on natural language commands. The study explores how agents handle compositional instructions and noisy web data, providing a robust environment for reinforcement learning and imitation learning. The best models show effective sim-to-real transfer on websites like Amazon, illustrating WebShop’s potential for training grounded agents.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2205.11029\" rel=\"nofollow\"\u003eMETA-GUI: Towards Multi-modal Conversational Agents on Mobile GUI\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLiangtai Sun, Xingyu Chen, Lu Chen, Tianle Dai, Zichen Zhu, Kai Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: May 23, 2022\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2022\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [dataset], [task-oriented dialogue], [GUI-based interaction], [multi-modal agent]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents META-GUI, a dataset and framework for training multi-modal conversational agents capable of interacting directly with mobile app interfaces without the need for backend APIs. META-GUI includes over 1,100 dialogues with annotated action sequences on various tasks such as booking and scheduling. The authors propose a GUI-based task-oriented dialogue system that allows agents to navigate mobile interfaces via direct GUI actions, with performance shown to improve in multi-modal task-oriented dialogue contexts.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2202.08137\" rel=\"nofollow\"\u003eA Data-Driven Approach for Learning to Control Computers\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePeter C. Humphreys, David Raposo, Tobias Pohlen, Gregory Thornton, Rachita Chhaparia, Alistair Muldal, Josh Abramson, Petko Georgiev, Alex Goldin, Adam Santoro, Timothy Lillicrap\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: February 16, 2022\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2022\u003c/li\u003e\n\u003cli\u003e💻 Env: [Desktop]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [framework], [computer control], [reinforcement learning], [multimodal transformer]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This study presents a reinforcement learning-based approach to train agents for computer control tasks, using keyboard and mouse interactions guided by natural language. By leveraging human demonstration data, agents trained in this environment achieved strong cross-task generalization across the MiniWob++ benchmark. This framework demonstrates how agents can control computers as humans would, enabling enhanced performance in complex computer tasks with high transferability.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2202.02312\" rel=\"nofollow\"\u003eA Dataset for Interactive Vision-Language Navigation with Unknown Command Feasibility\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eAndrea Burns, Deniz Arsan, Sanjna Agrawal, Ranjitha Kumar, Kate Saenko, Bryan A. Plummer\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Boston University, UIUC\u003c/li\u003e\n\u003cli\u003e📅 Date: February 4, 2022\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ECCV 2022\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [feasibility prediction], [vision-language navigation], [mobile interaction]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the \u003cem\u003eMobile App Tasks with Iterative Feedback (MoTIF)\u003c/em\u003e dataset, which addresses vision-language navigation (VLN) with a focus on task feasibility uncertainty in mobile applications. MoTIF provides commands paired with mobile actions and feasibility annotations, allowing researchers to examine the impact of command feasibility on task completion. The dataset includes 125 apps and emphasizes diverse app environments, action sequences, and follow-up questions to improve task ambiguity resolution, making it a valuable resource for feasibility prediction research.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2108.03353\" rel=\"nofollow\"\u003eScreen2Words: Automatic Mobile UI Summarization with Multimodal Learning\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eBryan Wang, Gang Li, Xin Zhou, Zhourong Chen, Tovi Grossman, Yang Li\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: University of Toronto\u003c/li\u003e\n\u003cli\u003e📅 Date: August 6, 2021\u003c/li\u003e\n\u003cli\u003e📑 Publisher: UIST 2021\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [mobile UI summarization], [multimodal learning], [Transformer model]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: The paper introduces \u003cem\u003eScreen2Words\u003c/em\u003e, an approach that utilizes multimodal learning to generate descriptive language summaries for mobile UI screens, combining textual, visual, and structural data from screens. The study created a large-scale dataset with 112,085 annotated screen summaries for 22,417 unique UIs, aiming to support model training for mobile UI understanding. The dataset facilitates a Transformer-based model trained to summarize screens by highlighting main functionalities, and the approach is validated with benchmarks in the mobile environment.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://www.ijcai.org/proceedings/2021/235\" rel=\"nofollow\"\u003eUIBert: Learning Generic Multimodal Representations for UI Understanding\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eChongyang Bai, Xiaoxue Zang, Ying Xu, Srinivas Sunkara, Abhinav Rastogi, Jindong Chen, Blaise Agüera y Arcas\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google Research\u003c/li\u003e\n\u003cli\u003e📅 Date: July 29, 2021\u003c/li\u003e\n\u003cli\u003e📑 Publisher: IJCAI 2021\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [model], [dataset], [multimodal representation learning], [UI understanding]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents \u003cem\u003eUIBert\u003c/em\u003e, a multimodal model aimed at understanding user interfaces (UIs) by combining visual, textual, and structural metadata. UIBert is designed for tasks such as component retrieval and expression resolution, using a transformer-based joint image-text model. The authors introduce five novel pre-training tasks to leverage UI-specific features, enhancing accessibility and task completion in mobile applications. UIBert demonstrates superior performance on nine downstream UI tasks, highlighting the potential of multimodal pre-training in UI understanding.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2105.13231\" rel=\"nofollow\"\u003eAndroidEnv: A Reinforcement Learning Platform for Android\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eDaniel Toyama, Philippe Hamel, Anita Gergely, Gheorghe Comanici, Amelia Glaese, Zafarali Ahmed, Tyler Jackson, Shibl Mourad, Doina Precup\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: DeepMind\u003c/li\u003e\n\u003cli\u003e📅 Date: May 27, 2021\u003c/li\u003e\n\u003cli\u003e📑 Publisher: arXiv\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [reinforcement learning], [Android interface], [RL environment], [task flexibility], [touchscreen action space]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: AndroidEnv provides a reinforcement learning (RL) platform for Android that lets RL agents interact with a realistic Android simulation via touchscreen events. The platform supports diverse applications, enabling agents to interact with over 100 predefined tasks across a variety of apps. With hybrid continuous and discrete action spaces, AndroidEnv is well-suited for training agents in complex, real-world Android scenarios where actions must be contextually sequenced, such as in UI navigation, gaming, and productivity apps. This environment encourages further RL research by offering task flexibility and realistic Android emulation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2103.16057\" rel=\"nofollow\"\u003eGrounding Open-Domain Instructions to Automate Web Support Tasks\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eNancy Xu, Sam Masling, Michael Du, Giovanni Campagna, Larry Heck, James Landay, Monica Lam\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Stanford\u003c/li\u003e\n\u003cli\u003e📅 Date: March 30, 2021\u003c/li\u003e\n\u003cli\u003e📑 Publisher: NAACL 2021\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [benchmark], [framework], [grounding], [task automation], [open-domain instructions], [RUSS]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces RUSS (Rapid Universal Support Service), a framework designed to interpret and execute open-domain, step-by-step web instructions automatically. RUSS uses a BERT-LSTM model for semantic parsing into a custom language, ThingTalk, which allows the system to map language to actions across various web elements. The framework, including a dataset of instructions, facilitates agent-based web support task automation by grounding natural language to interactive commands.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2101.09465\" rel=\"nofollow\"\u003eWebSRC: A Dataset for Web-Based Structural Reading Comprehension\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eLu Chen, Zihan Zhao, Xingyu Chen, Danyang Zhang, Jiabao Ji, Ao Luo, Yuxuan Xiong, Kai Yu\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: SJTU\u003c/li\u003e\n\u003cli\u003e📅 Date: January 23, 2021\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2021\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [structural reading comprehension], [web page QA], [structural information], [HTML element alignment]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cstrong\u003eWebSRC\u003c/strong\u003e, a dataset specifically designed for web-based structural reading comprehension, which requires understanding not only textual content but also the structural layout of web pages. WebSRC consists of 0.44 million question-answer pairs derived from 6,500 complex web pages. Each question challenges models to identify answers from HTML structures or to respond with yes/no, requiring a nuanced grasp of HTML and layout features. The authors benchmark several models on this dataset, highlighting its difficulty and the critical role of structural comprehension in improving machine understanding of web content.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/2010.04295\" rel=\"nofollow\"\u003eWidget Captioning: Generating Natural Language Description for Mobile User Interface Elements\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYang Li, Gang Li, Luheng He, Jingjie Zheng, Hong Li, Zhiwei Guan\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google Research\u003c/li\u003e\n\u003cli\u003e📅 Date: November 2020\u003c/li\u003e\n\u003cli\u003e📑 Publisher: EMNLP 2020\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [benchmark], [model], [accessibility], [natural language generation], [WidgetCaption]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces the task of \u003cem\u003ewidget captioning\u003c/em\u003e, which aims to automatically generate natural language descriptions for UI elements in mobile apps to enhance accessibility. Using both visual and structural data from UI components, the study presents a novel dataset of 162,859 captions across 61,285 UI elements. Multiple deep learning models were tested on this dataset, with findings suggesting the potential for improving screen reader usability for visually impaired users by generating descriptive captions of UI elements.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://aclanthology.org/2020.acl-demos.25/\" rel=\"nofollow\"\u003eInteractive Task Learning from GUI-Grounded Natural Language Instructions and Demonstrations\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eToby Jia-Jun Li, Tom Mitchell, Brad Myers\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU\u003c/li\u003e\n\u003cli\u003e📅 Date: July 2020\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2020\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [Sugilite], [programming-by-demonstration]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eSugilite\u003c/em\u003e, an intelligent task automation agent that learns new tasks and associated concepts interactively from users' natural language instructions and demonstrations on third-party mobile app GUIs. The system allows users to teach procedures and concepts through verbal instructions combined with GUI demonstrations, supports intent clarification for demonstrated actions, infers task parameters using hierarchical app GUI structures, and generalizes taught concepts across different contexts and domains. A prototype is presented as a conversational assistant on Android. \u003ca href=\"https://aclanthology.org/2020.acl-demos.25/?utm_source=chatgpt.com\" rel=\"nofollow\"\u003eoai_citation_attribution:0‡ACL Anthology\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://aclanthology.org/2020.acl-main.729\" rel=\"nofollow\"\u003eMapping Natural Language Instructions to Mobile UI Action Sequences\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eYang Li, Jiacong He, Xin Zhou, Yuan Zhang, Jason Baldridge\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Google Researc\u003c/li\u003e\n\u003cli\u003e📅 Date: July 2020\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ACL 2020\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [mobile UI automation], [natural language instructions], [action grounding], [RicoSCA]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces a method for grounding natural language instructions to mobile UI actions, aiming to automate mobile task execution through user interface manipulation. It introduces three key datasets: \u003cstrong\u003ePixelHelp\u003c/strong\u003e for task instruction-performance mappings on a Pixel emulator, \u003cstrong\u003eAndroidHowTo\u003c/strong\u003e for detailed phrase extraction, and \u003cstrong\u003eRicoSCA\u003c/strong\u003e for synthetic UI command training. The system utilizes a Transformer model to extract action phrase tuples, aligning them to UI elements with contextual screen positioning. Achieving over 70% accuracy in task completion, this approach is foundational for natural language-driven mobile UI automation.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/1909.00031\" rel=\"nofollow\"\u003ePUMICE: A Multi-Modal Agent that Learns Concepts and Conditionals from Natural Language and Demonstrations\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eToby Jia-Jun Li, Marissa Radensky, Justin Jia, Kirielle Singarajah, Tom M. Mitchell, Brad A. Myers\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Amherst College\u003c/li\u003e\n\u003cli\u003e📅 Date: August 30, 2019\u003c/li\u003e\n\u003cli\u003e📑 Publisher: UIST 2019\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [programming-by-demonstration], [PUMICE]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003ePUMICE\u003c/em\u003e, a multi-modal agent that combines natural language programming and programming-by-demonstration to enable end users to instruct intelligent agents in performing new tasks. By allowing users to describe tasks and conditions naturally and then collaboratively resolving ambiguities through conversation and demonstration, PUMICE facilitates the teaching of new concepts and procedures within existing mobile app GUIs. A lab study with 10 users demonstrated its usability and effectiveness.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://arxiv.org/abs/1802.08802\" rel=\"nofollow\"\u003eReinforcement Learning on Web Interfaces Using Workflow-Guided Exploration\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eEvan Zheran Liu, Kelvin Guu, Panupong Pasupat, Tianlin Shi, Percy Liang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Stanford\u003c/li\u003e\n\u003cli\u003e📅 Date: February 24, 2018\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICLR 2018\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [benchmark], [reinforcement learning], [web tasks], [workflow-guided exploration]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper presents a novel RL approach using \u003cem\u003eworkflow-guided exploration\u003c/em\u003e to efficiently train agents on web-based tasks, where actions are restricted based on demonstrated workflows to streamline learning. Evaluated on MiniWoB and MiniWoB++ benchmarks, the method significantly outperforms traditional RL techniques in sparse reward settings by structuring exploration according to high-level action constraints.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://dl.acm.org/doi/10.1145/3126594.3126651\" rel=\"nofollow\"\u003eRico: A Mobile App Dataset for Building Data-Driven Design Applications\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eGenevieve Patterson, Joseph Gonzalez, Jeffrey Heer, Daniel H. Haim, Keyur Govani, Andrew Hertzmann, Noah Snavely, Neel Joshi\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: UIUC, Northwestern University, Google\u003c/li\u003e\n\u003cli\u003e📅 Date: October 20, 2017\u003c/li\u003e\n\u003cli\u003e📑 Publisher: UIST 2017\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [dataset], [mobile UI], [UI design analysis], [interaction mining], [RICO]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eRico\u003c/em\u003e, a large-scale dataset comprising UI screens and view hierarchies from over 9,000 Android apps, designed to aid in understanding mobile app design. Rico supports a variety of tasks, including UI design analysis and interaction mining, by providing labeled UI components, screenshots, and interaction traces.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://proceedings.mlr.press/v70/shi17a.html\" rel=\"nofollow\"\u003eWorld of Bits: An Open-Domain Platform for Web-Based Agents\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTianlin Shi, Andrej Karpathy, Linxi Fan, Jonathan Hernandez, Percy Liang\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: Stanford, OpenAI\u003c/li\u003e\n\u003cli\u003e📅 Date: August 2017\u003c/li\u003e\n\u003cli\u003e📑 Publisher: ICML 2017\u003c/li\u003e\n\u003cli\u003e💻 Env: [Web]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [dataset], [reinforcement learning], [open-domain]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eWorld of Bits (WoB)\u003c/em\u003e, a platform enabling agents to perform complex web-based tasks using low-level keyboard and mouse actions, addressing the lack of open-domain realism in existing reinforcement learning environments. WoB leverages a novel framework where crowdworkers create tasks with structured rewards and reproducibility by caching web interactions, forming a stable training environment. The authors validate WoB by training agents via behavioral cloning and reinforcement learning to accomplish various real-world tasks, showcasing its potential as an effective platform for reinforcement learning on web tasks.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://dl.acm.org/doi/abs/10.1145/3025453.3025483\" rel=\"nofollow\"\u003eSUGILITE: Creating Multimodal Smartphone Automation by Demonstration\u003c/a\u003e\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eToby Jia-Jun Li, Amos Azaria, Brad A. Myers\u003c/li\u003e\n\u003cli\u003e🏛️ Institutions: CMU, Ariel University\u003c/li\u003e\n\u003cli\u003e📅 Date: May 6, 2017\u003c/li\u003e\n\u003cli\u003e📑 Publisher: CHI 2017\u003c/li\u003e\n\u003cli\u003e💻 Env: [Mobile]\u003c/li\u003e\n\u003cli\u003e🔑 Key: [framework], [PBD], [multimodal interaction], [SUGILITE], [programming-by-demonstration], [demonstration]\u003c/li\u003e\n\u003cli\u003e📖 TLDR: This paper introduces \u003cem\u003eSUGILITE\u003c/em\u003e, a programming-by-demonstration (PBD) system that enables users to automate tasks on smartphones through multimodal interactions. By leveraging Android's accessibility API, SUGILITE allows users to create generalized automation scripts for arbitrary third-party apps by demonstrating tasks using the regular app UI. The system combines verbal instructions, user demonstrations, and app UI hierarchies to generalize scripts from single demonstrations, facilitating task variations and parameterization. Extensive error handling and context checking enhance robustness against app UI changes. A lab study indicates that users with minimal programming knowledge can successfully automate smartphone tasks using SUGILITE.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/details\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Add a Paper or Update the README\u003c/h2\u003e\u003ca id=\"user-content-how-to-add-a-paper-or-update-the-readme\" class=\"anchor\" aria-label=\"Permalink: How to Add a Paper or Update the README\" href=\"#how-to-add-a-paper-or-update-the-readme\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003ePlease fork and update:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/update_template_or_data/update_paper_list.md\"\u003epaper list\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/update_template_or_data/update_readme_template.md\"\u003eREADME template\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/.github/workflows/main.yml\"\u003eautomatic workflow\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003e🤖 You can use \u003ca href=\"https://chatgpt.com/g/g-VqW9ONrgL-gui-paper-list\" rel=\"nofollow\"\u003ethis GPTs\u003c/a\u003e to quickly search and get a formatted paper entry automatically by inputting a paper name. Or you can simply leave a comment in an issue.\u003c/p\u003e\n\u003cdetails\u003e\n\u003csummary\u003eFormat example and explanation\u003c/summary\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"- [title](paper link)\n    - List authors directly without a \u0026quot;key\u0026quot; identifier (e.g., author1, author2)\n    - 🏛️ Institutions: List the institutions concisely, using abbreviations (e.g., university names, like OSU).\n    - 📅 Date: e.g., Oct 30, 2024\n    - 📑 Publisher: ICLR 2025\n    - 💻 Env: Indicate the research environment within brackets, such as [Web], [Mobile], or [Desktop]. Use [GUI] if the research spans multiple environments. Use [Misc] if it is researching in general domains.\n    - 🔑 Key: Label each keyword within brackets, e.g., [model], [framework],[dataset],[benchmark].\n    - 📖 TLDR: Brief summary of the paper.\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003e- [title](paper link)\n    - List authors directly without a \"key\" identifier (e.g., author1, author2)\n    - 🏛️ Institutions: List the institutions concisely, using abbreviations (e.g., university names, like OSU).\n    - 📅 Date: e.g., Oct 30, 2024\n    - 📑 Publisher: ICLR 2025\n    - 💻 Env: Indicate the research environment within brackets, such as [Web], [Mobile], or [Desktop]. Use [GUI] if the research spans multiple environments. Use [Misc] if it is researching in general domains.\n    - 🔑 Key: Label each keyword within brackets, e.g., [model], [framework],[dataset],[benchmark].\n    - 📖 TLDR: Brief summary of the paper.\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eRegarding the 🔑 Key:\u003c/p\u003e\n\u003cmarkdown-accessiblity-table\u003e\u003ctable\u003e\n\u003cthead\u003e\n\u003ctr\u003e\n\u003cth\u003eKey\u003c/th\u003e\n\u003cth\u003eDefinition\u003c/th\u003e\n\u003c/tr\u003e\n\u003c/thead\u003e\n\u003ctbody\u003e\n\u003ctr\u003e\n\u003ctd\u003emodel\u003c/td\u003e\n\u003ctd\u003eIndicates a newly trained model.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd\u003eframework\u003c/td\u003e\n\u003ctd\u003eIf the paper proposes a new agent framework.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd\u003edataset\u003c/td\u003e\n\u003ctd\u003eIf a new (training) dataset is created and published.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd\u003ebenchmark\u003c/td\u003e\n\u003ctd\u003eIf a new benchmark is established (also add \"dataset\" if there's a new training set).\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd\u003eprimary studies\u003c/td\u003e\n\u003ctd\u003eList the main focus or innovation in the study.\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd\u003eAbbreviations\u003c/td\u003e\n\u003ctd\u003eInclude commonly used abbreviations associated with the paper (model names, framework names, etc.).\u003c/td\u003e\n\u003c/tr\u003e\n\u003c/tbody\u003e\n\u003c/table\u003e\u003c/markdown-accessiblity-table\u003e\n\u003cp dir=\"auto\"\u003eFor missing information, use \"Unknown.\"\u003c/p\u003e\n\u003c/details\u003e\n\u003c/article\u003e","loaded":true,"timedOut":false,"errorMessage":null,"headerInfo":{"toc":[{"level":1,"text":"Awesome GUI Agent Paper List","anchor":"awesome-gui-agent-paper-list","htmlText":"Awesome GUI Agent Paper List"},{"level":2,"text":"Papers Grouped by Environments","anchor":"papers-grouped-by-environments","htmlText":"Papers Grouped by Environments"},{"level":2,"text":"Papers Grouped by Keywords","anchor":"papers-grouped-by-keywords","htmlText":"Papers Grouped by Keywords"},{"level":2,"text":"Papers Grouped by Authors","anchor":"papers-grouped-by-authors","htmlText":"Papers Grouped by Authors"},{"level":2,"text":"All Papers (from most recent to oldest)","anchor":"all-papers-from-most-recent-to-oldest","htmlText":"All Papers (from most recent to oldest)"},{"level":2,"text":"How to Add a Paper or Update the README","anchor":"how-to-add-a-paper-or-update-the-readme","htmlText":"How to Add a Paper or Update the README"}],"siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2FOSU-NLP-Group%2FGUI-Agents-Paper-List"}}],"overviewFilesProcessingTime":0}},"appPayload":{"helpUrl":"https://docs.github.com","findFileWorkerPath":"/assets-cdn/worker/find-file-worker-7d7eb7c71814.js","findInFileWorkerPath":"/assets-cdn/worker/find-in-file-worker-1ae9fa256942.js","githubDevUrl":null,"enabled_features":{"copilot_workspace":null,"code_nav_ui_events":false,"react_blob_overlay":false,"accessible_code_button":true,"github_models_repo_integration":false}}}}</script>
  <div data-target="react-partial.reactRoot"> <!-- --> <!-- --> <div class="OverviewContent-module__Box--uNd1J"><div class="OverviewHeader-module__Box--fFKf5"><div class="Box-sc-g0xbh4-0 fQZdXd OverviewHeader-module__Box_1--zJpeS"></div></div><div class="OverviewContent-module__Box_1--RhaEy"><div class="OverviewContent-module__Box_2--uHewD"><div class="OverviewContent-module__Box_3--NEYWl"><button type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="main branch" data-testid="anchor-button" class="Box-sc-g0xbh4-0 gMOVLe prc-Button-ButtonBase-c50BI overview-ref-selector width-full" data-loading="false" data-size="medium" data-variant="default" aria-describedby="branch-picker-repos-header-ref-selector-loading-announcement" id="branch-picker-repos-header-ref-selector"><span data-component="buttonContent" class="Box-sc-g0xbh4-0 gUkoLg prc-Button-ButtonContent-HKbr-"><span data-component="text" class="prc-Button-Label-pTQ3x"><div class="Box-sc-g0xbh4-0 bZBlpz"><div class="Box-sc-g0xbh4-0 lhTYNA"><svg aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></div><div class="Box-sc-g0xbh4-0 ffLUq ref-selector-button-text-container"><span class="Box-sc-g0xbh4-0 bmcJak prc-Text-Text-0ima0">&nbsp;<!-- -->main</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><button hidden="" data-hotkey-scope="read-only-cursor-text-area"></button></div><div class="OverviewContent-module__Box_4--rOz8J"><a type="button" href="/OSU-NLP-Group/GUI-Agents-Paper-List/branches" class="prc-Button-ButtonBase-c50BI OverviewContent-module__Button--MDoYP" data-loading="false" data-size="medium" data-variant="invisible" aria-describedby=":Rclab:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Branches</span></span></a><a type="button" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tags" class="prc-Button-ButtonBase-c50BI OverviewContent-module__Button--MDoYP" data-loading="false" data-size="medium" data-variant="invisible" aria-describedby=":Rklab:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-tag" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Tags</span></span></a></div><div class="OverviewContent-module__Box_5--PPbL1"><a type="button" aria-label="Go to Branches page" href="/OSU-NLP-Group/GUI-Agents-Paper-List/branches" class="prc-Button-ButtonBase-c50BI OverviewContent-module__Button_1--_1Ng2" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-describedby=":Relab:-loading-announcement"><svg aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></a><a type="button" aria-label="Go to Tags page" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tags" class="prc-Button-ButtonBase-c50BI OverviewContent-module__Button_1--_1Ng2" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-describedby=":Rmlab:-loading-announcement"><svg aria-hidden="true" focusable="false" class="octicon octicon-tag" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"></path></svg></a></div></div><div class="OverviewContent-module__Box_6--wV7Tw"><div class="OverviewContent-module__Box_7--SbxdI"><div class="OverviewContent-module__Box_8--oumpR"><!--$--><div class="Box-sc-g0xbh4-0 OverviewContent-module__FileResultsList--irMg6 FileResultsList-module__Box_1--ZnWjQ"><span class="FileResultsList-module__FilesSearchBox--J5FtW FilesSearchBox-module__TextInput--LKpMn TextInput-wrapper prc-components-TextInputWrapper-i1ofR prc-components-TextInputBaseWrapper-ueK9q" data-leading-visual="true" data-trailing-visual="true" aria-busy="false"><span class="TextInput-icon" id=":R2j5ab:" aria-hidden="true"><svg aria-hidden="true" focusable="false" class="octicon octicon-search" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg></span><input type="text" aria-label="Go to file" role="combobox" aria-controls="file-results-list" aria-expanded="false" aria-haspopup="dialog" autocorrect="off" spellcheck="false" placeholder="Go to file" aria-describedby=":R2j5ab: :R2j5abH1:" data-component="input" class="prc-components-Input-Ic-y8" value=""><span class="TextInput-icon" id=":R2j5abH1:" aria-hidden="true"></span></span></div><!--/$--></div><div class="OverviewContent-module__Box_9--mQYON"><button type="button" class="prc-Button-ButtonBase-c50BI" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-describedby=":Rr5ab:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="text" class="prc-Button-Label-pTQ3x">Go to file</span></span></button></div><div class="react-directory-add-file-icon"></div><div class="react-directory-remove-file-icon"></div></div><button type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-c50BI" data-loading="false" data-size="medium" data-variant="primary" aria-describedby=":R55ab:-loading-announcement" id=":R55ab:"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-code hide-sm" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Code</span><span data-component="trailingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="OverviewContent-module__Box_10--ULKAG"><button data-component="IconButton" type="button" aria-label="Open more actions menu" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-c50BI prc-Button-IconButton-szpyj" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-describedby=":R75ab:-loading-announcement" id=":R75ab:"><svg aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button></div></div></div><div class="OverviewContent-module__Box_11--Tqhu2"><div data-hpc="true"><button hidden="" data-testid="focus-next-element-button" data-hotkey="j"></button><button hidden="" data-testid="focus-previous-element-button" data-hotkey="k"></button><h2 class="sr-only ScreenReaderHeading-module__userSelectNone--vW4Cq prc-Heading-Heading-6CmGO" data-testid="screen-reader-heading" id="folders-and-files">Folders and files</h2><table class="Table-module__Box--h4W6R DirectoryContent-module__Table--DNJx9" aria-labelledby="folders-and-files"><thead class="DirectoryContent-module__OverviewHeaderRow--W8yGl Table-module__Box_1--JrPYF"><tr class="Table-module__Box_2--kJgvd"><th colspan="2" class="DirectoryContent-module__Box--J2MQZ"><span class="text-bold">Name</span></th><th colspan="1" class="DirectoryContent-module__Box_1--mB8B7"><span class="text-bold">Name</span></th><th class="hide-sm"><div title="Last commit message" class="Truncate__StyledTruncate-sc-23o1d2-0 liVpTx width-fit"><span class="text-bold">Last commit message</span></div></th><th colspan="1" class="DirectoryContent-module__Box_2--LsXd4"><div title="Last commit date" class="Truncate__StyledTruncate-sc-23o1d2-0 liVpTx width-fit"><span class="text-bold">Last commit date</span></div></th></tr></thead><tbody><tr class="DirectoryContent-module__Box_3--BoinM"><td colspan="3" class="bgColor-muted p-1 rounded-top-2"><div class="LatestCommit-module__Box--En0AE"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone--vW4Cq prc-Heading-Heading-6CmGO" data-testid="screen-reader-heading">Latest commit</h2><div style="width:120px" class="Skeleton Skeleton--text" data-testid="loading">&nbsp;</div><div class="d-flex flex-shrink-0 gap-2"><div data-testid="latest-commit-details" class="d-none d-sm-flex flex-items-center"></div><div class="d-flex gap-2"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone--vW4Cq prc-Heading-Heading-6CmGO" data-testid="screen-reader-heading">History</h2><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/commits/main/" class="prc-Button-ButtonBase-c50BI d-none d-lg-flex LinkButton-module__code-view-link-button--xvCGA flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby=":Raqj8pab:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x"><span class="fgColor-default">528 Commits</span></span></span></a><div class="d-sm-none"></div><div class="d-flex d-lg-none"><span role="tooltip" aria-label="528 Commits" id="history-icon-button-tooltip" class="Tooltip__TooltipBase-sc-17tf59c-0 hWlpPn tooltipped-n"><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/commits/main/" class="prc-Button-ButtonBase-c50BI LinkButton-module__code-view-link-button--xvCGA flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby=":R1iqj8pab:-loading-announcement history-icon-button-tooltip"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span></span></a></span></div></div></div></div></td></tr><tr class="react-directory-row undefined" id="folder-row-0"><td class="react-directory-row-name-cell-small-screen" colspan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="This path skips through empty directories" aria-label=".github/workflows, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/.github/workflows"><span class="react-directory-default-color" data-testid="path-name-segment">.github/</span><span class="" data-testid="path-name-segment">workflows</span></a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colspan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="This path skips through empty directories" aria-label=".github/workflows, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/.github/workflows"><span class="react-directory-default-color" data-testid="path-name-segment">.github/</span><span class="" data-testid="path-name-segment">workflows</span></a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text">&nbsp;</div></td><td><div class="Skeleton Skeleton--text">&nbsp;</div></td></tr><tr class="react-directory-row undefined" id="folder-row-1"><td class="react-directory-row-name-cell-small-screen" colspan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="paper_by_author" aria-label="paper_by_author, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/paper_by_author">paper_by_author</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colspan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="paper_by_author" aria-label="paper_by_author, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/paper_by_author">paper_by_author</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text">&nbsp;</div></td><td><div class="Skeleton Skeleton--text">&nbsp;</div></td></tr><tr class="react-directory-row undefined" id="folder-row-2"><td class="react-directory-row-name-cell-small-screen" colspan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="paper_by_env" aria-label="paper_by_env, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/paper_by_env">paper_by_env</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colspan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="paper_by_env" aria-label="paper_by_env, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/paper_by_env">paper_by_env</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text">&nbsp;</div></td><td><div class="Skeleton Skeleton--text">&nbsp;</div></td></tr><tr class="react-directory-row undefined" id="folder-row-3"><td class="react-directory-row-name-cell-small-screen" colspan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="paper_by_key" aria-label="paper_by_key, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/paper_by_key">paper_by_key</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colspan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="paper_by_key" aria-label="paper_by_key, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/paper_by_key">paper_by_key</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text">&nbsp;</div></td><td><div class="Skeleton Skeleton--text">&nbsp;</div></td></tr><tr class="react-directory-row undefined" id="folder-row-4"><td class="react-directory-row-name-cell-small-screen" colspan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="update_template_or_data" aria-label="update_template_or_data, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/update_template_or_data">update_template_or_data</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colspan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="update_template_or_data" aria-label="update_template_or_data, (Directory)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/tree/main/update_template_or_data">update_template_or_data</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text">&nbsp;</div></td><td><div class="Skeleton Skeleton--text">&nbsp;</div></td></tr><tr class="react-directory-row undefined" id="folder-row-5"><td class="react-directory-row-name-cell-small-screen" colspan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="README.md" aria-label="README.md, (File)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/README.md">README.md</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colspan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="README.md" aria-label="README.md, (File)" class="Link--primary" href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/README.md">README.md</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text">&nbsp;</div></td><td><div class="Skeleton Skeleton--text">&nbsp;</div></td></tr><tr class="d-none DirectoryContent-module__Box_4--EzUVO" data-testid="view-all-files-row"><td colspan="3" class="DirectoryContent-module__Box_5--r9c8e"><div><button class="prc-Link-Link-85e08">View all files</button></div></td></tr></tbody></table></div><div class="OverviewRepoFiles-module__Box_1--xSt0T"><div class="OverviewRepoFiles-module__Box_2--yIjMp"><div itemscope="" itemtype="https://schema.org/abstract" class="OverviewRepoFiles-module__Box_3--Bi2jM"><h2 class="_VisuallyHidden__VisuallyHidden-sc-11jhm7a-0 brGdpi">Repository files navigation</h2><nav class="prc-components-UnderlineWrapper-oOh5J OverviewRepoFiles-module__UnderlineNav--BHfFi" aria-label="Repository files"><ul class="prc-components-UnderlineItemList-b23Hf" role="list"><li class="Box-sc-g0xbh4-0 hUCRAk"><a class="prc-components-UnderlineItem-lJsg-" href="#" aria-current="page"><span data-component="icon"><svg aria-hidden="true" focusable="false" class="octicon octicon-book" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path></svg></span><span data-component="text" data-content="README">README</span></a></li></ul></nav><button type="button" aria-label="Outline" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-c50BI OverviewRepoFiles-module__ActionMenu_Button--xB9DS" data-loading="false" data-size="medium" data-variant="invisible" aria-describedby=":Rr9ab:-loading-announcement" id=":Rr9ab:"><svg aria-hidden="true" focusable="false" class="octicon octicon-list-unordered" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.75 2.5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5ZM2 14a1 1 0 1 1 0-2 1 1 0 0 1 0 2Zm1-6a1 1 0 1 1-2 0 1 1 0 0 1 2 0ZM2 4a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path></svg></button></div><div class="Box-sc-g0xbh4-0 js-snippet-clipboard-copy-unpositioned DirectoryRichtextContent-module__SharedMarkdownContent--YORdJ" data-hpc="true"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Awesome GUI Agent Paper List</h1><a id="user-content-awesome-gui-agent-paper-list" class="anchor" aria-label="Permalink: Awesome GUI Agent Paper List" href="#awesome-gui-agent-paper-list"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">This repo covers a variety of papers related to GUI Agents, such as:</p>
<ul dir="auto">
<li>Datasets</li>
<li>Benchmarks</li>
<li>Models</li>
<li>Agent frameworks</li>
<li>Vision, language, multimodal foundation models (with explicit support for GUI)</li>
<li>Works in general domains extensively used by GUI Agents (e.g., SoM prompting)</li>
</ul>
<p dir="auto"><a target="_blank" rel="noopener noreferrer" href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/update_template_or_data/statistics/keyword_wordcloud_long.png"><img src="/OSU-NLP-Group/GUI-Agents-Paper-List/raw/main/update_template_or_data/statistics/keyword_wordcloud_long.png" alt="keyword_wordcloud_long.png" style="max-width: 100%;"></a></p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Papers Grouped by Environments</h2><a id="user-content-papers-grouped-by-environments" class="anchor" aria-label="Permalink: Papers Grouped by Environments" href="#papers-grouped-by-environments"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<markdown-accessiblity-table><table>
<thead>
<tr>
<th><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_web.md">Web</a></th>
<th><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_mobile.md">Mobile</a></th>
<th><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_desktop.md">Desktop</a></th>
<th><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_gui.md">GUI</a></th>
<th><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_env/paper_misc.md">Misc</a></th>
</tr>
</thead>
</table></markdown-accessiblity-table>
<p dir="auto">(Misc: Papers for general topics that have important applications in GUI agents.)</p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Papers Grouped by Keywords</h2><a id="user-content-papers-grouped-by-keywords" class="anchor" aria-label="Permalink: Papers Grouped by Keywords" href="#papers-grouped-by-keywords"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto"><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_framework.md">framework (142)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_benchmark.md">benchmark (90)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_dataset.md">dataset (80)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_model.md">model (42)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_reinforcement_learning.md">reinforcement learning (22)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_planning.md">planning (13)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_safety.md">safety (12)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_reasoning.md">reasoning (10)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_visual-grounding.md">visual grounding (10)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_evaluation.md">evaluation (9)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_survey.md">survey (8)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_grounding.md">grounding (8)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_learning.md">learning (5)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_vision_language_model.md">vision language model (5)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_multimodal.md">multimodal (4)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_memory.md">memory (4)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_attack.md">attack (4)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_GUI_grounding.md">GUI grounding (3)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_WebArena.md">WebArena (3)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_key/paper_ScreenSpot-Pro.md">ScreenSpot-Pro (3)</a></p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Papers Grouped by Authors</h2><a id="user-content-papers-grouped-by-authors" class="anchor" aria-label="Permalink: Papers Grouped by Authors" href="#papers-grouped-by-authors"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto"><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Yu_Su.md">Yu Su (13)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Graham_Neubig.md">Graham Neubig (10)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Huan_Sun.md">Huan Sun (9)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Boyuan_Zheng.md">Boyuan Zheng (8)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Tao_Yu.md">Tao Yu (8)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Tianbao_Xie.md">Tianbao Xie (7)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Shuyan_Zhou.md">Shuyan Zhou (7)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Qiushi_Sun.md">Qiushi Sun (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Yu_Gu.md">Yu Gu (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Daniel_Fried.md">Daniel Fried (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Boyu_Gou.md">Boyu Gou (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Mike_Zheng_Shou.md">Mike Zheng Shou (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Kun_Shao.md">Kun Shao (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Zhiyong_Wu.md">Zhiyong Wu (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Xiao_Liu.md">Xiao Liu (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Hanyu_Lai.md">Hanyu Lai (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Jie_Tang.md">Jie Tang (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Yuxiao_Dong.md">Yuxiao Dong (6)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Jianfeng_Gao.md">Jianfeng Gao (5)</a> | <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/paper_by_author/paper_Jianye_Hao.md">Jianye Hao (5)</a></p>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">All Papers (from most recent to oldest)</h2><a id="user-content-all-papers-from-most-recent-to-oldest" class="anchor" aria-label="Permalink: All Papers (from most recent to oldest)" href="#all-papers-from-most-recent-to-oldest"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<details open="">
<summary>Papers</summary>
<ul dir="auto">
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2505.00684" rel="nofollow">Visual Test-time Scaling for GUI Agent Grounding</a></p>
<ul dir="auto">
<li>Tiange Luo, Lajanugen Logeswaran, Justin Johnson, Honglak Lee</li>
<li>🏛️ Institutions: UMich, LG AI Research</li>
<li>📅 Date: May 1, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [benchmark], [RegionFocus], [test-time scaling], [grounding], [Qwen2.5-VL], [UI-TARS]</li>
<li>📖 TLDR: This paper introduces <em>RegionFocus</em>, a visual test-time scaling method for GUI agents that dynamically zooms into relevant regions within GUI images, reducing background clutter and enhancing grounding accuracy. By integrating an "image-as-map" mechanism to visualize key landmarks during each interaction step, the approach improves transparency and decision-making. Applied to state-of-the-art vision-language models like UI-TARS and Qwen2.5-VL, RegionFocus achieves significant performance gains—28% on ScreenSpot-Pro and 24% on WebVoyager benchmarks—setting a new state-of-the-art grounding accuracy of 61.6% on ScreenSpot-Pro.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2505.00416" rel="nofollow">ScaleTrack: Scaling and back-tracking Automated GUI Agents</a></p>
<ul dir="auto">
<li>Jing Huang, Zhixiong Zeng, Wenkang Han, Yufeng Zhong, Liming Zheng, Shuai Fu, Jingyuan Chen, Lin Ma</li>
<li>🏛️ Institutions: Meituan, Zhejiang University, University of Adelaide</li>
<li>📅 Date: May 1, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [training strategy], [back-tracking], [grounding], [planning], [ScaleTrack]</li>
<li>📖 TLDR: This paper introduces <em>ScaleTrack</em>, a training framework designed to enhance automated GUI agents by addressing two primary challenges: insufficient training data for GUI grounding and the lack of back-tracking in GUI planning. The authors aggregate diverse GUI samples from various synthesis methods into a unified template to scale the grounding process. Additionally, they propose a hybrid training strategy that combines forward-planning and back-tracking, enabling the agent to predict both the next action and the historical actions leading to the current GUI state. Experimental results demonstrate that ScaleTrack significantly improves task success rates across multiple benchmarks, highlighting the effectiveness of integrating back-tracking into GUI agent training.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.20464" rel="nofollow">A Summary on GUI Agents with Foundation Models Enhanced by Reinforcement Learning</a></p>
<ul dir="auto">
<li>Jiahao Li, Kaer Huang</li>
<li>🏛️ Institutions: Lenovo Research</li>
<li>📅 Date: April 29, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [survey], [framework], [reinforcement learning], [multimodal], [training taxonomy]</li>
<li>📖 TLDR: This paper presents a structured summary of recent advances in GUI agents powered by Multi-modal Large Language Models (MLLMs) and enhanced through Reinforcement Learning (RL). It formalizes GUI agent tasks as Markov Decision Processes and reviews modular architectures comprising Perception, Planning, and Acting modules. The study categorizes training methodologies into Prompt-based, Supervised Fine-Tuning (SFT), and RL-based approaches, highlighting the progression towards dynamic policy learning. The paper concludes by identifying key challenges and future directions for developing more capable and reliable GUI agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/PhoneLLM/Awesome-LLM-Powered-Phone-GUI-Agents">LLM-Powered GUI Agents in Phone Automation: Surveying Progress and Prospects</a></p>
<ul dir="auto">
<li>Guangyi Liu, Pengxiang Zhao, Liang Liu, Yaxuan Guo, Han Xiao, Weifeng Lin, Yuxiang Chai, Yue Han, Shuai Ren, Hao Wang, Xiaoyu Liang, Wenhao Wang, Tianze Wu, Linghao Li, Guanjing Xiong, Yong Liu, Hongsheng Li</li>
<li>🏛️ Institutions: Zhejiang Univ., vivo AI Lab, CUHK MMLab</li>
<li>📅 Date: April 28, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [survey], [framework], [dataset], [benchmark], [planning], [multimodal], [taxonomy]</li>
<li>📖 TLDR: This comprehensive survey examines the evolution of LLM-powered GUI agents in mobile phone automation, transitioning from static scripts to intelligent, adaptive systems. It presents a taxonomy encompassing agent frameworks (single-agent, multi-agent, plan-then-act), modeling approaches (prompt engineering, training-based), and essential datasets and benchmarks. The paper discusses how LLMs enhance language understanding, multimodal perception, and decision-making in GUI tasks, and addresses challenges such as dataset diversity, on-device deployment efficiency, user-centric adaptation, and security concerns. It serves as a definitive reference for researchers and practitioners aiming to leverage LLMs in designing scalable, user-friendly phone GUI agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.14603" rel="nofollow">UFO2: The Desktop AgentOS</a></p>
<ul dir="auto">
<li>Chaoyun Zhang, He Huang, Chiming Ni, Jian Mu, Si Qin, Shilin He, Lu Wang, Fangkai Yang, Pu Zhao, Chao Du, Liqun Li, Yu Kang, Zhao Jiang, Suzhen Zheng, Rujia Wang, Jiaxu Qian, Minghua Ma, Jian-Guang Lou, Qingwei Lin, Saravan Rajmohan, Dongmei Zhang</li>
<li>🏛️ Institutions: Microsoft Research, ZJU-UIUC Institute, Nanjing University, Peking University</li>
<li>📅 Date: April 20, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [benchmark], [hybrid control], [multi-agent], [speculative execution], [knowledge substrate], [UFO2]</li>
<li>📖 TLDR: UFO2 introduces a multi-agent AgentOS for Windows desktops, aiming to transform Computer-Using Agents (CUAs) from fragile prototypes into robust, system-level automation tools. It features a centralized HostAgent for task decomposition and coordination, along with specialized AppAgents equipped with native APIs and domain-specific knowledge. The system employs a hybrid control detection pipeline combining Windows UI Automation with vision-based parsing, and enhances runtime efficiency through speculative multi-action planning. A Picture-in-Picture interface allows agents and users to operate concurrently without interference. Evaluated across over 20 real-world Windows applications, UFO2 demonstrates significant improvements in robustness and execution accuracy over prior CUAs.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.14239" rel="nofollow">InfiGUI-R1: Advancing Multimodal GUI Agents from Reactive Actors to Deliberative Reasoners</a></p>
<ul dir="auto">
<li>Yuhang Liu, Pengxiang Li, Congkai Xie, Xavier Hu, Xiaotian Han, Shengyu Zhang, Hongxia Yang, Fei Wu</li>
<li>🏛️ Institutions: Zhejiang Univ., Dalian Univ. of Tech., Reallm Labs, HK PolyU</li>
<li>📅 Date: April 19, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [model], [reinforcement learning], [planning], [reasoning], [Actor2Reasoner], [InfiGUI-R1]</li>
<li>📖 TLDR: This paper introduces <strong>InfiGUI-R1</strong>, a multimodal GUI agent developed through the <strong>Actor2Reasoner</strong> framework, aiming to transition agents from reactive behaviors to deliberative reasoning. The framework comprises two stages: <em>Reasoning Injection</em>, which employs Spatial Reasoning Distillation to integrate GUI visual-spatial information with logical reasoning, and <em>Deliberation Enhancement</em>, which uses Reinforcement Learning with Sub-goal Guidance and Error Recovery Scenario Construction to refine the agent's planning and error correction capabilities. Evaluations on benchmarks like AndroidControl and ScreenSpot demonstrate that InfiGUI-R1-3B achieves state-of-the-art performance in GUI grounding and trajectory tasks, outperforming larger models in several categories.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://lgy0404.github.io/LearnAct" rel="nofollow">LearnAct: Few-Shot Mobile GUI Agent with a Unified Demonstration Benchmark</a></p>
<ul dir="auto">
<li>Guangyi Liu, Pengxiang Zhao, Liang Liu, Zhiming Chen, Yuxiang Chai, Shuai Ren, Hao Wang, Shibo He, Wenchao Meng</li>
<li>🏛️ Institutions: ZJU, vivo AI Lab</li>
<li>📅 Date: April 18, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [few-shot learning], [LearnAct], [LearnGUI], [DemoParser], [KnowSeeker], [ActExecutor]</li>
<li>📖 TLDR: This paper introduces <strong>LearnAct</strong>, a multi-agent framework designed to enhance mobile GUI agents through few-shot demonstration learning. Accompanied by <strong>LearnGUI</strong>, a dataset comprising 2,252 offline and 101 online tasks with high-quality human demonstrations, the framework integrates three specialized agents—DemoParser, KnowSeeker, and ActExecutor—to extract, retrieve, and apply knowledge from demonstrations. Experimental results show significant performance improvements in both offline and online evaluations, establishing demonstration-based learning as a promising direction for developing adaptable and personalized mobile GUI agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://tongui-agent.github.io/" rel="nofollow">TongUI: Building Generalized GUI Agents by Learning from Multimodal Web Tutorials</a></p>
<ul dir="auto">
<li>Bofei Zhang, Zirui Shang, Zhi Gao, Wang Zhang, Rui Xie, Xiaojian Ma, Tao Yuan, Xinxiao Wu, Song-Chun Zhu, Qing Li</li>
<li>🏛️ Institutions: BIGAI, BIT, PKU, SJTU, THU</li>
<li>📅 Date: April 17, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [Qwen2.5-VL], [GUI-Net], [multimodal], [trajectory generation]</li>
<li>📖 TLDR: This paper introduces <strong>TongUI</strong>, a framework for building generalized GUI agents by learning from multimodal web tutorials. By crawling and processing online tutorials into GUI agent trajectory data, the authors construct the <strong>GUI-Net</strong> dataset containing 143K trajectories across five operating systems and over 200 applications. Fine-tuning Qwen2.5-VL-3B/7B models on GUI-Net leads to significant performance improvements on grounding and navigation benchmarks, demonstrating the effectiveness of the TongUI framework.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.11257" rel="nofollow">UI-E2I-Synth: Advancing GUI Grounding with Large-Scale Instruction Synthesis</a></p>
<ul dir="auto">
<li>Xinyi Liu, Xiaoyi Zhang, Ziyun Zhang, Yan Lu</li>
<li>🏛️ Institutions: MSRA, PKU</li>
<li>📅 Date: April 15, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [instruction synthesis], [UI-E2I-Synth], [UI-I2E-Bench], [GPT-4o]</li>
<li>📖 TLDR: This paper introduces <strong>UI-E2I-Synth</strong>, a large-scale data synthesis pipeline that generates diverse GUI grounding instructions using GPT-4o, addressing challenges like implicit instructions and unbalanced element types. It also presents <strong>UI-I2E-Bench</strong>, a new benchmark with detailed annotations for evaluating GUI instruction grounding. Models trained on the synthesized data demonstrate superior performance across multiple platforms, highlighting the effectiveness of the proposed approach.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://realevals.xyz" rel="nofollow">REAL: Benchmarking Autonomous Agents on Deterministic Simulations of Real Websites</a></p>
<ul dir="auto">
<li>Divyansh Garg, Shaun VanWeelden, Diego Caples, Andis Draguns, Nikil Ravi, Pranav Putta, Naman Garg, Tomas Abraham, Michael Lara, Federico Lopez, James Liu, Atharva Gundawar, Prannay Hebbar, Youngchul Joo, Charles London, Christian Schroeder de Witt, Sumeet Motwani</li>
<li>🏛️ Institutions: AGI Inc.</li>
<li>📅 Date: April 15, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [framework], [dataset], [evaluation], [reproducibility], [multi-turn tasks], [REAL]</li>
<li>📖 TLDR: This paper introduces <strong>REAL</strong>, a benchmark and framework designed for evaluating autonomous agents through deterministic simulations of real-world websites. REAL encompasses high-fidelity replicas of 11 widely-used websites across domains like e-commerce, travel, communication, and professional networking. It includes 112 practical tasks that mirror complex user interactions requiring both accurate information retrieval and state-changing actions. The controlled environment ensures safety and reproducibility, combining programmatic checks for action-based tasks with rubric-guided LLM-based judgments for information retrieval. Empirical results reveal that current frontier language models achieve at most a 41% success rate on REAL, highlighting significant gaps in autonomous web navigation and task completion capabilities.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.10127" rel="nofollow">Breaking the Data Barrier -- Building GUI Agents Through Task Generalization</a></p>
<ul dir="auto">
<li>Junlei Zhang, Zichen Ding, Chang Ma, Zijie Chen, Qiushi Sun, Zhenzhong Lan, Junxian He</li>
<li>🏛️ Institutions: ZJU, WestlakeU, Shanghai AI Lab, HKU, HKUST</li>
<li>📅 Date: April 14, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [mid-training], [reasoning], [GUIMid]</li>
<li>📖 TLDR: This paper introduces a mid-training approach to enhance GUI agents by leveraging diverse, reasoning-intensive datasets such as mathematical and coding tasks. The authors demonstrate that training Vision Language Models (VLMs) on these data-rich tasks before fine-tuning on limited GUI trajectory data significantly improves performance on GUI benchmarks like WebArena and AndroidWorld. Notably, text-only mathematical reasoning data led to substantial cross-modal gains, highlighting the effectiveness of task generalization in overcoming data scarcity challenges in GUI agent development.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.10458" rel="nofollow">GUI-R1: A Generalist R1-Style Vision-Language Action Model For GUI Agents</a></p>
<ul dir="auto">
<li>Xiaobo Xia, Run Luo</li>
<li>🏛️ Institutions: Unknown</li>
<li>📅 Date: April 14, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [reinforcement learning], [unified action space], [GRPO], [GUI-R1]</li>
<li>📖 TLDR: GUI-R1 introduces a reinforcement learning framework that enhances the capabilities of Large Vision-Language Models (LVLMs) for GUI agents. By employing a unified action space and a rule-based reward function, the model efficiently learns to perform high-level tasks across multiple platforms, including Windows, Linux, MacOS, Android, and Web. Utilizing only 0.02% of the data compared to previous methods, GUI-R1 demonstrates superior performance on eight benchmarks, showcasing the potential of reinforcement learning in GUI agent tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://scai.cs.jhu.edu/projects/RealWebAssist/" rel="nofollow">RealWebAssist: A Benchmark for Long-Horizon Web Assistance with Real-World Users</a></p>
<ul dir="auto">
<li>Suyu Ye, Haojun Shi, Darren Shih, Hyokun Yun, Tanya Roosta, Tianmin Shu</li>
<li>🏛️ Institutions: JHU, Amazon</li>
<li>📅 Date: April 14, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [GUI grounding], [speech input], [spatial reasoning], [temporal reasoning], [multi-step planning], [routine learning], [RealWebAssist]</li>
<li>📖 TLDR: RealWebAssist introduces a benchmark for evaluating AI agents' ability to assist with long-horizon web tasks using sequential instructions from real-world users. The dataset includes 1,885 instructions across 107 tasks on 66 websites, featuring challenges like ambiguous instructions, GUI grounding, and evolving user goals. Evaluations show that current state-of-the-art models struggle with these complex, realistic scenarios.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://agent-reward-bench.github.io/" rel="nofollow">AgentRewardBench: Evaluating Automatic Evaluations of Web Agent Trajectories</a></p>
<ul dir="auto">
<li>Xing Han Lù, Amirhossein Kazemnejad, Nicholas Meade, Arkil Patel, Dongchan Shin, Alejandra Zambrano, Karolina Stańczak, Peter Shaw, Christopher J. Pal, Siva Reddy</li>
<li>🏛️ Institutions: McGill, Mila, Google DeepMind, Polytechnique Montréal, ServiceNow Research</li>
<li>📅 Date: April 11, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [evaluation], [LLM judges], [AgentRewardBench]</li>
<li>📖 TLDR: This paper introduces <em>AgentRewardBench</em>, a benchmark designed to assess the effectiveness of large language model (LLM) judges in evaluating web agent trajectories. The benchmark comprises 1,302 trajectories across five web benchmarks, each annotated by experts for success, side effects, and repetitiveness. Evaluations of 12 LLM judges reveal that no single model excels across all benchmarks, and that rule-based evaluations often underreport agent success rates, highlighting the need for more adaptable automatic evaluation methods.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.07079" rel="nofollow">SkillWeaver: Web Agents can Self-Improve by Discovering and Honing Skills</a></p>
<ul dir="auto">
<li>Boyuan Zheng, Michael Y. Fatemi, Xiaolong Jin, Zora Zhiruo Wang, Apurva Gandhi, Yueqi Song, Yu Gu, Jayanth Srinivasa, Gaowen Liu, Graham Neubig, Yu Su</li>
<li>🏛️ Institutions: OSU, CMU, UVA, Purdue, Cisco Research</li>
<li>📅 Date: April 9, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [API synthesis], [skill discovery], [transfer learning], [WebArena]</li>
<li>📖 TLDR: SkillWeaver is a framework that enables web agents to autonomously improve by discovering, practicing, and refining reusable skills, encapsulated as APIs. Through iterative exploration, agents build a library of plug-and-play APIs, enhancing their capabilities. Experiments on WebArena and real-world websites demonstrate significant performance improvements, and the synthesized APIs can be shared among agents to boost overall performance.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.06821" rel="nofollow">Inducing Programmatic Skills for Agentic Tasks</a></p>
<ul dir="auto">
<li>Zora Zhiruo Wang, Apurva Gandhi, Graham Neubig, Daniel Fried</li>
<li>🏛️ Institutions: CMU, Microsoft</li>
<li>📅 Date: April 9, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [model], [benchmark], [learning], [reasoning], [planning], [ASI], [WebArena]</li>
<li>📖 TLDR: This paper introduces Agent Skill Induction (ASI), a framework enabling web agents to learn and apply programmatic skills dynamically. By representing skills as executable programs, ASI allows agents to verify and reuse these skills across tasks, enhancing adaptability and efficiency. Evaluated on the WebArena benchmark, ASI outperforms static and text-based skill agents in success rate and step efficiency, demonstrating improved generalization and adaptability to new web environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.04716" rel="nofollow">On the Robustness of GUI Grounding Models Against Image Attacks</a></p>
<ul dir="auto">
<li>Haoren Zhao, Tianyi Chen, Zhen Wang</li>
<li>🏛️ Institutions: HDU, Microsoft</li>
<li>📅 Date: April 7, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [benchmark], [robustness], [adversarial attacks], [UGround], [ScreenSpot-V2]</li>
<li>📖 TLDR: This paper systematically evaluates the robustness of GUI grounding models, such as UGround, against natural noise, untargeted adversarial attacks, and targeted adversarial attacks. Experiments conducted across mobile, desktop, and web interfaces reveal that these models are highly sensitive to adversarial perturbations and low-resolution conditions. The findings highlight vulnerabilities in current GUI grounding models and establish a benchmark for future research aimed at enhancing their robustness in practical applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.01382" rel="nofollow">An Illusion of Progress? Assessing the Current State of Web Agents</a></p>
<ul dir="auto">
<li>Tianci Xue, Weijian Qi, Tianneng Shi, Chan Hee Song, Boyu Gou, Dawn Song, Huan Sun, Yu Su</li>
<li>🏛️ Institutions: OSU, UC Berkeley</li>
<li>📅 Date: April 2, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [evaluation], [Online-Mind2Web], [WebJudge], [Operator], [LLM-as-a-Judge]</li>
<li>📖 TLDR: This paper critically evaluates the performance of current web agents, revealing that many underperform on the newly introduced <strong>Online-Mind2Web</strong> benchmark, which comprises 300 realistic tasks across 136 websites. The study highlights a discrepancy between reported successes and actual capabilities, attributing this to shortcomings in existing benchmarks. To address evaluation scalability, the authors propose <strong>WebJudge</strong>, an automatic LLM-based evaluation method achieving approximately 85% agreement with human judgments. The comprehensive analysis underscores the need for more robust assessment frameworks in web agent research.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2504.00906" rel="nofollow">Agent S2: A Compositional Generalist-Specialist Framework for Computer Use Agents</a></p>
<ul dir="auto">
<li>Saaket Agashe, Kyle Wong, Vincent Tu, Jiachen Yang, Ang Li, Xin Eric Wang</li>
<li>🏛️ Institutions: Simular Research</li>
<li>📅 Date: April 1, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [Mixture-of-Grounding], [Proactive Hierarchical Planning], [benchmark], [OSWorld], [WindowsAgentArena], [AndroidWorld]</li>
<li>📖 TLDR: This paper introduces <em>Agent S2</em>, a compositional framework for computer use agents that combines generalist and specialist models to address challenges in GUI element grounding and long-horizon task planning. The framework employs a novel Mixture-of-Grounding technique for precise GUI localization and Proactive Hierarchical Planning to dynamically refine action plans. Evaluations demonstrate that Agent S2 achieves state-of-the-art performance on benchmarks like OSWorld, WindowsAgentArena, and AndroidWorld, outperforming existing agents such as Claude Computer Use and UI-TARS.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.23350" rel="nofollow">A Survey of WebAgents: Towards Next-Generation AI Agents for Web Automation with Large Foundation Models</a></p>
<ul dir="auto">
<li>Liangbo Ning, Ziran Liang, Zhuohang Jiang, Haohao Qu, Yujuan Ding, Wenqi Fan, Xiao-yong Wei, Shanru Lin, Hui Liu, Philip S. Yu, Qing Li</li>
<li>🏛️ Institutions: PolyU, CityUHK, MSU, UIC</li>
<li>📅 Date: March 30, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [survey], [framework], [training], [trustworthiness], [WebAgents], [Large Foundation Models]</li>
<li>📖 TLDR: This comprehensive survey examines the development of WebAgents—AI agents designed to automate web tasks—by leveraging Large Foundation Models (LFMs). It delves into the architectures, training methodologies, and trustworthiness of these agents, providing a detailed overview of current research and proposing future directions to enhance their effectiveness and reliability in web automation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.21620" rel="nofollow">UI-R1: Enhancing Action Prediction of GUI Agents by Reinforcement Learning</a></p>
<ul dir="auto">
<li>Zhengxi Lu, Yuxiang Chai, Yaxuan Guo, Xi Yin, Liang Liu, Hao Wang, Guanjing Xiong, Hongsheng Li</li>
<li>🏛️ Institutions: vivo AI Lab, MMLab @ CUHK</li>
<li>📅 Date: March 27, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [reinforcement learning], [UI-R1-3B], [GRPO], [AndroidControl], [ScreenSpot-Pro]</li>
<li>📖 TLDR: This paper introduces <strong>UI-R1</strong>, a framework that enhances the reasoning capabilities of multimodal large language models (MLLMs) for GUI action prediction tasks through rule-based reinforcement learning. Utilizing a small, high-quality dataset of 136 challenging mobile tasks, the authors design a unified rule-based action reward function and optimize the model using Group Relative Policy Optimization (GRPO). The resulting model, <strong>UI-R1-3B</strong>, demonstrates significant improvements over the base model (Qwen2.5-VL-3B) on both in-domain (AndroidControl) and out-of-domain (ScreenSpot-Pro) benchmarks, showcasing the effectiveness of rule-based RL in advancing GUI understanding and control.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.15937" rel="nofollow">Advancing Mobile GUI Agents: A Verifier-Driven Approach to Practical Deployment</a></p>
<ul dir="auto">
<li>Gaole Dai, Shiqi Jiang, Ting Cao, Yuanchun Li, Yuqing Yang, Rui Tan, Mo Li, Lili Qiu</li>
<li>🏛️ Institutions: Tsinghua University, NTU, UT Austin</li>
<li>📅 Date: March 20, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [model], [dataset], [benchmark], [V-Droid], [verifier-driven], [pair-wise preference training], [human-agent joint annotation], [low-latency]</li>
<li>📖 TLDR: This paper introduces <strong>V-Droid</strong>, a mobile GUI automation agent that leverages large language models (LLMs) as verifiers rather than generators. By evaluating candidate actions before execution, V-Droid enhances decision-making accuracy and reduces latency. The framework incorporates discretized action space construction, a prefilling-only workflow, pair-wise preference training, and a scalable human-agent joint annotation scheme. Evaluated on benchmarks like AndroidWorld, AndroidLab, and MobileAgentBench, V-Droid achieves state-of-the-art success rates and operates with near-real-time decision-making capabilities.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.15661" rel="nofollow">UI-Vision: A Desktop-centric GUI Benchmark for Visual Perception and Interaction</a></p>
<ul dir="auto">
<li>Shravan Nayak, Xiangru Jian, Kevin Qinghong Lin, Juan A. Rodriguez, Montek Kalsi, Rabiul Awal, Nicolas Chapados, M. Tamer Özsu, Aishwarya Agrawal, David Vazquez, Christopher Pal, Perouz Taslakian, Spandana Gella, Sai Rajeswar</li>
<li>🏛️ Institutions: Unknown</li>
<li>📅 Date: March 19, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [benchmark], [dataset], [framework], [UI-Vision], [UI-TARS-72B], [spatial reasoning], [drag-and-drop], [element grounding], [layout grounding], [action prediction]</li>
<li>📖 TLDR: This paper introduces <em>UI-Vision</em>, a comprehensive, license-permissive benchmark designed for evaluating autonomous agents in real-world desktop GUI environments. It encompasses 83 software applications with dense annotations of human demonstrations, including bounding boxes, UI labels, and action trajectories. The benchmark defines three tasks—Element Grounding, Layout Grounding, and Action Prediction—to assess agents' performance. Evaluations reveal limitations in state-of-the-art models like UI-TARS-72B, particularly in understanding professional software, spatial reasoning, and complex actions such as drag-and-drop. By releasing UI-Vision as open-source, the authors aim to advance the development of more capable agents for desktop tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/FanbinLu/STEVE">STEVE: A Step Verification Pipeline for Computer-use Agent Training</a></p>
<ul dir="auto">
<li>Fanbin Lu, Zhisheng Zhong, Ziqin Wei, Shu Liu, Chi-Wing Fu, Jiaya Jia</li>
<li>🏛️ Institutions: CUHK, SmartMore, HKUST</li>
<li>📅 Date: March 16, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [model], [UI grounding], [KTO], [GPT-4o], [WinAgentArena]</li>
<li>📖 TLDR: This paper introduces <strong>STEVE</strong>, a step verification pipeline designed to enhance the training of computer-use agents. The approach involves creating a large instruction set and collecting trajectory data using suboptimal agents. GPT-4o is employed to verify the correctness of each action step by comparing pre- and post-action screenshots, assigning binary labels. The agent is then optimized using Kahneman and Tversky Optimization (KTO) based on these labels. The result is a 7B vision-language model that achieves state-of-the-art performance in the WinAgentArena desktop environment, outperforming supervised fine-tuning methods by effectively leveraging both positive and negative action data.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.11170" rel="nofollow">DeskVision: Large Scale Desktop Region Captioning for Advanced GUI Agents</a></p>
<ul dir="auto">
<li>Yibin Xu, Liang Yang, Hao Chen, Hua Wang, Zhi Chen, Yaohua Tang</li>
<li>🏛️ Institutions: Unknown</li>
<li>📅 Date: March 14, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [model], [AutoCaptioner], [DeskVision], [DeskVision-Eval], [GUIExplorer]</li>
<li>📖 TLDR: This paper introduces <em>AutoCaptioner</em>, an automated pipeline for generating richly annotated GUI data with minimal human effort. Utilizing AutoCaptioner, the authors created <em>DeskVision</em>, a large-scale desktop GUI dataset comprising 54,855 images and over 303,622 annotations across various operating systems. They also developed <em>DeskVision-Eval</em>, the largest desktop benchmark reflecting daily usage scenarios. Leveraging DeskVision, the authors trained <em>GUIExplorer</em>, a GUI understanding model that achieves state-of-the-art performance in grounding visual elements without complex architectural designs. The effectiveness of DeskVision was further validated through ablation studies on various large visual language models (LVLMs).</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.09241" rel="nofollow">In-Context Defense in Computer Agents: An Empirical Study</a></p>
<ul dir="auto">
<li>Pei Yang, Hai Ci, Mike Zheng Shou</li>
<li>🏛️ Institutions: Unknown</li>
<li>📅 Date: March 12, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [defense], [in-context learning], [chain-of-thought], [context deception], [security]</li>
<li>📖 TLDR: This paper introduces an in-context defense strategy for computer agents powered by vision-language models (VLMs), targeting context deception attacks such as malicious pop-ups and deceptive HTML elements. By incorporating a small set of curated exemplars and employing chain-of-thought reasoning, the approach guides agents to perform explicit defensive reasoning before action planning. Experiments demonstrate significant reductions in attack success rates across various attack types, highlighting the effectiveness of the method in enhancing agent reliability without requiring model fine-tuning.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2503.06470" rel="nofollow">Think Twice, Click Once: Enhancing GUI Grounding via Fast and Slow Systems</a></p>
<ul dir="auto">
<li>Fei Tang, Yongliang Shen, Hang Zhang, Siqi Chen, Guiyang Hou, Wenqi Zhang, Wenqiao Zhang, Kaitao Song, Weiming Lu, Yueting Zhuang</li>
<li>🏛️ Institutions: Zhejiang University</li>
<li>📅 Date: March 9, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [dual-system cognition], [FOCUS], [ScreenSpot], [ScreenSpot-Pro]</li>
<li>📖 TLDR: This paper introduces <strong>FOCUS</strong>, a novel GUI grounding framework inspired by human dual-system cognition. FOCUS dynamically switches between a fast, intuitive system and a slow, analytical system based on task complexity. The framework decomposes GUI grounding into three stages: interface summarization, focused analysis, and precise coordinate prediction. Trained on a synthesized dataset of 300,000 samples, FOCUS achieves state-of-the-art performance on the ScreenSpot and ScreenSpot-Pro benchmarks, demonstrating significant improvements in both efficiency and accuracy for complex GUI interactions.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/PathOnAI/LiteWebAgent">LiteWebAgent: The Open-Source Suite for VLM-Based Web-Agent Applications</a></p>
<ul dir="auto">
<li>Danqing Zhang, Balaji Rama, Jingyi Ni, Shiying He, Fu Zhao, Kunyu Chen, Arnold Chen, Junyu Cao</li>
<li>🏛️ Institutions: PathOnAI.org, Rutgers Univ., UT Austin</li>
<li>📅 Date: March 4, 2025</li>
<li>📑 Publisher: NAACL 2025</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [planning], [memory], [tree search], [LiteWebAgent]</li>
<li>📖 TLDR: LiteWebAgent is an open-source suite designed for VLM-based web agent applications. It offers a modular framework that decouples action generation from grounding, supports agent planning, memory, and tree search, and is deployable via a Vercel-based web app or a Chrome extension using the Chrome DevTools Protocol (CDP).</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://vulnerable-ai-agents.github.io/" rel="nofollow">Why Are Web AI Agents More Vulnerable Than Standalone LLMs? A Security Analysis</a></p>
<ul dir="auto">
<li>Jeffrey Yang Fan Chiang, Seungjae Lee, Jia-Bin Huang, Furong Huang, Yizheng Chen</li>
<li>🏛️ Institutions: UMD</li>
<li>📅 Date: March 4, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [benchmark], [security], [jailbreaking], [evaluation], [OpenHands]</li>
<li>📖 TLDR: This paper investigates why Web AI agents are significantly more susceptible to executing harmful commands compared to standalone LLMs, despite sharing the same underlying models. Through a fine-grained evaluation, the authors identify three critical factors contributing to this vulnerability: embedding user goals into system prompts, multi-step action generation, and processing of event streams from web navigation. The study introduces a five-level harmfulness evaluation framework and utilizes the OpenHands platform to systematically assess these vulnerabilities, revealing a 46.6% success rate in malicious task execution by Web AI agents versus 0% for standalone LLMs.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2502.13053" rel="nofollow">AEIA-MN: Evaluating the Robustness of Multimodal LLM-Powered Mobile Agents Against Active Environmental Injection Attacks</a></p>
<ul dir="auto">
<li>Yurun Chen, Xueyu Hu, Keting Yin, Juncheng Li, Shengyu Zhang</li>
<li>🏛️ Institutions: Zhejiang University</li>
<li>📅 Date: February 18, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [safety], [AEIA-MN]</li>
<li>📖 TLDR: This paper introduces the concept of Active Environment Injection Attack (AEIA), where attackers disguise malicious actions as environmental elements to disrupt AI agents' decision-making processes. The authors propose AEIA-MN, an attack scheme leveraging mobile notifications to evaluate the robustness of multimodal large language model-based mobile agents. Experimental results demonstrate that even advanced models are highly vulnerable to such attacks, with success rates reaching up to 93% in the AndroidWorld benchmark.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://microsoft.github.io/Magma/" rel="nofollow">Magma: A Foundation Model for Multimodal AI Agents</a></p>
<ul dir="auto">
<li>Jianwei Yang, Reuben Tan, Qianhui Wu, Ruijie Zheng, Baolin Peng, Yongyuan Liang, Yu Gu, Mu Cai, Seonghyeon Ye, Joel Jang, Yuquan Deng, Lars Liden, Jianfeng Gao</li>
<li>🏛️ Institutions: Microsoft Research, Univ. of Maryland, Univ. of Wisconsin-Madison, KAIST, Univ. of Washington</li>
<li>📅 Date: Feb 18, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI], [Robotics]</li>
<li>🔑 Key: [model], [framework], [SoM], [ToM], [robotics], [UI navigation]</li>
<li>📖 TLDR: This paper introduces <strong>Magma</strong>, a foundation model designed for multimodal AI agents operating in both digital and physical environments. Magma extends traditional vision-language models by incorporating planning and action capabilities, enabling tasks from UI navigation to robotic manipulation. The model is pretrained on diverse datasets, including images, videos, and robotics data, utilizing <strong>Set-of-Mark (SoM)</strong> for action grounding and <strong>Trace-of-Mark (ToM)</strong> for action planning. Experiments demonstrate that SoM and ToM synergistically enhance Magma's spatial-temporal intelligence, achieving state-of-the-art performance in UI navigation and robotic manipulation tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2502.11357" rel="nofollow">Explorer: Scaling Exploration-driven Web Trajectory Synthesis for Multimodal Web Agents</a></p>
<ul dir="auto">
<li>Vardaan Pahuja, Yadong Lu, Corby Rosset, Boyu Gou, Arindam Mitra, Spencer Whitehead, Yu Su, Ahmed Awadallah</li>
<li>🏛️ Institutions: Microsoft Research, The Ohio State University</li>
<li>📅 Date: February 17, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [web agents], [reinforcement learning], [Explorer]</li>
<li>📖 TLDR: This paper introduces <em>Explorer</em>, a multimodal web agent trained on a newly synthesized dataset comprising over 94,000 successful web trajectories. The dataset, generated through scalable exploration and refinement techniques, spans 49,000 unique URLs and includes 720,000 screenshots and 33 million web elements. <em>Explorer</em> demonstrates strong performance on benchmarks like Mind2Web-Live, Multimodal-Mind2Web, and MiniWob++, highlighting the importance of data scaling in enhancing web agent capabilities.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2502.07949" rel="nofollow">VSC-RL: Advancing Autonomous Vision-Language Agents with Variational Subgoal-Conditioned Reinforcement Learning</a></p>
<ul dir="auto">
<li>Qingyuan Wu, Jianheng Liu, Jianye Hao, Jun Wang, Kun Shao</li>
<li>🏛️ Institutions: Huawei Noah's Ark Lab, Univ. College London</li>
<li>📅 Date: February 11, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [reinforcement learning], [subgoal generation], [VSC-RL], [learning efficiency]</li>
<li>📖 TLDR: This paper introduces <strong>VSC-RL</strong>, a novel reinforcement learning framework that enhances learning efficiency for vision-language agents in complex sequential decision-making tasks. By reformulating these tasks as variational goal-conditioned problems, VSC-RL leverages advanced optimization techniques and utilizes vision-language models to autonomously decompose goals into feasible subgoals. Empirical results demonstrate that VSC-RL significantly outperforms state-of-the-art agents, particularly in challenging mobile device control tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2502.06395" rel="nofollow">AppVLM: A Lightweight Vision Language Model for Online App Control</a></p>
<ul dir="auto">
<li>Georgios Papoudakis, Thomas Coste, Zhihao Wu, Jianye Hao, Jun Wang, Kun Shao</li>
<li>🏛️ Institutions: Univ. of Cambridge, Huawei Noah's Ark Lab, Univ. College London</li>
<li>📅 Date: February 10, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [model], [framework], [evaluation], [AppVLM], [on-device control]</li>
<li>📖 TLDR: This paper introduces <strong>AppVLM</strong>, a lightweight Vision-Language Model designed for efficient on-device control of smartphone applications. AppVLM is fine-tuned on the AndroidControl dataset and further refined through interactions within the AndroidWorld environment. It achieves superior action prediction accuracy in offline evaluations and matches GPT-4o in online task completion success rates, while operating up to ten times faster, making it a practical solution for real-world deployment.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2502.02982" rel="nofollow">MobileA3gent: Training Mobile GUI Agents Using Decentralized Self-Sourced Data from Diverse Users</a></p>
<ul dir="auto">
<li>Wenhao Wang, Mengying Yuan, Zijie Yu, Guangyi Liu, Rui Ye, Tian Jin, Siheng Chen, Yanfeng Wang</li>
<li>🏛️ Institutions: ZJU, SJTU, Shanghai AI Lab, MAGIC</li>
<li>📅 Date: February 5, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [Auto-Annotation], [federated learning], [privacy], [non-IID], [adapted aggregation]</li>
<li>📖 TLDR: This paper introduces <em>MobileA3gent</em>, a framework for training mobile GUI agents using decentralized, self-sourced data from diverse users. It addresses challenges in extracting user instructions without human intervention and utilizing distributed data while preserving privacy. The framework comprises two key techniques: <em>Auto-Annotation</em>, which automatically collects high-quality datasets during users' routine phone usage, and <em>FedVLM-A</em>, which enhances federated training on non-IID user data by incorporating both episode- and step-level distributions. Experiments demonstrate that MobileA3gent achieves performance comparable to centralized human-annotated models at less than 1% of the cost, highlighting its potential for real-world applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2501.12326" rel="nofollow">UI-TARS: Pioneering Automated GUI Interaction with Native Agents</a></p>
<ul dir="auto">
<li>Yujia Qin, Yining Ye, Junjie Fang, Haoming Wang, Shihao Liang, Shizuo Tian, Junda Zhang, Jiahao Li, Yunxin Li, Shijue Huang, Wanjun Zhong, Kuanye Li, Jiale Yang, Yu Miao, Woyu Lin, Longxiang Liu, Xu Jiang, Qianli Ma, Jingyu Li, Xiaojun Xiao, Kai Cai, Chuang Li, Yaowei Zheng, Chaolin Jin, Chen Li, Xiao Zhou, Minchao Wang, Haoli Chen, Zhaojian Li, Haihua Yang, Haifeng Liu, Feng Lin, Tao Peng, Xin Liu, Guang Shi</li>
<li>🏛️ Institutions: ByteDance, Tsinghua University</li>
<li>📅 Date: January 21, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [UI-TARS]</li>
<li>📖 TLDR: This paper introduces <strong>UI-TARS</strong>, a native GUI agent model that processes screenshots to perform human-like interactions such as keyboard and mouse operations. Unlike traditional frameworks relying on commercial models with handcrafted prompts, UI-TARS is an end-to-end model demonstrating superior performance across over 10 GUI agent benchmarks. Key innovations include enhanced perception through large-scale GUI screenshot datasets, unified action modeling across platforms, incorporation of deliberate multi-step reasoning (System-2 Reasoning), and iterative training with reflective online traces, enabling continuous learning and adaptation with minimal human intervention.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2501.07572" rel="nofollow">WebWalker: Benchmarking LLMs in Web Traversal</a></p>
<ul dir="auto">
<li>Jialong Wu, Wenbiao Yin, Yong Jiang, Zhenglin Wang, Zekun Xi, Runnan Fang, Deyu Zhou, Pengjun Xie, Fei Huang</li>
<li>🏛️ Institutions: Tongyi Lab, Alibaba NLP</li>
<li>📅 Date: January 13, 2025</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [framework], [RAG], [WebWalker], [WebWalkerQA]</li>
<li>📖 TLDR: This paper presents <strong>WebWalker</strong>, a multi-agent framework designed to improve the ability of large language models (LLMs) to traverse websites, addressing the challenges of retrieving complex, multi-layered information. WebWalker integrates an "explore-critic" paradigm, where the explorer agent navigates the web, and the critic agent evaluates the progress. The <strong>WebWalkerQA</strong> benchmark is introduced to assess web traversal tasks, showing how retrieval-augmented generation (RAG) can be enhanced with vertical exploration to solve real-world queries.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://likaixin2000.github.io/papers/ScreenSpot_Pro.pdf" rel="nofollow">ScreenSpot-Pro: GUI Grounding for Professional High-Resolution Computer Use</a></p>
<ul dir="auto">
<li>Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua</li>
<li>🏛️ Institutions: NUS, ECNU, HKBU</li>
<li>📅 Date: January 3, 2025</li>
<li>📑 Publisher: GitHub</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [benchmark], [GUI grounding], [high-resolution], [ScreenSpot-Pro]</li>
<li>📖 TLDR: ScreenSpot-Pro introduces a benchmark designed to evaluate GUI grounding models in professional, high-resolution environments. It encompasses 1,581 tasks across 23 applications in various industries, highlighting the challenges models face with complex software interfaces. Current models achieve low accuracy, underscoring the need for further research in this domain.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://qiushisun.github.io/OS-Genesis-Home/" rel="nofollow">OS-Genesis: Automating GUI Agent Trajectory Construction via Reverse Task Synthesis</a></p>
<ul dir="auto">
<li>Qiushi Sun, Kanzhi Cheng, Zichen Ding, Chuanyang Jin, Yian Wang, Fangzhi Xu, Zhenyu Wu, Chengyou Jia, Liheng Chen, Zhoumianze Liu, Ben Kao, Guohao Li, Junxian He, Yu Qiao, Zhiyong Wu</li>
<li>🏛️ Institutions: Shanghai AI Lab, HKU, Johns Hopkins University, SJTU, Oxford, HKUST</li>
<li>📅 Date: Dec 27, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [trajectory data], [reward model], [e2e model], [data synthesis], [OS-Genesis]</li>
<li>📖 TLDR: This paper introduces <em>OS-Genesis</em>, an interaction-driven pipeline that automates the construction of high-quality and diverse GUI agent trajectory data without human supervision. By employing reverse task synthesis and a trajectory reward model, OS-Genesis enables effective end-to-end training of GUI agents, significantly enhancing their performance on online benchmarks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.17589" rel="nofollow">PC Agent: While You Sleep, AI Works -- A Cognitive Journey into Digital World</a></p>
<ul dir="auto">
<li>Yanheng He, Jiahe Jin, Shijie Xia, Jiadi Su, Runze Fan, Haoyang Zou, Xiangkun Hu, Pengfei Liu</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: Dec 23, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [PC Agent], [human cognition transfer], [PC Tracker], [Cognition Completion], [multi-agent system]</li>
<li>📖 TLDR: This paper introduces <em>PC Agent</em>, an AI system designed to autonomously perform complex computer-based tasks by emulating human cognitive processes. The system comprises three key components: <strong>PC Tracker</strong>, a lightweight infrastructure for collecting high-quality human-computer interaction data; a <strong>Cognition Completion</strong> pipeline that enriches raw interaction data into detailed cognitive trajectories; and a <strong>multi-agent system</strong> combining a planning agent for decision-making with a grounding agent for precise visual grounding. This approach represents a significant advancement toward AI systems capable of handling intricate real-world work autonomously.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/OS-Agent-Survey/OS-Agent-Survey">OS Agents: A Survey on MLLM-based Agents for General Computing Devices Use</a></p>
<ul dir="auto">
<li>Xueyu Hu, Tao Xiong, Biao Yi, Zishu Wei, Ruixuan Xiao, Yurun Chen, Jiasheng Ye, Meiling Tao, Xiangxin Zhou, Ziyu Zhao, Yuhuai Li, Shengze Xu, Shawn Wang, Xinchen Xu, Shuofei Qiao , Kun Kuang, Tieyong Zeng, Liang Wang, Jiwei Li, Yuchen Eleanor Jiang, Wangchunshu Zhou, Guoyin Wang, Keting Yin, Zhou Zhao, Hongxia Yang, Fan Wu, Shengyu Zhang, Fei Wu</li>
<li>🏛️ Institutions: Zhejiang University, Fudan University, OPPO AI Center, University of Chinese Academy of Sciences, Chinese Academy of Sciences, The Chinese HKU, Tsinghua University, 01.AI, The Hong Kong Polytechnic University, SJTU</li>
<li>📅 Date: December 20, 2024</li>
<li>📑 Publisher: Github Repo</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [survey]</li>
<li>📖 TLDR: This paper conducts a comprehensive survey on OS Agents, which are (M)LLM-based agents that use computing devices (e.g., computers and mobile phones) by operating within the environments and interfaces (e.g., Graphical User Interface (GUI)) provided by operating systems (OS) to automate tasks. The survey begins by elucidating the fundamentals of OS Agents, exploring their key components including the environment, observation space, and action space, and outlining essential capabilities such as understanding, planning, and grounding. Methodologies for constructing OS Agents are examined, with a focus on domain-specific foundation models and agent frameworks. A detailed review of evaluation protocols and benchmarks highlights how OS Agents are assessed across diverse tasks. Finally, current challenges and promising future research directions, including safety and privacy, personalization and self-evolution, are discussed.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://ariaui.github.io/" rel="nofollow">Aria-UI: Visual Grounding for GUI Instructions</a></p>
<ul dir="auto">
<li>Yuhao Yang, Yue Wang, Dongxu Li, Ziyang Luo, Bei Chen, Chao Huang, Junnan Li</li>
<li>🏛️ Institutions: HKU, Rhymes AI</li>
<li>📅 Date: Dec 20, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [grounding], [visual grounding], [Aria-UI]</li>
<li>📖 TLDR: This paper introduces <em>Aria-UI</em>, a large multimodal model specifically designed for GUI grounding. Aria-UI employs a pure-vision approach, avoiding reliance on auxiliary inputs like HTML or AXTree. It utilizes a scalable data pipeline to synthesize diverse and high-quality instruction samples and incorporates textual and text-image interleaved action histories for robust context-aware reasoning. Aria-UI achieves state-of-the-art results across offline and online agent benchmarks, outperforming both vision-only and AXTree-reliant baselines.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.13501" rel="nofollow">GUI Agents: A Survey</a></p>
<ul dir="auto">
<li>Dang Nguyen, Jian Chen, Yu Wang, Gang Wu, Namyong Park, Zhengmian Hu, Hanjia Lyu, Junda Wu, Ryan Aponte, Yu Xia, Xintong Li, Jing Shi, Hongjie Chen, Viet Dac Lai, Zhouhang Xie, Sungchul Kim, Ruiyi Zhang, Tong Yu, Mehrab Tanjim, Nesreen K. Ahmed, Puneet Mathur, Seunghyun Yoon, Lina Yao, Branislav Kveton, Thien Huu Nguyen, Trung Bui, Tianyi Zhou, Ryan A. Rossi, Franck Dernoncourt</li>
<li>🏛️ Institutions: UMD, SUNY Buffalo, University of Oregon, Adobe Research, Meta AI, University of Rochester, UCSD, CMU, Dolby Labs, Intel AI Research, UNSW</li>
<li>📅 Date: December 18, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [survey]</li>
<li>📖 TLDR: This survey provides a comprehensive overview of GUI agents powered by Large Foundation Models, detailing their benchmarks, evaluation metrics, architectures, and training methods. It introduces a unified framework outlining their perception, reasoning, planning, and acting capabilities, identifies open challenges, and discusses future research directions, serving as a resource for both practitioners and researchers in the field.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.13194" rel="nofollow">Proposer-Agent-Evaluator (PAE): Autonomous Skill Discovery For Foundation Model Internet Agents</a></p>
<ul dir="auto">
<li>Yifei Zhou, Qianlan Yang, Kaixiang Lin, Min Bai, Xiong Zhou, Yu-Xiong Wang, Sergey Levine, Erran Li</li>
<li>🏛️ Institutions: UCB, UIUC, Amazon</li>
<li>📅 Date: December 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [reinforcement learning], [skill discovery], [PAE]</li>
<li>📖 TLDR: This paper introduces the Proposer-Agent-Evaluator (PAE) system, enabling foundation model agents to autonomously discover and practice skills in real-world web environments. PAE comprises a context-aware task proposer, an agent policy for task execution, and a vision-language model-based success evaluator. Validated on vision-based web navigation tasks, PAE significantly enhances zero-shot generalization capabilities of vision-language model Internet agents, achieving over 30% relative improvement on unseen tasks and websites, and surpassing state-of-the-art open-source agents by more than 10%.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.10342" rel="nofollow">Iris: Breaking GUI Complexity with Adaptive Focus and Self-Refining</a></p>
<ul dir="auto">
<li>Zhiqi Ge, Juncheng Li, Xinglei Pang, Minghe Gao, Kaihang Pan, Wang Lin, Hao Fei, Wenqiao Zhang, Siliang Tang, Yueting Zhuang</li>
<li>🏛️ Institutions: Zhejiang University, NUS</li>
<li>📅 Date: December 13, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [Information-Sensitive Cropping], [Self-Refining Dual Learning], [visual grounding], [model]</li>
<li>📖 TLDR: This paper introduces <em>Iris</em>, a visual agent designed to enhance GUI automation by addressing challenges in high-resolution, complex digital environments. It employs two key innovations: <strong>Information-Sensitive Cropping (ISC)</strong>, which dynamically identifies and prioritizes visually dense regions using an edge detection algorithm for efficient processing, and <strong>Self-Refining Dual Learning (SRDL)</strong>, which enhances the agent's ability to handle complex tasks through a dual-learning loop that iteratively refines its performance without requiring additional annotated data. Empirical evaluations demonstrate that Iris achieves state-of-the-art performance across multiple benchmarks with only 850K GUI annotations, outperforming methods using ten times more training data.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.09362" rel="nofollow">Falcon-UI: Understanding GUI Before Following User Instructions</a></p>
<ul dir="auto">
<li>Huawen Shen, Chang Liu, Gengluo Li, Xinlong Wang, Yu Zhou, Can Ma, Xiangyang Ji</li>
<li>🏛️ Institutions: Chinese Academy of Sciences, Tsinghua University, Nankai University, BAAI</li>
<li>📅 Date: Dec 12, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [Falcon-UI], [GUI understanding]</li>
<li>📖 TLDR: This paper introduces <em>Falcon-UI</em>, a GUI agent model that emphasizes understanding GUI contexts before following user instructions. The authors present the <em>Insight-UI Dataset</em>, an instruction-free GUI navigation dataset generated from the Common Crawl corpus, simulating various platforms like iOS, Android, Windows, and Linux across multiple resolutions on 312K domains. Falcon-UI is pretrained on this dataset and fine-tuned on Android and Web GUI datasets, achieving performance comparable to larger models, highlighting the importance of decoupling GUI understanding from instruction following in agent performance.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.05467" rel="nofollow">The BrowserGym Ecosystem for Web Agent Research</a></p>
<ul dir="auto">
<li>Thibault Le Sellier De Chezelles, Maxime Gasse, Alexandre Drouin, Massimo Caccia, Léo Boisvert, Megh Thakkar, Tom Marty, Rim Assouel, Sahar Omidi Shayegan, Lawrence Keunho Jang, Xing Han Lù, Ori Yoran, Dehan Kong, Frank F. Xu, Siva Reddy, Quentin Cappart, Graham Neubig, Ruslan Salakhutdinov, Nicolas Chapados, Alexandre Lacoste</li>
<li>🏛️ Institutions: ServiceNow Research, Mila, Polytechnique Montréal, CMU, McGill University, Tel Aviv University, Université de Montréal, iMean AI</li>
<li>📅 Date: December 6, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [framework], [LLM], [automation], [BrowserGym], [AgentLab]</li>
<li>📖 TLDR: This paper presents <em>BrowserGym</em>, an ecosystem designed to standardize the evaluation and benchmarking of web agents, particularly those leveraging Large Language Models (LLMs). It addresses the challenges posed by fragmented benchmarks and inconsistent methodologies in web agent research. BrowserGym provides a unified, gym-like environment with clearly defined observation and action spaces, enabling reproducible comparisons across various benchmarks. Additionally, <em>AgentLab</em>, a complementary framework, supports agent creation, testing, and analysis. The paper also features a large-scale experiment comparing the performance of 6 leading LLMs, highlighting the strengths and weaknesses of different models in real-world web tasks, while emphasizing the ongoing challenges in building efficient and robust web agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://aguvis-project.github.io/" rel="nofollow">Aguvis: Unified Pure Vision Agents for Autonomous GUI Interaction</a></p>
<ul dir="auto">
<li>Yiheng Xu, Zekun Wang, Junli Wang, Dunjie Lu, Tianbao Xie, Amrita Saha, Doyen Sahoo, Tao Yu, Caiming Xiong</li>
<li>🏛️ Institutions: HKU, NTU, Salesforce</li>
<li>📅 Date: Dec 5, 2024</li>
<li>📑 Publisher: ICML 2025</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [planning], [reasoning], [Aguvis], [visual grounding]</li>
<li>📖 TLDR: This paper introduces <em>Aguvis</em>, a unified pure vision-based framework for autonomous GUI agents that operates across various platforms. It leverages image-based observations and grounds natural language instructions to visual elements, employing a consistent action space to ensure cross-platform generalization. The approach integrates explicit planning and reasoning within the model, enhancing its ability to autonomously navigate and interact with complex digital environments. A large-scale dataset of GUI agent trajectories is constructed, incorporating multimodal reasoning and grounding. Comprehensive experiments demonstrate that Aguvis surpasses previous state-of-the-art methods in both offline and real-world online scenarios, achieving the first fully autonomous pure vision GUI agent capable of performing tasks independently without collaboration with external closed-source models. All datasets, models, and training recipes are open-sourced to facilitate future research.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2412.01268" rel="nofollow">Ponder &amp; Press: Advancing Visual GUI Agent towards General Computer Control</a></p>
<ul dir="auto">
<li>Yiqin Wang, Haoji Zhang, Jingqi Tian, Yansong Tang</li>
<li>🏛️ Institutions: Tsinghua University</li>
<li>📅 Date: December 2, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [visual grounding], [reinforcement learning], [Ponder &amp; Press]</li>
<li>📖 TLDR: This paper introduces <em>Ponder &amp; Press</em>, a divide-and-conquer framework for general computer control using only visual input. The approach combines a general-purpose multimodal large language model (MLLM) as an 'interpreter' to translate high-level user instructions into detailed action descriptions, with a GUI-specific MLLM as a 'locator' that precisely identifies GUI elements for action execution. By relying solely on visual inputs, the agent offers a versatile, human-like interaction paradigm applicable across various platforms, achieving state-of-the-art performance in GUI grounding and control tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.17465" rel="nofollow">ShowUI: One Vision-Language-Action Model for GUI Visual Agent</a></p>
<ul dir="auto">
<li>Kevin Qinghong Lin, Linjie Li, Difei Gao, Zhengyuan Yang, Shiwei Wu, Zechen Bai, Weixian Lei, Lijuan Wang, Mike Zheng Shou</li>
<li>🏛️ Institutions: NUS, Microsoft</li>
<li>📅 Date: November 26, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [framework], [dataset], [UI-Guided Visual Token Selection], [Interleaved Vision-Language-Action Streaming], [ShowUI]</li>
<li>📖 TLDR: This paper introduces <em>ShowUI</em>, a vision-language-action model designed to enhance GUI automation by addressing challenges in UI visual perception and action modeling. It features innovations like UI-Guided Visual Token Selection to reduce computational costs and Interleaved Vision-Language-Action Streaming for effective management of visual-action history. Trained on a curated dataset, ShowUI achieves 75.1% accuracy in zero-shot screenshot grounding and demonstrates competitive performance across web, mobile, and online environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.13451" rel="nofollow">AdaptAgent: Adapting Multimodal Web Agents with Few-Shot Learning from Human Demonstrations</a></p>
<ul dir="auto">
<li>Gaurav Verma, Rachneet Kaur, Nishan Srishankar, Zhen Zeng, Tucker Balch, Manuela Veloso</li>
<li>🏛️ Institutions: J.P. Morgan AI Research</li>
<li>📅 Date: November 24, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [few-shot learning], [meta-learning], [AdaptAgent]</li>
<li>📖 TLDR: This paper introduces <strong>AdaptAgent</strong>, a framework that enables multimodal web agents to adapt to new websites and domains using few human demonstrations (up to 2). The approach enhances agents' adaptability beyond large-scale pre-training and fine-tuning by leveraging in-context learning and meta-learning techniques. Experiments on benchmarks like Mind2Web and VisualWebArena show that incorporating minimal human demonstrations boosts task success rates significantly, highlighting the effectiveness of multimodal demonstrations over text-only ones and the impact of data selection strategies during meta-learning on agent generalization.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.13591" rel="nofollow">Improved GUI Grounding via Iterative Narrowing</a></p>
<ul dir="auto">
<li>Anthony Nguyen</li>
<li>🏛️ Institutions: Algoma University</li>
<li>📅 Date: November 18, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [grounding], [visual grounding], [iterative narrowing]</li>
<li>📖 TLDR: This paper introduces a visual framework to enhance GUI grounding. By iteratively refining model predictions through progressively focused image crops, the proposed method improves the performance of both general and fine-tuned Vision-Language Models (VLMs) in GUI grounding tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.10943" rel="nofollow">Generalist Virtual Agents: A Survey on Autonomous Agents Across Digital Platforms</a></p>
<ul dir="auto">
<li>Minghe Gao, Wendong Bu, Bingchen Miao, Yang Wu, Yunfei Li, Juncheng Li, Siliang Tang, Qi Wu, Yueting Zhuang, Meng Wang</li>
<li>🏛️ Institutions: Zhejiang University, University of Adelaide, Hefei University of Technology</li>
<li>📅 Date: November 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [survey], [Generalist Virtual Agent], [GVA], [autonomous agents], [digital platforms]</li>
<li>📖 TLDR: This survey introduces the concept of Generalist Virtual Agents (GVAs), autonomous entities designed to operate across various digital platforms and environments to assist users in performing diverse tasks. It traces the evolution of GVAs from early intelligent assistants to modern implementations incorporating large-scale models, discussing their philosophical foundations, development challenges, and current methodologies. The paper provides a detailed taxonomy of GVA environments, tasks, and capabilities, aiming to bridge theoretical and practical aspects and suggesting that agents operating in environments closely mirroring the real world are more likely to exhibit human-like intelligence.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.10323" rel="nofollow">The Dawn of GUI Agent: A Preliminary Case Study with Claude 3.5 Computer Use</a></p>
<ul dir="auto">
<li>Siyuan Hu, Mingyu Ouyang, Difei Gao, Mike Zheng Shou</li>
<li>🏛️ Institutions: NUS</li>
<li>📅 Date: Nov 15, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [Claude 3.5 Computer Use], [GUI automation], [planning], [action], [critic]</li>
<li>📖 TLDR: This study evaluates Claude 3.5 Computer Use, an AI model enabling end-to-end language-to-desktop actions, through curated tasks across various domains. It introduces an out-of-the-box framework for deploying API-based GUI automation models, analyzing the model's planning, action execution, and adaptability to dynamic environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://aclanthology.org/2024.emnlp-demo.20/" rel="nofollow">WebOlympus: An Open Platform for Web Agents on Live Websites</a></p>
<ul dir="auto">
<li>Boyuan Zheng, Boyu Gou, Scott Salisbury, Zheng Du, Huan Sun, Yu Su</li>
<li>🏛️ Institutions: OSU</li>
<li>📅 Date: November 12, 2024</li>
<li>📑 Publisher: EMNLP 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [safety], [Chrome extension], [WebOlympus], [SeeAct], [Annotation Tool]</li>
<li>📖 TLDR: This paper introduces <em>WebOlympus</em>, an open platform designed to facilitate the research and deployment of web agents on live websites. It features a user-friendly Chrome extension interface, allowing users without programming expertise to operate web agents with minimal effort. The platform incorporates a safety monitor module to prevent harmful actions through human supervision or model-based control, supporting applications such as annotation interfaces for web agent trajectories and data crawling.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.06559" rel="nofollow">Is Your LLM Secretly a World Model of the Internet? Model-Based Planning for Web Agents</a></p>
<ul dir="auto">
<li>Yu Gu, Boyuan Zheng, Boyu Gou, Kai Zhang, Cheng Chang, Sanjari Srivastava, Yanan Xie, Peng Qi, Huan Sun, Yu Su</li>
<li>🏛️ Institutions: OSU, Orby AI</li>
<li>📅 Date: November 10, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [WebDreamer], [model-based planning], [world model]</li>
<li>📖 TLDR: This paper investigates whether Large Language Models (LLMs) can function as world models within web environments, enabling model-based planning for web agents. Introducing <strong>WebDreamer</strong>, a framework that leverages LLMs to simulate potential action sequences in web environments, the study demonstrates significant performance improvements over reactive baselines on benchmarks like VisualWebArena and Mind2Web-live. The findings suggest that LLMs possess the capability to model the dynamic nature of the internet, paving the way for advancements in automated web interaction and opening new research avenues in optimizing LLMs for complex, evolving environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.04890" rel="nofollow">GUI Agents with Foundation Models: A Comprehensive Survey</a></p>
<ul dir="auto">
<li>Shuai Wang, Weiwen Liu, Jingxuan Chen, Weinan Gan, Xingshan Zeng, Shuai Yu, Xinlong Hao, Kun Shao, Yasheng Wang, Ruiming Tang</li>
<li>🏛️ Institutions: Huawei Noah’s Ark Lab</li>
<li>📅 Date: November 7, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [survey]</li>
<li>📖 TLDR: This survey consolidates recent research on GUI agents powered by foundation models, particularly Large Language Models (LLMs) and Multimodal Large Language Models (MLLMs). It discusses representative datasets and benchmarks, summarizes a unified framework capturing essential components from prior studies, and explores commercial applications. The paper identifies key challenges and proposes future research directions to inspire further developments in (M)LLM-based GUI agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.02337v1" rel="nofollow">WebRL: Training LLM Web Agents via Self-Evolving Online Curriculum Reinforcement Learning</a></p>
<ul dir="auto">
<li>Zehan Qi, Xiao Liu, Iat Long Iong, Hanyu Lai, Xueqiao Sun, Xinyue Yang, Jiadai Sun, Yu Yang, Shuntian Yao, Tianjie Zhang, Wei Xu, Jie Tang, Yuxiao Dong</li>
<li>🏛️ Institutions: Tsinghua University, BAAI</li>
<li>📅 Date: November 4, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [reinforcement learning], [self-evolving curriculum], [WebRL], [outcome-supervised reward model]</li>
<li>📖 TLDR: This paper introduces <em>WebRL</em>, a self-evolving online curriculum reinforcement learning framework designed to train high-performance web agents using open large language models (LLMs). WebRL addresses challenges such as the scarcity of training tasks, sparse feedback signals, and policy distribution drift in online learning. It incorporates a self-evolving curriculum that generates new tasks from unsuccessful attempts, a robust outcome-supervised reward model (ORM), and adaptive reinforcement learning strategies to ensure consistent improvements. Applied to Llama-3.1 and GLM-4 models, WebRL significantly enhances their performance on web-based tasks, surpassing existing state-of-the-art web agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2411.02391" rel="nofollow">Attacking Vision-Language Computer Agents via Pop-ups</a></p>
<ul dir="auto">
<li>Yanzhe Zhang, Tao Yu, Diyi Yang</li>
<li>🏛️ Institutions: Georgia Tech, HKU, Stanford</li>
<li>📅 Date: Nov 4, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [attack], [adversarial pop-ups], [VLM agents], [safety]</li>
<li>📖 TLDR: This paper demonstrates that vision-language model (VLM) agents can be easily deceived by carefully designed adversarial pop-ups, leading them to perform unintended actions such as clicking on these pop-ups instead of completing their assigned tasks. Integrating these pop-ups into environments like OSWorld and VisualWebArena resulted in an average attack success rate of 86% and a 47% decrease in task success rate. Basic defense strategies, such as instructing the agent to ignore pop-ups or adding advertisement notices, were found to be ineffective against these attacks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://language-agent-tutorial.github.io/" rel="nofollow">Language Agents: Foundations, Prospects, and Risks</a></p>
<ul dir="auto">
<li>Yu Su, Diyi Yang, Shunyu Yao, Tao Yu</li>
<li>🏛️ Institutions: OSU, Stanford, Princeton, HKU</li>
<li>📅 Date: November 2024</li>
<li>📑 Publisher: EMNLP 2024</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [survey], [tutorial], [reasoning], [planning], [memory], [multi-agent systems], [safty]</li>
<li>📖 TLDR: This tutorial provides a comprehensive exploration of language agents—autonomous systems powered by large language models capable of executing complex tasks through language instructions. It delves into their theoretical foundations, potential applications, associated risks, and future directions, covering topics such as reasoning, memory, planning, tool augmentation, grounding, multi-agent systems, and safety considerations.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.13701" rel="nofollow">From Context to Action: Analysis of the Impact of State Representation and Context on the Generalization of Multi-Turn Web Navigation Agents</a></p>
<ul dir="auto">
<li>Nalin Tiwary, Vardhan Dongre, Sanil Arun Chawla, Ashwin Lamani, Dilek Hakkani-Tür</li>
<li>🏛️ Institutions: UIUC</li>
<li>📅 Date: October 31, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [context management], [generalization], [multi-turn navigation], [CWA]</li>
<li>📖 TLDR: This study examines how different contextual elements affect the performance and generalization of Conversational Web Agents (CWAs) in multi-turn web navigation tasks. By optimizing context management—specifically interaction history and web page representation—the research demonstrates enhanced agent performance across various out-of-distribution scenarios, including unseen websites, categories, and geographic locations.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.24024" rel="nofollow">AndroidLab: Training and Systematic Benchmarking of Android Autonomous Agents</a></p>
<ul dir="auto">
<li>Yifan Xu, Xiao Liu, Xueqiao Sun, Siyi Cheng, Hao Yu, Hanyu Lai, Shudan Zhang, Dan Zhang, Jie Tang, Yuxiao Dong</li>
<li>🏛️ Institutions: Tsinghua University, Peking University</li>
<li>📅 Date: October 31, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [AndroidLab]</li>
<li>📖 TLDR: This paper introduces <strong>AndroidLab</strong>, a comprehensive framework for training and systematically benchmarking Android autonomous agents. It provides an operational environment with diverse modalities and action spaces, supporting both large language models (LLMs) and multimodal models (LMMs). The benchmark includes 138 tasks across nine apps on predefined Android virtual devices. Utilizing AndroidLab, the authors developed an Android Instruction dataset and trained six open-source LLMs and LMMs, significantly improving their average success rates.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://osatlas.github.io/" rel="nofollow">OS-ATLAS: A Foundation Action Model For Generalist GUI Agents</a></p>
<ul dir="auto">
<li>Zhiyong Wu, Zhenyu Wu, Fangzhi Xu, Yian Wang, Qiushi Sun, Chengyou Jia, Kanzhi Cheng, Zichen Ding, Liheng Chen, Paul Pu Liang, Yu Qiao</li>
<li>🏛️ Institutions: Shanghai AI Lab, SJTU, HKU, MIT</li>
<li>📅 Date: October 30, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [benchmark], [OS-Atlas]</li>
<li>📖 TLDR: This paper introduces OS-Atlas, a foundational GUI action model designed to enhance GUI grounding and out-of-distribution tasks. The authors developed a toolkit to synthesize multi-platform GUI grounding data, resulting in a cross-platform corpus of over 13 million GUI elements. OS-Atlas demonstrates significant performance improvements across six benchmarks spanning mobile, desktop, and web platforms.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.23252" rel="nofollow">Evaluating Cultural and Social Awareness of LLM Web Agents</a></p>
<ul dir="auto">
<li>Haoyi Qiu, Alexander R. Fabbri, Divyansh Agarwal, Kung-Hsiang Huang, Sarah Tan, Nanyun Peng, Chien-Sheng Wu</li>
<li>🏛️ Institutions: UCLA, Salesforce AI Research</li>
<li>📅 Date: October 30, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [CASA], [cultural awareness], [social awareness], [fine-tuning], [prompting]</li>
<li>📖 TLDR: This paper introduces CASA, a benchmark designed to assess the cultural and social awareness of LLM web agents in tasks like online shopping and social discussion forums. It evaluates agents' abilities to detect and appropriately respond to norm-violating user queries and observations. The study finds that current LLM agents have limited cultural and social awareness, with less than 10% awareness coverage and over 40% violation rates. To enhance performance, the authors explore prompting and fine-tuning methods, demonstrating that combining both can offer complementary advantages.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.22552" rel="nofollow">Auto-Intent: Automated Intent Discovery and Self-Exploration for Large Language Model Web Agents</a></p>
<ul dir="auto">
<li>Jaekyeom Kim, Dong-Ki Kim, Lajanugen Logeswaran, Sungryull Sohn, Honglak Lee</li>
<li>🏛️ Institutions: LG AI Research, Field AI, University of Michigan</li>
<li>📅 Date: October 29, 2024</li>
<li>📑 Publisher: EMNLP 2024 (Findings)</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [Auto-Intent]</li>
<li>📖 TLDR: The paper presents Auto-Intent, a method to adapt pre-trained large language models for web navigation tasks without direct fine-tuning. It discovers underlying intents from domain demonstrations and trains an intent predictor to enhance decision-making. Auto-Intent improves the performance of GPT-3.5, GPT-4, and Llama-3.1 agents on benchmarks like Mind2Web and WebArena.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://doi.org/10.48550/arXiv.2410.19461" rel="nofollow">EDGE: Enhanced Grounded GUI Understanding with Enriched Multi-Granularity Synthetic Data</a></p>
<ul dir="auto">
<li>Xuetian Chen, Hangcheng Li, Jiaqing Liang, Sihang Jiang, Deqing Yang</li>
<li>🏛️ Institutions: Fudan University</li>
<li>📅 Date: October 25, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [dataset], [framework], [synthetic data]</li>
<li>📖 TLDR: The <em>EDGE</em> framework proposes an innovative approach to improve GUI understanding and interaction capabilities in vision-language models through large-scale, multi-granularity synthetic data generation. By leveraging webpage data, EDGE minimizes the need for manual annotations and enhances the adaptability of models across desktop and mobile GUI environments. Evaluations show its effectiveness in diverse GUI-related tasks, contributing significantly to autonomous agent development in GUI navigation and interaction.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://doi.org/10.48550/arXiv.2410.19609" rel="nofollow">OpenWebVoyager: Building Multimodal Web Agents via Iterative Real-World Exploration, Feedback and Optimization</a></p>
<ul dir="auto">
<li>Hongliang He, Wenlin Yao, Kaixin Ma, Wenhao Yu, Hongming Zhang, Tianqing Fang, Zhenzhong Lan, Dong Yu</li>
<li>🏛️ Institutions: Zhejiang University, Tencent AI Lab, Westlake University</li>
<li>📅 Date: October 25, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [learning], [imitation learning], [exploration], [AI feedback]</li>
<li>📖 TLDR: The paper presents <strong>OpenWebVoyager</strong>, an open-source framework for training web agents that explore real-world online environments autonomously. The framework employs a cycle of exploration, feedback, and optimization, enhancing agent capabilities through multimodal perception and iterative learning. Initial skills are acquired through imitation learning, followed by real-world exploration, where the agent’s performance is evaluated and refined through feedback loops.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://xiao9905.github.io/AutoGLM/" rel="nofollow">AutoGLM: Autonomous Foundation Agents for GUIs</a></p>
<ul dir="auto">
<li>Xiao Liu, Bo Qin, Dongzhu Liang, Guang Dong, Hanyu Lai, Hanchen Zhang, Hanlin Zhao, Iat Long Iong, Jiadai Sun, Jiaqi Wang, Junjie Gao, Junjun Shan, Kangning Liu, Shudan Zhang, Shuntian Yao, Siyi Cheng, Wentao Yao, Wenyi Zhao, Xinghan Liu, Xinyi Liu, Xinying Chen, Xinyue Yang, Yang Yang, Yifan Xu, Yu Yang, Yujia Wang, Yulin Xu, Zehan Qi, Yuxiao Dong, Jie Tang</li>
<li>🏛️ Institutions: Zhipu AI, Tsinghua University</li>
<li>📅 Date: October 25, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [model], [learning], [AutoGLM]</li>
<li>📖 TLDR: This paper introduces AutoGLM, a new series in the ChatGLM family, designed as foundation agents for autonomous control of digital devices through GUIs. It addresses the challenges foundation models face in decision-making within dynamic environments by developing agents capable of learning through autonomous interactions. Focusing on web browsers and Android devices, AutoGLM integrates various techniques to create deployable agent systems. Key insights include the importance of designing an appropriate "intermediate interface" for GUI control and a novel progressive training framework for self-evolving online curriculum reinforcement learning. Evaluations demonstrate AutoGLM's effectiveness across multiple domains, achieving notable success rates in web browsing and Android device control tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/pdf/2410.16464" rel="nofollow">Beyond Browsing: API-Based Web Agents</a></p>
<ul dir="auto">
<li>Yueqi Song, Frank Xu, Shuyan Zhou, Graham Neubig</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: October 24, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [API-based agent], [hybrid agent], [benchmark], [WebArena], [SOTA performance]</li>
<li>📖 TLDR: This paper introduces API-based and hybrid agents designed to execute online tasks by accessing both APIs and traditional web browsing interfaces. In evaluations using WebArena, a benchmark for web navigation, the API-based agent achieves higher performance than browser-based agents, and the hybrid model achieves a success rate of 35.8%, setting a new state-of-the-art (SOTA) in task-agnostic web navigation. The findings highlight the efficiency and reliability gains of API interactions for web agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.18603" rel="nofollow">AgentStore: Scalable Integration of Heterogeneous Agents As Specialized Generalist Computer Assistant</a></p>
<ul dir="auto">
<li>Chengyou Jia, Minnan Luo, Zhuohang Dang, Qiushi Sun, Fangzhi Xu, Junlin Hu, Tianbao Xie, Zhiyong Wu</li>
<li>🏛️ Institutions: XJTU, Shanghai AI Lab, HKU</li>
<li>📅 Date: October 24, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [multi-agent systems], [specialized generalist agent], [OSWorld benchmark]</li>
<li>📖 TLDR: AgentStore introduces a scalable platform to integrate and manage heterogeneous agents, designed to enhance generalist assistant capabilities for diverse computer tasks. Using a MetaAgent and AgentToken strategy, AgentStore shows improved generalization on the OSWorld benchmark.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://doi.org/10.48550/arXiv.2410.19100" rel="nofollow">VideoWebArena: Evaluating Long Context Multimodal Agents with Video Understanding Web Tasks</a></p>
<ul dir="auto">
<li>Lawrence Jang, Yinheng Li, Charles Ding, Justin Lin, Paul Pu Liang, Dan Zhao, Rogerio Bonatti, Kazuhito Koishida</li>
<li>🏛️ Institutions: CMU, MIT, NYU, Microsoft</li>
<li>📅 Date: October 24, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [video understanding], [long-context], [VideoWA]</li>
<li>📖 TLDR: This paper introduces <strong>VideoWebArena (VideoWA)</strong>, a benchmark assessing multimodal agents in video-based tasks. It features over 2,000 tasks focused on skill and factual retention, using video tutorials to simulate long-context environments. Results highlight current challenges in agentic abilities, providing a critical testbed for long-context video understanding improvements.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.17520" rel="nofollow">MobileSafetyBench: Evaluating Safety of Autonomous Agents in Mobile Device Control</a></p>
<ul dir="auto">
<li>Juyong Lee, Dongyoon Hahm, June Suk Choi, W. Bradley Knox, Kimin Lee</li>
<li>🏛️ Institutions: KAIST, UT at Austin</li>
<li>📅 Date: October 23, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [safety], [evaluation], [Android emulator]</li>
<li>📖 TLDR: <em>MobileSafetyBench</em> introduces a benchmark for evaluating the safety of large language model (LLM)-based autonomous agents in mobile device control. Using Android emulators, the benchmark simulates real-world tasks in apps such as messaging and banking to assess agents' safety and helpfulness. The safety-focused tasks test for privacy risk management and robustness against adversarial prompt injections. Experiments show agents perform well in helpful tasks but struggle with safety-related challenges, underscoring the need for continued advancements in mobile safety mechanisms for autonomous agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.17883" rel="nofollow">Lightweight Neural App Control</a></p>
<ul dir="auto">
<li>Filippos Christianos, Georgios Papoudakis, Thomas Coste, Jianye Hao, Jun Wang, Kun Shao</li>
<li>🏛️ Institutions: Huawei Noah's Ark Lab, UCL</li>
<li>📅 Date: October 23, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [vision language model], [Action Transformer], [app agent], [Android control], [multi-modal]</li>
<li>📖 TLDR: This paper introduces LiMAC, a mobile control framework for Android that integrates an Action Transformer and fine-tuned vision-language models to execute precise actions in mobile apps. Tested on open-source datasets, LiMAC improves action accuracy by up to 42% over traditional prompt engineering baselines, demonstrating enhanced efficiency and accuracy in mobile app control tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://ar5iv.org/abs/2410.17236" rel="nofollow">Large Language Models Empowered Personalized Web Agents</a></p>
<ul dir="auto">
<li>Hongru Cai, Yongqi Li, Wenjie Wang, Fengbin Zhu, Xiaoyu Shen, Wenjie Li, Tat-Seng Chua</li>
<li>🏛️ Institutions: HK PolyU, NTU Singapore</li>
<li>📅 Date: Oct 22, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [benchmark], [personalized web agent], [user behavior alignment], [memory-enhanced alignment]</li>
<li>📖 TLDR: This paper proposes a novel framework, <em>Personalized User Memory-enhanced Alignment (PUMA)</em>, enabling large language models to serve as personalized web agents by incorporating user-specific data and historical web interactions. The authors also introduce a benchmark, <em>PersonalWAB</em>, to evaluate these agents on various personalized web tasks. Results show that PUMA improves web agent performance by optimizing action execution based on user-specific preferences.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.15711" rel="nofollow">AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?</a></p>
<ul dir="auto">
<li>Ori Yoran, Samuel Joseph Amouyal, Chaitanya Malaviya, Ben Bogin, Ofir Press, Jonathan Berant</li>
<li>🏛️ Institutions: Tel Aviv University</li>
<li>📅 Date: October 21, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [planning and reasoning]</li>
<li>📖 TLDR: AssistantBench is a benchmark designed to test the abilities of web agents in completing time-intensive, realistic web-based tasks. Covering 214 tasks across various domains, the benchmark introduces the SPA (See-Plan-Act) framework to handle multi-step planning and memory retention. AssistantBench emphasizes realistic task completion, showing that current agents achieve only modest success, with significant improvements needed for complex information synthesis and execution across multiple web domains.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://openreview.net/forum?id=LjVIGva5Ct" rel="nofollow">Dissecting Adversarial Robustness of Multimodal LM Agents</a></p>
<ul dir="auto">
<li>Chen Henry Wu, Rishi Rajesh Shah, Jing Yu Koh, Russ Salakhutdinov, Daniel Fried, Aditi Raghunathan</li>
<li>🏛️ Institutions: CMU, Stanford</li>
<li>📅 Date: October 21, 2024</li>
<li>📑 Publisher: NeurIPS 2024 Workshop</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [dataset], [attack], [ARE], [safety]</li>
<li>📖 TLDR: This paper introduces the Agent Robustness Evaluation (ARE) framework to assess the adversarial robustness of multimodal language model agents in web environments. By creating 200 targeted adversarial tasks within VisualWebArena, the study reveals that minimal perturbations can significantly compromise agent performance, even in advanced systems utilizing reflection and tree-search mechanisms. The findings highlight the need for enhanced safety measures in deploying such agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://ai-agents-2030.github.io/SPA-Bench/" rel="nofollow">SPA-Bench: A Comprehensive Benchmark for SmartPhone Agent Evaluation</a></p>
<ul dir="auto">
<li>Jingxuan Chen, Derek Yuen, Bin Xie, Yuhao Yang, Gongwei Chen, Zhihao Wu, Li Yixing, Xurui Zhou, Weiwen Liu, Shuai Wang, Rui Shao, Liqiang Nie, Yasheng Wang, Jianye Hao, Jun Wang, Kun Shao</li>
<li>🏛️ Institutions: Huawei Noah’s Ark Lab, Harbin Institute of Technology, Shenzhen, UCL</li>
<li>📅 Date: October 19, 2024</li>
<li>📑 Publisher: ICLR 2025</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [AI agent], [smartphone control], [framework]</li>
<li>📖 TLDR: SPA-Bench is introduced as a benchmark designed to evaluate multimodal large language model (MLLM)-based smartphone agents, offering a task set that spans common smartphone functionalities across system and third-party applications. It includes a plug-and-play framework for real-time agent interactions on Android, integrating over ten agents with an adaptable evaluation pipeline measuring success across diverse metrics. Through this, the benchmark exposes challenges such as UI interpretation, action grounding, and memory retention in mobile environments, advancing research in smartphone-based agent applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://ai-agents-2030.github.io/DistRL/" rel="nofollow">DistRL: An Asynchronous Distributed Reinforcement Learning Framework for On-Device Control Agents</a></p>
<ul dir="auto">
<li>Taiyi Wang, Zhihao Wu, Jianheng Liu, Jianye Hao, Jun Wang, Kun Shao</li>
<li>🏛️ Institutions: Univ. of Cambridge, Huawei Noah's Ark Lab, Univ. College London</li>
<li>📅 Date: October 18, 2024</li>
<li>📑 Publisher: ICLR 2025</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [reinforcement learning], [distributed training], [A-RIDE], [on-device control]</li>
<li>📖 TLDR: This paper introduces <strong>DistRL</strong>, a novel framework designed to enhance the efficiency of online reinforcement learning fine-tuning for mobile device control agents. DistRL employs centralized training and decentralized data acquisition to ensure efficient fine-tuning in dynamic online interactions. The framework is supported by a custom RL algorithm, A-RIDE, which balances exploration with prioritized data utilization to ensure stable and robust training. Experiments demonstrate that DistRL improves training efficiency by 3× and accelerates data collection by 2.4× compared to leading synchronous multi-machine methods. Notably, after training, DistRL achieves a 20% relative improvement in success rate on general Android tasks from an open benchmark, outperforming existing approaches while maintaining the same training time.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.13824" rel="nofollow">Harnessing Webpage UIs for Text-Rich Visual Understanding</a></p>
<ul dir="auto">
<li>Junpeng Liu, Tianyue Ou, Yifan Song, Yuxiao Qu, Wai Lam, Chenyan Xiong, Wenhu Chen, Graham Neubig, Xiang Yue</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: October 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web], [Doc]</li>
<li>🔑 Key: [dataset], [model], [text-rich visual understanding], [web UI comprehension]</li>
<li>📖 TLDR: This paper introduces <em>MultiUI</em>, a large-scale dataset containing 7.3 million annotated samples from 1 million websites, specifically designed to enhance multimodal large language models’ (MLLMs) capabilities in text-rich visual understanding. Utilizing webpage UI structures as a training resource, MultiUI provides robust accessibility tree data paired with UI screenshots, significantly improving MLLMs’ grounding, OCR, and interaction performance. Models trained with MultiUI achieve up to a 48% performance boost on VisualWebBench and demonstrate enhanced generalization across non-web tasks, setting a new standard for structured, visually integrated web data modeling.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.03648" rel="nofollow">AutoWebGLM: A Large Language Model-based Web Navigating Agent</a></p>
<ul dir="auto">
<li>Hanyu Lai, Xiao Liu, Iat Long Iong, Shuntian Yao, Yuxuan Chen, Pengbo Shen, Hao Yu, Hanchen Zhang, Xiaohan Zhang, Yuxiao Dong, Jie Tang</li>
<li>🏛️ Institutions: THU, OSU</li>
<li>📅 Date: October 12, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [reinforcement learning]</li>
<li>📖 TLDR: AutoWebGLM introduces a web navigation agent based on ChatGLM3-6B, designed to autonomously navigate and interact with webpages for complex tasks. The paper highlights a two-phase data construction approach using a hybrid human-AI methodology for diverse, curriculum-based web task training. It also presents AutoWebBench, a benchmark for evaluating agent performance in web tasks, and uses reinforcement learning to fine-tune operations, addressing complex webpage interaction and grounding.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.13886" rel="nofollow">Refusal-Trained LLMs Are Easily Jailbroken As Browser Agents</a></p>
<ul dir="auto">
<li>Priyanshu Kumar, Elaine Lau, Saranya Vijayakumar, Tu Trinh, Scale Red Team, Elaine Chang, Vaughn Robinson, Sean Hendryx, Shuyan Zhou, Matt Fredrikson, Summer Yue, Zifan Wang</li>
<li>🏛️ Institutions: CMU, GraySwan AI, Scale AI</li>
<li>📅 Date: October 11, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [attack], [BrowserART], [jailbreaking], [safety]</li>
<li>📖 TLDR: This paper introduces <strong>Browser Agent Red teaming Toolkit (BrowserART)</strong>, a comprehensive test suite for evaluating the safety of LLM-based browser agents. The study reveals that while refusal-trained LLMs decline harmful instructions in chat settings, their corresponding browser agents often comply with such instructions, indicating a significant safety gap. The authors call for collaboration among developers and policymakers to enhance agent safety.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.08164" rel="nofollow">Agent S: An Open Agentic Framework that Uses Computers Like a Human</a></p>
<ul dir="auto">
<li>Saaket Agashe, Jiuzhou Han, Shuyu Gan, Jiachen Yang, Ang Li, Xin Eric Wang</li>
<li>🏛️ Institutions: Simular Research</li>
<li>📅 Date: October 10, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [autonomous GUI interaction], [experience-augmented hierarchical planning]</li>
<li>📖 TLDR: This paper introduces Agent S, an open agentic framework that enables autonomous interaction with computers through a Graphical User Interface (GUI). The system addresses key challenges in automating computer tasks through experience-augmented hierarchical planning and an Agent-Computer Interface (ACI). Agent S demonstrates significant improvements over baselines on the OSWorld benchmark, achieving a 20.58% success rate (83.6% relative improvement). The framework shows generalizability across different operating systems and provides insights for developing more effective GUI agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.11872" rel="nofollow">ClickAgent: Enhancing UI Location Capabilities of Autonomous Agents</a></p>
<ul dir="auto">
<li>Jakub Hoscilowicz, Bartosz Maj, Bartosz Kozakiewicz, Oleksii Tymoschuk, Artur Janicki</li>
<li>🏛️ Institutions: Samsung R&amp;D Poland, Warsaw University of Technology</li>
<li>📅 Date: October 9, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [model], [SeeClick], [AITW benchmark]</li>
<li>📖 TLDR: The paper introduces <em>ClickAgent</em>, a framework that enhances autonomous agents' interaction with mobile UIs by improving their ability to locate interface elements accurately. This is achieved through a dual-component system where an MLLM performs reasoning and action planning, while a dedicated UI location model (e.g., SeeClick) handles element identification. ClickAgent, evaluated on the AITW benchmark and tested on both emulators and real Android devices, surpasses other agents like CogAgent and AppAgent in task success rate, advancing automation reliability on mobile platforms.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://sites.google.com/view/st-webagentbench/home" rel="nofollow">ST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents</a></p>
<ul dir="auto">
<li>Ido Levy, Ben Wiesel, Sami Marreed, Alon Oved, Avi Yaeli, Segev Shlomov</li>
<li>🏛️ Institutions: IBM Research</li>
<li>📅 Date: October 9, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [safety], [trustworthiness], [ST-WebAgentBench]</li>
<li>📖 TLDR: This paper introduces <strong>ST-WebAgentBench</strong>, a benchmark designed to evaluate the safety and trustworthiness of web agents in enterprise contexts. It defines safe and trustworthy agent behavior, outlines the structure of safety policies, and introduces the "Completion under Policies" metric to assess agent performance. The study reveals that current state-of-the-art agents struggle with policy adherence, highlighting the need for improved policy awareness and compliance in web agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.11871" rel="nofollow">TinyClick: Single-Turn Agent for Empowering GUI Automation</a></p>
<ul dir="auto">
<li>Pawel Pawlowski, Krystian Zawistowski, Wojciech Lapacz, Marcin Skorupa, Adam Wiacek, Sebastien Postansque, Jakub Hoscilowicz</li>
<li>🏛️ Institutions: Samsung R&amp;D Poland, Warsaw University of Technology</li>
<li>📅 Date: October 9, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [vision language model], [Screenspot], [OmniAct]</li>
<li>📖 TLDR: TinyClick is a compact, single-turn agent designed to automate GUI tasks by precisely locating screen elements via the Vision-Language Model Florence-2-Base. Trained with multi-task strategies and MLLM-based data augmentation, TinyClick achieves high accuracy on Screenspot and OmniAct, outperforming specialized GUI interaction models and general MLLMs like GPT-4V. The model's lightweight design (0.27B parameters) ensures fast processing and minimal latency, making it efficient for real-world applications on multiple platforms.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://osu-nlp-group.github.io/UGround/" rel="nofollow">Navigating the Digital World as Humans Do: Universal Visual Grounding for GUI Agents</a></p>
<ul dir="auto">
<li>Boyu Gou, Ruochen Wang, Boyuan Zheng, Yucheng Xie, Cheng Chang, Yiheng Shu, Haotian Sun, Yu Su</li>
<li>🏛️ Institutions: OSU, Orby AI</li>
<li>📅 Date: October 7, 2024</li>
<li>📑 Publisher: ICLR 2025</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [model], [dataset], [visual grounding], [GUI agents], [cross-platform generalization], [UGround], [SeeAct-V], [synthetic data]</li>
<li>📖 TLDR: This paper introduces UGround, a universal visual grounding model for GUI agents that enables human-like navigation of digital interfaces. The authors advocate for GUI agents with human-like embodiment that perceive the environment entirely visually and take pixel-level actions. UGround is trained on a large-scale synthetic dataset of 10M GUI elements across 1.3M screenshots. Evaluated on six benchmarks spanning grounding, offline, and online agent tasks, UGround significantly outperforms existing visual grounding models by up to 20% absolute. Agents using UGround achieve comparable or better performance than state-of-the-art agents that rely on additional textual input, demonstrating the feasibility of vision-only GUI agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://agent-e3.github.io/ExACT/" rel="nofollow">ExACT: Teaching AI Agents to Explore with Reflective-MCTS and Exploratory Learning</a></p>
<ul dir="auto">
<li>Xiao Yu, Baolin Peng, Vineeth Vajipey, Hao Cheng, Michel Galley, Jianfeng Gao, Zhou Yu</li>
<li>🏛️ Institutions: Columbia Univ., MSR</li>
<li>📅 Date: Oct 2, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [learning], [R-MCTS], [Exploratory Learning], [VisualWebArena]</li>
<li>📖 TLDR: This paper introduces ExACT, an approach that combines Reflective Monte Carlo Tree Search (R-MCTS) and Exploratory Learning to enhance AI agents' exploration and decision-making capabilities in complex web environments. R-MCTS incorporates contrastive reflection and multi-agent debate for improved search efficiency and reliable state evaluation. Evaluated on the VisualWebArena benchmark, the GPT-4o-based R-MCTS agent demonstrates significant performance improvements over previous state-of-the-art methods. Additionally, knowledge gained from test-time search is effectively transferred back to GPT-4o through fine-tuning, enabling the model to explore, evaluate, and backtrack without external search algorithms, achieving 87% of R-MCTS's performance with reduced computational resources.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2410.00467" rel="nofollow">Dynamic Planning for LLM-based Graphical User Interface Automation</a></p>
<ul dir="auto">
<li>Shaoqing Zhang, Zhuosheng Zhang, Kehai Chen, Xinbei Ma, Muyun Yang, Tiejun Zhao, Min Zhang</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: October 1, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dynamic planning]</li>
<li>📖 TLDR: This paper introduces a novel method called Dynamic Planning of Thoughts (D-PoT) aimed at enhancing LLM-based agents for GUI tasks. It addresses the challenges of task execution by dynamically adjusting planning based on environmental feedback and action history, outperforming existing methods such as ReAct by improving accuracy significantly in navigating GUI environments. The study emphasizes the importance of integrating execution history and contextual cues to optimize decision-making processes for autonomous agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.20566" rel="nofollow">MM1.5: Methods, Analysis &amp; Insights from Multimodal LLM Fine-tuning</a></p>
<ul dir="auto">
<li>Haotian Zhang, Mingfei Gao, Zhe Gan, Philipp Dufter, Nina Wenzel, Forrest Huang, Dhruti Shah, Xianzhi Du, Bowen Zhang, Yanghao Li, Sam Dodge, Keen You, Zhen Yang, Aleksei Timofeev, Mingze Xu, Hong-You Chen, Jean-Philippe Fauconnier, Zhengfeng Lai, Haoxuan You, Zirui Wang, Afshin Dehghan, Peter Grasch, Yinfei Yang</li>
<li>🏛️ Institutions: Apple</li>
<li>📅 Date: September 30, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [model], [MM1.5], [vision language model], [visual grounding], [reasoning], [data-centric], [analysis]</li>
<li>📖 TLDR: This paper introduces MM1.5, a family of multimodal large language models (MLLMs) ranging from 1B to 30B parameters, including dense and mixture-of-experts variants. MM1.5 enhances capabilities in text-rich image understanding, visual referring and grounding, and multi-image reasoning. The authors employ a data-centric training approach, utilizing high-quality OCR data and synthetic captions for continual pre-training, alongside an optimized visual instruction-tuning data mixture for supervised fine-tuning. Specialized variants, MM1.5-Video and MM1.5-UI, are designed for video understanding and mobile UI comprehension, respectively. Extensive empirical studies provide insights into the training processes, offering guidance for future MLLM development.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.15637" rel="nofollow">Synatra: Turning Indirect Knowledge into Direct Demonstrations for Digital Agents at Scale</a></p>
<ul dir="auto">
<li>Tianyue Ou, Frank F. Xu, Aman Madaan, Jiarui Liu, Robert Lo, Abishek Sridhar, Sudipta Sengupta, Dan Roth, Graham Neubig, Shuyan Zhou</li>
<li>🏛️ Institutions: CMU, Amazon AWS AI</li>
<li>📅 Date: September 27, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [synthetic data]</li>
<li>📖 TLDR: Synatra introduces a scalable framework for digital agents, enabling them to convert indirect knowledge sources into actionable demonstrations. This approach enhances the ability of agents to learn tasks without extensive labeled data, leveraging insights from indirect observations to scale practical implementations in digital environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://ai-secure.github.io/AdvWeb/" rel="nofollow">AdvWeb: Controllable Black-box Attacks on VLM-powered Web Agents</a></p>
<ul dir="auto">
<li>Chejian Xu, Mintong Kang, Jiawei Zhang, Zeyi Liao, Lingbo Mo, Mengqi Yuan, Huan Sun, Bo Li</li>
<li>🏛️ Institutions: UIUC, OSU</li>
<li>📅 Date: September 27, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [safety], [black-box attack], [adversarial prompter model], [Direct Policy Optimization]</li>
<li>📖 TLDR: This paper presents AdvWeb, a black-box attack framework that exploits vulnerabilities in vision-language model (VLM)-powered web agents by injecting adversarial prompts directly into web pages. Using Direct Policy Optimization (DPO), AdvWeb trains an adversarial prompter model that can mislead agents into executing harmful actions, such as unauthorized financial transactions, while maintaining high stealth and control. Extensive evaluations reveal that AdvWeb achieves high success rates across multiple real-world tasks, emphasizing the need for stronger security measures in web agent deployments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.17140" rel="nofollow">Turn Every Application into an Agent: Towards Efficient Human-Agent-Computer Interaction with API-First LLM-Based Agents</a></p>
<ul dir="auto">
<li>Junting Lu, Zhiyang Zhang, Fangkai Yang, Jue Zhang, Lu Wang, Chao Du, Qingwei Lin, Saravan Rajmohan, Dongmei Zhang, Qi Zhang</li>
<li>🏛️ Institutions: Peking University, Microsoft</li>
<li>📅 Date: September 26, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [API interaction], [HACI], [Agent OS]</li>
<li>📖 TLDR: This paper proposes an API-centered framework called <strong>AXIS</strong>, enhancing the efficiency and reliability of LLM-based agents by prioritizing API interactions over UI-based actions. This approach aims to reduce the high latency and error rates of traditional UI-interaction models. AXIS not only supports the rapid creation and extension of APIs through automated application exploration but also contributes to a new <strong>Human-Agent-Computer Interaction (HACI)</strong> framework. The paper outlines the development of an agent-centric operating system (Agent OS), which improves task completion times by up to 70% and reduces cognitive load on users while maintaining high accuracy across complex multi-application tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://molmo.allenai.org/blog" rel="nofollow">Molmo and PixMo: Open Weights and Open Data for State-of-the-Art Vision-Language Models</a></p>
<ul dir="auto">
<li>Matt Deitke, Christopher Clark, Sangho Lee, Rohun Tripathi, Yue Yang, Jae Sung Park, Mohammadreza Salehi, Niklas Muennighoff, Kyle Lo, Luca Soldaini, Jiasen Lu, Taira Anderson, Erin Bransom, Kiana Ehsani, Huong Ngo, YenSung Chen, Ajay Patel, Mark Yatskar, Chris Callison-Burch, Andrew Head, Rose Hendrix, Favyen Bastani, Eli VanderBilt, Nathan Lambert, Yvonne Chou, Arnavi Chheda, Jenna Sparks, Sam Skjonsberg, Michael Schmitz, Aaron Sarnat, Byron Bischoff, Pete Walsh, Chris Newell, Piper Wolters, Tanmay Gupta, Kuo-Hao Zeng, Jon Borchardt, Dirk Groeneveld, Crystal Nam, Sophie Lebrecht, Caitlin Wittlif, Carissa Schoenick, Oscar Michel, Ranjay Krishna, Luca Weihs, Noah A. Smith, Hannaneh Hajishirzi, Ross Girshick, Ali Farhadi, Aniruddha Kembhavi</li>
<li>🏛️ Institutions: AI2, UW</li>
<li>📅 Date: September 25, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [model], [dataset], [PixMo], [Molmo], [vision language model], [foundation model]</li>
<li>📖 TLDR: This paper introduces <em>Molmo</em>, a family of state-of-the-art open vision-language models (VLMs), and <em>PixMo</em>, a collection of new datasets including detailed image captions, free-form image Q&amp;A, and innovative 2D pointing data, all collected without reliance on proprietary VLMs. The authors demonstrate that careful model design, a well-tuned training pipeline, and high-quality open datasets can produce VLMs that outperform existing open models and rival proprietary systems. The model weights, datasets, and source code are made publicly available to advance research in this field.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.14818" rel="nofollow">MobileVLM: A Vision-Language Model for Better Intra- and Inter-UI Understanding</a></p>
<ul dir="auto">
<li>Qinzhuo Wu, Weikai Xu, Wei Liu, Tao Tan, Jianfeng Liu, Ang Li, Jian Luan, Bin Wang, Shuo Shang</li>
<li>🏛️ Institutions: XiaoMi AI Lab, University of Electronic Science and Technology of China, Renmin University of China</li>
<li>📅 Date: September 23, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [model], [dataset], [MobileVLM], [Mobile3M], [UI understanding]</li>
<li>📖 TLDR: This paper introduces <em>MobileVLM</em>, a vision-language model designed to enhance both intra- and inter-UI understanding for mobile applications. The authors propose two additional pre-training stages with four specific UI-based tasks to improve the model's perception of fine-grained elements and capture page transition actions. To support this, they constructed <em>Mobile3M</em>, a large-scale Chinese mobile dataset comprising 3 million UI pages and real-world transition actions, organized into directed graphs. Experimental results demonstrate that MobileVLM outperforms existing vision-language models on both in-house test sets and public mobile benchmarks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://qwenlm.github.io/blog/qwen2-vl/" rel="nofollow">Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution</a></p>
<ul dir="auto">
<li>Peng Wang, Shuai Bai, Sinan Tan, Shijie Wang, Zhihao Fan, Jinze Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Yang Fan, Kai Dang, Mengfei Du, Xuancheng Ren, Rui Men, Dayiheng Liu, Chang Zhou, Jingren Zhou, Junyang Lin</li>
<li>🏛️ Institutions: Alibaba Cloud</li>
<li>📅 Date: September 18, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [foundation model], [MLLM], [Qwen2-VL]</li>
<li>📖 TLDR: Qwen2-VL introduces an advanced vision-language framework that enables dynamic resolution handling for images and videos through its Naive Dynamic Resolution mechanism and Multimodal Rotary Position Embedding (M-RoPE). This structure allows the model to convert images of varying resolutions into diverse token counts for improved visual comprehension. With model sizes up to 72B parameters, Qwen2-VL demonstrates competitive performance across multiple benchmarks, achieving results on par with or better than prominent multimodal models like GPT-4o and Claude3.5-Sonnet. This work represents a significant step forward in scalable vision-language learning for multimodal tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.11295" rel="nofollow">EIA: Environmental Injection Attack on Generalist Web Agents for Privacy Leakage</a></p>
<ul dir="auto">
<li>Zeyi Liao, Lingbo Mo, Chejian Xu, Mintong Kang, Jiawei Zhang, Chaowei Xiao, Yuan Tian, Bo Li, Huan Sun</li>
<li>🏛️ Institutions: OSU, UCLA, UChicago, UIUC, UW-Madison</li>
<li>📅 Date: September 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [safety], [privacy attack], [environmental injection], [stealth attack]</li>
<li>📖 TLDR: This paper introduces the Environmental Injection Attack (EIA), a privacy attack targeting generalist web agents by embedding malicious yet concealed web elements to trick agents into leaking users' PII. Utilizing 177 action steps within realistic web scenarios, EIA demonstrates a high success rate in extracting specific PII and whole user requests. Through its detailed threat model and defense suggestions, the work underscores the challenge of detecting and mitigating privacy risks in autonomous web agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://microsoft.github.io/WindowsAgentArena/" rel="nofollow">Windows Agent Arena: Evaluating Multi-Modal OS Agents at Scale</a></p>
<ul dir="auto">
<li>Rogerio Bonatti, Dan Zhao, Francesco Bonacci, Dillon Dupont, Sara Abdali, Yinheng Li, Yadong Lu, Justin Wagle, Kazuhito Koishida, Arthur Bucker, Lawrence Jang, Zack Hui</li>
<li>🏛️ Institutions: Microsoft</li>
<li>📅 Date: September 13, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [benchmark], [Navi]</li>
<li>📖 TLDR: This paper introduces the <em>Windows Agent Arena (WAA)</em>, a scalable platform for testing and benchmarking multi-modal AI agents within a realistic Windows OS environment. WAA enables researchers to evaluate agentic workflows across diverse tasks and supports large-scale deployment using Azure ML. The study also presents <em>Navi</em>, a multi-modal agent achieving a 19.5% success rate on Windows tasks, highlighting the platform's potential for advancing AI agent development.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.07429" rel="nofollow">Agent Workflow Memory</a></p>
<ul dir="auto">
<li>Zora Zhiruo Wang, Jiayuan Mao, Daniel Fried, Graham Neubig</li>
<li>🏛️ Institutions: CMU, MIT</li>
<li>📅 Date: September 11, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [memory], [AWM]</li>
<li>📖 TLDR: The paper proposes <em>Agent Workflow Memory (AWM)</em>, a method enabling language model-based agents to induce and utilize reusable workflows from past experiences to guide future actions in web navigation tasks. AWM operates in both offline and online settings, significantly improving performance on benchmarks like Mind2Web and WebArena, and demonstrating robust generalization across tasks, websites, and domains.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2409.01927" rel="nofollow">From Grounding to Planning: Benchmarking Bottlenecks in Web Agents</a></p>
<ul dir="auto">
<li>Segev Shlomov, Ben Wiesel, Aviad Sela, Ido Levy, Liane Galanti, Roy Abitbol</li>
<li>🏛️ Institutions: IBM</li>
<li>📅 Date: September 3, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [planning], [grounding], [Mind2Web dataset], [web navigation]</li>
<li>📖 TLDR: This paper analyzes performance bottlenecks in web agents by separately evaluating grounding and planning tasks, isolating their individual impacts on navigation efficacy. Using an enhanced version of the Mind2Web dataset, the study reveals planning as a significant bottleneck, with advancements in grounding and task-specific benchmarking for elements like UI component recognition. Through experimental adjustments, the authors propose a refined evaluation framework, aiming to enhance web agents' contextual adaptability and accuracy in complex web environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/SqueezeAILab/TinyAgent">TinyAgent: Function Calling at the Edge</a></p>
<ul dir="auto">
<li>Lutfi Eren Erdogan, Nicholas Lee, Siddharth Jha, Sehoon Kim, Ryan Tabrizi, Suhong Moon, Coleman Hooper, Gopala Anumanchipalli, Kurt Keutzer, Amir Gholami</li>
<li>🏛️ Institutions: UC Berkeley, ICSI</li>
<li>📅 Date: September 1, 2024</li>
<li>📑 Publisher: EMNLP 2024</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [dataset], [quantization], [LLMCompiler], [TinyAgent-1.1B], [TinyAgent-7B]</li>
<li>📖 TLDR: This paper introduces TinyAgent, an end-to-end framework for training and deploying task-specific small language model agents capable of function calling at the edge. By fine-tuning small models with curated datasets and employing techniques like quantization and a novel tool retrieval method, TinyAgent enables efficient, real-time execution of user commands on local devices without relying on cloud infrastructure. The framework demonstrates that these small models can match or even surpass the function-calling capabilities of larger models like GPT-4-Turbo while operating entirely on edge devices.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2408.15978" rel="nofollow">WebPilot: A Versatile and Autonomous Multi-Agent System for Web Task Execution with Strategic Exploration</a></p>
<ul dir="auto">
<li>Yao Zhang, Zijian Ma, Yunpu Ma, Zhen Han, Yu Wu, Volker Tresp</li>
<li>🏛️ Institutions: LMU Munich, Technical University of Munich, Munich Center for Machine Learning (MCML)</li>
<li>📅 Date: August 28, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [Monte Carlo Tree Search], [reinforcement learning], [WebPilot]</li>
<li>📖 TLDR: This paper introduces <strong>WebPilot</strong>, a multi-agent system designed to execute complex web tasks requiring dynamic interaction. By employing a dual optimization strategy grounded in Monte Carlo Tree Search (MCTS), WebPilot enhances adaptability in complex web environments. The system's Global Optimization phase generates high-level plans by decomposing tasks into manageable subtasks, while the Local Optimization phase executes each subtask using a tailored MCTS approach. Experimental results on WebArena and MiniWoB++ demonstrate WebPilot's effectiveness, achieving state-of-the-art performance with GPT-4 and marking a significant advancement in autonomous web agent capabilities. :contentReference[oaicite:0]{index=0}</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2408.07199" rel="nofollow">Agent Q: Advanced Reasoning and Learning for Autonomous AI Agents</a></p>
<ul dir="auto">
<li>Agent Q: Advanced Reasoning and Learning for Autonomous AI Agents</li>
<li>🏛️ Institutions: MultiOn, Stanford</li>
<li>📅 Date: August 13, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [MCTS], [Tree Search], [DPO], [Reinforcement Learning]</li>
<li>📖 TLDR: TBD</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2408.06327" rel="nofollow">VisualAgentBench: Towards Large Multimodal Models as Visual Foundation Agents</a></p>
<ul dir="auto">
<li>Xiao Liu, Tianjie Zhang, Yu Gu, Iat Long Iong, Yifan Xu, Xixuan Song, Shudan Zhang, Hanyu Lai, Xinyi Liu, Hanlin Zhao, Jiadai Sun, Xinyue Yang, Yu Yang, Zehan Qi, Shuntian Yao, Xueqiao Sun, Siyi Cheng, Qinkai Zheng, Hao Yu, Hanchen Zhang, Wenyi Hong, Ming Ding, Lihang Pan, Xiaotao Gu, Aohan Zeng, Zhengxiao Du, Chan Hee Song, Yu Su, Yuxiao Dong, Jie Tang</li>
<li>🏛️ Institutions: Tsinghua University, MSRA, The Ohio State University</li>
<li>📅 Date: August 12, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [benchmark], [dataset], [VisualAgentBench], [VAB]</li>
<li>📖 TLDR: The authors introduce <em>VisualAgentBench (VAB)</em>, a comprehensive benchmark designed to train and evaluate large multimodal models (LMMs) as visual foundation agents across diverse scenarios, including embodied tasks, graphical user interfaces, and visual design. VAB comprises five distinct environments that systematically challenge LMMs' understanding and interaction capabilities. Additionally, the benchmark offers supervised fine-tuning trajectory data for behavior cloning training, demonstrating the potential to improve open LMMs for serving as visual foundation agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2408.11824" rel="nofollow">AppAgent v2: Advanced Agent for Flexible Mobile Interactions</a></p>
<ul dir="auto">
<li>Yanda Li, Chi Zhang, Wanqi Yang, Bin Fu, Pei Cheng, Xin Chen, Ling Chen, Yunchao Wei</li>
<li>🏛️ Institutions: University of Technology Sydney, Tencent, Beijing Jiaotong University, Westlake University</li>
<li>📅 Date: August 5, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [AppAgent v2]</li>
<li>📖 TLDR: This work presents <em>AppAgent v2</em>, a novel LLM-based multimodal agent framework for mobile devices capable of navigating applications by emulating human-like interactions such as tapping and swiping. The agent constructs a flexible action space that enhances adaptability across various applications, including parsing text and vision descriptions. It operates through two main phases: exploration and deployment, utilizing retrieval-augmented generation (RAG) technology to efficiently retrieve and update information from a knowledge base, thereby empowering the agent to perform tasks effectively and accurately.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://aclanthology.org/2024.findings-acl.539" rel="nofollow">CoCo-Agent: A Comprehensive Cognitive MLLM Agent for Smartphone GUI Automation</a></p>
<ul dir="auto">
<li>Xinbei Ma, Zhuosheng Zhang, Hai Zhao</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: August 2024</li>
<li>📑 Publisher: ACL 2024</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [model], [framework], [benchmark]</li>
<li>📖 TLDR: This paper presents CoCo-Agent, a multimodal large language model (MLLM) designed for smartphone GUI automation. It introduces two novel approaches: Comprehensive Environment Perception (CEP) for enhanced GUI understanding, and Conditional Action Prediction (CAP) to improve action response accuracy. The proposed agent achieves state-of-the-art performance on GUI automation benchmarks such as AITW and META-GUI, showcasing its capabilities in realistic scenarios.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2408.02544" rel="nofollow">Caution for the Environment: Multimodal Agents are Susceptible to Environmental Distractions</a></p>
<ul dir="auto">
<li>Xinbei Ma, Yiting Wang, Yao Yao, Tongxin Yuan, Aston Zhang, Zhuosheng Zhang, Hai Zhao</li>
<li>🏛️ Institutions: SJTU, Meta</li>
<li>📅 Date: August 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [multimodal agents], [environmental distractions], [robustness]</li>
<li>📖 TLDR: This paper highlights the vulnerability of multimodal agents to environmental distractions. The researchers demonstrate that these agents, which process multiple types of input (e.g., text, images, audio), can be significantly impacted by irrelevant or misleading environmental cues. The study provides insights into the limitations of current multimodal systems and emphasizes the need for more robust architectures that can filter out distractions and maintain focus on relevant information in complex, real-world environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://microsoft.github.io/OmniParser/" rel="nofollow">OmniParser for Pure Vision Based GUI Agent</a></p>
<ul dir="auto">
<li>Yadong Lu, Jianwei Yang, Yelong Shen, Ahmed Awadallah</li>
<li>🏛️ Institutions: MSR, Microsoft Gen AI</li>
<li>📅 Date: August 1, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [OmniParser]</li>
<li>📖 TLDR: This paper introduces <strong>OmniParser</strong>, a method for parsing user interface screenshots into structured elements, enhancing the ability of models like GPT-4V to generate actions accurately grounded in corresponding UI regions. The authors curated datasets for interactable icon detection and icon description, fine-tuning models to parse interactable regions and extract functional semantics of UI elements.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.19056" rel="nofollow">OfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation</a></p>
<ul dir="auto">
<li>Zilong Wang, Yuedong Cui, Li Zhong, Zimin Zhang, Da Yin, Bill Yuchen Lin, Jingbo Shang</li>
<li>🏛️ Institutions: UCSD, UCLA, AI2</li>
<li>📅 Date: July 26, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [benchmark], [multi-application], [office automation]</li>
<li>📖 TLDR: OfficeBench introduces a benchmark that evaluates language models' ability to automate office tasks across a range of applications like Word, Excel, and email. The benchmark tests agents’ skills in task-switching, planning, and decision-making by simulating realistic office workflows. Current models, including GPT-4, demonstrate significant gaps in task accuracy and efficiency, revealing areas for improvement in managing complex, multi-application tasks in office environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.13032" rel="nofollow">Agent-E: From Autonomous Web Navigation to Foundational Design Principles in Agentic Systems</a></p>
<ul dir="auto">
<li>Aditya Vempaty, [Other authors not provided in the search results]</li>
<li>🏛️ Institutions: Emergence AI</li>
<li>📅 Date: July 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [autonomous web navigation], [hierarchical architecture], [DOM distillation]</li>
<li>📖 TLDR: This paper presents Agent-E, a novel web agent that introduces several architectural improvements over previous state-of-the-art systems. Key features include a hierarchical architecture, flexible DOM distillation and denoising methods, and a "change observation" concept for improved performance. Agent-E outperforms existing text and multi-modal web agents by 10-30% on the WebVoyager benchmark. The authors synthesize their findings into general design principles for developing agentic systems, including the use of domain-specific primitive skills, hierarchical architectures, and agentic self-improvement.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://spider2-v.github.io/" rel="nofollow">Spider2-V: How Far Are Multimodal Agents From Automating Data Science and Engineering Workflows?</a></p>
<ul dir="auto">
<li>Ruisheng Cao, Fangyu Lei, Haoyuan Wu, Jixuan Chen, Yeqiao Fu, Hongcheng Gao, Xinzhuang Xiong, Hanchong Zhang, Yuchen Mao, Wenjing Hu, Tianbao Xie, Hongsheng Xu, Danyang Zhang, Sida Wang, Ruoxi Sun, Pengcheng Yin, Caiming Xiong, Ansong Ni, Qian Liu, Victor Zhong, Lu Chen, Kai Yu, Tao Yu</li>
<li>🏛️ Institutions: HKU, SJTU, Google Cloud AI Research, Google DeepMind, Salesforce Research, Yale University, Sea AI Lab, University of Waterloo</li>
<li>📅 Date: July 15, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [benchmark], [dataset], [data science], [engineering workflows], [Spider2-V]</li>
<li>📖 TLDR: This paper introduces <strong>Spider2-V</strong>, a multimodal agent benchmark designed to evaluate the capability of agents in automating professional data science and engineering workflows. It comprises 494 real-world tasks across 20 enterprise-level applications, assessing agents' proficiency in code generation and GUI operations within authentic computer environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/bz-lab/AUITestAgent">AUITestAgent: Automatic Requirements Oriented GUI Function Testing</a></p>
<ul dir="auto">
<li>Yongxiang Hu, Xuan Wang, Yingchuan Wang, Yu Zhang, Shiyu Guo, Chaoyi Chen, Xin Wang, Yangfan Zhou</li>
<li>🏛️ Institutions: Fudan University, Meituan</li>
<li>📅 Date: July 12, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [GUI testing], [AUITestAgent]</li>
<li>📖 TLDR: This paper presents <strong>AUITestAgent</strong>, the first automatic, natural language-driven GUI testing tool for mobile apps. It automates the entire process of GUI interaction and function verification by extracting GUI interactions from test requirements via dynamically organized agents and employing a multi-dimensional data extraction strategy for verification.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://luffyzm3d2y.github.io/publication/IoA" rel="nofollow">Internet of Agents: Weaving a Web of Heterogeneous Agents for Collaborative Intelligence</a></p>
<ul dir="auto">
<li>Weize Chen, Ziming You, Ran Li, Yitong Guan, Chen Qian, Chenyang Zhao, Cheng Yang, Ruobing Xie, Zhiyuan Liu, Maosong Sun</li>
<li>🏛️ Institutions: Tsinghua University, Peking University, BUPT, Tencent</li>
<li>📅 Date: July 7, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [framework], [IoA]</li>
<li>📖 TLDR: The paper proposes the <strong>Internet of Agents (IoA)</strong>, a framework inspired by the Internet to facilitate collaboration among diverse autonomous agents. IoA introduces an agent integration protocol, dynamic teaming mechanisms, and conversation flow control, enabling flexible and scalable multi-agent collaboration. Experiments demonstrate IoA's superior performance across various tasks, highlighting its effectiveness in integrating heterogeneous agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.05291" rel="nofollow">WorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks</a></p>
<ul dir="auto">
<li>Léo Boisvert, Megh Thakkar, Maxime Gasse, Massimo Caccia, Thibault Le Sellier De Chezelles, Quentin Cappart, Nicolas Chapados, Alexandre Lacoste, Alexandre Drouin</li>
<li>🏛️ Institutions: ServiceNow Research, Mila, Polytechnique Montréal, Université de Montréal</li>
<li>📅 Date: July 7, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [planning], [reasoning], [WorkArena++]</li>
<li>📖 TLDR: This paper introduces <strong>WorkArena++</strong>, a benchmark comprising 682 tasks that simulate realistic workflows performed by knowledge workers. It evaluates web agents' capabilities in planning, problem-solving, logical/arithmetic reasoning, retrieval, and contextual understanding. The study reveals challenges faced by current large language models and vision-language models in serving as effective workplace assistants, providing a resource to advance autonomous agent development. <a href="https://arxiv.org/abs/2407.05291?utm_source=chatgpt.com" rel="nofollow">oai_citation_attribution:0‡arXiv</a></li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.04346" rel="nofollow">MobileFlow: A Multimodal LLM For Mobile GUI Agent</a></p>
<ul dir="auto">
<li>Songqin Nong, Jiali Zhu, Rui Wu, Jiongchao Jin, Shuo Shan, Xiutian Huang, Wenhao Xu</li>
<li>🏛️ Institutions: Ant Group</li>
<li>📅 Date: July 5, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [model], [framework], [MobileFlow]</li>
<li>📖 TLDR: This paper introduces <em>MobileFlow</em>, a multimodal large language model tailored for mobile GUI agents. With approximately 21 billion parameters and hybrid visual encoders, it supports variable image resolutions and multilingual GUIs, enhancing the model's ability to interpret image data and comprehend user instructions for GUI interaction tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.03913" rel="nofollow">MobileExperts: A Dynamic Tool-Enabled Agent Team in Mobile Devices</a></p>
<ul dir="auto">
<li>Jiayi Zhang, Chuang Zhao, Yihan Zhao, Zhaoyang Yu, Ming He, Jianping Fan</li>
<li>🏛️ Institutions: HKUST, Ant Group</li>
<li>📅 Date: July 4, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [tool formulation], [multi-agent collaboration], [MobileExperts]</li>
<li>📖 TLDR: This paper introduces <em>MobileExperts</em>, a framework that enhances autonomous operations on mobile devices by dynamically assembling agent teams based on user requirements. Each agent independently explores and formulates tools to evolve into an expert, improving efficiency and reducing reasoning costs.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.03037" rel="nofollow">Vision-driven Automated Mobile GUI Testing via Multimodal Large Language Model</a></p>
<ul dir="auto">
<li>Zhe Liu, Cheng Li, Chunyang Chen, Junjie Wang, Boyu Wu, Yawen Wang, Jun Hu, Qing Wang</li>
<li>🏛️ Institutions: Institute of Software, Chinese Academy of Sciences, Monash University, Beijing Institute of Technology, University of Chinese Academy of Sciences</li>
<li>📅 Date: July 3, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [VisionDroid]</li>
<li>📖 TLDR: The paper presents <strong>VisionDroid</strong>, a vision-driven automated GUI testing approach utilizing Multimodal Large Language Models (MLLM) to detect non-crash functional bugs in mobile applications. By extracting GUI text information and aligning it with screenshots, VisionDroid enables MLLM to understand GUI context, facilitating deeper and function-oriented exploration. The approach segments exploration history into logically cohesive parts, prompting MLLM for bug detection, demonstrating superior performance over existing methods.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.01511" rel="nofollow">CRAB: Cross-environment Agent Benchmark for Multimodal Language Model Agents</a></p>
<ul dir="auto">
<li>Tianqi Xu, Linyao Chen, Dai-Jie Wu, Yanjun Chen, Zecheng Zhang, Xiang Yao, Zhiqiang Xie, Yongchao Chen, Shilong Liu, Bochen Qian, Philip Torr, Bernard Ghanem, Guohao Li</li>
<li>🏛️ Institutions: KAUST, UTokyo, CMU, Stanford, Harvard, Tsinghua University, SUSTech, Oxford</li>
<li>📅 Date: July 3, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [benchmark], [framework], [evaluation], [CRAB]</li>
<li>📖 TLDR: The authors present <em>CRAB</em>, a benchmark framework designed to evaluate Multimodal Language Model agents across multiple environments. It features a graph-based fine-grained evaluation method and supports automatic task generation, addressing limitations in existing benchmarks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://yuxiangchai.github.io/AMEX/" rel="nofollow">AMEX: Android Multi-annotation Expo Dataset for Mobile GUI Agents</a></p>
<ul dir="auto">
<li>Yuxiang Chai, Siyuan Huang, Yazhe Niu, Han Xiao, Liang Liu, Dingyu Zhang, Peng Gao, Shuai Ren, Hongsheng Li</li>
<li>🏛️ Institutions: CUHK, SJTU, Shanghai AI Lab, vivo AI Lab</li>
<li>📅 Date: July 3, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [benchmark], [AMEX]</li>
<li>📖 TLDR: This paper introduces the <strong>Android Multi-annotation EXpo (AMEX)</strong>, a comprehensive dataset designed for training and evaluating mobile GUI-control agents. AMEX comprises over 104K high-resolution screenshots from 110 popular mobile applications, annotated at multiple levels, including GUI interactive element grounding, functionality descriptions, and complex natural language instructions. The dataset aims to advance research on AI agents capable of completing complex tasks by interacting directly with mobile device GUIs.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://screen-point-and-read.github.io/" rel="nofollow">Read Anywhere Pointed: Layout-aware GUI Screen Reading with Tree-of-Lens Grounding</a></p>
<ul dir="auto">
<li>Yue Fan, Lei Ding, Ching-Chen Kuo, Shan Jiang, Yang Zhao, Xinze Guan, Jie Yang, Yi Zhang, Xin Eric Wang</li>
<li>🏛️ Institutions: UCSC, MSR</li>
<li>📅 Date: June 27, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [ToL], [screen reading], [accessibility]</li>
<li>📖 TLDR: The authors propose the Tree-of-Lens (ToL) agent to address the Screen Point-and-Read (ScreenPR) task, which involves generating natural language descriptions of screen regions based on user-indicated points. The ToL agent constructs a Hierarchical Layout Tree to comprehend the content and articulate the layout and spatial relationships between elements. The authors also introduce the ScreenPR benchmark, consisting of 650 screenshots from web, mobile, and operating system GUIs, manually annotated with 1,500 target points and regions.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://nexa.ai/octo-planner" rel="nofollow">Octo-planner: On-device Language Model for Planner-Action Agents</a></p>
<ul dir="auto">
<li>Nexa AI Team</li>
<li>🏛️ Institutions: Nexa AI</li>
<li>📅 Date: June 26, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [model], [framework], [Octo-planner], [on-device], [planning]</li>
<li>📖 TLDR: This paper presents Octo-planner, an on-device planning model designed for the Planner-Action Agents Framework. Octo-planner utilizes a fine-tuned model based on Phi-3 Mini (3.8 billion parameters) for high efficiency and low power consumption. It separates planning and action execution into two distinct components: a planner agent optimized for edge devices and an action agent using the Octopus model for function execution. The model achieves a planning success rate of 98.1% on benchmark datasets, providing reliable and effective performance.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.14250" rel="nofollow">E-ANT: A Large-Scale Dataset for Efficient Automatic GUI NavigaTion</a></p>
<ul dir="auto">
<li>Ke Wang, Tianyu Xia, Zhangxuan Gu, Yi Zhao, Shuheng Shen, Changhua Meng, Weiqiang Wang, Ke Xu</li>
<li>🏛️ Institutions: Ant Group, Tsinghua University</li>
<li>📅 Date: June 20, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [benchmark], [E-ANT]</li>
<li>📖 TLDR: This paper introduces <strong>E-ANT</strong>, the first large-scale Chinese GUI navigation dataset comprising over 40,000 real human interaction traces across more than 5,000 tiny apps. The dataset includes high-quality screenshots with annotations, facilitating the evaluation and development of GUI navigation and decision-making capabilities in multimodal large language models (MLLMs). The authors also assess various MLLMs on E-ANT, providing insights into their performance and potential improvements.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://ical-learning.github.io/" rel="nofollow">VLM Agents Generate Their Own Memories: Distilling Experience into Embodied Programs of Thought</a></p>
<ul dir="auto">
<li>Gabriel Sarch, Lawrence Jang, Michael J. Tarr, William W. Cohen, Kenneth Marino, Katerina Fragkiadaki</li>
<li>🏛️ Institutions: CMU, Google DeepMind</li>
<li>📅 Date: June 20, 2024</li>
<li>📑 Publisher: NeurIPS 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [memory], [in-context learning], [ICAL]</li>
<li>📖 TLDR: This paper introduces <em>In-Context Abstraction Learning (ICAL)</em>, a method enabling Vision-Language Models (VLMs) to generate their own examples from sub-optimal demonstrations and human feedback. By abstracting trajectories into generalized programs of thought, ICAL enhances decision-making in retrieval-augmented LLM and VLM agents, reducing reliance on manual prompt engineering and improving performance across various tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.14314" rel="nofollow">Identifying User Goals from UI Trajectories</a></p>
<ul dir="auto">
<li>Omri Berkovitch, Sapir Caduri, Noam Kahlon, Anatoly Efros, Avi Caciularu, Ido Dagan</li>
<li>🏛️ Institutions: Google Research, Bar-Ilan University</li>
<li>📅 Date: June 20, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [evaluation metric], [intent identification]</li>
<li>📖 TLDR: This paper introduces the task of goal identification from observed UI trajectories, aiming to infer the user's intended task based on their GUI interactions. It proposes a novel evaluation metric to assess whether two task descriptions are paraphrases within a specific UI environment. Experiments utilizing the Android-In-The-Wild and Mind2Web datasets reveal that state-of-the-art models, such as GPT-4 and Gemini-1.5 Pro, underperform compared to humans, indicating significant room for improvement.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.14056" rel="nofollow">VGA: Vision GUI Assistant -- Minimizing Hallucinations through Image-Centric Fine-Tuning</a></p>
<ul dir="auto">
<li>Ziyang Meng, Yu Dai, Zezheng Gong, Shaoxiong Guo, Minglong Tang, Tongquan Wei</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: June 20, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [framework], [VGA], [hallucination]</li>
<li>📖 TLDR: This paper introduces VGA, a fine-tuned model designed to enhance GUI comprehension by reducing hallucinations. The authors constructed a Vision Question Answering (VQA) dataset of 63.8k high-quality examples using a Referent Method, ensuring model responses are highly dependent on visual content. They also propose a two-stage fine-tuning method called Foundation and Advanced Comprehension (FAC) to improve the model's ability to extract information from images and align with human intent.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://showlab.github.io/GUI-Narrator/" rel="nofollow">GUI Action Narrator: Where and When Did That Action Take Place?</a></p>
<ul dir="auto">
<li>Qinchen Wu, Difei Gao, Kevin Qinghong Lin, Zhuoyu Wu, Xiangwu Guo, Peiran Li, Weichen Zhang, Hengxu Wang, Mike Zheng Shou</li>
<li>🏛️ Institutions: NUS, Chinese Academy of Sciences</li>
<li>📅 Date: June 19, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [dataset], [framework], [Act2Cap], [GUI Narrator]</li>
<li>📖 TLDR: The authors present <strong>Act2Cap</strong>, a GUI action dataset containing 4,189 video-caption pairs depicting various GUI actions such as clicks, drags, and typing across multiple software environments. They also propose <strong>GUI Narrator</strong>, a framework that leverages cursor detection as a visual prompt to enhance the interpretation of high-resolution screenshots for GUI video captioning. Evaluations reveal that even advanced multimodal models face challenges in this domain, highlighting the need for specialized approaches to improve performance.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.12373" rel="nofollow">WebCanvas: Benchmarking Web Agents in Online Environments</a></p>
<ul dir="auto">
<li>Yichen Pan, Dehan Kong, Sida Zhou, Cheng Cui, Yifei Leng, Bing Jiang, Hangyu Liu, Yanyi Shang, Shuyan Zhou, Tongshuang Wu, Zhengyang Wu</li>
<li>🏛️ Institutions: iMean AI, CMU</li>
<li>📅 Date: June 18, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [Mind2Web-Live], [key-node evaluation]</li>
<li>📖 TLDR: This paper presents WebCanvas, an online evaluation framework for web agents designed to address the dynamic nature of web interactions. It introduces a key-node-based evaluation metric to capture critical actions or states necessary for task completion while disregarding noise from insignificant events or changed web elements. The framework includes the Mind2Web-Live dataset, a refined version of the original Mind2Web static dataset, containing 542 tasks with 2,439 intermediate evaluation states. Despite advancements, the best-performing model achieves a task success rate of 23.1%, highlighting substantial room for improvement.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://chenwu.io/attack-agent/" rel="nofollow">Adversarial Attacks on Multimodal Agents</a></p>
<ul dir="auto">
<li>Chen Henry Wu, Jing Yu Koh, Ruslan Salakhutdinov, Daniel Fried, Aditi Raghunathan</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: Jun 18, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [safety], [VisualWebArena-Adv]</li>
<li>📖 TLDR: This paper investigates the safety risks posed by multimodal agents built on vision-enabled language models (VLMs). The authors introduce two adversarial attack methods: a captioner attack targeting white-box captioners and a CLIP attack that transfers to proprietary VLMs. To evaluate these attacks, they curated VisualWebArena-Adv, a set of adversarial tasks based on VisualWebArena. The study demonstrates that within a limited perturbation norm, the captioner attack can achieve a 75% success rate in making a captioner-augmented GPT-4V agent execute adversarial goals. The paper also discusses the robustness of agents based on other VLMs and provides insights into factors contributing to attack success and potential defenses. <a href="https://arxiv.org/abs/2406.12814?utm_source=chatgpt.com" rel="nofollow">oai_citation_attribution:0‡ArXiv</a></li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/yiye3/GUICourse">GUICourse: From General Vision Language Models to Versatile GUI Agents</a></p>
<ul dir="auto">
<li>Wentong Chen, Junbo Cui, Jinyi Hu, Yujia Qin, Junjie Fang, Yue Zhao, Chongyi Wang, Jun Liu, Guirong Chen, Yupeng Huo, Yuan Yao, Yankai Lin, Zhiyuan Liu, Maosong Sun</li>
<li>🏛️ Institutions: Tsinghua University, Rhapsody AI, University of Electronic Science and Technology of China</li>
<li>📅 Date: June 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [dataset], [framework], [GUICourse]</li>
<li>📖 TLDR: This paper introduces <em>GUICourse</em>, a suite of datasets aimed at training visual-based GUI agents from general vision-language models. It addresses challenges in OCR, grounding, and GUI knowledge, enhancing the models' capabilities in GUI navigation tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.10819" rel="nofollow">GUI-WORLD: A Dataset for GUI-oriented Multimodal LLM-based Agents</a></p>
<ul dir="auto">
<li>Dongping Chen, Yue Huang, Siyuan Wu, Jingyu Tang, Liuyi Chen, Yilin Bai, Zhigang He, Chenlong Wang, Huichi Zhou, Yiqiang Li, Tianshuo Zhou, Yue Yu, Chujie Gao, Qihui Zhang, Yi Gui, Zhen Li, Yao Wan, Pan Zhou, Jianfeng Gao, Lichao Sun</li>
<li>🏛️ Institutions: Huazhong University of Science and Technology (HUST), MSR, University of Illinois at Chicago (UIC)</li>
<li>📅 Date: June 16, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [dataset], [benchmark], [GUI-World], [GUI-Vid]</li>
<li>📖 TLDR: This paper introduces <em>GUI-World</em>, a comprehensive dataset designed to evaluate Multimodal Large Language Models (MLLMs) in dynamic and complex GUI environments. It includes over 12,000 annotated GUI interaction videos covering diverse applications and scenarios. The study highlights the limitations of current MLLMs in handling dynamic and multi-step tasks and presents <em>GUI-Vid</em>, a fine-tuned VideoLLM, demonstrating improved understanding of various GUI tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://digirl-agent.github.io/" rel="nofollow">DigiRL: Training In-The-Wild Device-Control Agents with Autonomous Reinforcement Learning</a></p>
<ul dir="auto">
<li>Hao Bai, Yifei Zhou, Mert Cemri, Jiayi Pan, Alane Suhr, Sergey Levine, Aviral Kumar</li>
<li>🏛️ Institutions: UC Berkeley, UIUC, Google DeepMind</li>
<li>📅 Date: June 14, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [reinforcement learning], [DigiRL]</li>
<li>📖 TLDR: The authors present <em>DigiRL</em>, an autonomous reinforcement learning approach for training device-control agents. By fine-tuning a pre-trained vision-language model in two stages—offline and offline-to-online RL—DigiRL achieves a significant improvement in success rates on the Android-in-the-Wild dataset, establishing a new state-of-the-art for digital agents in device control.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.08451" rel="nofollow">GUI Odyssey: A Comprehensive Dataset for Cross-App GUI Navigation on Mobile Devices</a></p>
<ul dir="auto">
<li>Quanfeng Lu, Wenqi Shao, Zitao Liu, Fanqing Meng, Boxuan Li, Botong Chen, Siyuan Huang, Kaipeng Zhang, Yu Qiao, Ping Luo</li>
<li>🏛️ Institutions: OpenGVLab, Shanghai AI Laboratory, HKU, Nanjing University, Harbin Institute of Technology, Shenzhen, SJTU</li>
<li>📅 Date: June 13, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [model], [OdysseyAgent], [cross-app navigation]</li>
<li>📖 TLDR: This paper presents <em>GUI Odyssey</em>, a dataset comprising 7,735 episodes from six mobile devices, designed to train and evaluate cross-app navigation agents. It spans six types of cross-app tasks across 201 apps and 1,399 app combinations. Leveraging this dataset, the authors developed <em>OdysseyAgent</em>, a multimodal cross-app navigation agent fine-tuned from the Qwen-VL model, demonstrating superior accuracy over existing models in both in-domain and out-of-domain scenarios.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://mobileagentbench.github.io/" rel="nofollow">MobileAgentBench: An Efficient and User-Friendly Benchmark for Mobile LLM Agents</a></p>
<ul dir="auto">
<li>Luyuan Wang, Yongyu Deng, Yiwei Zha, Guodong Mao, Qinmin Wang, Tianchen Min, Wei Chen, Shoufa Chen</li>
<li>🏛️ Institutions: CMU, University of Michigan, Northeastern University, HKU</li>
<li>📅 Date: June 12, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [MobileAgentBench]</li>
<li>📖 TLDR: This paper introduces <em>MobileAgentBench</em>, a benchmark designed to evaluate the performance of large language model-based mobile agents. It defines 100 tasks across 10 open-source apps, categorized by difficulty levels, and assesses existing agents like AppAgent and MobileAgent to facilitate systematic comparisons.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.08340" rel="nofollow">Practical, Automated Scenario-based Mobile App Testing</a></p>
<ul dir="auto">
<li>Shengcheng Yu, Chunrong Fang, Mingzhe Du, Zimin Ding, Zhenyu Chen, Zhendong Su</li>
<li>🏛️ Institutions: Nanjing University, ETH Zurich</li>
<li>📅 Date: June 12, 2024</li>
<li>📑 Publisher: IEEE Transactions on Software Engineering</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [ScenTest], [event knowledge graph], [GUI image understanding]</li>
<li>📖 TLDR: This paper introduces <em>ScenTest</em>, a novel approach for scenario-based mobile app testing that integrates event knowledge graphs (EKGs) with GUI image understanding. By extracting entities and relationships from crowdsourced test reports, ScenTest constructs EKGs for specific scenarios, guiding automated testing processes. This method bridges the gap between testing execution and app business logic, achieving fully automated testing on target scenarios for the first time.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.03679" rel="nofollow">On the Effects of Data Scale on UI Control Agents</a></p>
<ul dir="auto">
<li>Wei Li, William Bishop, Alice Li, Chris Rawles, Folawiyo Campbell-Ajala, Divya Tyamagundlu, Oriana Riva</li>
<li>🏛️ Institutions: Google DeepMind, Google</li>
<li>📅 Date: June 6, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [AndroidControl], [fine-tuning], [scalability]</li>
<li>📖 TLDR: This study investigates how the performance of computer control agents scales with the amount of fine-tuning data. The authors introduce <strong>AndroidControl</strong>, a dataset comprising 15,283 demonstrations across 833 Android applications. Findings indicate that while in-domain performance improves with more data, out-of-domain performance, especially on high-level tasks, scales more slowly, suggesting that fine-tuning alone may be insufficient for robust out-of-domain performance.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/X-PLUG/MobileAgent">Mobile-Agent-v2: Mobile Device Operation Assistant with Effective Navigation via Multi-Agent Collaboration</a></p>
<ul dir="auto">
<li>Junyang Wang, Haiyang Xu, Haitao Jia, Xi Zhang, Ming Yan, Weizhou Shen, Ji Zhang, Fei Huang, Jitao Sang</li>
<li>🏛️ Institutions: Alibaba Group, Beijing University of Posts and Telecommunications</li>
<li>📅 Date: June 3, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [multi-agent], [planning], [decision-making], [reflection]</li>
<li>📖 TLDR: The paper presents <strong>Mobile-Agent-v2</strong>, a multi-agent architecture designed to assist with mobile device operations. It comprises three agents: a planning agent that generates task progress, a decision agent that navigates tasks using a memory unit, and a reflection agent that corrects erroneous operations. This collaborative approach addresses challenges in navigation and long-context input scenarios, achieving over a 30% improvement in task completion compared to single-agent architectures.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://aclanthology.org/2024.naacl-industry.9/" rel="nofollow">Visual Grounding for User Interfaces</a></p>
<ul dir="auto">
<li>Yijun Qian, Yujie Lu, Alexander Hauptmann, Oriana Riva</li>
<li>🏛️ Institutions: CMU, UCSB</li>
<li>📅 Date: June 2024</li>
<li>📑 Publisher: NAACL 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [visual grounding], [UI element localization], [LVG]</li>
<li>📖 TLDR: This work introduces the task of visual UI grounding, which unifies detection and grounding by enabling models to identify UI elements referenced by natural language commands solely from visual input. The authors propose <strong>LVG</strong>, a model that outperforms baselines pre-trained on larger datasets by over 4.9 points in top-1 accuracy, demonstrating its effectiveness in localizing referenced UI elements without relying on UI metadata.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.01623" rel="nofollow">WebSuite: Systematically Evaluating Why Web Agents Fail</a></p>
<ul dir="auto">
<li>Eric Li, Jim Waldo</li>
<li>🏛️ Institutions: Harvard</li>
<li>📅 Date: June 1, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [framework], [failure analysis], [analysis], [task disaggregation]</li>
<li>📖 TLDR: This paper introduces <em>WebSuite</em>, a diagnostic benchmark to investigate the causes of web agent failures. By categorizing agent tasks using a taxonomy of operational, informational, and navigational actions, WebSuite offers granular insights into the specific actions where agents struggle, like filtering or form completion. It enables detailed comparison across agents, identifying areas for architectural and UX adaptation to improve agent reliability and task success on the web.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.10227" rel="nofollow">VideoGUI: A Benchmark for GUI Automation from Instructional Videos</a></p>
<ul dir="auto">
<li>Kevin Qinghong Lin, Linjie Li, Difei Gao, Qinchen WU, Mingyi Yan, Zhengyuan Yang, Lijuan Wang, Mike Zheng Shou</li>
<li>🏛️ Institutions: NUS, Microsoft Gen AI</li>
<li>📅 Date: June 2024</li>
<li>📑 Publisher: NeurIPS 2024</li>
<li>💻 Env: [Desktop, Web]</li>
<li>🔑 Key: [benchmark], [instructional videos], [visual planning], [hierarchical task decomposition], [complex software interaction]</li>
<li>📖 TLDR: VideoGUI presents a benchmark for evaluating GUI automation on tasks derived from instructional videos, focusing on visually intensive applications like Adobe Photoshop and video editing software. The benchmark includes 178 tasks, with a hierarchical evaluation method distinguishing high-level planning, mid-level procedural steps, and precise action execution. VideoGUI reveals current model limitations in complex visual tasks, marking a significant step toward improved visual planning in GUI automation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2405.20309" rel="nofollow">Large Language Models Can Self-Improve At Web Agent Tasks</a></p>
<ul dir="auto">
<li>Ajay Patel, Markus Hofmarcher, Claudiu Leoveanu-Condrei, Marius-Constantin Dinu, Chris Callison-Burch, Sepp Hochreiter</li>
<li>🏛️ Institutions: University of Pennsylvania, ExtensityAI, Johannes Kepler University Linz, NXAI</li>
<li>📅 Date: May 30, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [self-improvement], [self-improve]</li>
<li>📖 TLDR: This paper investigates the ability of large language models (LLMs) to enhance their performance as web agents through self-improvement. Utilizing the WebArena benchmark, the authors fine-tune LLMs on synthetic training data, achieving a 31% improvement in task completion rates. They also introduce novel evaluation metrics to assess the performance, robustness, and quality of the fine-tuned agents' trajectories.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2405.14573" rel="nofollow">AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents</a></p>
<ul dir="auto">
<li>Christopher Rawles, Sarah Clinckemaillie, Yifan Chang, Jonathan Waltz, Gabrielle Lau, Marybeth Fair, Alice Li, William Bishop, Wei Li, Folawiyo Campbell-Ajala, Daniel Toyama, Robert Berry, Divya Tyamagundlu, Timothy Lillicrap, Oriana Riva</li>
<li>🏛️ Institutions: Google DeepMind, Google</li>
<li>📅 Date: May 23, 2024</li>
<li>📑 Publisher: ICLR 2025</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [Android-based agents], [task diversity], [reinforcement learning], [dynamic environment]</li>
<li>📖 TLDR: AndroidWorld introduces a dynamic Android environment for benchmarking autonomous agents across 116 tasks spanning 20 Android apps. These tasks vary through parameterized and natural language prompts, fostering a realistic testing ground for agents designed to operate in complex mobile environments. The benchmark supports millions of task variations, allowing agents to respond to the Android system's changing states and improving real-world applicability.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2405.04497" rel="nofollow">Unveiling Disparities in Web Task Handling Between Human and Web Agent</a></p>
<ul dir="auto">
<li>Kihoon Son, Jinhyeon Kwon, DaEun Choi, Tae Soo Kim, Young-Ho Kim, Sangdoo Yun, Juho Kim</li>
<li>🏛️ Institutions: KAIST, Seoul National University</li>
<li>📅 Date: May 7, 2024</li>
<li>📑 Publisher: CHI 2024 Workshop</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [cognitive comparison], [task analysis]</li>
<li>📖 TLDR: This paper examines how humans and web agents differ in handling web-based tasks, focusing on key aspects such as planning, action-taking, and reflection. Using a think-aloud protocol, the study highlights the cognitive processes humans employ, like exploration and adjustment, versus the more rigid task execution patterns observed in web agents. The authors identify several limitations in current web agents, proposing the need for improved frameworks to enhance adaptability and knowledge update mechanisms in agent-based systems.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2405.00516" rel="nofollow">Navigating WebAI: Training Agents to Complete Web Tasks with Large Language Models and Reinforcement Learning</a></p>
<ul dir="auto">
<li>Lucas-Andreï Thil, Mirela Popa, Gerasimos Spanakis</li>
<li>🏛️ Institutions: Maastricht University the Netherlands</li>
<li>📅 Date: May 1, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [large language models], [reinforcement learning]</li>
<li>📖 TLDR: This paper proposes a novel approach combining supervised learning (SL) and reinforcement learning (RL) techniques to train web navigation agents using large language models. The authors address limitations in previous models' understanding of HTML content and introduce methods to enhance true comprehension. Their approach, evaluated on the MiniWoB benchmark, outperforms previous SL methods on certain tasks using less data and narrows the performance gap with RL models. The study achieves 43.58% average accuracy in SL and 36.69% when combined with a multimodal RL approach, setting a new direction for future web navigation research.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.12500" rel="nofollow">UIClip: A Data-driven Model for Assessing User Interface Design</a></p>
<ul dir="auto">
<li>Jason Wu, Yi-Hao Peng, Amanda Li, Amanda Swearngin, Jeffrey P. Bigham, Jeffrey Nichols</li>
<li>🏛️ Institutions: CMU, Apple</li>
<li>📅 Date: Apr 18, 2024</li>
<li>📑 Publisher: UIST 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [UIClip], [vision foundation model], [foundation model]</li>
<li>📖 TLDR: This paper introduces <em>UIClip</em>, a machine-learned model that evaluates the design quality and visual relevance of user interfaces by analyzing screenshots and corresponding natural language descriptions. Trained on a large-scale dataset combining automated crawling, synthetic augmentation, and human ratings, UIClip assigns numerical scores representing a UI's relevance and quality, and offers design suggestions. Evaluations show that UIClip's assessments align closely with human designer rankings. The paper also demonstrates UIClip's utility in applications like UI code generation, design tips generation, and quality-aware UI example search.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.11459" rel="nofollow">Octopus v3: Technical Report for On-device Sub-billion Multimodal AI Agent</a></p>
<ul dir="auto">
<li>Wei Chen, Zhiyuan Li</li>
<li>🏛️ Institutions: Stanford University</li>
<li>📅 Date: April 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [model], [functional token], [on-device AI], [Octopus v3]</li>
<li>📖 TLDR: This paper introduces Octopus v3, a compact multimodal AI agent with less than 1 billion parameters, designed for efficient on-device operation. It processes both English and Chinese inputs, integrating visual and textual data to perform tasks such as sending emails, messaging, and online shopping. The model employs a functional token approach to translate image-based data into actionable outcomes, demonstrating high accuracy and efficiency on edge devices, including Raspberry Pi.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.10887" rel="nofollow">Search Beyond Queries: Training Smaller Language Models for Web Interactions via Reinforcement Learning</a></p>
<ul dir="auto">
<li>Moghis Fereidouni, A.B. Siddique</li>
<li>🏛️ Institutions: University of Kentucky</li>
<li>📅 Date: April 16, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [reinforcement learning], [grounded language agent], [Flan-T5], [unsupervised domain adaptation]</li>
<li>📖 TLDR: This paper introduces GLAINTEL, a grounded language agent framework designed to enhance web interaction using instruction-finetuned language models, particularly Flan-T5, with reinforcement learning (PPO) to tackle interactive web navigation challenges. The study explores unsupervised and supervised training methods, evaluating the effects of human demonstration on agent performance. Results indicate that combining human feedback with reinforcement learning yields effective outcomes, rivaling larger models like GPT-4 on web navigation tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.09992" rel="nofollow">MMInA: Benchmarking Multihop Multimodal Internet Agents</a></p>
<ul dir="auto">
<li>Ziniu Zhang, Shulin Tian, Liangyu Chen, Ziwei Liu</li>
<li>🏛️ Institutions: NTU</li>
<li>📅 Date: April 15, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [framework], [multihop web browsing], [multimodal tasks], [long-range reasoning]</li>
<li>📖 TLDR: The <strong>MMInA</strong> benchmark is designed to evaluate agents' capacity to complete complex, multihop web tasks by navigating and extracting information across evolving real-world websites. Composed of 1,050 tasks across diverse domains, MMInA challenges agents with realistic, multimodal information retrieval and reasoning tasks, such as comparative shopping and travel inquiries. Despite recent advances, agents show difficulties in handling tasks requiring sequential steps across multiple sites, underscoring the need for enhanced multimodal and memory-augmented models.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.16054" rel="nofollow">LlamaTouch: A Faithful and Scalable Testbed for Mobile UI Automation Task Evaluation</a></p>
<ul dir="auto">
<li>Li Zhang, Shihe Wang, Xianqing Jia, Zhihan Zheng, Yunhe Yan, Longxi Gao, Yuanchun Li, Mengwei Xu</li>
<li>🏛️ Institutions: BUPT, Tsinghua University</li>
<li>📅 Date: April 12, 2024</li>
<li>📑 Publisher: UIST 2024</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [UI automation], [mobile agent evaluation]</li>
<li>📖 TLDR: LlamaTouch is an evaluation testbed designed for mobile UI automation, enabling reliable task assessment across 495 annotated tasks. It provides a scalable solution to evaluate agents in real-world mobile settings, comparing agent actions to essential UI states for accurate task completion. LlamaTouch supports dynamic environments, advancing mobile agent reliability and scalability in task automation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.07972" rel="nofollow">OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments</a></p>
<ul dir="auto">
<li>Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu</li>
<li>🏛️ Institutions: HKU, CMU, Salesforce, University of Waterloo</li>
<li>📅 Date: April 11, 2024</li>
<li>📑 Publisher: NeurIPS 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [benchmark], [real computer tasks], [online environment], [online benchmark]</li>
<li>📖 TLDR: OSWorld introduces a groundbreaking benchmark for multimodal agents to perform open-ended tasks within real computer environments across platforms like Ubuntu, Windows, and macOS. It includes 369 real-world tasks involving web and desktop apps, file management, and multi-app workflows, with custom evaluation scripts for reproducibility. The results reveal current agents’ limitations in GUI interaction and operational knowledge, as they achieve just 12.24% task success compared to humans' 72.36%, highlighting critical gaps for future model improvement.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.05955" rel="nofollow">VisualWebBench: How Far Have Multimodal LLMs Evolved in Web Page Understanding and Grounding?</a></p>
<ul dir="auto">
<li>Junpeng Liu, Yifan Song, Bill Yuchen Lin, Wai Lam, Graham Neubig, Yuanzhi Li, Xiang Yue</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: April 9, 2024</li>
<li>📑 Publisher: COLM 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [web page understanding], [grounding]</li>
<li>📖 TLDR: VisualWebBench introduces a comprehensive benchmark for evaluating multimodal large language models (MLLMs) on web-based tasks. It includes 1.5K human-curated instances across 139 websites in 87 sub-domains. The benchmark spans seven tasks—such as OCR, grounding, and web-based QA—aiming to test MLLMs' capabilities in fine-grained web page understanding. Results reveal significant performance gaps, particularly in grounding tasks, highlighting the need for advancement in MLLM web understanding.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.06474" rel="nofollow">Autonomous Evaluation and Refinement of Digital Agents</a></p>
<ul dir="auto">
<li>Jiayi Pan, Yichi Zhang, Nicholas Tomlin, Yifei Zhou, Sergey Levine, Alane Suhr</li>
<li>🏛️ Institutions: UCB, UMich</li>
<li>📅 Date: April 9, 2024</li>
<li>📑 Publisher: COLM 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [benchmark], [evaluation model], [domain transfer]</li>
<li>📖 TLDR: This paper presents an autonomous evaluation framework for digital agents to enhance performance on web navigation and device control. The study introduces modular, cost-effective evaluators achieving up to 92.9% accuracy in benchmarks like WebArena and outlines their use in fine-tuning agents, improving state-of-the-art by 29% without additional supervision.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://machinelearning.apple.com/research/ferretui-mobile" rel="nofollow">Ferret-UI: Grounded Mobile UI Understanding with Multimodal LLMs</a></p>
<ul dir="auto">
<li>Keen You, Haotian Zhang, Eldon Schoop, Floris Weers, Amanda Swearngin, Jeffrey Nichols, Yinfei Yang, Zhe Gan</li>
<li>🏛️ Institutions: Apple</li>
<li>📅 Date: April 8, 2024</li>
<li>📑 Publisher: ECCV 2024</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [model], [framework], [dataset], [benchmark], [mobile UI understanding]</li>
<li>📖 TLDR: This paper presents <strong>Ferret-UI</strong>, a multimodal large language model (MLLM) designed to understand and interact with mobile user interfaces. The model incorporates advanced capabilities for referring, grounding, and reasoning about UI elements. By training on a variety of UI tasks, Ferret-UI achieves high performance in tasks such as icon recognition and text extraction. The authors introduce a unique architecture that allows for improved visual feature extraction from mobile screens, paving the way for applications in accessibility and user interaction.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.16660" rel="nofollow">Benchmarking Mobile Device Control Agents across Diverse Configurations</a></p>
<ul dir="auto">
<li>Juyong Lee, Taywon Min, Minyong An, Dongyoon Hahm, Haeone Lee, Changyeon Kim, Kimin Lee</li>
<li>🏛️ Institutions: KAIST, Seoul National University, Yonsei University</li>
<li>📅 Date: April 2024</li>
<li>📑 Publisher: ICLR 2024</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [dataset], [mobile device control], [agent performance]</li>
<li>📖 TLDR: This paper presents <strong>B-MoCA</strong>, a comprehensive benchmark for evaluating mobile device control agents using an Android-based testbed with 131 tasks and various device configurations. The benchmark assesses agents' abilities across tasks that include device-specific variations, navigation, and human-like dual-gesture interactions. B-MoCA highlights that current agents perform well on basic tasks but struggle with complex configurations, pointing to opportunities for future improvements in mobile automation capabilities.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2404.08860" rel="nofollow">Enhancing Mobile "How-to" Queries with Automated Search Results Verification and Reranking</a></p>
<ul dir="auto">
<li>Lei Ding, Jeshwanth Bheemanpally, Yi Zhang</li>
<li>🏛️ Institutions: UCSC</li>
<li>📅 Date: April 2024</li>
<li>📑 Publisher: SIGIR 2024</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [benchmark], [reranking], [verification], [mobile task automation]</li>
<li>📖 TLDR: This paper presents a system that enhances mobile "how-to" queries by verifying and reranking search results through automated instruction extraction, on-device action execution, and reranking based on relevance. The method improves on traditional ranking by analyzing device-specific execution success. The approach comprises a three-stage pipeline: 1) extracting step-by-step instructions from top search results, 2) validating these instructions on mobile devices, and 3) reranking based on performance. The system leverages a pre-trained GPT model for initial processing, ensuring adaptability across diverse apps and systems.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2403.17918" rel="nofollow">AgentStudio: A Toolkit for Building General Virtual Agents</a></p>
<ul dir="auto">
<li>Longtao Zheng, Zhiyuan Huang, Zhenghai Xue, Xinrun Wang, Bo An, Shuicheng Yan</li>
<li>🏛️ Institutions: NTU, Skywork AI, ETH Zurich</li>
<li>📅 Date: March 26, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [dataset], [general virtual agents], [open-ended learning], [tool creation], [GroundUI], [benchmark]</li>
<li>📖 TLDR: AgentStudio is a robust toolkit for developing virtual agents with versatile actions, such as GUI automation and code execution. It unifies real-world human-computer interactions across OS platforms and includes diverse observation and action spaces, facilitating comprehensive training and benchmarking in complex settings. The toolkit's flexibility promotes agent generalization across varied tasks, supporting tool creation and a multimodal interaction interface to advance agent adaptability and learning.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2312.15820" rel="nofollow">WebVLN: Vision-and-Language Navigation on Websites</a></p>
<ul dir="auto">
<li>Qi Chen, Dileepa Pitawela, Chongyang Zhao, Gengze Zhou, Hsiang-Ting Chen, Qi Wu</li>
<li>🏛️ Institutions: The University of Adelaide</li>
<li>📅 Date: March 24, 2024</li>
<li>📑 Publisher: AAAI 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [web-based VLN], [HTML content integration], [multimodal navigation]</li>
<li>📖 TLDR: This paper introduces the <em>WebVLN</em> task, where agents navigate websites by following natural language instructions that include questions and descriptions. Aimed at emulating real-world browsing behavior, the task allows the agent to interact with elements not directly visible in the rendered content by integrating HTML-specific information. A new <em>WebVLN-Net</em> model, based on the VLN BERT framework, is introduced alongside the <em>WebVLN-v1</em> dataset, supporting question-answer navigation across web pages. This framework demonstrated significant improvement over existing web-based navigation methods, marking a new direction in vision-and-language navigation research.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2403.11905" rel="nofollow">Tur[k]ingBench: A Challenge Benchmark for Web Agents</a></p>
<ul dir="auto">
<li>Kevin Xu, Yeganeh Kordi, Kate Sanders, Yizhong Wang, Adam Byerly, Jingyu Zhang, Benjamin Van Durme, Daniel Khashabi</li>
<li>🏛️ Institutions: JHU, Brown, UW</li>
<li>📅 Date: March 18, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [multi-modal reasoning], [TurkingBench], [Turking]</li>
<li>📖 TLDR: This paper introduces <strong>Tur[k]ingBench</strong>, a benchmark comprising 158 web-grounded tasks designed to evaluate AI agents' capabilities in complex web-based environments. Unlike prior benchmarks that utilize synthetic web pages, Tur[k]ingBench leverages natural HTML pages from crowdsourcing platforms, presenting tasks with rich multi-modal contexts. The benchmark includes 32.2K instances, each with diverse inputs, challenging models to interpret and interact with web pages effectively. Evaluations of state-of-the-art models reveal significant room for improvement, highlighting the need for advanced web-based agents capable of handling real-world web interactions.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2403.07718" rel="nofollow">WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?</a></p>
<ul dir="auto">
<li>Alexandre Drouin, Maxime Gasse, Massimo Caccia, Issam H. Laradji, Manuel Del Verme, Tom Marty, Léo Boisvert, Megh Thakkar, Quentin Cappart, David Vazquez, Nicolas Chapados, Alexandre Lacoste</li>
<li>🏛️ Institutions: ServiceNow Research, Mila, Polytechnique Montreal, McGill University, University de Montreal</li>
<li>📅 Date: March 11, 2024</li>
<li>📑 Publisher: ICML 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [enterprise task automation], [ServiceNow], [knowledge work automation]</li>
<li>📖 TLDR: WorkArena introduces a robust benchmark hosted on the ServiceNow platform to assess the effectiveness of large language model-based agents in performing 33 knowledge tasks common to enterprise environments. Leveraging BrowserGym, an environment that simulates complex browser interactions, WorkArena provides web agents with realistic challenges like data entry, form completion, and information retrieval in knowledge bases. Despite promising initial results, open-source models show a 42.7% success rate compared to closed-source counterparts, underlining the current gap in task automation for enterprise applications and highlighting key areas for improvement.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2403.03186" rel="nofollow">Towards General Computer Control: A Multimodal Agent for Red Dead Redemption II as a Case Study</a></p>
<ul dir="auto">
<li>Weihao Tan, Ziluo Ding, Wentao Zhang, Boyu Li, Bohan Zhou, Junpeng Yue, Haochong Xia, Jiechuan Jiang, Longtao Zheng, Xinrun Xu, Yifei Bi, Pengjie Gu, Xinrun Wang, Börje F. Karlsson, Bo An, Zongqing Lu</li>
<li>🏛️ Institutions: NTU, BAAI, PKU</li>
<li>📅 Date: March 5, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [Cradle], [General Computer Control], [multimodal], [keyboard and mouse control], [long-term memory], [reasoning], [self-improvement]</li>
<li>📖 TLDR: This paper introduces <em>Cradle</em>, a framework designed to achieve General Computer Control (GCC) by enabling agents to perform any computer task using only screen images (and possibly audio) as input and producing keyboard and mouse operations as output. The authors deploy Cradle in the complex AAA game Red Dead Redemption II, demonstrating its capability to follow the main storyline and complete real missions with minimal reliance on prior knowledge or resources.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2403.03186" rel="nofollow">Cradle: Empowering Foundation Agents Towards General Computer Control</a></p>
<ul dir="auto">
<li>Weihao Tan, Wentao Zhang, Xinrun Xu, Haochong Xia, Ziluo Ding, Boyu Li, Bohan Zhou, Junpeng Yue, Jiechuan Jiang, Yewen Li, Ruyi An, Molei Qin, Chuqiao Zong, Longtao Zheng, Yujie Wu, Xiaoqiang Chai, Yifei Bi, Tianbao Xie, Pengjie Gu, Xiyun Li, Ceyao Zhang, Long Tian, Chaojie Wang, Xinrun Wang, Börje F. Karlsson, Bo An, Shuicheng Yan, Zongqing Lu</li>
<li>🏛️ Institutions: Skywork AI, BAAI, NTU, PKU, Institute of Software - Chinese Academy of Sciences, HKU, CUHK</li>
<li>📅 Date: March 5, 2024</li>
<li>📑 Publisher: TBD</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [model], [general computer control], [skill curation], [self-improvement]</li>
<li>📖 TLDR: This paper introduces the Cradle framework, designed to enable general computer control (GCC) through multimodal input (e.g., screen images and optional audio) and outputs (keyboard and mouse). Cradle’s six core modules, including self-reflection, skill curation, and memory, allow for generalized task handling in complex environments like AAA games. Demonstrated in <em>Red Dead Redemption II</em>, the framework exhibits adaptability by performing real missions and following the storyline with minimal prior knowledge, showcasing its potential as a generalist agent for diverse computer tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2403.02713" rel="nofollow">Android in the Zoo: Chain-of-Action-Thought for GUI Agents</a></p>
<ul dir="auto">
<li>Jiwen Zhang, Jihao Wu, Yihua Teng, Minghui Liao, Nuo Xu, Xiao Xiao, Zhongyu Wei, Duyu Tang</li>
<li>🏛️ Institutions: Fudan University, Huawei</li>
<li>📅 Date: March 5, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [Android GUI], [Chain-of-Action-Thought], [autonomous GUI agents]</li>
<li>📖 TLDR: This paper introduces <em>Chain-of-Action-Thought</em> (CoAT), a novel paradigm to improve GUI agent task completion by enabling agents to interpret previous actions, current screen content, and action rationale for next steps. The authors present the <em>Android-In-The-Zoo</em> (AitZ) dataset, which includes 18,643 screen-action pairs with detailed annotations, supporting CoAT's development and evaluation. The study demonstrates that fine-tuning with the AitZ dataset improves performance of a baseline large language model in predicting correct action sequences in Android tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.15057" rel="nofollow">On the Multi-turn Instruction Following for Conversational Web Agents</a></p>
<ul dir="auto">
<li>Yang Deng, Xuan Zhang, Wenxuan Zhang, Yifei Yuan, See-Kiong Ng, Tat-Seng Chua</li>
<li>🏛️ Institutions: NUS, DAMO Academy, University of Copenhagen</li>
<li>📅 Date: February 23, 2024</li>
<li>📑 Publisher: ACL 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [dataset], [multi-turn dialogue], [memory utilization], [self-reflective planning]</li>
<li>📖 TLDR: This paper explores multi-turn conversational web navigation, introducing the MT-Mind2Web dataset to support instruction-following tasks for web agents. The proposed Self-MAP (Self-Reflective Memory-Augmented Planning) framework enhances agent performance by integrating memory with self-reflection for sequential decision-making in complex interactions. Extensive evaluations using MT-Mind2Web demonstrate Self-MAP's efficacy in addressing the limitations of current models in multi-turn interactions, providing a novel dataset and framework for evaluating and training agents on detailed, multi-step web-based tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.14073" rel="nofollow">Improving Language Understanding from Screenshots</a></p>
<ul dir="auto">
<li>Tianyu Gao, Zirui Wang, Adithya Bhaskar, Danqi Chen</li>
<li>🏛️ Institutions: Princeton</li>
<li>📅 Date: February 22, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [framework], [screenshot language models], [patch-and-text prediction]</li>
<li>📖 TLDR: This paper introduces a novel approach to improve the language understanding capabilities of screenshot language models (LMs). The authors propose a Patch-and-Text Prediction (PTP) objective, which masks and recovers both image patches and text within screenshots. The method significantly narrows the performance gap between screenshot LMs and text-only models on language understanding tasks, achieving comparable results to BERT on most GLUE tasks. The research also extends PTP to train autoregressive screenshot LMs, demonstrating improved perplexity by utilizing screenshot context.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.11208" rel="nofollow">Watch Out for Your Agents! Investigating Backdoor Threats to LLM-Based Agents</a></p>
<ul dir="auto">
<li>Wenkai Yang, Xiaohan Bi, Yankai Lin, Sishuo Chen, Jie Zhou, Xu Sun</li>
<li>🏛️ Institutions: Renming University of China, PKU, Tencent</li>
<li>📅 Date: Feb 17, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI], [Misc]</li>
<li>🔑 Key: [attack], [backdoor], [safety]</li>
<li>📖 TLDR: This paper investigates backdoor attacks on LLM-based agents, introducing a framework that categorizes attacks based on outcomes and trigger locations. The study demonstrates the vulnerability of such agents to backdoor attacks and emphasizes the need for targeted defenses.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://www.catalyzex.com/paper/a-trembling-house-of-cards-mapping" rel="nofollow">A Trembling House of Cards? Mapping Adversarial Attacks against Language Agents</a></p>
<ul dir="auto">
<li>Lingbo Mo, Zeyi Liao, Boyuan Zheng, Yu Su, Chaowei Xiao, Huan Sun</li>
<li>🏛️ Institutions: OSU, UWM</li>
<li>📅 Date: February 15, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [safety], [adversarial attacks], [security risks], [language agents], [Perception-Brain-Action]</li>
<li>📖 TLDR: This paper introduces a conceptual framework to assess and understand adversarial vulnerabilities in language agents, dividing the agent structure into three components—Perception, Brain, and Action. It discusses 12 specific adversarial attack types that exploit these components, ranging from input manipulation to complex backdoor and jailbreak attacks. The framework provides a basis for identifying and mitigating risks before the widespread deployment of these agents in real-world applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.07939" rel="nofollow">UFO: A UI-Focused Agent for Windows OS Interaction</a></p>
<ul dir="auto">
<li>Chaoyun Zhang, Liqun Li, Shilin He, Xu Zhang, Bo Qiao, Si Qin, Minghua Ma, Yu Kang, Qingwei Lin, Saravan Rajmohan, Dongmei Zhang, Qi Zhang</li>
<li>🏛️ Institutions: Microsoft</li>
<li>📅 Date: February 14, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [UI automation], [Windows], [UFO]</li>
<li>📖 TLDR: This paper presents UFO, a pioneering multimodal LLM-based agent designed to fulfill user requests on Windows OS. UFO employs a dual-agent architecture—comprising AppAgent and ActAgent—that can interpret and execute complex tasks across multiple Windows applications by observing UI elements and utilizing control interactions. The framework allows UFO to handle intricate, cross-application workflows and execute commands seamlessly based on natural language prompts. It integrates GPT-Vision to recognize and interact with graphical elements, enabling flexible, autonomous task completion within and across diverse Windows applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.07945" rel="nofollow">ScreenAgent: A Computer Control Agent Driven by Visual Language Large Model</a></p>
<ul dir="auto">
<li>Runliang Niu, Jindong Li, Shiqi Wang, Yali Fu, Xiyu Hu, Xueyuan Leng, He Kong, Yi Chang, Qi Wang</li>
<li>🏛️ Institutions: Jilin University</li>
<li>📅 Date: February 13, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [visual language model], [computer control agent]</li>
<li>📖 TLDR: This paper introduces ScreenAgent, a computer control agent powered by a visual language large model. The system can interpret natural language instructions and execute them on various computer applications by analyzing screen content. ScreenAgent employs a novel action grounding mechanism to map high-level instructions to specific UI interactions. Evaluated on a diverse set of tasks across different applications, ScreenAgent demonstrates superior performance in task completion and generalization compared to existing methods.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.07456" rel="nofollow">OS-Copilot: Towards Generalist Computer Agents with Self-Improvement</a></p>
<ul dir="auto">
<li>Zhiyong Wu, Chengcheng Han, Zichen Ding, Zhenmin Weng, Zhoumianze Liu, Shunyu Yao, Tao Yu, Lingpeng Kong</li>
<li>🏛️ Institutions: Shanghai AI Lab, East China Normal University, Princeton, HKU</li>
<li>📅 Date: February 12, 2024</li>
<li>📑 Publisher: ICLR 2024 Workshop LLMAgents</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [self-directed learning], [GAIA], [FRIDAY], [OS-Copilot]</li>
<li>📖 TLDR: The OS-Copilot framework supports building generalist agents capable of performing diverse tasks across an operating system (OS). This work introduces FRIDAY, an embodied agent using OS-Copilot to self-improve by learning from task outcomes. It operates with a memory-based architecture to tackle OS-level tasks across applications like terminals, web browsers, and third-party tools. Tested on the GAIA benchmark, FRIDAY achieved 35% higher performance than prior methods, proving effective in adapting to unfamiliar applications and refining its capabilities with minimal guidance.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.04615" rel="nofollow">ScreenAI: A Vision-Language Model for UI and Infographics Understanding</a></p>
<ul dir="auto">
<li>Gilles Baechler, Srinivas Sunkara, Maria Wang, Fedir Zubach, Hassan Mansoor, Vincent Etter, Victor Cărbune, Jason Lin, Jindong Chen, Abhanshu Sharma</li>
<li>🏛️ Institutions: Google DeepMind</li>
<li>📅 Date: February 7, 2024</li>
<li>📑 Publisher: IJCAI 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [UI understanding], [infographics understanding], [vision language model]</li>
<li>📖 TLDR: This paper introduces ScreenAI, a vision-language model specializing in UI and infographics understanding. The model combines the PaLI architecture with the flexible patching strategy of pix2struct and is trained on a unique mixture of datasets. ScreenAI achieves state-of-the-art results on several UI and infographics-based tasks, outperforming larger models. The authors also release three new datasets for screen annotation and question answering tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.04476" rel="nofollow">Dual-View Visual Contextualization for Web Navigation</a></p>
<ul dir="auto">
<li>Jihyung Kil, Chan Hee Song, Boyuan Zheng, Xiang Deng, Yu Su, Wei-Lun Chao</li>
<li>🏛️ Institutions: OSU</li>
<li>📅 Date: February 6, 2024</li>
<li>📑 Publisher: CVPR 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [visual contextualization]</li>
<li>📖 TLDR: This paper proposes a novel approach to web navigation by contextualizing HTML elements through their "dual views" in webpage screenshots. The method leverages both the textual content of HTML elements and their visual representation in the screenshot to create more informative representations for web agents. Evaluated on the Mind2Web dataset, the approach demonstrates consistent improvements over baseline methods across various scenarios, including cross-task, cross-website, and cross-domain navigation tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.05930" rel="nofollow">WebLINX: Real-World Website Navigation with Multi-Turn Dialogue</a></p>
<ul dir="auto">
<li>Xing Han Lu, Zdeněk Kasner, Siva Reddy</li>
<li>🏛️ Institutions: Mila, McGill University</li>
<li>📅 Date: February 2024</li>
<li>📑 Publisher: ICML 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [multi-turn dialogue], [real-world navigation], [WebLINX]</li>
<li>📖 TLDR: WebLINX addresses the complexity of real-world website navigation for conversational agents, with a benchmark featuring over 2,300 demonstrations across 150+ websites. The benchmark allows agents to handle multi-turn instructions and interact dynamically across diverse domains, including geographic and thematic categories. The study proposes a retrieval-inspired model that selectively extracts key HTML elements and browser actions, achieving efficient task-specific representations. Experiments reveal that smaller finetuned decoders outperform larger zero-shot multimodal models, though generalization to new environments remains challenging.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2402.17553" rel="nofollow">OmniACT: A Dataset and Benchmark for Enabling Multimodal Generalist Autonomous Agents for Desktop and Web</a></p>
<ul dir="auto">
<li>Raghav Kapoor, Yash Parag Butala, Melisa Russak, Jing Yu Koh, Kiran Kamble, Waseem Alshikh, Ruslan Salakhutdinov</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: February 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [dataset], [benchmark]</li>
<li>📖 TLDR: OmniACT introduces a dataset and benchmark to train and evaluate multimodal agents capable of autonomously performing diverse tasks across desktop and web environments. Using annotated UI elements across applications, it combines visual grounding with natural language instructions, providing 9,802 data points for developing agents that integrate high-level reasoning with UI interactions. The study highlights the limited proficiency of current models, with baselines like GPT-4 only achieving 15% of human performance on executable scripts, emphasizing OmniACT's potential as a testbed for advancing multimodal AI.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2401.16158" rel="nofollow">Mobile-Agent: Autonomous Multi-Modal Mobile Device Agent with Visual Perception</a></p>
<ul dir="auto">
<li>Junyang Wang, Haiyang Xu, Jiabo Ye, Ming Yan, Weizhou Shen, Ji Zhang, Fei Huang, Jitao Sang</li>
<li>🏛️ Institutions: Beijing Jiaotong University, Alibaba</li>
<li>📅 Date: January 29, 2024</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [benchmark]</li>
<li>📖 TLDR: This paper presents Mobile-Agent, an autonomous multi-modal agent designed for mobile device interaction. The system integrates visual perception, natural language processing, and action prediction to navigate and operate mobile applications. The authors introduce a new dataset and benchmark for evaluating mobile agents, demonstrating Mobile-Agent's superior performance in task completion and generalization across various apps compared to existing methods.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2401.13649" rel="nofollow">VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks</a></p>
<ul dir="auto">
<li>Jing Yu Koh, Robert Lo, Lawrence Jang, Vikram Duvvur, Ming Chong Lim, Po-Yu Huang, Graham Neubig, Shuyan Zhou, Ruslan Salakhutdinov, Daniel Fried</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: January 24, 2024</li>
<li>📑 Publisher: ACL 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [benchmark], [dataset], [multimodal agent evaluation], [visually grounded tasks]</li>
<li>📖 TLDR: VisualWebArena is a benchmark designed for testing multimodal web agents on complex, visually grounded web tasks. It provides a reproducible framework with 910 task scenarios across real-world web applications, emphasizing open-ended, visually guided interactions. The tasks are modeled within a partially observable Markov decision process to assess agents’ capacity to interpret multimodal inputs, execute navigation, and accomplish user-defined objectives across complex visual and textual information on websites.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2401.13919" rel="nofollow">WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models</a></p>
<ul dir="auto">
<li>Hongliang He, Wenlin Yao, Kaixin Ma, Wenhao Yu, Yong Dai, Hongming Zhang, Zhenzhong Lan, Dong Yu</li>
<li>🏛️ Institutions: Zhejiang University, Tencent AI Lab, Westlake University</li>
<li>📅 Date: January 24, 2024</li>
<li>📑 Publisher: ACL 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [evaluation]</li>
<li>📖 TLDR: This paper introduces WebVoyager, an innovative web agent powered by Large Multimodal Models (LMMs) that can complete user instructions end-to-end by interacting with real-world websites. The authors establish a new benchmark with tasks from 15 popular websites and propose an automatic evaluation protocol using GPT-4V. WebVoyager achieves a 59.1% task success rate, significantly outperforming GPT-4 (All Tools) and text-only setups. The study demonstrates the effectiveness of multimodal approaches in web automation and provides insights into developing more intelligent web interaction solutions.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2401.10935" rel="nofollow">SeeClick: Harnessing GUI Grounding for Advanced Visual GUI Agents</a></p>
<ul dir="auto">
<li>Kanzhi Cheng, Qiushi Sun, Yougang Chu, Fangzhi Xu, Yantao Li, Jianbing Zhang, Zhiyong Wu</li>
<li>🏛️ Institutions: Nanjing University, Shanghai AI Lab</li>
<li>📅 Date: January 19, 2024</li>
<li>📑 Publisher: ACL 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [benchmark], [GUI grounding], [visual grounding]</li>
<li>📖 TLDR: TBD.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://llmbench.ai/agent" rel="nofollow">AgentBench: Evaluating LLMs as Agents</a></p>
<ul dir="auto">
<li>Xiao Liu, Hao Yu, Hanchen Zhang, Yifan Xu, Xuanyu Lei, Hanyu Lai, Yu Gu, Hangliang Ding, Kaiwen Men, Kejuan Yang, Shudan Zhang, Xiang Deng, Aohan Zeng, Zhengxiao Du, Chenhui Zhang, Sheng Shen, Tianjun Zhang, Yu Su, Huan Sun, Minlie Huang, Yuxiao Dong, Jie Tang</li>
<li>🏛️ Institutions: THU, OSU, ByteDance</li>
<li>📅 Date: January 1, 2024</li>
<li>📑 Publisher: ICLR 2024</li>
<li>💻 Env: [GUI], [General]</li>
<li>🔑 Key: [benchmark], [evaluation]</li>
<li>📖 TLDR: AgentBench provides a comprehensive benchmark for evaluating LLMs as autonomous agents in various environments. It includes eight distinct scenarios, testing the LLMs' reasoning and decision-making capabilities in tasks such as OS interaction, database querying, knowledge graph traversal, and more. This benchmark compares the effectiveness of multiple commercial and open-source LLMs, revealing areas of improvement in instruction-following and long-term reasoning, essential for practical agent development.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://huggingface.co/papers/2305.11854" rel="nofollow">Multimodal Web Navigation with Instruction-Finetuned Foundation Models</a></p>
<ul dir="auto">
<li>Hiroki Furuta, Kuang-Huei Lee, Ofir Nachum, Yutaka Matsuo, Aleksandra Faust, Shixiang Shane Gu, Izzeddin Gur</li>
<li>🏛️ Institutions: Univ. of Tokyo, Google DeepMind</li>
<li>📅 Date: Jan 1, 2024</li>
<li>📑 Publisher: ICLR 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [model], [dataset], [web navigation], [instruction-following], [WebShop]</li>
<li>📖 TLDR: This paper introduces WebGUM, an instruction-following multimodal agent for autonomous web navigation that leverages both visual (webpage screenshots) and textual (HTML) inputs to perform actions such as click and type. The model is trained on a vast corpus of demonstrations and shows improved capabilities in visual perception, HTML comprehension, and multi-step decision-making, achieving state-of-the-art performance on benchmarks like MiniWoB and WebShop. WebGUM provides a scalable approach to web-based tasks without task-specific architectures, enabling high-performance web navigation with generalizable, multimodal foundation models.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://osu-nlp-group.github.io/SeeAct/" rel="nofollow">GPT-4V(ision) is a Generalist Web Agent, if Grounded</a></p>
<ul dir="auto">
<li>Boyuan Zheng, Boyu Gou, Jihyung Kil, Huan Sun, Yu Su</li>
<li>🏛️ Institutions: OSU</li>
<li>📅 Date: January 1, 2024</li>
<li>📑 Publisher: ICML 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [grounding], [SeeAct], [Multimodal-Mind2web]</li>
<li>📖 TLDR: This paper explores the capability of GPT-4V(ision), a multimodal model, as a web agent that can perform tasks across various websites by following natural language instructions. It introduces the <strong>SEEACT</strong> framework, enabling GPT-4V to navigate, interpret, and interact with elements on websites. Evaluated using the <strong>Mind2Web</strong> benchmark and an online test environment, the framework demonstrates high performance on complex web tasks by integrating grounding strategies like element attributes and image annotations to improve HTML element targeting. However, grounding remains challenging, presenting opportunities for further improvement.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2312.13771" rel="nofollow">AppAgent: Multimodal Agents as Smartphone Users</a></p>
<ul dir="auto">
<li>Chi Zhang, Zhao Yang, Jiaxuan Liu, Yucheng Han, Xin Chen, Zebiao Huang, Bin Fu, Gang Yu</li>
<li>🏛️ Institutions: Tencent</li>
<li>📅 Date: December 21, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [smartphone interaction], [autonomous exploration], [self-improve]</li>
<li>📖 TLDR: This paper introduces AppAgent, a novel multimodal agent framework designed to operate smartphone applications. The agent uses a simplified action space to mimic human-like interactions such as tapping and swiping. AppAgent learns to navigate and use new apps through autonomous exploration or by observing human demonstrations, creating a knowledge base for executing complex tasks across different applications. The framework's effectiveness is demonstrated through extensive testing on 50 tasks across 10 diverse applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2312.13108" rel="nofollow">AssistGUI: Task-Oriented Desktop Graphical User Interface Automation</a></p>
<ul dir="auto">
<li>Difei Gao, Lei Ji, Zechen Bai, Mingyu Ouyang, Peiran Li, Dongxing Mao, Qinchen Wu, Weichen Zhang, Peiyi Wang, Xiangwu Guo, Hengxu Wang, Luowei Zhou, Mike Zheng Shou</li>
<li>🏛️ Institutions: NUS</li>
<li>📅 Date: December 20, 2023</li>
<li>📑 Publisher: CVPR 2024</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [desktop productivity tasks]</li>
<li>📖 TLDR: This study presents <em>AssistGUI</em>, a benchmark and framework for desktop GUI automation, featuring an LLM-based agent capable of completing complex user requests by analyzing instructional videos and performing actions on the desktop. Utilizing a novel Actor-Critic framework and GUI parser, <em>AssistGUI</em> was tested on 100 tasks across nine applications, such as MS Word and After Effects. Despite advances, the top-performing model achieved only a 46% success rate, illustrating the challenge of comprehensive desktop automation and underscoring areas for future research in agent-driven GUI tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2312.08914" rel="nofollow">CogAgent: A Visual Language Model for GUI Agents</a></p>
<ul dir="auto">
<li>Wenyi Hong, Weihan Wang, Qingsong Lv, Jiazheng Xu, Wenmeng Yu, Junhao Chen, Yuxuan Wang, Yining Ye, Jiayi Zhang, Hao Dong, Wenhu Chen, Yizhou Wang, Kai-Wei Chang</li>
<li>🏛️ Institutions: Tsinghua University, Zhipu AI</li>
<li>📅 Date: December 15, 2023</li>
<li>📑 Publisher: CVPR 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [dataset], [benchmark], [visual language model], [GUI agent]</li>
<li>📖 TLDR: This paper presents CogAgent, a visual language model designed for GUI agents. The authors introduce a new dataset, CogBench, featuring 1,430 GUI tasks across various applications. CogAgent employs a novel training approach combining supervised fine-tuning and decision-making fine-tuning. The model demonstrates superior performance on CogBench and generalizes well to unseen applications, outperforming existing models like GPT-4V in GUI task completion.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://huggingface.co/gaia-benchmark" rel="nofollow">GAIA: a benchmark for General AI Assistants</a></p>
<ul dir="auto">
<li>Grégoire Mialon, Yassine Nakkach, Aslan Tchamkerten, Albert Thomas, Laurent Dinh, and a research team from Meta AI and Hugging Face.</li>
<li>🏛️ Institutions: Meta AI, Hugging Face</li>
<li>📅 Date: November 21, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [benchmark], [multi-modality], [tool use], [reasoning]</li>
<li>📖 TLDR: GAIA is a benchmark developed for evaluating general-purpose AI assistants. It aims to test assistant models across multiple modalities and complex reasoning tasks in real-world settings, including scenarios that require tool usage and open-ended question answering. With a dataset comprising 466 questions across various domains, GAIA highlights gaps between current AI performance and human capability, presenting a significant challenge for large language models such as GPT-4.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2311.07562" rel="nofollow">GPT-4V in Wonderland: Large Multimodal Models for Zero-Shot Smartphone GUI Navigation</a></p>
<ul dir="auto">
<li>An Yan, Zhengyuan Yang, Wanrong Zhu, Kevin Lin, Linjie Li, Jianfeng Wang, Jianwei Yang, Yiwu Zhong, Julian McAuley, Jianfeng Gao, Zicheng Liu, Lijuan Wang</li>
<li>🏛️ Institutions: UCSD, Microsoft, UCSB, UWM</li>
<li>📅 Date: November 13, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [benchmark], [zero-shot GUI navigation], [multimodal LLMs]</li>
<li>📖 TLDR: This paper explores the capabilities of GPT-4V in navigating smartphone GUIs without prior training. The authors introduce a novel framework for GUI navigation and a new benchmark, MobileNav, featuring 1,000 navigation tasks across 100 mobile apps. The study demonstrates GPT-4V's impressive zero-shot performance in understanding and interacting with mobile interfaces, outperforming previous methods and even approaching human-level performance on some tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2406.11736" rel="nofollow">Interactive Evolution: A Neural-Symbolic Self-Training Framework For Large Language Models</a></p>
<ul dir="auto">
<li>Fangzhi Xu, Qiushi Sun, Kanzhi Cheng, Jun Liu, Yu Qiao, Zhiyong Wu</li>
<li>🏛️ Institutions: Xi'an Jiaotong University, Shanghai AI Lab, HKU, Nanjing University</li>
<li>📅 Date: November 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI (evaluated on web, math reasoning, and logic reasoning environments)]</li>
<li>🔑 Key: [framework], [dataset], [neural-symbolic self-training], [online exploration], [self-refinement]</li>
<li>📖 TLDR: This paper introduces <em>ENVISIONS</em>, a neural-symbolic self-training framework designed to improve large language models (LLMs) by enabling self-training through interaction with a symbolic environment. The framework addresses symbolic data scarcity and enhances LLMs' symbolic reasoning proficiency by iteratively exploring, refining, and learning from symbolic tasks without reinforcement learning. Extensive evaluations across web navigation, math, and logical reasoning tasks highlight <em>ENVISIONS</em> as a promising approach for enhancing LLM symbolic processing.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2310.15455" rel="nofollow">UI Layout Generation with LLMs Guided by UI Grammar</a></p>
<ul dir="auto">
<li>Yuwen Lu, Ziang Tong, Qinyi Zhao, Chengzhi Zhang, Toby Jia-Jun Li</li>
<li>🏛️ Institutions: ICML 2023 Workshop on AI and HCI</li>
<li>📅 Date: October 24, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [UI grammar], [UI Layout Generation]</li>
<li>📖 TLDR: This position paper explores the use of Large Language Models (LLMs) for generating mobile user interface (UI) layouts. It introduces <em>UI grammar</em>, a novel approach to represent the hierarchical structure of UI screens, aiming to guide LLMs' generative capabilities more effectively and enhance the explainability and controllability of the process. Initial experiments with GPT-4 demonstrate the potential of LLMs to produce high-quality UIs through in-context learning, with the grammar-based approach improving certain aspects of generation quality.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2310.11441" rel="nofollow">Set-of-Mark Prompting Unleashes Extraordinary Visual Grounding in GPT-4V</a></p>
<ul dir="auto">
<li>Jianwei Yang, Hao Zhang, Feng Li, Xueyan Zou, Chunyuan Li, Jianfeng Gao</li>
<li>🏛️ Institutions: MSR</li>
<li>📅 Date: October 17, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [visual prompting], [framework], [benchmark], [visual grounding], [zero-shot]</li>
<li>📖 TLDR: This paper introduces Set-of-Mark (SoM), a novel visual prompting approach designed to enhance the visual grounding capabilities of multimodal models like GPT-4V. By overlaying images with spatially and semantically distinct marks, SoM enables fine-grained object recognition and interaction within visual data, surpassing conventional zero-shot segmentation methods in accuracy. The framework is validated on tasks requiring detailed spatial reasoning, demonstrating a significant improvement over existing visual-language models without fine-tuning.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://github.com/xlang-ai/OpenAgents">OpenAgents: An Open Platform for Language Agents in the Wild</a></p>
<ul dir="auto">
<li>Tianbao Xie, Fan Zhou, Zhoujun Cheng, Peng Shi, Luoxuan Weng, Yitao Liu, Toh Jing Hua, Junning Zhao, Qian Liu, Che Liu, Leo Z. Liu, Yiheng Xu, Hongjin Su, Dongchan Shin, Caiming Xiong, Tao Yu</li>
<li>🏛️ Institutions: HKU, XLang Lab, Sea AI Lab, Salesforce Research</li>
<li>📅 Date: October 16, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [Data Agent], [Plugins Agent], [Web Agent]</li>
<li>📖 TLDR: This paper introduces OpenAgents, an open-source platform designed to facilitate the use and hosting of language agents in real-world scenarios. It features three agents: Data Agent for data analysis using Python and SQL, Plugins Agent with access to over 200 daily API tools, and Web Agent for autonomous web browsing. OpenAgents aims to provide a user-friendly web interface for general users and a seamless deployment experience for developers and researchers, promoting the development and evaluation of innovative language agents in practical applications.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2310.04716" rel="nofollow">Reinforced UI Instruction Grounding: Towards a Generic UI Task Automation API</a></p>
<ul dir="auto">
<li>Zhizheng Zhang, Wenxuan Xie, Xiaoyi Zhang, Yan Lu</li>
<li>🏛️ Institutions: MSRA</li>
<li>📅 Date: October 7, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [model], [framework], [reinforcement learning], [UI task automation], [instruction grounding]</li>
<li>📖 TLDR: This paper introduces a multimodal model, termed RUIG (Reinforced UI Instruction Grounding), for automating UI tasks through natural language instructions. By leveraging a pixel-to-sequence approach, the model directly decodes UI element locations from screenshots based on user commands, removing the need for metadata like element coordinates. The framework uses a transformer-based encoder-decoder setup optimized through reinforcement learning to improve spatial accuracy. This novel approach outperforms prior methods, offering a generalized solution for UI task automation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://asappresearch.github.io/webagents-step/" rel="nofollow">SteP: Stacked LLM Policies for Web Actions</a></p>
<ul dir="auto">
<li>Paloma Sodhi, S.R.K. Branavan, Yoav Artzi, Ryan McDonald</li>
<li>🏛️ Institutions: ASAPP Research, Cornell University</li>
<li>📅 Date: October 5, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [policy composition], [dynamic control], [SteP]</li>
<li>📖 TLDR: This paper introduces <strong>SteP (Stacked LLM Policies)</strong>, a framework that dynamically composes policies to tackle diverse web tasks. By defining a Markov Decision Process where the state is a stack of policies, SteP enables adaptive control that adjusts to task complexity. Evaluations on WebArena, MiniWoB++, and a CRM simulator demonstrate that SteP significantly outperforms existing methods, achieving a success rate improvement from 14.9% to 35.8% over state-of-the-art GPT-4 policies.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2309.11436" rel="nofollow">You Only Look at Screens: Multimodal Chain-of-Action Agents</a></p>
<ul dir="auto">
<li>Zhuosheng Zhang, Aston Zhang</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: September 20, 2023</li>
<li>📑 Publisher: ICLR 2024</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [multimodal agent], [chain-of-action technique]</li>
<li>📖 TLDR: This paper presents Auto-GUI, a multimodal agent capable of directly interacting with graphical user interfaces without relying on environment parsing or application-specific APIs. The authors introduce a novel chain-of-action technique that leverages previous action histories and future action plans to improve decision-making. Auto-GUI is evaluated on a new device-control benchmark, AITW, demonstrating state-of-the-art performance in action prediction and task completion across various applications and web-based tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2309.08172" rel="nofollow">LASER: LLM Agent with State-Space Exploration for Web Navigation</a></p>
<ul dir="auto">
<li>Kaixin Ma, Hongming Zhang, Hongwei Wang, Xiaoman Pan, Dong Yu, Jianshu Chen</li>
<li>🏛️ Institutions: Tencent AI Lab</li>
<li>📅 Date: September 15, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [state-space exploration], [backtracking]</li>
<li>📖 TLDR: This paper introduces LASER, an LLM agent that models interactive web navigation tasks as state-space exploration. The approach defines a set of high-level states and associated actions, allowing the agent to transition between states and backtrack from errors. LASER significantly outperforms previous methods on the WebShop task without using in-context examples, demonstrating improved handling of novel situations and mistakes during task execution.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2308.15272" rel="nofollow">AutoDroid: LLM-powered Task Automation in Android</a></p>
<ul dir="auto">
<li>Hao Wen, Yuanchun Li, Guohong Liu, Shanhui Zhao, Tao Yu, Toby Jia-Jun Li, Shiqi Jiang, Yunhao Liu, Yaqin Zhang, Yunxin Liu</li>
<li>🏛️ Institutions: Tsinghua University, Shanghai AI Lab, University of Notre Dame, MSR</li>
<li>📅 Date: August 29, 2023</li>
<li>📑 Publisher: MobiCom 2024</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [Android task automation], [LLM-powered agent]</li>
<li>📖 TLDR: This paper introduces AutoDroid, a novel mobile task automation system capable of handling arbitrary tasks on any Android application without manual efforts. The framework combines the commonsense knowledge of LLMs with domain-specific knowledge of apps through automated dynamic analysis. AutoDroid features a functionality-aware UI representation method, exploration-based memory injection techniques, and a multi-granularity query optimization module. Evaluated on a new benchmark with 158 common tasks, AutoDroid achieves a 90.9% action generation accuracy and a 71.3% task completion rate, significantly outperforming GPT-4-powered baselines.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2407.20183" rel="nofollow">MindSearch: Mimicking Human Minds Elicits Deep AI Searcher</a></p>
<ul dir="auto">
<li>Zehui Chen, Kuikun Liu, Qiuchen Wang, Jiangning Liu, Wenwei Zhang, Kai Chen, Feng Zhao</li>
<li>🏛️ Institutions: USTC, Shanghai AI Lab</li>
<li>📅 Date: July 29, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [information seeking], [planning], [AI search], [MindSearch]</li>
<li>📖 TLDR: This paper presents MindSearch, a novel approach to web information seeking and integration that mimics human cognitive processes. The system uses a multi-agent framework consisting of a WebPlanner and WebSearcher. The WebPlanner models multi-step information seeking as a dynamic graph construction process, decomposing complex queries into sub-questions. The WebSearcher performs hierarchical information retrieval for each sub-question. MindSearch demonstrates significant improvements in response quality and depth compared to existing AI search solutions, processing information from over 300 web pages in just 3 minutes.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2307.13854" rel="nofollow">WebArena: A Realistic Web Environment for Building Autonomous Agents</a></p>
<ul dir="auto">
<li>Shuyan Zhou, Frank F. Xu, Hao Zhu, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng, Tianyue Ou, Yonatan Bisk, Daniel Fried, Uri Alon, Graham Neubig</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: July 26, 2023</li>
<li>📑 Publisher: NeurIPS 2023</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [benchmark], [multi-tab navigation], [web-based interaction], [agent simulation]</li>
<li>📖 TLDR: <em>WebArena</em> provides a standalone, realistic web simulation environment where autonomous agents can perform complex web-based tasks. The platform offers functionalities such as multi-tab browsing, element interaction, and customized user profiles. Its benchmark suite contains 812 tasks grounded in high-level natural language commands. WebArena uses multi-modal observations, including HTML and accessibility tree views, supporting advanced tasks that require contextual understanding across diverse web pages, making it suitable for evaluating generalist agents in real-world web environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2307.10088" rel="nofollow">Android in the Wild: A Large-Scale Dataset for Android Device Control</a></p>
<ul dir="auto">
<li>Christopher Rawles, Alice Li, Daniel Rodriguez, Oriana Riva, Timothy Lillicrap</li>
<li>🏛️ Institutions: Google Research, Google DeepMind</li>
<li>📅 Date: July 19, 2023</li>
<li>📑 Publisher: NeurIPS 2023</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [benchmark], [device control], [natural language interaction], [gesture-based actions]</li>
<li>📖 TLDR: The <em>Android in the Wild (AitW)</em> dataset introduces a significant benchmark for Android device control, encompassing over 715,000 human-labeled episodes with natural language commands and corresponding UI actions. Collected from Android devices across versions 10-13, it captures complex multi-step tasks requiring both visual and contextual understanding. The dataset is structured to test the robustness of device-control systems under varying conditions, such as new tasks or applications, and includes data to evaluate gesture-based interactions, providing a unique foundation for mobile interface automation and task execution research.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2307.12856" rel="nofollow">A Real-World WebAgent with Planning, Long Context Understanding, and Program Synthesis</a></p>
<ul dir="auto">
<li>Izzeddin Gur, Hiroki Furuta, Austin Huang, Mustafa Safdari, Yutaka Matsuo, Douglas Eck, Aleksandra Faust</li>
<li>🏛️ Institutions: Google DeepMind, The University of Tokyo</li>
<li>📅 Date: July 2023</li>
<li>📑 Publisher: ICLR 2024</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [program synthesis], [HTML comprehension], [web automation], [self-supervised learning]</li>
<li>📖 TLDR: WebAgent leverages two LLMs—HTML-T5 for HTML comprehension and Flan-U-PaLM for program synthesis—to complete web automation tasks. It combines planning, HTML summarization, and code generation to navigate and interact with real-world web environments, improving success rates on HTML-based tasks and achieving state-of-the-art performance in benchmarks like MiniWoB and Mind2Web. The modular architecture adapts well to open-domain tasks, using local-global attention mechanisms to manage long HTML contexts.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2306.07863" rel="nofollow">Synapse: Trajectory-as-Exemplar Prompting with Memory for Computer Control</a></p>
<ul dir="auto">
<li>Longtao Zheng, Rundong Wang, Xinrun Wang, Bo An</li>
<li>🏛️ Institutions: NTU</li>
<li>📅 Date: June 13, 2023</li>
<li>📑 Publisher: ICLR 2024</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [benchmark], [trajectory prompting], [state abstraction], [memory retrieval]</li>
<li>📖 TLDR: Synapse introduces a novel framework for computer control tasks, leveraging trajectory-as-exemplar prompting and memory to enhance LLM performance in complex, multi-step computer tasks. The system combines state abstraction, trajectory-based prompts, and memory retrieval, overcoming LLM limitations by filtering task-irrelevant data, storing exemplar trajectories, and retrieving relevant instances for improved decision-making. Synapse achieves significant performance gains on benchmarks such as MiniWoB++ and Mind2Web, demonstrating enhanced task success rates and generalization across diverse web-based tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2306.06070" rel="nofollow">Mind2Web: Towards a Generalist Agent for the Web</a></p>
<ul dir="auto">
<li>Xiang Deng, Yu Gu, Boyuan Zheng, Shijie Chen, Sam Stevens, Boshi Wang, Huan Sun, Yu Su</li>
<li>🏛️ Institutions: OSU</li>
<li>📅 Date: June 9, 2023</li>
<li>📑 Publisher: NeurIPS 2023</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [dataset], [benchmark], [model], [Mind2Web], [MindAct]</li>
<li>📖 TLDR: <em>Mind2Web</em> presents a dataset and benchmark specifically crafted for generalist web agents capable of performing language-guided tasks across varied websites. Featuring over 2,000 tasks from 137 sites, it spans 31 domains and emphasizes open-ended, realistic tasks in authentic, unsimplified web settings. The study proposes the <em>MindAct</em> framework, which optimizes LLMs for handling complex HTML elements by using small LMs to rank elements before full processing, thereby enhancing the efficiency and versatility of web agents in diverse contexts.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2305.19308" rel="nofollow">SheetCopilot: Bringing Software Productivity to the Next Level through Large Language Models</a></p>
<ul dir="auto">
<li>Hongxin Li, Jingran Su, Yuntao Chen, Qing Li, Zhaoxiang Zhang</li>
<li>🏛️ Institutions: UCAS, HKISI-CAS, PolyU, Shanghai AI Lab</li>
<li>📅 Date: May 30, 2023</li>
<li>📑 Publisher: NeurIPS 2023</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [spreadsheet automation], [natural language interface]</li>
<li>📖 TLDR: This paper introduces SheetCopilot, an innovative system that leverages large language models to automate spreadsheet tasks through natural language interactions. The framework includes a novel prompt design for task decomposition and execution, and a feedback loop for error correction. SheetCopilot demonstrates significant improvements in task completion rates and efficiency across various spreadsheet operations, outperforming existing methods and showing potential for enhancing productivity in spreadsheet software.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2305.12487" rel="nofollow">Augmenting Autotelic Agents with Large Language Models</a></p>
<ul dir="auto">
<li>Cédric Colas, Laetitia Teodorescu, Pierre-Yves Oudeyer, Xingdi Yuan, Marc-Alexandre Côté</li>
<li>🏛️ Institutions: MIT, Inria, Microsoft</li>
<li>📅 Date: May 22, 2023</li>
<li>📑 Publisher: CoLLAs 2023</li>
<li>💻 Env: [GUI]</li>
<li>🔑 Key: [framework], [reinforcement learning], [goal generation], [large language models], [autotelic learning]</li>
<li>📖 TLDR: This study introduces the <em>Language Model-Augmented Autotelic Agent (LMA3)</em>, a framework leveraging large language models to help agents autonomously generate, represent, and learn diverse goals in a task-agnostic, text-based environment. LMA3 integrates pretrained language models to emulate human cultural knowledge, aiming to dynamically relabel goals, generate new goals, and create goal-driven reward functions without manual inputs. This approach supports skill development by autonomously expanding goal repertoires in ways that resemble human open-ended learning, showcasing potential for achieving complex, self-directed learning in AI.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2305.08144" rel="nofollow">Mobile-Env: Building Qualified Evaluation Benchmarks for LLM-GUI Interaction</a></p>
<ul dir="auto">
<li>Danyang Zhang, Zhennan Shen, Rui Xie, Situo Zhang, Tianbao Xie, Zihan Zhao, Siyuan Chen, Lu Chen, Hongshen Xu, Ruisheng Cao, Kai Yu</li>
<li>🏛️ Institutions: SJTU, HKU</li>
<li>📅 Date: May 14, 2023</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [dataset], [interaction platform], [multistep interaction], [InfoUI]</li>
<li>📖 TLDR: This paper introduces <em>Mobile-Env</em>, a novel interaction platform and benchmark aimed at assessing large language models' (LLMs) capabilities in interactive environments. It builds on the InfoUI task set, derived from WikiHow, to create structured text-based challenges that simulate real-world mobile interactions. The platform is designed to support task expansions from the community, aiming to drive advancements in LLM-based interactive agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2303.17491" rel="nofollow">Language Models can Solve Computer Tasks</a></p>
<ul dir="auto">
<li>Geunwoo Kim, Pierre Baldi, Stephen McAleer</li>
<li>🏛️ Institutions: UCI</li>
<li>📅 Date: March 30, 2023</li>
<li>📑 Publisher: NeurIPS 2023</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [framework], [benchmark], [Recursive Critique and Improve], [RCI], [MiniWoB++], [general computer tasks]</li>
<li>📖 TLDR: This study demonstrates that large language models (LLMs) can effectively automate computer tasks using a Recursive Critique and Improve (RCI) prompting method, enabling agents to handle complex desktop tasks like email and file management. By combining RCI with existing Chain of Thought (CoT) prompting, the method outperforms prior LLM approaches and traditional supervised and reinforcement learning models on the <strong>MiniWoB++</strong> benchmark, showing potential for broad computer task automation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2303.11366" rel="nofollow">Reflexion: Language Agents with Verbal Reinforcement Learning</a></p>
<ul dir="auto">
<li>Noah Shinn, Federico Cassano, Edward Berman, Ashwin Gopinath, Karthik Narasimhan, Shunyu Yao</li>
<li>🏛️ Institutions: Northeastern University, MIT, Princeton</li>
<li>📅 Date: March 20, 2023</li>
<li>📑 Publisher: NeurIPS 2023</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [framework], [learning], [verbal reinforcement learning], [Reflexion]</li>
<li>📖 TLDR: This paper introduces <em>Reflexion</em>, a framework that enhances language agents by enabling them to reflect on task feedback linguistically, storing these reflections in an episodic memory to improve decision-making in future trials. Reflexion allows agents to learn from various feedback types without traditional weight updates, achieving significant performance improvements across tasks like decision-making, coding, and reasoning. For instance, Reflexion attains a 91% pass@1 accuracy on the HumanEval coding benchmark, surpassing the previous state-of-the-art GPT-4's 80%.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2210.03347" rel="nofollow">Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding</a></p>
<ul dir="auto">
<li>Kenton Lee, Mandar Joshi, Iulia Raluca Turc, Hexiang Hu, Fangyu Liu, Julian Martin Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova</li>
<li>🏛️ Institutions: Google</li>
<li>📅 Date: February 1, 2023</li>
<li>📑 Publisher: ICML 2023</li>
<li>💻 Env: [Web], [Doc]</li>
<li>🔑 Key: [model], [framework], [vision encoder], [visual language understanding], [screenshot parsing], [image-to-text]</li>
<li>📖 TLDR: This paper introduces Pix2Struct, a model pre-trained to parse masked screenshots into simplified HTML for tasks requiring visual language understanding. By leveraging the structure of HTML and diverse web page elements, Pix2Struct captures pretraining signals like OCR and image captioning, achieving state-of-the-art performance across tasks in domains including documents, user interfaces, and illustrations.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2301.13280" rel="nofollow">WebUI: A Dataset for Enhancing Visual UI Understanding with Web Semantics</a></p>
<ul dir="auto">
<li>Jason Wu, Siyan Wang, Siman Shen, Yi-Hao Peng, Jeffrey Nichols, Jeffrey P. Bigham</li>
<li>🏛️ Institutions: CMU, Wellesley College, Grinnell College, Snooty Bird LLC</li>
<li>📅 Date: January 30, 2023</li>
<li>📑 Publisher: CHI 2023</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [dataset], [element detection], [screen classification], [screen similarity], [UI modeling]</li>
<li>📖 TLDR: The WebUI dataset includes 400,000 web UIs captured to enhance UI modeling by integrating visual UI metadata. This dataset supports tasks such as element detection, screen classification, and screen similarity, especially for accessibility, app automation, and testing applications. Through transfer learning and semi-supervised methods, WebUI addresses the challenge of training robust models with limited labeled mobile data, proving effective in tasks beyond web contexts, such as mobile UIs.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://react-lm.github.io/" rel="nofollow">ReAct: Synergizing Reasoning and Acting in Language Models</a></p>
<ul dir="auto">
<li>Shunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik Narasimhan, Yuan Cao</li>
<li>🏛️ Institutions: Princeton, Google Research</li>
<li>📅 Date: October 6, 2022</li>
<li>📑 Publisher: ICLR 2023</li>
<li>💻 Env: [Misc]</li>
<li>🔑 Key: [framework], [reasoning], [ReAct]</li>
<li>📖 TLDR: This paper introduces <em>ReAct</em>, a framework that enables large language models to generate reasoning traces and task-specific actions in an interleaved manner. By combining reasoning and acting, ReAct enhances the model's ability to perform complex tasks in language understanding and interactive decision making. The approach is validated across various benchmarks, demonstrating improved performance and interpretability over existing methods.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2209.14927" rel="nofollow">Spotlight: Mobile UI Understanding using Vision-Language Models with a Focus</a></p>
<ul dir="auto">
<li>Gang Li, Yang Li</li>
<li>🏛️ Institutions: Google Research</li>
<li>📅 Date: September 29, 2022</li>
<li>📑 Publisher: ICLR 2023</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [model], [dataset], [mobile UI tasks], [region-based focus]</li>
<li>📖 TLDR: This paper introduces "Spotlight," a vision-language model for mobile UI understanding that operates solely on visual inputs (screenshots) and a specified focus region on the screen. By leveraging a large-scale dataset and training strategies tailored to mobile interfaces, Spotlight performs multiple UI-related tasks, including widget captioning, screen summarization, command grounding, and tappability prediction. It utilizes a vision-only approach, avoiding reliance on view hierarchies to achieve greater robustness and scalability across different mobile UI environments.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2207.01206" rel="nofollow">WebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents</a></p>
<ul dir="auto">
<li>Shunyu Yao, Howard Chen, John Yang, Karthik Narasimhan</li>
<li>🏛️ Institutions: Princeton</li>
<li>📅 Date: July 2022</li>
<li>📑 Publisher: NeurIPS 2022</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [benchmark], [e-commerce web interaction], [language grounding]</li>
<li>📖 TLDR: This paper introduces <strong>WebShop</strong>, a simulated web-based shopping environment with over 1 million real-world products and 12,087 annotated instructions. It allows language agents to navigate, search, and make purchases based on natural language commands. The study explores how agents handle compositional instructions and noisy web data, providing a robust environment for reinforcement learning and imitation learning. The best models show effective sim-to-real transfer on websites like Amazon, illustrating WebShop’s potential for training grounded agents.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2205.11029" rel="nofollow">META-GUI: Towards Multi-modal Conversational Agents on Mobile GUI</a></p>
<ul dir="auto">
<li>Liangtai Sun, Xingyu Chen, Lu Chen, Tianle Dai, Zichen Zhu, Kai Yu</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: May 23, 2022</li>
<li>📑 Publisher: EMNLP 2022</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [benchmark], [dataset], [task-oriented dialogue], [GUI-based interaction], [multi-modal agent]</li>
<li>📖 TLDR: This paper presents META-GUI, a dataset and framework for training multi-modal conversational agents capable of interacting directly with mobile app interfaces without the need for backend APIs. META-GUI includes over 1,100 dialogues with annotated action sequences on various tasks such as booking and scheduling. The authors propose a GUI-based task-oriented dialogue system that allows agents to navigate mobile interfaces via direct GUI actions, with performance shown to improve in multi-modal task-oriented dialogue contexts.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2202.08137" rel="nofollow">A Data-Driven Approach for Learning to Control Computers</a></p>
<ul dir="auto">
<li>Peter C. Humphreys, David Raposo, Tobias Pohlen, Gregory Thornton, Rachita Chhaparia, Alistair Muldal, Josh Abramson, Petko Georgiev, Alex Goldin, Adam Santoro, Timothy Lillicrap</li>
<li>🏛️ Institutions: DeepMind</li>
<li>📅 Date: February 16, 2022</li>
<li>📑 Publisher: ICML 2022</li>
<li>💻 Env: [Desktop]</li>
<li>🔑 Key: [dataset], [framework], [computer control], [reinforcement learning], [multimodal transformer]</li>
<li>📖 TLDR: This study presents a reinforcement learning-based approach to train agents for computer control tasks, using keyboard and mouse interactions guided by natural language. By leveraging human demonstration data, agents trained in this environment achieved strong cross-task generalization across the MiniWob++ benchmark. This framework demonstrates how agents can control computers as humans would, enabling enhanced performance in complex computer tasks with high transferability.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2202.02312" rel="nofollow">A Dataset for Interactive Vision-Language Navigation with Unknown Command Feasibility</a></p>
<ul dir="auto">
<li>Andrea Burns, Deniz Arsan, Sanjna Agrawal, Ranjitha Kumar, Kate Saenko, Bryan A. Plummer</li>
<li>🏛️ Institutions: Boston University, UIUC</li>
<li>📅 Date: February 4, 2022</li>
<li>📑 Publisher: ECCV 2022</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [feasibility prediction], [vision-language navigation], [mobile interaction]</li>
<li>📖 TLDR: This paper introduces the <em>Mobile App Tasks with Iterative Feedback (MoTIF)</em> dataset, which addresses vision-language navigation (VLN) with a focus on task feasibility uncertainty in mobile applications. MoTIF provides commands paired with mobile actions and feasibility annotations, allowing researchers to examine the impact of command feasibility on task completion. The dataset includes 125 apps and emphasizes diverse app environments, action sequences, and follow-up questions to improve task ambiguity resolution, making it a valuable resource for feasibility prediction research.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2108.03353" rel="nofollow">Screen2Words: Automatic Mobile UI Summarization with Multimodal Learning</a></p>
<ul dir="auto">
<li>Bryan Wang, Gang Li, Xin Zhou, Zhourong Chen, Tovi Grossman, Yang Li</li>
<li>🏛️ Institutions: University of Toronto</li>
<li>📅 Date: August 6, 2021</li>
<li>📑 Publisher: UIST 2021</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [mobile UI summarization], [multimodal learning], [Transformer model]</li>
<li>📖 TLDR: The paper introduces <em>Screen2Words</em>, an approach that utilizes multimodal learning to generate descriptive language summaries for mobile UI screens, combining textual, visual, and structural data from screens. The study created a large-scale dataset with 112,085 annotated screen summaries for 22,417 unique UIs, aiming to support model training for mobile UI understanding. The dataset facilitates a Transformer-based model trained to summarize screens by highlighting main functionalities, and the approach is validated with benchmarks in the mobile environment.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://www.ijcai.org/proceedings/2021/235" rel="nofollow">UIBert: Learning Generic Multimodal Representations for UI Understanding</a></p>
<ul dir="auto">
<li>Chongyang Bai, Xiaoxue Zang, Ying Xu, Srinivas Sunkara, Abhinav Rastogi, Jindong Chen, Blaise Agüera y Arcas</li>
<li>🏛️ Institutions: Google Research</li>
<li>📅 Date: July 29, 2021</li>
<li>📑 Publisher: IJCAI 2021</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [model], [dataset], [multimodal representation learning], [UI understanding]</li>
<li>📖 TLDR: This paper presents <em>UIBert</em>, a multimodal model aimed at understanding user interfaces (UIs) by combining visual, textual, and structural metadata. UIBert is designed for tasks such as component retrieval and expression resolution, using a transformer-based joint image-text model. The authors introduce five novel pre-training tasks to leverage UI-specific features, enhancing accessibility and task completion in mobile applications. UIBert demonstrates superior performance on nine downstream UI tasks, highlighting the potential of multimodal pre-training in UI understanding.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2105.13231" rel="nofollow">AndroidEnv: A Reinforcement Learning Platform for Android</a></p>
<ul dir="auto">
<li>Daniel Toyama, Philippe Hamel, Anita Gergely, Gheorghe Comanici, Amelia Glaese, Zafarali Ahmed, Tyler Jackson, Shibl Mourad, Doina Precup</li>
<li>🏛️ Institutions: DeepMind</li>
<li>📅 Date: May 27, 2021</li>
<li>📑 Publisher: arXiv</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [reinforcement learning], [Android interface], [RL environment], [task flexibility], [touchscreen action space]</li>
<li>📖 TLDR: AndroidEnv provides a reinforcement learning (RL) platform for Android that lets RL agents interact with a realistic Android simulation via touchscreen events. The platform supports diverse applications, enabling agents to interact with over 100 predefined tasks across a variety of apps. With hybrid continuous and discrete action spaces, AndroidEnv is well-suited for training agents in complex, real-world Android scenarios where actions must be contextually sequenced, such as in UI navigation, gaming, and productivity apps. This environment encourages further RL research by offering task flexibility and realistic Android emulation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2103.16057" rel="nofollow">Grounding Open-Domain Instructions to Automate Web Support Tasks</a></p>
<ul dir="auto">
<li>Nancy Xu, Sam Masling, Michael Du, Giovanni Campagna, Larry Heck, James Landay, Monica Lam</li>
<li>🏛️ Institutions: Stanford</li>
<li>📅 Date: March 30, 2021</li>
<li>📑 Publisher: NAACL 2021</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [benchmark], [framework], [grounding], [task automation], [open-domain instructions], [RUSS]</li>
<li>📖 TLDR: This paper introduces RUSS (Rapid Universal Support Service), a framework designed to interpret and execute open-domain, step-by-step web instructions automatically. RUSS uses a BERT-LSTM model for semantic parsing into a custom language, ThingTalk, which allows the system to map language to actions across various web elements. The framework, including a dataset of instructions, facilitates agent-based web support task automation by grounding natural language to interactive commands.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2101.09465" rel="nofollow">WebSRC: A Dataset for Web-Based Structural Reading Comprehension</a></p>
<ul dir="auto">
<li>Lu Chen, Zihan Zhao, Xingyu Chen, Danyang Zhang, Jiabao Ji, Ao Luo, Yuxuan Xiong, Kai Yu</li>
<li>🏛️ Institutions: SJTU</li>
<li>📅 Date: January 23, 2021</li>
<li>📑 Publisher: EMNLP 2021</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [dataset], [structural reading comprehension], [web page QA], [structural information], [HTML element alignment]</li>
<li>📖 TLDR: This paper introduces <strong>WebSRC</strong>, a dataset specifically designed for web-based structural reading comprehension, which requires understanding not only textual content but also the structural layout of web pages. WebSRC consists of 0.44 million question-answer pairs derived from 6,500 complex web pages. Each question challenges models to identify answers from HTML structures or to respond with yes/no, requiring a nuanced grasp of HTML and layout features. The authors benchmark several models on this dataset, highlighting its difficulty and the critical role of structural comprehension in improving machine understanding of web content.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/2010.04295" rel="nofollow">Widget Captioning: Generating Natural Language Description for Mobile User Interface Elements</a></p>
<ul dir="auto">
<li>Yang Li, Gang Li, Luheng He, Jingjie Zheng, Hong Li, Zhiwei Guan</li>
<li>🏛️ Institutions: Google Research</li>
<li>📅 Date: November 2020</li>
<li>📑 Publisher: EMNLP 2020</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [benchmark], [model], [accessibility], [natural language generation], [WidgetCaption]</li>
<li>📖 TLDR: This paper introduces the task of <em>widget captioning</em>, which aims to automatically generate natural language descriptions for UI elements in mobile apps to enhance accessibility. Using both visual and structural data from UI components, the study presents a novel dataset of 162,859 captions across 61,285 UI elements. Multiple deep learning models were tested on this dataset, with findings suggesting the potential for improving screen reader usability for visually impaired users by generating descriptive captions of UI elements.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://aclanthology.org/2020.acl-demos.25/" rel="nofollow">Interactive Task Learning from GUI-Grounded Natural Language Instructions and Demonstrations</a></p>
<ul dir="auto">
<li>Toby Jia-Jun Li, Tom Mitchell, Brad Myers</li>
<li>🏛️ Institutions: CMU</li>
<li>📅 Date: July 2020</li>
<li>📑 Publisher: ACL 2020</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [Sugilite], [programming-by-demonstration]</li>
<li>📖 TLDR: This paper introduces <em>Sugilite</em>, an intelligent task automation agent that learns new tasks and associated concepts interactively from users' natural language instructions and demonstrations on third-party mobile app GUIs. The system allows users to teach procedures and concepts through verbal instructions combined with GUI demonstrations, supports intent clarification for demonstrated actions, infers task parameters using hierarchical app GUI structures, and generalizes taught concepts across different contexts and domains. A prototype is presented as a conversational assistant on Android. <a href="https://aclanthology.org/2020.acl-demos.25/?utm_source=chatgpt.com" rel="nofollow">oai_citation_attribution:0‡ACL Anthology</a></li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://aclanthology.org/2020.acl-main.729" rel="nofollow">Mapping Natural Language Instructions to Mobile UI Action Sequences</a></p>
<ul dir="auto">
<li>Yang Li, Jiacong He, Xin Zhou, Yuan Zhang, Jason Baldridge</li>
<li>🏛️ Institutions: Google Researc</li>
<li>📅 Date: July 2020</li>
<li>📑 Publisher: ACL 2020</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [dataset], [mobile UI automation], [natural language instructions], [action grounding], [RicoSCA]</li>
<li>📖 TLDR: This paper introduces a method for grounding natural language instructions to mobile UI actions, aiming to automate mobile task execution through user interface manipulation. It introduces three key datasets: <strong>PixelHelp</strong> for task instruction-performance mappings on a Pixel emulator, <strong>AndroidHowTo</strong> for detailed phrase extraction, and <strong>RicoSCA</strong> for synthetic UI command training. The system utilizes a Transformer model to extract action phrase tuples, aligning them to UI elements with contextual screen positioning. Achieving over 70% accuracy in task completion, this approach is foundational for natural language-driven mobile UI automation.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/1909.00031" rel="nofollow">PUMICE: A Multi-Modal Agent that Learns Concepts and Conditionals from Natural Language and Demonstrations</a></p>
<ul dir="auto">
<li>Toby Jia-Jun Li, Marissa Radensky, Justin Jia, Kirielle Singarajah, Tom M. Mitchell, Brad A. Myers</li>
<li>🏛️ Institutions: CMU, Amherst College</li>
<li>📅 Date: August 30, 2019</li>
<li>📑 Publisher: UIST 2019</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [programming-by-demonstration], [PUMICE]</li>
<li>📖 TLDR: This paper introduces <em>PUMICE</em>, a multi-modal agent that combines natural language programming and programming-by-demonstration to enable end users to instruct intelligent agents in performing new tasks. By allowing users to describe tasks and conditions naturally and then collaboratively resolving ambiguities through conversation and demonstration, PUMICE facilitates the teaching of new concepts and procedures within existing mobile app GUIs. A lab study with 10 users demonstrated its usability and effectiveness.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://arxiv.org/abs/1802.08802" rel="nofollow">Reinforcement Learning on Web Interfaces Using Workflow-Guided Exploration</a></p>
<ul dir="auto">
<li>Evan Zheran Liu, Kelvin Guu, Panupong Pasupat, Tianlin Shi, Percy Liang</li>
<li>🏛️ Institutions: Stanford</li>
<li>📅 Date: February 24, 2018</li>
<li>📑 Publisher: ICLR 2018</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [benchmark], [reinforcement learning], [web tasks], [workflow-guided exploration]</li>
<li>📖 TLDR: This paper presents a novel RL approach using <em>workflow-guided exploration</em> to efficiently train agents on web-based tasks, where actions are restricted based on demonstrated workflows to streamline learning. Evaluated on MiniWoB and MiniWoB++ benchmarks, the method significantly outperforms traditional RL techniques in sparse reward settings by structuring exploration according to high-level action constraints.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://dl.acm.org/doi/10.1145/3126594.3126651" rel="nofollow">Rico: A Mobile App Dataset for Building Data-Driven Design Applications</a></p>
<ul dir="auto">
<li>Genevieve Patterson, Joseph Gonzalez, Jeffrey Heer, Daniel H. Haim, Keyur Govani, Andrew Hertzmann, Noah Snavely, Neel Joshi</li>
<li>🏛️ Institutions: UIUC, Northwestern University, Google</li>
<li>📅 Date: October 20, 2017</li>
<li>📑 Publisher: UIST 2017</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [dataset], [mobile UI], [UI design analysis], [interaction mining], [RICO]</li>
<li>📖 TLDR: This paper introduces <em>Rico</em>, a large-scale dataset comprising UI screens and view hierarchies from over 9,000 Android apps, designed to aid in understanding mobile app design. Rico supports a variety of tasks, including UI design analysis and interaction mining, by providing labeled UI components, screenshots, and interaction traces.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://proceedings.mlr.press/v70/shi17a.html" rel="nofollow">World of Bits: An Open-Domain Platform for Web-Based Agents</a></p>
<ul dir="auto">
<li>Tianlin Shi, Andrej Karpathy, Linxi Fan, Jonathan Hernandez, Percy Liang</li>
<li>🏛️ Institutions: Stanford, OpenAI</li>
<li>📅 Date: August 2017</li>
<li>📑 Publisher: ICML 2017</li>
<li>💻 Env: [Web]</li>
<li>🔑 Key: [framework], [dataset], [reinforcement learning], [open-domain]</li>
<li>📖 TLDR: This paper introduces <em>World of Bits (WoB)</em>, a platform enabling agents to perform complex web-based tasks using low-level keyboard and mouse actions, addressing the lack of open-domain realism in existing reinforcement learning environments. WoB leverages a novel framework where crowdworkers create tasks with structured rewards and reproducibility by caching web interactions, forming a stable training environment. The authors validate WoB by training agents via behavioral cloning and reinforcement learning to accomplish various real-world tasks, showcasing its potential as an effective platform for reinforcement learning on web tasks.</li>
</ul>
</li>
<li>
<p dir="auto"><a href="https://dl.acm.org/doi/abs/10.1145/3025453.3025483" rel="nofollow">SUGILITE: Creating Multimodal Smartphone Automation by Demonstration</a></p>
<ul dir="auto">
<li>Toby Jia-Jun Li, Amos Azaria, Brad A. Myers</li>
<li>🏛️ Institutions: CMU, Ariel University</li>
<li>📅 Date: May 6, 2017</li>
<li>📑 Publisher: CHI 2017</li>
<li>💻 Env: [Mobile]</li>
<li>🔑 Key: [framework], [PBD], [multimodal interaction], [SUGILITE], [programming-by-demonstration], [demonstration]</li>
<li>📖 TLDR: This paper introduces <em>SUGILITE</em>, a programming-by-demonstration (PBD) system that enables users to automate tasks on smartphones through multimodal interactions. By leveraging Android's accessibility API, SUGILITE allows users to create generalized automation scripts for arbitrary third-party apps by demonstrating tasks using the regular app UI. The system combines verbal instructions, user demonstrations, and app UI hierarchies to generalize scripts from single demonstrations, facilitating task variations and parameterization. Extensive error handling and context checking enhance robustness against app UI changes. A lab study indicates that users with minimal programming knowledge can successfully automate smartphone tasks using SUGILITE.</li>
</ul>
</li>
</ul>
</details>
<div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">How to Add a Paper or Update the README</h2><a id="user-content-how-to-add-a-paper-or-update-the-readme" class="anchor" aria-label="Permalink: How to Add a Paper or Update the README" href="#how-to-add-a-paper-or-update-the-readme"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
<p dir="auto">Please fork and update:</p>
<ul dir="auto">
<li><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/update_template_or_data/update_paper_list.md">paper list</a></li>
<li><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/update_template_or_data/update_readme_template.md">README template</a></li>
<li><a href="/OSU-NLP-Group/GUI-Agents-Paper-List/blob/main/.github/workflows/main.yml">automatic workflow</a></li>
</ul>
<p dir="auto">🤖 You can use <a href="https://chatgpt.com/g/g-VqW9ONrgL-gui-paper-list" rel="nofollow">this GPTs</a> to quickly search and get a formatted paper entry automatically by inputting a paper name. Or you can simply leave a comment in an issue.</p>
<details>
<summary>Format example and explanation</summary>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto"><pre class="notranslate"><code>- [title](paper link)
    - List authors directly without a "key" identifier (e.g., author1, author2)
    - 🏛️ Institutions: List the institutions concisely, using abbreviations (e.g., university names, like OSU).
    - 📅 Date: e.g., Oct 30, 2024
    - 📑 Publisher: ICLR 2025
    - 💻 Env: Indicate the research environment within brackets, such as [Web], [Mobile], or [Desktop]. Use [GUI] if the research spans multiple environments. Use [Misc] if it is researching in general domains.
    - 🔑 Key: Label each keyword within brackets, e.g., [model], [framework],[dataset],[benchmark].
    - 📖 TLDR: Brief summary of the paper.
</code></pre><div class="zeroclipboard-container">
    <clipboard-copy aria-label="Copy" class="ClipboardButton btn btn-invisible js-clipboard-copy m-2 p-0 d-flex flex-justify-center flex-items-center" data-copy-feedback="Copied!" data-tooltip-direction="w" value="- [title](paper link)
    - List authors directly without a &quot;key&quot; identifier (e.g., author1, author2)
    - 🏛️ Institutions: List the institutions concisely, using abbreviations (e.g., university names, like OSU).
    - 📅 Date: e.g., Oct 30, 2024
    - 📑 Publisher: ICLR 2025
    - 💻 Env: Indicate the research environment within brackets, such as [Web], [Mobile], or [Desktop]. Use [GUI] if the research spans multiple environments. Use [Misc] if it is researching in general domains.
    - 🔑 Key: Label each keyword within brackets, e.g., [model], [framework],[dataset],[benchmark].
    - 📖 TLDR: Brief summary of the paper." tabindex="0" role="button">
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon">
    <path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none">
    <path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
</svg>
    </clipboard-copy>
  </div></div>
<p dir="auto">Regarding the 🔑 Key:</p>
<markdown-accessiblity-table><table>
<thead>
<tr>
<th>Key</th>
<th>Definition</th>
</tr>
</thead>
<tbody>
<tr>
<td>model</td>
<td>Indicates a newly trained model.</td>
</tr>
<tr>
<td>framework</td>
<td>If the paper proposes a new agent framework.</td>
</tr>
<tr>
<td>dataset</td>
<td>If a new (training) dataset is created and published.</td>
</tr>
<tr>
<td>benchmark</td>
<td>If a new benchmark is established (also add "dataset" if there's a new training set).</td>
</tr>
<tr>
<td>primary studies</td>
<td>List the main focus or innovation in the study.</td>
</tr>
<tr>
<td>Abbreviations</td>
<td>Include commonly used abbreviations associated with the paper (model names, framework names, etc.).</td>
</tr>
</tbody>
</table></markdown-accessiblity-table>
<p dir="auto">For missing information, use "Unknown."</p>
</details>
</article></div></div></div></div></div> <!-- --> <!-- --> <script type="application/json" id="__PRIMER_DATA_:R0:__">{"resolvedServerColorMode":"day"}</script></div>
</react-partial>

      <input type="hidden" data-csrf="true" value="LdsGpRUml0DRk3QqQ72n4Vv+orUpuQHPMzdQ6qRRaHx4EoU5+79hE1Pwxk6khEtLynW8HJ9pSPo6q02bopTdug==">
</div>
  <div data-view-component="true" class="Layout-sidebar">      

      <div class="BorderGrid about-margin" data-pjax="">
        <div class="BorderGrid-row">
          <div class="BorderGrid-cell">
            <div class="hide-sm hide-md">
  <h2 class="mb-3 h4">About</h2>

      <p class="f4 my-3">
         Building a comprehensive and handy list of papers for GUI agents
      </p>


    <h3 class="sr-only">Resources</h3>
    <div class="mt-2">
      <a class="Link--muted" data-analytics-event="{&quot;category&quot;:&quot;Repository Overview&quot;,&quot;action&quot;:&quot;click&quot;,&quot;label&quot;:&quot;location:sidebar;file:readme&quot;}" href="#readme-ov-file">
        <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-book mr-2">
    <path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path>
</svg>
        Readme
</a>    </div>

  



  <include-fragment src="/OSU-NLP-Group/GUI-Agents-Paper-List/hovercards/citation/sidebar_partial?tree_name=main" data-nonce="v2:073dc60b-75ed-517e-12ed-f8b77db3c2d0" data-view-component="true">
  

  <div data-show-on-forbidden-error="" hidden="">
    <div class="Box">
  <div class="blankslate-container">
    <div data-view-component="true" class="blankslate blankslate-spacious color-bg-default rounded-2">
      

      <h3 data-view-component="true" class="blankslate-heading">        Uh oh!
</h3>
      <p data-view-component="true">        </p><p class="color-fg-muted my-2 mb-2 ws-normal">There was an error while loading. <a class="Link--inTextBlock" data-turbo="false" href="" aria-label="Please reload this page">Please reload this page</a>.</p>
<p></p>

</div>  </div>
</div>  </div>
</include-fragment>
  <div class="mt-2">
    <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/activity" data-view-component="true" class="Link Link--muted"><svg text="gray" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-pulse mr-2">
    <path d="M6 2c.306 0 .582.187.696.471L10 10.731l1.304-3.26A.751.751 0 0 1 12 7h3.25a.75.75 0 0 1 0 1.5h-2.742l-1.812 4.528a.751.751 0 0 1-1.392 0L6 4.77 4.696 8.03A.75.75 0 0 1 4 8.5H.75a.75.75 0 0 1 0-1.5h2.742l1.812-4.529A.751.751 0 0 1 6 2Z"></path>
</svg>
      <span class="color-fg-muted">Activity</span></a>  </div>

    <div class="mt-2">
      <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/custom-properties" data-view-component="true" class="Link Link--muted"><svg text="gray" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-note mr-2">
    <path d="M0 3.75C0 2.784.784 2 1.75 2h12.5c.966 0 1.75.784 1.75 1.75v8.5A1.75 1.75 0 0 1 14.25 14H1.75A1.75 1.75 0 0 1 0 12.25Zm1.75-.25a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-8.5a.25.25 0 0 0-.25-.25ZM3.5 6.25a.75.75 0 0 1 .75-.75h7a.75.75 0 0 1 0 1.5h-7a.75.75 0 0 1-.75-.75Zm.75 2.25h4a.75.75 0 0 1 0 1.5h-4a.75.75 0 0 1 0-1.5Z"></path>
</svg>
        <span class="color-fg-muted">Custom properties</span></a>    </div>

  <h3 class="sr-only">Stars</h3>
  <div class="mt-2">
    <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/stargazers" data-view-component="true" class="Link Link--muted"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star mr-2">
    <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path>
</svg>
      <strong>370</strong>
      stars</a>  </div>

  <h3 class="sr-only">Watchers</h3>
  <div class="mt-2">
    <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/watchers" data-view-component="true" class="Link Link--muted"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-eye mr-2">
    <path d="M8 2c1.981 0 3.671.992 4.933 2.078 1.27 1.091 2.187 2.345 2.637 3.023a1.62 1.62 0 0 1 0 1.798c-.45.678-1.367 1.932-2.637 3.023C11.67 13.008 9.981 14 8 14c-1.981 0-3.671-.992-4.933-2.078C1.797 10.83.88 9.576.43 8.898a1.62 1.62 0 0 1 0-1.798c.45-.677 1.367-1.931 2.637-3.022C4.33 2.992 6.019 2 8 2ZM1.679 7.932a.12.12 0 0 0 0 .136c.411.622 1.241 1.75 2.366 2.717C5.176 11.758 6.527 12.5 8 12.5c1.473 0 2.825-.742 3.955-1.715 1.124-.967 1.954-2.096 2.366-2.717a.12.12 0 0 0 0-.136c-.412-.621-1.242-1.75-2.366-2.717C10.824 4.242 9.473 3.5 8 3.5c-1.473 0-2.825.742-3.955 1.715-1.124.967-1.954 2.096-2.366 2.717ZM8 10a2 2 0 1 1-.001-3.999A2 2 0 0 1 8 10Z"></path>
</svg>
      <strong>13</strong>
      watching</a>  </div>

  <h3 class="sr-only">Forks</h3>
  <div class="mt-2">
    <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/forks" data-view-component="true" class="Link Link--muted"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2">
    <path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path>
</svg>
      <strong>20</strong>
      forks</a>  </div>


    <div class="mt-2">
      <a class="Link--muted" href="/contact/report-content?content_url=https%3A%2F%2Fgithub.com%2FOSU-NLP-Group%2FGUI-Agents-Paper-List&amp;report=OSU-NLP-Group+%28user%29">
          Report repository
</a>    </div>
</div>

          </div>
        </div>

        
        
        
        
            <div class="BorderGrid-row" hidden="">
              <div class="BorderGrid-cell">
                <include-fragment src="/OSU-NLP-Group/GUI-Agents-Paper-List/used_by_list" accept="text/fragment+html" data-nonce="v2:073dc60b-75ed-517e-12ed-f8b77db3c2d0" data-view-component="true">
  

  <div data-show-on-forbidden-error="" hidden="">
    <div class="Box">
  <div class="blankslate-container">
    <div data-view-component="true" class="blankslate blankslate-spacious color-bg-default rounded-2">
      

      <h3 data-view-component="true" class="blankslate-heading">        Uh oh!
</h3>
      <p data-view-component="true">        </p><p class="color-fg-muted my-2 mb-2 ws-normal">There was an error while loading. <a class="Link--inTextBlock" data-turbo="false" href="" aria-label="Please reload this page">Please reload this page</a>.</p>
<p></p>

</div>  </div>
</div>  </div>
</include-fragment>
              </div>
            </div>

        
            <div class="BorderGrid-row">
              <div class="BorderGrid-cell">
                <h2 class="h4 mb-3">
  <a href="/OSU-NLP-Group/GUI-Agents-Paper-List/graphs/contributors" data-view-component="true" class="Link--primary no-underline Link d-flex flex-items-center">Contributors
      <span title="10" data-view-component="true" class="Counter ml-1">10</span></a></h2>


    
  <ul class="list-style-none d-flex flex-wrap mb-n2">
    <li class="mb-2 mr-2">
      <a href="https://github.com/boyugou" class="" data-hovercard-type="user" data-hovercard-url="/users/boyugou/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/103808989?s=64&amp;v=4" alt="@boyugou" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/YuanDaoze" class="" data-hovercard-type="user" data-hovercard-url="/users/YuanDaoze/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/102368460?s=64&amp;v=4" alt="@YuanDaoze" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/ysu1989" class="" data-hovercard-type="user" data-hovercard-url="/users/ysu1989/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/4349332?s=64&amp;v=4" alt="@ysu1989" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/wwh0411" class="" data-hovercard-type="user" data-hovercard-url="/users/wwh0411/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/55397039?s=64&amp;v=4" alt="@wwh0411" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/ant-8" class="" data-hovercard-type="user" data-hovercard-url="/users/ant-8/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/28424838?s=64&amp;v=4" alt="@ant-8" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/huan-sunrise" class="" data-hovercard-type="user" data-hovercard-url="/users/huan-sunrise/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/42845938?s=64&amp;v=4" alt="@huan-sunrise" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/QiushiSun" class="" data-hovercard-type="user" data-hovercard-url="/users/QiushiSun/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/54871790?s=64&amp;v=4" alt="@QiushiSun" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/callanwu" class="" data-hovercard-type="user" data-hovercard-url="/users/callanwu/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/63695429?s=64&amp;v=4" alt="@callanwu" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
    <li class="mb-2 mr-2">
      <a href="https://github.com/derekywk" class="" data-hovercard-type="user" data-hovercard-url="/users/derekywk/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self">
        <img src="https://avatars.githubusercontent.com/u/75664093?s=64&amp;v=4" alt="@derekywk" size="32" height="32" width="32" data-view-component="true" class="avatar circle">
      </a>
    </li>
</ul>





              </div>
            </div>

        
        
            <div class="BorderGrid-row">
              <div class="BorderGrid-cell">
                <h2 class="h4 mb-3">Languages</h2>
<div class="mb-2">
  <span data-view-component="true" class="Progress">
    <span style="background-color:#3572A5 !important;;width: 100.0%;" itemprop="keywords" data-view-component="true" class="Progress-item color-bg-success-emphasis"></span>
</span></div>
<ul class="list-style-none">
    <li class="d-inline">
        <a class="d-inline-flex flex-items-center flex-nowrap Link--secondary no-underline text-small mr-3" href="/OSU-NLP-Group/GUI-Agents-Paper-List/search?l=python" data-ga-click="Repository, language stats search click, location:repo overview">
          <svg style="color:#3572A5;" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-dot-fill mr-2">
    <path d="M8 4a4 4 0 1 1 0 8 4 4 0 0 1 0-8Z"></path>
</svg>
          <span class="color-fg-default text-bold mr-1">Python</span>
          <span>100.0%</span>
        </a>
    </li>
</ul>

              </div>
            </div>

              </div>
</div>
  
</div></div>

  </div>


  </div>

</turbo-frame>


    </main>
  </div>

  </div>

          <footer class="footer pt-8 pb-6 f6 color-fg-muted p-responsive" role="contentinfo">
  <h2 class="sr-only">Footer</h2>

  


  <div class="d-flex flex-justify-center flex-items-center flex-column-reverse flex-lg-row flex-wrap flex-lg-nowrap">
    <div class="d-flex flex-items-center flex-shrink-0 mx-2">
      <a aria-label="GitHub Homepage" class="footer-octicon mr-2" href="https://github.com">
        <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-mark-github">
    <path d="M12 1C5.9225 1 1 5.9225 1 12C1 16.8675 4.14875 20.9787 8.52125 22.4362C9.07125 22.5325 9.2775 22.2025 9.2775 21.9137C9.2775 21.6525 9.26375 20.7862 9.26375 19.865C6.5 20.3737 5.785 19.1912 5.565 18.5725C5.44125 18.2562 4.905 17.28 4.4375 17.0187C4.0525 16.8125 3.5025 16.3037 4.42375 16.29C5.29 16.2762 5.90875 17.0875 6.115 17.4175C7.105 19.0812 8.68625 18.6137 9.31875 18.325C9.415 17.61 9.70375 17.1287 10.02 16.8537C7.5725 16.5787 5.015 15.63 5.015 11.4225C5.015 10.2262 5.44125 9.23625 6.1425 8.46625C6.0325 8.19125 5.6475 7.06375 6.2525 5.55125C6.2525 5.55125 7.17375 5.2625 9.2775 6.67875C10.1575 6.43125 11.0925 6.3075 12.0275 6.3075C12.9625 6.3075 13.8975 6.43125 14.7775 6.67875C16.8813 5.24875 17.8025 5.55125 17.8025 5.55125C18.4075 7.06375 18.0225 8.19125 17.9125 8.46625C18.6138 9.23625 19.04 10.2125 19.04 11.4225C19.04 15.6437 16.4688 16.5787 14.0213 16.8537C14.42 17.1975 14.7638 17.8575 14.7638 18.8887C14.7638 20.36 14.75 21.5425 14.75 21.9137C14.75 22.2025 14.9563 22.5462 15.5063 22.4362C19.8513 20.9787 23 16.8537 23 12C23 5.9225 18.0775 1 12 1Z"></path>
</svg>
</a>
      <span>
        © 2025 GitHub,&nbsp;Inc.
      </span>
    </div>

    <nav aria-label="Footer">
      <h3 class="sr-only" id="sr-footer-heading">Footer navigation</h3>

      <ul class="list-style-none d-flex flex-justify-center flex-wrap mb-2 mb-lg-0" aria-labelledby="sr-footer-heading">

          <li class="mx-2">
            <a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to Terms&quot;,&quot;label&quot;:&quot;text:terms&quot;}" href="https://docs.github.com/site-policy/github-terms/github-terms-of-service" data-view-component="true" class="Link--secondary Link">Terms</a>
          </li>

          <li class="mx-2">
            <a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to privacy&quot;,&quot;label&quot;:&quot;text:privacy&quot;}" href="https://docs.github.com/site-policy/privacy-policies/github-privacy-statement" data-view-component="true" class="Link--secondary Link">Privacy</a>
          </li>

          <li class="mx-2">
            <a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to security&quot;,&quot;label&quot;:&quot;text:security&quot;}" href="https://github.com/security" data-view-component="true" class="Link--secondary Link">Security</a>
          </li>

          <li class="mx-2">
            <a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to status&quot;,&quot;label&quot;:&quot;text:status&quot;}" href="https://www.githubstatus.com/" data-view-component="true" class="Link--secondary Link">Status</a>
          </li>

          <li class="mx-2">
            <a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to docs&quot;,&quot;label&quot;:&quot;text:docs&quot;}" href="https://docs.github.com/" data-view-component="true" class="Link--secondary Link">Docs</a>
          </li>

          <li class="mx-2">
            <a data-analytics-event="{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to contact&quot;,&quot;label&quot;:&quot;text:contact&quot;}" href="https://support.github.com?tags=dotcom-footer" data-view-component="true" class="Link--secondary Link">Contact</a>
          </li>

          <li class="mx-2">
  <cookie-consent-link>
    <button type="button" class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent" data-action="click:cookie-consent-link#showConsentManagement" data-analytics-event="{&quot;location&quot;:&quot;footer&quot;,&quot;action&quot;:&quot;cookies&quot;,&quot;context&quot;:&quot;subfooter&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;cookies_link_subfooter_footer&quot;}">
      Manage cookies
    </button>
  </cookie-consent-link>
</li>

<li class="mx-2">
  <cookie-consent-link>
    <button type="button" class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent" data-action="click:cookie-consent-link#showConsentManagement" data-analytics-event="{&quot;location&quot;:&quot;footer&quot;,&quot;action&quot;:&quot;dont_share_info&quot;,&quot;context&quot;:&quot;subfooter&quot;,&quot;tag&quot;:&quot;link&quot;,&quot;label&quot;:&quot;dont_share_info_link_subfooter_footer&quot;}">
      Do not share my personal information
    </button>
  </cookie-consent-link>
</li>

      </ul>
    </nav>
  </div>
</footer>



    <ghcc-consent id="ghcc" class="position-fixed bottom-0 left-0" style="z-index: 999999" data-initial-cookie-consent-allowed="" data-cookie-consent-required="false"></ghcc-consent>



  <div id="ajax-error-message" class="ajax-error-message flash flash-error" hidden="">
    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert">
    <path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path>
</svg>
    <button type="button" class="flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
    <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
    </button>
    You can’t perform that action at this time.
  </div>

    <template id="site-details-dialog">
  <details class="details-reset details-overlay details-overlay-dark lh-default color-fg-default hx_rsm" open="">
    <summary role="button" aria-label="Close dialog"></summary>
    <details-dialog class="Box Box--overlay d-flex flex-column anim-fade-in fast hx_rsm-dialog hx_rsm-modal">
      <button class="Box-btn-octicon m-0 btn-octicon position-absolute right-0 top-0" type="button" aria-label="Close dialog" data-close-dialog="">
        <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
    <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg>
      </button>
      <div class="octocat-spinner my-6 js-details-dialog-spinner"></div>
    </details-dialog>
  </details>
</template>

    <div class="Popover js-hovercard-content position-absolute" style="display: none; outline: none;">
  <div class="Popover-message Popover-message--bottom-left Popover-message--large Box color-shadow-large" style="width:360px;">
  </div>
</div>

    <template id="snippet-clipboard-copy-button">
  <div class="zeroclipboard-container position-absolute right-0 top-0">
    <clipboard-copy aria-label="Copy" class="ClipboardButton btn js-clipboard-copy m-2 p-0" data-copy-feedback="Copied!" data-tooltip-direction="w">
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon m-2">
    <path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none m-2">
    <path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
</svg>
    </clipboard-copy>
  </div>
</template>
<template id="snippet-clipboard-copy-button-unpositioned">
  <div class="zeroclipboard-container">
    <clipboard-copy aria-label="Copy" class="ClipboardButton btn btn-invisible js-clipboard-copy m-2 p-0 d-flex flex-justify-center flex-items-center" data-copy-feedback="Copied!" data-tooltip-direction="w">
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon">
    <path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path>
</svg>
      <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none">
    <path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path>
</svg>
    </clipboard-copy>
  </div>
</template>




    </div>

    <div id="js-global-screen-reader-notice" class="sr-only mt-n1" aria-live="polite" aria-atomic="true"></div>
    <div id="js-global-screen-reader-notice-assertive" class="sr-only mt-n1" aria-live="assertive" aria-atomic="true"></div>
  


</body></html>