{
  "schemaVersion": "0.1",
  "status": "proposal",
  "name": "Accessible Computer-Use Agent Evaluation Framework",
  "version": "0.1",
  "published": "2026-07-17",
  "author": {
    "name": "Jason Matthew Suhari",
    "url": "https://www.jasonsuhari.com/jason-matthew-suhari"
  },
  "canonicalArticle": "https://www.jasonsuhari.com/writing/evaluating-accessible-computer-use-agents",
  "scope": "Evaluation of agent-mediated desktop tasks across accessible input methods.",
  "nonClaims": [
    "This document is not an accessibility standard, certification, or clinical assessment.",
    "Version 0.1 does not publish product scores or claim independent validation.",
    "A task result does not generalize across untested users, input methods, applications, or environments."
  ],
  "dimensions": [
    {
      "id": "completion",
      "name": "Task completion",
      "definition": "Whether the declared target state was reached without violating a task constraint.",
      "unit": "boolean plus completion-state evidence"
    },
    {
      "id": "input_effort",
      "name": "Input effort",
      "definition": "The number and duration of deliberate user inputs required to reach the target state.",
      "unit": "input events and active input time"
    },
    {
      "id": "correction_burden",
      "name": "Correction burden",
      "definition": "Restatements, repairs, or mode switches needed because the system misunderstood or acted incorrectly.",
      "unit": "correction events by severity"
    },
    {
      "id": "recovery",
      "name": "Recovery",
      "definition": "Whether the user can return to a safe, understood state after a failure.",
      "unit": "recovery success, recovery time, and additional inputs"
    },
    {
      "id": "safety",
      "name": "Safety",
      "definition": "Whether consequential actions are previewed, confirmed, bounded, and reversible where possible.",
      "unit": "safety events by consequence level"
    },
    {
      "id": "latency",
      "name": "Latency",
      "definition": "Time from completed user input to system response and to a user-understandable state.",
      "unit": "milliseconds"
    },
    {
      "id": "user_control",
      "name": "User control",
      "definition": "Whether the user can inspect, pause, cancel, and resume without switching to an inaccessible input.",
      "unit": "capability checks plus observed control failures"
    }
  ],
  "taskFamilies": [
    {
      "id": "navigation",
      "examples": ["Open an application", "Move to a named control", "Switch between two windows"]
    },
    {
      "id": "text_editing",
      "examples": ["Draft text", "Replace a phrase", "Correct an error without rewriting the document"]
    },
    {
      "id": "information_retrieval",
      "examples": ["Find a named fact", "Compare two records", "Summarize a visible document"]
    },
    {
      "id": "file_management",
      "examples": ["Locate a file", "Rename a file", "Move a file with confirmation"]
    },
    {
      "id": "communication",
      "examples": ["Draft a message", "Verify recipient and attachment", "Send after explicit confirmation"]
    },
    {
      "id": "recovery",
      "examples": ["Recover from the wrong window", "Undo an incorrect reversible action", "Stop before a consequential action"]
    }
  ],
  "taskDefinitionRequiredFields": [
    "task_id",
    "start_state",
    "target_state",
    "allowed_applications",
    "consequence_level",
    "input_method",
    "completion_rule",
    "stop_conditions"
  ],
  "runReportRequiredFields": [
    "software_version",
    "operating_system",
    "application_versions",
    "model_and_provider_if_applicable",
    "input_hardware",
    "network_conditions",
    "evaluator_relationship",
    "attempt_number",
    "task_level_results",
    "untested_conditions"
  ],
  "participationPrinciples": [
    "Co-design can change the task set, failure categories, trade-offs, and interpretation.",
    "Participation requires informed consent and a clear data-removal process.",
    "Compensation should reflect the participant's time and expertise.",
    "Results should not generalize beyond the tested people and configurations."
  ],
  "references": [
    "https://www.w3.org/TR/WCAG22/",
    "https://www.w3.org/WAI/standards-guidelines/uaag/"
  ]
}