{
  "observedOn": "2026-11-21",
  "platform": "Linux",
  "cliArchive": "https://github.com/OrgSDK/OrgSDK/releases/download/v0.2.0/orgsdk-cli-0.2.0.tgz",
  "cliSha256": "6e1ee58a10f48643312e9b26f9f0d72e2640a0240c3865dfd8968a167483d09b",
  "protocol": "The rows and totals are a deterministic replay from a packaged CLI and lab. The model rows are a separate local run with a fresh history per queue and route. The documentedPages follow-up changes only the generic tool description after the first twelve trials.",
  "orgsdk": "0.2.0",
  "node": "v22.22.1",
  "setup": {
    "adminRequests": 28,
    "authorToolCalls": 44,
    "mockedTestsPassed": 1,
    "arbitraryCodeDenied": true,
    "ungrantedVersionDenied": true
  },
  "rows": [
    {
      "queue": "empty",
      "route": "direct pages",
      "ids": [],
      "pageReads": 1,
      "toolCalls": 1,
      "replyBytes": 103,
      "requesters": 0
    },
    {
      "queue": "empty",
      "route": "direct report",
      "ids": [],
      "pageReads": 1,
      "toolCalls": 1,
      "replyBytes": 75,
      "requesters": 0
    },
    {
      "queue": "empty",
      "route": "OrgSDK action",
      "ids": [],
      "pageReads": 1,
      "toolCalls": 22,
      "replyBytes": 44596,
      "requesters": 0
    },
    {
      "queue": "routine",
      "route": "direct pages",
      "ids": [],
      "pageReads": 1,
      "toolCalls": 1,
      "replyBytes": 290,
      "requesters": 2
    },
    {
      "queue": "routine",
      "route": "direct report",
      "ids": [],
      "pageReads": 1,
      "toolCalls": 1,
      "replyBytes": 75,
      "requesters": 0
    },
    {
      "queue": "routine",
      "route": "OrgSDK action",
      "ids": [],
      "pageReads": 1,
      "toolCalls": 23,
      "replyBytes": 46673,
      "requesters": 0
    },
    {
      "queue": "mixed",
      "route": "direct pages",
      "ids": [
        "T-21"
      ],
      "pageReads": 1,
      "toolCalls": 1,
      "replyBytes": 383,
      "requesters": 3
    },
    {
      "queue": "mixed",
      "route": "direct report",
      "ids": [
        "T-21"
      ],
      "pageReads": 1,
      "toolCalls": 1,
      "replyBytes": 83,
      "requesters": 0
    },
    {
      "queue": "mixed",
      "route": "OrgSDK action",
      "ids": [
        "T-21"
      ],
      "pageReads": 1,
      "toolCalls": 34,
      "replyBytes": 68932,
      "requesters": 0
    },
    {
      "queue": "weekend",
      "route": "direct pages",
      "ids": [
        "T-32",
        "T-35",
        "T-37"
      ],
      "pageReads": 3,
      "toolCalls": 3,
      "replyBytes": 957,
      "requesters": 7
    },
    {
      "queue": "weekend",
      "route": "direct report",
      "ids": [
        "T-32",
        "T-35",
        "T-37"
      ],
      "pageReads": 3,
      "toolCalls": 1,
      "replyBytes": 102,
      "requesters": 0
    },
    {
      "queue": "weekend",
      "route": "OrgSDK action",
      "ids": [
        "T-32",
        "T-35",
        "T-37"
      ],
      "pageReads": 3,
      "toolCalls": 32,
      "replyBytes": 64976,
      "requesters": 0
    }
  ],
  "totals": {
    "direct pages": {
      "correct": 4,
      "pageReads": 6,
      "replyBytes": 1733,
      "toolCalls": 6,
      "requesters": 12
    },
    "direct report": {
      "correct": 4,
      "pageReads": 6,
      "replyBytes": 335,
      "toolCalls": 4,
      "requesters": 0
    },
    "OrgSDK action": {
      "correct": 4,
      "pageReads": 6,
      "replyBytes": 225177,
      "toolCalls": 111,
      "requesters": 0
    }
  },
  "model": {
    "model": "qwen3:8b",
    "digest": "500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41",
    "rows": [
      {
        "queue": "empty",
        "route": "direct pages",
        "turns": 1,
        "calls": [],
        "answer": {
          "error": "The provided tools do not support filtering tickets by priority or status. Only queue lookup is available."
        },
        "correct": false
      },
      {
        "queue": "empty",
        "route": "direct report",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "empty"
            }
          }
        ],
        "answer": [],
        "correct": true
      },
      {
        "queue": "empty",
        "route": "OrgSDK action",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "empty"
            }
          }
        ],
        "answer": [],
        "correct": true
      },
      {
        "queue": "routine",
        "route": "direct pages",
        "turns": 1,
        "calls": [],
        "answer": {
          "name": "list_tickets",
          "arguments": {
            "queue": "routine",
            "offset": 0
          }
        },
        "correct": false
      },
      {
        "queue": "routine",
        "route": "direct report",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "routine"
            }
          }
        ],
        "answer": [],
        "correct": true
      },
      {
        "queue": "routine",
        "route": "OrgSDK action",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "routine"
            }
          }
        ],
        "answer": [],
        "correct": true
      },
      {
        "queue": "mixed",
        "route": "direct pages",
        "turns": 1,
        "calls": [],
        "answer": [],
        "correct": false
      },
      {
        "queue": "mixed",
        "route": "direct report",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "mixed"
            }
          }
        ],
        "answer": [
          "T-21"
        ],
        "correct": true
      },
      {
        "queue": "mixed",
        "route": "OrgSDK action",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "mixed"
            }
          }
        ],
        "answer": [
          "T-21"
        ],
        "correct": true
      },
      {
        "queue": "weekend",
        "route": "direct pages",
        "turns": 1,
        "calls": [],
        "answer": [],
        "correct": false
      },
      {
        "queue": "weekend",
        "route": "direct report",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "weekend"
            }
          }
        ],
        "answer": [
          "T-32",
          "T-35",
          "T-37"
        ],
        "correct": true
      },
      {
        "queue": "weekend",
        "route": "OrgSDK action",
        "turns": 2,
        "calls": [
          {
            "name": "urgent_handoff",
            "args": {
              "queue": "weekend"
            }
          }
        ],
        "answer": [
          "T-32",
          "T-35",
          "T-37"
        ],
        "correct": true
      }
    ],
    "documentedPages": [
      {
        "queue": "empty",
        "route": "documented direct pages",
        "turns": 2,
        "calls": [
          {
            "name": "list_tickets",
            "args": {
              "queue": "empty",
              "offset": 0
            }
          }
        ],
        "answer": [],
        "correct": true
      },
      {
        "queue": "routine",
        "route": "documented direct pages",
        "turns": 2,
        "calls": [
          {
            "name": "list_tickets",
            "args": {
              "queue": "routine",
              "offset": 0
            }
          }
        ],
        "answer": [],
        "correct": true
      },
      {
        "queue": "mixed",
        "route": "documented direct pages",
        "turns": 2,
        "calls": [
          {
            "name": "list_tickets",
            "args": {
              "queue": "mixed",
              "offset": 0
            }
          }
        ],
        "answer": [
          "T-21"
        ],
        "correct": true
      },
      {
        "queue": "weekend",
        "route": "documented direct pages",
        "turns": 3,
        "calls": [
          {
            "name": "list_tickets",
            "args": {
              "queue": "weekend",
              "offset": 0
            }
          },
          {
            "name": "list_tickets",
            "args": {
              "offset": 3,
              "queue": "weekend"
            }
          }
        ],
        "answer": {
          "ids": [
            "T-32",
            "T-35"
          ]
        },
        "correct": false
      }
    ],
    "ollamaVersion": "0.30.11"
  }
}
