gitea/agent-runtime
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
agent-runtime/testcases/results_v2.json
Nico 925fff731f v0.17.0: User expectation tracking, PA retry loop, machine state in PA context 
- Memorizer tracks user_expectation (conversational/delegated/waiting_input/observing)
- Output node adjusts phrasing per expectation
- PA retry loop: reformulates job on expert failure (all retries exhausted or tool skip)
- Machine state in PA context: get_machine_summary includes current state, buttons, stored data
- Expert writes to machine state via update_machine + transition_machine
- Expanded baked schema coverage
- Awareness panel shows color-coded expectation state
- Dashboard and workspace component updates

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-30 19:03:07 +02:00

1033 lines
33 KiB
JSON
Raw Blame History

{
"timestamp": "2026-03-29 12:22:42",
"testcases": {
"Button Persistence": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Create buttons",
"check": "send: create two buttons: Poodle Bark and Bolo",
"status": "PASS",
"detail": "response: "
},
{
"step": "Create buttons",
"check": "actions: length >= 2",
"status": "FAIL",
"detail": "0 actions < 2"
},
{
"step": "Create buttons",
"check": "actions: any action contains \"poodle\" or \"Poodle\"",
"status": "FAIL",
"detail": "none of ['poodle', 'Poodle'] found in 0 buttons"
},
{
"step": "Create buttons",
"check": "actions: any action contains \"bolonka\" or \"Bolonka\"",
"status": "FAIL",
"detail": "none of ['bolonka', 'Bolonka'] found in 0 buttons"
},
{
"step": "Ask unrelated question (buttons must survive)",
"check": "send: what time is it?",
"status": "PASS",
"detail": "response: Alright, I've created two bark buttons for you: Poodle Bark and Bolonka Bark. 🐶 "
},
{
"step": "Ask unrelated question (buttons must survive)",
"check": "response: contains \":\" or \"time\" or \"clock\"",
"status": "PASS",
"detail": "found ':'"
},
{
"step": "Ask unrelated question (buttons must survive)",
"check": "actions: any action contains \"poodle\" or \"Poodle\"",
"status": "PASS",
"detail": "found 'poodle' in actions"
},
{
"step": "Ask unrelated question (buttons must survive)",
"check": "actions: any action contains \"bolonka\" or \"Bolonka\"",
"status": "PASS",
"detail": "found 'bolonka' in actions"
},
{
"step": "Ask another question (buttons still there)",
"check": "send: say hello in German",
"status": "PASS",
"detail": "response: Hallo! Or, if you prefer something more formal, Guten Tag!\n"
},
{
"step": "Ask another question (buttons still there)",
"check": "response: contains \"Hallo\" or \"hallo\" or \"German\"",
"status": "PASS",
"detail": "found 'Hallo'"
},
{
"step": "Ask another question (buttons still there)",
"check": "actions: any action contains \"poodle\" or \"Poodle\"",
"status": "PASS",
"detail": "found 'poodle' in actions"
},
{
"step": "Explicitly replace buttons",
"check": "send: remove all buttons and create one button",
"status": "PASS",
"detail": "response: All buttons have been removed, and a new button called \"Reset\" has been created."
},
{
"step": "Explicitly replace buttons",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "1 actions >= 1"
},
{
"step": "Explicitly replace buttons",
"check": "actions: any action contains \"reset\" or \"Reset\"",
"status": "PASS",
"detail": "found 'reset' in actions"
}
],
"Counter State": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Create counter",
"check": "send: create a counter starting at 0 with incr",
"status": "PASS",
"detail": "response: "
},
{
"step": "Create counter",
"check": "response: contains \"counter\" or \"count\"",
"status": "FAIL",
"detail": "none of ['counter', 'count'] found in: "
},
{
"step": "Create counter",
"check": "actions: length >= 2",
"status": "FAIL",
"detail": "1 actions < 2"
},
{
"step": "Create counter",
"check": "actions: any action contains \"increment\" or \"inc\" or \"plus\" or \"add\"",
"status": "FAIL",
"detail": "none of ['increment', 'inc', 'plus', 'add'] found in 1 buttons"
},
{
"step": "Create counter",
"check": "actions: any action contains \"decrement\" or \"dec\" or \"minus\" or \"sub\"",
"status": "FAIL",
"detail": "none of ['decrement', 'dec', 'minus', 'sub'] found in 1 buttons"
},
{
"step": "Check state",
"check": "state: topic contains \"counter\" or \"count\" or \"button\"",
"status": "PASS",
"detail": "topic=creating reset button contains 'button'"
},
{
"step": "Ask for current value",
"check": "send: what is the current count?",
"status": "PASS",
"detail": "response: "
},
{
"step": "Ask for current value",
"check": "response: contains \"0\" or \"zero\"",
"status": "FAIL",
"detail": "none of ['0', 'zero'] found in: "
},
{
"step": "Increment",
"check": "action matching 'inc'",
"status": "FAIL",
"detail": "no action matching 'inc' in ['reset']"
},
{
"step": "Increment",
"check": "response: contains \"1\" or \"one\" or \"increment\" or \"Navigated\"",
"status": "FAIL",
"detail": "none of ['1', 'one', 'increment', 'Navigated'] found in: "
},
{
"step": "Increment again",
"check": "action matching 'inc'",
"status": "FAIL",
"detail": "no action matching 'inc' in ['reset']"
},
{
"step": "Increment again",
"check": "response: contains \"2\" or \"two\" or \"increment\" or \"Navigated\"",
"status": "FAIL",
"detail": "none of ['2', 'two', 'increment', 'Navigated'] found in: "
},
{
"step": "Decrement",
"check": "action matching 'dec'",
"status": "FAIL",
"detail": "no action matching 'dec' in ['reset']"
},
{
"step": "Decrement",
"check": "response: contains \"1\" or \"one\" or \"decrement\" or \"Navigated\"",
"status": "FAIL",
"detail": "none of ['1', 'one', 'decrement', 'Navigated'] found in: "
},
{
"step": "Verify memorizer tracks it",
"check": "state: topic contains \"count\"",
"status": "FAIL",
"detail": "topic=creating reset button doesn't contain any of ['count']"
}
],
"DB Exploration": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Query renders table in workspace",
"check": "send: show me 5 customers from the database",
"status": "PASS",
"detail": "response: Here are 5 customers from the database. The detailed data is shown in the table."
},
{
"step": "Query renders table in workspace",
"check": "trace: has tool_call",
"status": "PASS",
"detail": "found event 'tool_call'"
},
{
"step": "Query renders table in workspace",
"check": "actions: has table",
"status": "PASS",
"detail": "table found: 23 cols, 5 rows"
},
{
"step": "Query renders table in workspace",
"check": "response: not contains \"---|\" or \"| ID\"",
"status": "PASS",
"detail": "none of ['---|', '| ID'] found (as expected)"
},
{
"step": "Chat summarizes, does not dump data",
"check": "response: contains \"customer\" or \"Kunde\" or \"5\" or \"table\"",
"status": "PASS",
"detail": "found 'customer'"
},
{
"step": "Chat summarizes, does not dump data",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 81 > 10"
},
{
"step": "Thinker builds exploration UI (not describes it)",
"check": "send: select customer 2 Kathrin Jager, add but",
"status": "PASS",
"detail": "response: Okay, I've selected customer Kathrin Jager. You can now explore her objects and "
},
{
"step": "Thinker builds exploration UI (not describes it)",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "5 actions >= 1"
},
{
"step": "Thinker builds exploration UI (not describes it)",
"check": "response: not contains \"UI team\" or \"will add\" or \"will create\"",
"status": "PASS",
"detail": "none of ['UI team', 'will add', 'will create'] found (as expected)"
},
{
"step": "Error recovery on bad query",
"check": "send: SELECT * FROM nichtexistiert LIMIT 5",
"status": "PASS",
"detail": "response: I encountered an error. The table `eras2_production.nichtexistiert` does not exi"
},
{
"step": "Error recovery on bad query",
"check": "trace: has tool_call",
"status": "PASS",
"detail": "found event 'tool_call'"
},
{
"step": "Error recovery on bad query",
"check": "response: not contains \"1146\"",
"status": "PASS",
"detail": "none of ['1146'] found (as expected)"
},
{
"step": "Error recovery on bad query",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 277 > 10"
}
],
"Director Node": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Casual chat establishes mode",
"check": "send: hey, just hanging out, what's up?",
"status": "PASS",
"detail": "response: Alright! Since we're exploring Kathrin Jager's profile, would you like to see he"
},
{
"step": "Casual chat establishes mode",
"check": "response: length > 5",
"status": "PASS",
"detail": "length 107 > 5"
},
{
"step": "Casual chat establishes mode",
"check": "trace: has director_updated",
"status": "FAIL",
"detail": "no 'director_updated' event in trace"
},
{
"step": "Director picks up frustration",
"check": "send: ugh this is so annoying, nothing makes s",
"status": "PASS",
"detail": "response: "
},
{
"step": "Director picks up frustration",
"check": "response: length > 10",
"status": "FAIL",
"detail": "length 0 <= 10"
},
{
"step": "Director picks up frustration",
"check": "trace: has director_updated",
"status": "FAIL",
"detail": "no 'director_updated' event in trace"
},
{
"step": "Switch to building mode",
"check": "send: ok let's build a todo list app",
"status": "PASS",
"detail": "response: Frustration noted! Let's ditch that and build a fun todo app instead. Start addi"
},
{
"step": "Switch to building mode",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 90 > 10"
},
{
"step": "Switch to building mode",
"check": "trace: has director_updated",
"status": "FAIL",
"detail": "no 'director_updated' event in trace"
}
],
"Pub Conversation": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Set the scene",
"check": "send: Hey, Alice and I are heading to the pub ",
"status": "PASS",
"detail": "response: Nice! Have a great time at the pub tonight. Let me know if you need anything whe"
},
{
"step": "Set the scene",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 147 > 10"
},
{
"step": "Set the scene",
"check": "state: situation contains \"pub\" or \"Alice\"",
"status": "PASS",
"detail": "situation=at a pub with Alice later tonight contains 'pub'"
},
{
"step": "Language switch to German",
"check": "send: Wir sind jetzt im Biergarten angekommen",
"status": "PASS",
"detail": "response: Super! Viel Spaß im Biergarten mit Alice! Genießt die Zeit. 🍺\n"
},
{
"step": "Language switch to German",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 62 > 10"
},
{
"step": "Language switch to German",
"check": "state: language is \"de\" or \"mixed\"",
"status": "PASS",
"detail": "language=mixed"
},
{
"step": "Context awareness",
"check": "send: Was sollen wir bestellen?",
"status": "PASS",
"detail": "response: Perfekt fürs Biergarten-Feeling: Probiert ein kühles Weizenbier oder ein Helles "
},
{
"step": "Context awareness",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 168 > 10"
},
{
"step": "Context awareness",
"check": "state: topic contains \"bestell\" or \"order\" or \"pub\" or \"Biergarten\"",
"status": "PASS",
"detail": "topic=ordering drinks at the Biergarten contains 'order'"
},
{
"step": "Alice speaks",
"check": "send: Alice says: I'll have a Hefeweizen pleas",
"status": "PASS",
"detail": "response: Super Wahl! Hefeweizen passt perfekt zu einem Biergarten. Prost! 🍺\n"
},
{
"step": "Alice speaks",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 67 > 10"
},
{
"step": "Alice speaks",
"check": "state: facts any contains \"Alice\" or \"Hefeweizen\"",
"status": "PASS",
"detail": "found 'Alice' in facts"
},
{
"step": "Ask for time (tool use)",
"check": "send: wie spaet ist es eigentlich?",
"status": "PASS",
"detail": "response: Es ist jetzt 12:17 Uhr. Zeit fürs Mittagessen im Biergarten! ☀️\n"
},
{
"step": "Ask for time (tool use)",
"check": "response: matches \\d{1,2}:\\d{2}",
"status": "PASS",
"detail": "matched /\\d{1,2}:\\d{2}/"
},
{
"step": "Back to English",
"check": "send: Let's switch to English, what was the la",
"status": "PASS",
"detail": "response: Switching to English!\n\nAlice said she'd have a Hefeweizen. \n"
},
{
"step": "Back to English",
"check": "state: language is \"en\" or \"mixed\"",
"status": "PASS",
"detail": "language=mixed"
},
{
"step": "Back to English",
"check": "response: contains \"Alice\" or \"Hefeweizen\"",
"status": "PASS",
"detail": "found 'Alice'"
},
{
"step": "Mood check",
"check": "send: This is really fun!",
"status": "PASS",
"detail": "response: Glad you're enjoying it! 🎉\n"
},
{
"step": "Mood check",
"check": "state: user_mood is \"happy\" or \"playful\" or \"excited\"",
"status": "PASS",
"detail": "user_mood=happy"
}
],
"Reflex Path": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Greeting triggers reflex",
"check": "send: hey!",
"status": "PASS",
"detail": "response: Hey! 😊 What's up?\n"
},
{
"step": "Greeting triggers reflex",
"check": "response: length > 2",
"status": "PASS",
"detail": "length 18 > 2"
},
{
"step": "Greeting triggers reflex",
"check": "trace: has reflex_path",
"status": "PASS",
"detail": "found event 'reflex_path'"
},
{
"step": "Thanks triggers reflex",
"check": "send: thanks",
"status": "PASS",
"detail": "response: You're welcome! How can I help you further?\n"
},
{
"step": "Thanks triggers reflex",
"check": "response: length > 2",
"status": "PASS",
"detail": "length 44 > 2"
},
{
"step": "Thanks triggers reflex",
"check": "trace: has reflex_path",
"status": "PASS",
"detail": "found event 'reflex_path'"
},
{
"step": "Complex request does NOT trigger reflex",
"check": "send: explain how neural networks work in deta",
"status": "PASS",
"detail": "response: Alright, let's break down neural networks!\n\n**(1) The Basic Unit: The Neuron**\n\n"
},
{
"step": "Complex request does NOT trigger reflex",
"check": "response: length > 20",
"status": "PASS",
"detail": "length 1624 > 20"
},
{
"step": "Complex request does NOT trigger reflex",
"check": "trace: input.analysis.intent is \"question\" or \"request\"",
"status": "PASS",
"detail": "input.analysis.intent=request"
},
{
"step": "Complex request does NOT trigger reflex",
"check": "trace: has decided",
"status": "PASS",
"detail": "found event 'decided'"
}
],
"S3* Audit Corrections": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Tool calls produce results (baseline)",
"check": "send: create two buttons: Alpha and Beta",
"status": "PASS",
"detail": "response: OK. I've created the Alpha and Beta buttons.\n"
},
{
"step": "Tool calls produce results (baseline)",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "2 actions >= 1"
},
{
"step": "Tool calls produce results (baseline)",
"check": "actions: any action contains \"alpha\" or \"Alpha\"",
"status": "PASS",
"detail": "found 'alpha' in actions"
},
{
"step": "Dashboard mismatch triggers re-emit",
"check": "send: I see nothing on my dashboard, fix it",
"status": "PASS",
"detail": "response: My apologies! I'm re-creating the Alpha and Beta buttons. You should see them no"
},
{
"step": "Dashboard mismatch triggers re-emit",
"check": "response: not contains \"sorry\" or \"apologize\"",
"status": "PASS",
"detail": "none of ['sorry', 'apologize'] found (as expected)"
},
{
"step": "Dashboard mismatch triggers re-emit",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "2 actions >= 1"
},
{
"step": "DB error triggers retry with corrected SQL",
"check": "send: SELECT * FROM NichtExistent LIMIT 5",
"status": "PASS",
"detail": "response: That table doesn't exist.\n\nCould you clarify which table you'd like to query? Pe"
},
{
"step": "DB error triggers retry with corrected SQL",
"check": "trace: has tool_call",
"status": "PASS",
"detail": "found event 'tool_call'"
},
{
"step": "DB error triggers retry with corrected SQL",
"check": "response: not contains \"1146\"",
"status": "PASS",
"detail": "none of ['1146'] found (as expected)"
},
{
"step": "DB error triggers retry with corrected SQL",
"check": "response: length > 10",
"status": "PASS",
"detail": "length 111 > 10"
},
{
"step": "Complex request gets Director plan",
"check": "send: investigate which customers have the mos",
"status": "PASS",
"detail": "response: Sorry, I ran into an error when trying to determine which customers have the mos"
},
{
"step": "Complex request gets Director plan",
"check": "trace: has director_plan",
"status": "FAIL",
"detail": "no 'director_plan' event in trace"
},
{
"step": "Complex request gets Director plan",
"check": "trace: has tool_call",
"status": "PASS",
"detail": "found event 'tool_call'"
},
{
"step": "Complex request gets Director plan",
"check": "response: length > 20",
"status": "PASS",
"detail": "length 209 > 20"
}
],
"State Machines": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Create a machine",
"check": "send: create a navigation machine called \"nav\"",
"status": "PASS",
"detail": "response: Okay, a navigation machine named `nav` has been created. It starts in the \"main\""
},
{
"step": "Create a machine",
"check": "trace: has tool_call create_machine",
"status": "PASS",
"detail": "found create_machine via machine_created event"
},
{
"step": "Create a machine",
"check": "trace: machine_created id=\"nav\"",
"status": "PASS",
"detail": "machine 'nav' created"
},
{
"step": "Verify machine renders",
"check": "send: what machines are on my dashboard?",
"status": "PASS",
"detail": "response: The machine `nav` is currently on the dashboard.\n"
},
{
"step": "Verify machine renders",
"check": "response: contains \"nav\" or \"machine\"",
"status": "PASS",
"detail": "found 'nav'"
},
{
"step": "Navigate via button click (local transition)",
"check": "action matching 'menu_1'",
"status": "FAIL",
"detail": "no action matching 'menu_1' in []"
},
{
"step": "Navigate via button click (local transition)",
"check": "trace: has machine_transition",
"status": "FAIL",
"detail": "no 'machine_transition' event in trace"
},
{
"step": "Navigate via button click (local transition)",
"check": "trace: no thinker",
"status": "PASS",
"detail": "no 'thinker' event (as expected)"
},
{
"step": "Add a state to existing machine",
"check": "send: add a state \"sub3\" to the nav machine wi",
"status": "PASS",
"detail": "response: The \"nav\" machine now has a \"sub3\" state. It includes a Back button and displays"
},
{
"step": "Add a state to existing machine",
"check": "trace: has tool_call add_state",
"status": "FAIL",
"detail": "no tool_call 'add_state' in trace"
},
{
"step": "Reset machine",
"check": "send: reset the nav machine to its initial sta",
"status": "PASS",
"detail": "response: The \"nav\" machine has been reset to its initial state.\n"
},
{
"step": "Reset machine",
"check": "trace: has tool_call reset_machine",
"status": "PASS",
"detail": "found reset_machine via machine_reset event"
},
{
"step": "Reset machine",
"check": "response: contains \"main\" or \"reset\" or \"initial\"",
"status": "PASS",
"detail": "found 'reset'"
},
{
"step": "Create second machine alongside first",
"check": "send: create a counter machine called \"clicks\"",
"status": "PASS",
"detail": "response: A counter machine called \"clicks\" has been created, starting at \"zero\" with a \"C"
},
{
"step": "Create second machine alongside first",
"check": "trace: has tool_call create_machine",
"status": "PASS",
"detail": "found create_machine via machine_created event"
},
{
"step": "Create second machine alongside first",
"check": "trace: machine_created id=\"clicks\"",
"status": "PASS",
"detail": "machine 'clicks' created"
},
{
"step": "Both machines coexist",
"check": "send: what machines are running?",
"status": "PASS",
"detail": "response: The machines 'nav' and 'clicks' are running. nav is in the main state, and click"
},
{
"step": "Both machines coexist",
"check": "response: contains \"nav\"",
"status": "PASS",
"detail": "found 'nav'"
},
{
"step": "Both machines coexist",
"check": "response: contains \"click\"",
"status": "PASS",
"detail": "found 'click'"
},
{
"step": "Destroy one machine",
"check": "send: destroy the clicks machine",
"status": "PASS",
"detail": "response: The `clicks` machine has been destroyed.\n"
},
{
"step": "Destroy one machine",
"check": "trace: has tool_call destroy_machine",
"status": "PASS",
"detail": "found destroy_machine via machine_destroyed event"
},
{
"step": "Destroy one machine",
"check": "send: what machines are running?",
"status": "PASS",
"detail": "response: Only the 'nav' machine is currently running, and it's in the main state.\n"
},
{
"step": "Destroy one machine",
"check": "response: contains \"nav\"",
"status": "PASS",
"detail": "found 'nav'"
}
],
"Structured Input Analysis": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Social greeting",
"check": "send: hi there!",
"status": "PASS",
"detail": "response: Hello! 👋\n"
},
{
"step": "Social greeting",
"check": "response: length > 3",
"status": "PASS",
"detail": "length 9 > 3"
},
{
"step": "Social greeting",
"check": "trace: input.analysis.intent is \"social\"",
"status": "PASS",
"detail": "input.analysis.intent=social"
},
{
"step": "Social greeting",
"check": "trace: input.analysis.complexity is \"trivial\"",
"status": "PASS",
"detail": "input.analysis.complexity=trivial"
},
{
"step": "Simple request",
"check": "send: create a counter starting at 0",
"status": "PASS",
"detail": "response: "
},
{
"step": "Simple request",
"check": "response: length > 10",
"status": "FAIL",
"detail": "length 0 <= 10"
},
{
"step": "Simple request",
"check": "trace: input.analysis.intent is \"request\" or \"action\"",
"status": "PASS",
"detail": "input.analysis.intent=request"
},
{
"step": "Simple request",
"check": "trace: input.analysis.complexity is \"simple\" or \"complex\"",
"status": "PASS",
"detail": "input.analysis.complexity=simple"
},
{
"step": "German question",
"check": "send: Wie spaet ist es?",
"status": "PASS",
"detail": "response: "
},
{
"step": "German question",
"check": "response: length > 5",
"status": "FAIL",
"detail": "length 0 <= 5"
},
{
"step": "German question",
"check": "trace: input.analysis.language is \"de\"",
"status": "PASS",
"detail": "input.analysis.language=de"
},
{
"step": "German question",
"check": "trace: input.analysis.intent is \"question\"",
"status": "PASS",
"detail": "input.analysis.intent=question"
},
{
"step": "Frustrated tone",
"check": "send: this is broken, nothing works and I'm si",
"status": "PASS",
"detail": "response: "
},
{
"step": "Frustrated tone",
"check": "response: length > 10",
"status": "FAIL",
"detail": "length 0 <= 10"
},
{
"step": "Frustrated tone",
"check": "trace: input.analysis.tone is \"frustrated\" or \"urgent\"",
"status": "PASS",
"detail": "input.analysis.tone=frustrated"
},
{
"step": "Simple acknowledgment",
"check": "send: ok thanks bye",
"status": "PASS",
"detail": "response: "
},
{
"step": "Simple acknowledgment",
"check": "trace: input.analysis.intent is \"social\"",
"status": "FAIL",
"detail": "input.analysis.intent=feedback, expected one of ['social']"
},
{
"step": "Simple acknowledgment",
"check": "trace: input.analysis.complexity is \"trivial\"",
"status": "FAIL",
"detail": "input.analysis.complexity=simple, expected one of ['trivial']"
}
],
"Dashboard Feedback (S3*)": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Thinker sees buttons in dashboard",
"check": "send: create two buttons: hello and world",
"status": "PASS",
"detail": "response: Two buttons \"hello\" and \"world\" have been created.\n"
},
{
"step": "Thinker sees buttons in dashboard",
"check": "actions: length >= 2",
"status": "PASS",
"detail": "2 actions >= 2"
},
{
"step": "Thinker sees buttons in dashboard",
"check": "send: what buttons can you see in my dashboard",
"status": "PASS",
"detail": "response: Currently, the dashboard shows two buttons: \"Hello\" and \"World\".\n"
},
{
"step": "Thinker sees buttons in dashboard",
"check": "response: contains \"Hello\" or \"hello\"",
"status": "PASS",
"detail": "found 'Hello'"
},
{
"step": "Thinker sees buttons in dashboard",
"check": "response: contains \"World\" or \"world\"",
"status": "PASS",
"detail": "found 'World'"
},
{
"step": "Thinker detects empty dashboard",
"check": "send: I see nothing in my dashboard, what happ",
"status": "PASS",
"detail": "response: Sorry about that! The buttons didn't render properly the first time. I've just r"
},
{
"step": "Thinker detects empty dashboard",
"check": "response: contains \"button\" or \"fix\" or \"restore\" or \"create\" or \"empty\"",
"status": "PASS",
"detail": "found 'button'"
},
{
"step": "Dashboard state flows to thinker context",
"check": "send: create a counter starting at 5",
"status": "PASS",
"detail": "response: A counter starting at 5 has been created. You can use the + and - buttons to adj"
},
{
"step": "Dashboard state flows to thinker context",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "2 actions >= 1"
},
{
"step": "Dashboard state flows to thinker context",
"check": "send: what does my dashboard show?",
"status": "PASS",
"detail": "response: The dashboard shows a counter at 5, with +1 and -1 buttons to adjust it and a Re"
},
{
"step": "Dashboard state flows to thinker context",
"check": "response: contains \"5\" or \"count\"",
"status": "PASS",
"detail": "found '5'"
}
],
"Dashboard Mismatch Recovery": [
{
"step": "Setup",
"check": "clear",
"status": "PASS",
"detail": "cleared"
},
{
"step": "Create buttons",
"check": "send: create two buttons: red and blue",
"status": "PASS",
"detail": "response: Done! There are now two buttons, one red and one blue. Please click one and tell"
},
{
"step": "Create buttons",
"check": "actions: length >= 2",
"status": "PASS",
"detail": "2 actions >= 2"
},
{
"step": "Dashboard empty — Thinker re-emits",
"check": "send: I clicked red but nothing happened",
"status": "PASS",
"detail": "response: Thanks for letting us know. There was a glitch, but it should be fixed now! Plea"
},
{
"step": "Dashboard empty — Thinker re-emits",
"check": "response: contains \"button\" or \"red\" or \"blue\"",
"status": "PASS",
"detail": "found 'button'"
},
{
"step": "Dashboard empty — Thinker re-emits",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "3 actions >= 1"
},
{
"step": "Create counter",
"check": "send: create a counter starting at 0",
"status": "PASS",
"detail": "response: Okay, a counter has been initialized to 0.\n"
},
{
"step": "Create counter",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "3 actions >= 1"
},
{
"step": "Counter missing from dashboard — Thinker recovers",
"check": "send: the dashboard is broken, I only see old ",
"status": "PASS",
"detail": "response: I'm on it! Force-reloading the interface. Should be fixed in a moment.\n"
},
{
"step": "Counter missing from dashboard — Thinker recovers",
"check": "response: contains \"counter\" or \"count\" or \"fix\" or \"recreat\" or \"refresh\" or \"button\" or \"update\"",
"status": "PASS",
"detail": "found 'fix'"
},
{
"step": "Counter missing from dashboard — Thinker recovers",
"check": "actions: length >= 1",
"status": "PASS",
"detail": "3 actions >= 1"
}
]
},
"summary": {
"passed": 139,
"failed": 28
}
}
Reference in New Issue View Git Blame Copy Permalink