diff --git a/crates/tinymemory-integrations/examples/memory_eval/agent.rs b/crates/tinymemory-integrations/examples/memory_eval/agent.rs index 3fe57d18..61ad7b3f 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/agent.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/agent.rs @@ -15,11 +15,17 @@ //! The extractive answer is a stand-in for a model reading the pack. It //! scores what a model would see, not how well some model reasons. -use std::time::Instant; +use std::collections::HashSet; +use std::time::{Duration as StdDuration, Instant}; use chrono::{DateTime, Duration, Utc}; -use tinymemory_api::ToolCallRef; -use tinymemory_tools::{AgentMemory, BackgroundJob, ContextPack, PostTurn, PreTurn}; +use tinymemory_api::{Error, ToolCallRef}; +use tinymemory_tools::{AgentMemory, BackgroundJob, PostTurn, PreTurn, TurnContext}; + +/// OpenHuman's default deadline for the pack before a model turn. +pub(crate) const PRE_TURN_TIMEOUT: StdDuration = StdDuration::from_millis(1_500); +/// OpenHuman's maximum length of one logged tool result. +const MAX_TOOL_LINE_CHARS: usize = 240; /// A scripted tool call and the result the "tool" returns. #[derive(Debug, Clone, Copy)] @@ -39,6 +45,14 @@ pub(crate) struct TurnRecord { pub(crate) post_ms: f64, /// Whether the user turn was logged. pub(crate) logged: bool, + /// Whether OpenHuman would have continued with an empty pack. + pub(crate) timed_out: bool, + /// Tokens in the pack injected into the simulated model prompt. + pub(crate) pack_tokens: usize, + /// Repeated bullet lines in the injected pack. + pub(crate) duplicate_lines: usize, + /// Bullet lines in the injected pack. + pub(crate) pack_lines: usize, /// The jobs `post_turn` handed back. pub(crate) jobs: Vec, /// How many tool calls the reply made. @@ -53,6 +67,8 @@ pub(crate) struct ScriptedAgent { /// How many of the thread's turns stay in the prompt. window: u32, clock: Option>, + openhuman: bool, + pending: Vec>>, } impl ScriptedAgent { @@ -65,7 +81,28 @@ impl ScriptedAgent { next: 0, window, clock: None, + openhuman: false, + pending: Vec::new(), + } + } + + /// Mirror OpenHuman's dated hook, deadline, and reply logging. + pub(crate) fn openhuman(mut self) -> Self { + self.openhuman = true; + self + } + + /// Wait for pre-turn tasks whose host deadline expired. OpenHuman leaves + /// those tasks running, so their accepted user turns may still land. + pub(crate) async fn flush(&mut self) -> Result { + let mut logged = 0; + for task in self.pending.drain(..) { + let context = task + .await + .map_err(|error| Error::Unavailable(format!("pre-turn task failed: {error}")))??; + logged += usize::from(context.logged.is_some()); } + Ok(logged) } /// Timestamps the thread's turns from `at`, a minute apart. @@ -97,10 +134,46 @@ impl ScriptedAgent { pre.in_prompt_from = self.in_prompt_from(); pre.at = self.tick(); let started = Instant::now(); - let context = self.memory.pre_turn(pre).await?; + let context = if self.openhuman { + let memory = self.memory.clone(); + let mut task = + tokio::spawn( + async move { memory.pre_turn_dated(pre, false, async { None }).await }, + ); + match tokio::time::timeout(PRE_TURN_TIMEOUT, &mut task).await { + Ok(result) => Some(result.map_err(|error| { + Error::Unavailable(format!("pre-turn task failed: {error}")) + })??), + Err(_) => { + self.pending.push(task); + None + } + } + } else { + Some(self.memory.pre_turn(pre).await?) + }; let pre_ms = ms(started); - let reply = reply(&context.pack, text, tools); + let markdown = context + .as_ref() + .map_or("", |value| value.pack.markdown.as_str()); + let mut seen = HashSet::new(); + let lines: Vec<&str> = markdown + .lines() + .filter(|line| line.starts_with("- ")) + .collect(); + let duplicate_lines = lines.iter().filter(|line| !seen.insert(**line)).count(); + let prompt = if self.openhuman && !markdown.is_empty() { + format!("\n{markdown}\n\n\n{text}") + } else { + markdown.to_string() + }; + let answer = reply(&prompt, text, if self.openhuman { &[] } else { tools }); + let reply = if self.openhuman { + logged_reply(&answer, tools) + } else { + answer + }; let mut post = PostTurn::new(&self.thread, self.next + 1, reply); post.tool_calls = tools .iter() @@ -118,7 +191,11 @@ impl ScriptedAgent { Ok(TurnRecord { pre_ms, post_ms, - logged: context.logged.is_some(), + logged: context.as_ref().is_some_and(|value| value.logged.is_some()), + timed_out: context.is_none(), + pack_tokens: context.as_ref().map_or(0, |value| value.pack.tokens), + duplicate_lines, + pack_lines: lines.len(), jobs: report.jobs, tool_calls: tools.len(), }) @@ -127,13 +204,13 @@ impl ScriptedAgent { /// The agent's reply: tool results first, then an answer or an /// acknowledgement. -fn reply(pack: &ContextPack, text: &str, tools: &[ToolStep]) -> String { +fn reply(markdown: &str, text: &str, tools: &[ToolStep]) -> String { let mut lines: Vec = tools .iter() .map(|step| format!("{} returned: {}", step.name, step.result)) .collect(); if text.trim_end().ends_with('?') { - lines.push(match answer(&pack.markdown, text) { + lines.push(match answer(markdown, text) { Some(line) => format!("Going by memory: {line}"), None => NOT_IN_MEMORY.to_string(), }); @@ -143,6 +220,28 @@ fn reply(pack: &ContextPack, text: &str, tools: &[ToolStep]) -> String { lines.join("\n") } +/// The reply OpenHuman stores: the model's text and bounded tool-result lines. +fn logged_reply(text: &str, tools: &[ToolStep]) -> String { + let lines: Vec = tools + .iter() + .filter_map(|step| { + let result = step.result.split_whitespace().collect::>().join(" "); + (!result.is_empty()).then(|| { + format!( + "- {} → {}", + step.name, + result.chars().take(MAX_TOOL_LINE_CHARS).collect::() + ) + }) + }) + .collect(); + if lines.is_empty() { + text.trim().to_string() + } else { + format!("{}\n\nTools:\n{}", text.trim(), lines.join("\n")) + } +} + /// The agent's reply when nothing in the pack answers. const NOT_IN_MEMORY: &str = "I don't have that in memory."; @@ -189,3 +288,7 @@ pub(crate) fn answer(markdown: &str, question: &str) -> Option { pub(crate) fn ms(started: Instant) -> f64 { started.elapsed().as_secs_f64() * 1e3 } + +#[cfg(test)] +#[path = "agent_tests.rs"] +mod tests; diff --git a/crates/tinymemory-integrations/examples/memory_eval/agent_tests.rs b/crates/tinymemory-integrations/examples/memory_eval/agent_tests.rs new file mode 100644 index 00000000..90b09b2e --- /dev/null +++ b/crates/tinymemory-integrations/examples/memory_eval/agent_tests.rs @@ -0,0 +1,87 @@ +//! Regression coverage for the OpenHuman turn mirror. + +use super::*; +use std::sync::Arc; +use tinymemory_api::conformance::ReferenceEngine; +use tinymemory_api::{LearningKind, ListRequest, MemoryEngine, MemoryMeta, StoreItem}; +use tinymemory_tools::MemoryLayout; + +#[test] +fn logged_reply_keeps_bounded_tool_results() { + let long = "a".repeat(MAX_TOOL_LINE_CHARS + 20); + let result = logged_reply( + " Done. ", + &[ + ToolStep { + name: "read", + result: "file contents\n changed", + }, + ToolStep { + name: "empty", + result: " ", + }, + ], + ); + assert_eq!(result, "Done.\n\nTools:\n- read → file contents changed"); + assert_eq!( + logged_reply( + "Done.", + &[ToolStep { + name: "read", + result: Box::leak(long.into_boxed_str()) + }] + ) + .lines() + .last() + .unwrap() + .chars() + .count(), + "- read → ".chars().count() + MAX_TOOL_LINE_CHARS + ); +} + +#[tokio::test] +async fn openhuman_turn_logs_tool_result_without_echoing_the_pack() { + let engine = Arc::new(ReferenceEngine::new()); + let layout = MemoryLayout::default(); + engine + .store(StoreItem::learning( + "The release word is copper-lantern.", + LearningKind::Fact, + 1.0, + MemoryMeta { + namespace: layout.learnings().clone(), + ..MemoryMeta::default() + }, + )) + .await + .unwrap(); + let memory = AgentMemory::new(engine.clone(), layout.clone(), "agent") + .unwrap() + .with_policy(crate::RecallPolicy { + team_limit: 0, + ..crate::RecallPolicy::default() + }); + let mut agent = ScriptedAgent::new(memory, "thread", 8).openhuman(); + let record = agent + .user( + "What is the release word?", + &[ToolStep { + name: "read_file", + result: "retry logic is in src/retry.rs", + }], + ) + .await + .unwrap(); + assert!(!record.timed_out); + assert!(record.logged); + assert!(record.pack_tokens > 0); + + let exported = engine + .export(ListRequest::new(layout.holistic_filter(), 100)) + .await + .unwrap(); + let stored = serde_json::to_string(&exported.items).unwrap(); + assert!(stored.contains("read_file → retry logic is in src/retry.rs")); + assert!(!stored.contains("")); +} diff --git a/crates/tinymemory-integrations/examples/memory_eval/data/loop_guard.json b/crates/tinymemory-integrations/examples/memory_eval/data/loop_guard.json new file mode 100644 index 00000000..96374bb0 --- /dev/null +++ b/crates/tinymemory-integrations/examples/memory_eval/data/loop_guard.json @@ -0,0 +1,5 @@ +{ + "turns": 500, + "user_text": "Continue the coding session.", + "learning": "The project release word is copper-lantern." +} diff --git a/crates/tinymemory-integrations/examples/memory_eval/kpi.rs b/crates/tinymemory-integrations/examples/memory_eval/kpi.rs index 4aaf9244..1ca7b830 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/kpi.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/kpi.rs @@ -455,6 +455,17 @@ pub(crate) fn compute( .collect() }; let pre_turn = Latency::of(&samples(&|step| step.starts_with("pre_turn"))); + let pre_turn_timeouts = reports.iter().map(|r| r.pre_turn_timeouts).sum::() + + reports + .iter() + .flat_map(|r| &r.probes) + .filter(|probe| probe.timed_out) + .count(); + let probe_pre_turns = reports + .iter() + .flat_map(|r| &r.probes) + .filter(|probe| probe.via.starts_with("pre_turn")) + .count(); let probe = Latency::of(&samples(&|step| step.starts_with("probe "))); let total = |step: &str| { timings @@ -477,6 +488,20 @@ pub(crate) fn compute( Unit::Ms, Lower, ), + Kpi::new( + "latency", + "pre_turn p99", + (pre_turn.n > 0).then_some(pre_turn.p99), + Unit::Ms, + Lower, + ), + Kpi::rate( + "latency", + "pre_turn timeout rate", + pre_turn_timeouts, + pre_turn.n + probe_pre_turns, + Lower, + ), Kpi::new( "latency", "probe p50", diff --git a/crates/tinymemory-integrations/examples/memory_eval/kpi_tests.rs b/crates/tinymemory-integrations/examples/memory_eval/kpi_tests.rs index 593c2941..033745d8 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/kpi_tests.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/kpi_tests.rs @@ -27,6 +27,7 @@ fn report(name: &'static str, probes: Vec, conflicts: &[&str]) -> S about: "", writes: 0, tool_calls: 0, + pre_turn_timeouts: 0, settle_ms: 0.0, synthesis: Synthesis { captured: Captured { @@ -173,6 +174,20 @@ fn latency_reads_the_pre_turn_and_probe_samples() { assert_eq!(value(&kpis, "pack hit"), None); } +#[test] +fn timeout_rate_counts_scripted_turns_and_probe_turns() { + let mut timed_out = probed("brain_lookup", "recall", false); + timed_out.timed_out = true; + let on_time = probed("brain_lookup", "synthesis", true); + let mut report = report("brain_lookup", vec![timed_out, on_time], &[]); + report.pre_turn_timeouts = 1; + let mut timings = Timings::default(); + timings.add("pre_turn (log + recall)", 1_500.0); + timings.add("pre_turn (log + recall)", 200.0); + let kpis = compute(&[report], None, &timings); + assert_eq!(value(&kpis, "pre_turn timeout rate"), Some(50.0)); +} + #[test] fn values_format_by_unit() { assert_eq!(format(62.4, Unit::Pct), "62%"); diff --git a/crates/tinymemory-integrations/examples/memory_eval/loop_guard.rs b/crates/tinymemory-integrations/examples/memory_eval/loop_guard.rs new file mode 100644 index 00000000..ec0ab9d2 --- /dev/null +++ b/crates/tinymemory-integrations/examples/memory_eval/loop_guard.rs @@ -0,0 +1,125 @@ +//! A data-driven long thread that checks recall packs never become stored turns. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use tinymemory_api::{ + ForgetTarget, LearningKind, ListRequest, MemoryEngine, MemoryMeta, StoreItem, +}; +use tinymemory_tools::{AgentMemory, RecallPolicy}; + +use crate::agent::ScriptedAgent; +use crate::{Error, layout}; + +#[derive(Deserialize)] +struct Script { + turns: usize, + user_text: String, + learning: String, +} + +/// Metrics for the 500-turn feedback-loop probe. +#[derive(Serialize)] +pub(crate) struct LoopGuardReport { + turns: usize, + timeouts: usize, + nonempty_packs: usize, + first_50_mean_tokens: f64, + last_50_mean_tokens: f64, + first_50_duplicate_rate: f64, + last_50_duplicate_rate: f64, + pub(crate) echoed_items: usize, +} + +fn mean(values: impl Iterator) -> f64 { + let values: Vec = values.collect(); + values.iter().sum::() as f64 / values.len().max(1) as f64 +} + +/// Replay the fixture and inspect every stored item for injected pack tags. +pub(crate) async fn run( + engine: Arc, + run: u64, + policy: &RecallPolicy, +) -> Result { + let script: Script = serde_json::from_str(include_str!("data/loop_guard.json"))?; + let layout = layout(run, "loop_guard", "main")?; + let meta = MemoryMeta { + namespace: layout.learnings().clone(), + ..MemoryMeta::default() + }; + engine + .store(StoreItem::learning( + script.learning.clone(), + LearningKind::Fact, + 1.0, + meta, + )) + .await?; + let memory = + AgentMemory::new(engine.clone(), layout.clone(), "loop-agent")?.with_policy(policy.clone()); + let runner = memory.background(); + let mut agent = ScriptedAgent::new(memory, "loop-thread", 8).openhuman(); + let mut tokens = Vec::with_capacity(script.turns); + let mut duplicates = Vec::with_capacity(script.turns); + let mut timeouts = 0; + let mut nonempty_packs = 0; + for _ in 0..script.turns { + let record = agent.user(&script.user_text, &[]).await?; + timeouts += usize::from(record.timed_out); + nonempty_packs += usize::from(record.pack_tokens > 0); + tokens.push(record.pack_tokens); + duplicates.push((record.duplicate_lines, record.pack_lines)); + for job in record.jobs { + runner.run(job).await?; + } + } + agent.flush().await?; + let mut echoed_items = 0; + let mut cursor = None; + loop { + let mut req = ListRequest::new(layout.holistic_filter(), 100); + req.cursor = cursor; + let page = engine.export(req).await?; + echoed_items += page + .items + .iter() + .filter(|item| { + serde_json::to_string(&item.item) + .is_ok_and(|body| body.contains("")) + }) + .count(); + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + } + if std::env::var("CORTEX_DB_KEEP").is_err() { + engine + .forget(ForgetTarget::Filter(layout.holistic_filter())) + .await?; + } + let span = script.turns.min(50); + let rate = |slice: &[(usize, usize)]| { + let repeated = slice + .iter() + .map(|(duplicates, _)| duplicates) + .sum::(); + let lines = slice.iter().map(|(_, lines)| lines).sum::(); + if lines == 0 { + 0.0 + } else { + 100.0 * repeated as f64 / lines as f64 + } + }; + Ok(LoopGuardReport { + turns: script.turns, + timeouts, + nonempty_packs, + first_50_mean_tokens: mean(tokens.iter().take(span).copied()), + last_50_mean_tokens: mean(tokens.iter().rev().take(span).copied()), + first_50_duplicate_rate: rate(&duplicates[..span]), + last_50_duplicate_rate: rate(&duplicates[script.turns - span..]), + echoed_items, + }) +} diff --git a/crates/tinymemory-integrations/examples/memory_eval/main.rs b/crates/tinymemory-integrations/examples/memory_eval/main.rs index 16948d73..f99a5dfb 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/main.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/main.rs @@ -44,6 +44,10 @@ //! - `--label `: name the run in the report. //! - `--llm`: also have a model answer every probe from its pack (see //! `llm`). +//! - `--host openhuman`: run scripted turns through OpenHuman's dated +//! pre-turn hook with its 1500 ms deadline, policy and logged tool results. +//! - `--loop-guard`: replay a JSON-scripted 500-turn thread and inspect all +//! stored items for an injected `` tag. //! //! `memory_eval compare …` compares the reports of several runs //! instead (see `compare`). A run records the CortexDB flag profile it ran @@ -58,6 +62,7 @@ mod compare; mod inspect; mod kpi; mod llm; +mod loop_guard; mod scenarios; mod score; @@ -81,7 +86,7 @@ use tinymemory_tools::{ MemoryLayout, PreTurn, RecallPolicy, SessionStart, }; -use agent::{ScriptedAgent, ms}; +use agent::{PRE_TURN_TIMEOUT, ScriptedAgent, ms}; use inspect::{Captured, Derived, Inspector, Usage}; use llm::Llm; use scenarios::{MAIN, Probe, Scenario, Step, Via}; @@ -111,6 +116,8 @@ struct Args { json: Option, label: String, llm: bool, + host: String, + loop_guard: bool, } fn args() -> Result { @@ -125,6 +132,8 @@ fn args() -> Result { json: None, label: String::new(), llm: false, + host: "scripted".into(), + loop_guard: false, }; let mut raw = std::env::args().skip(1); while let Some(flag) = raw.next() { @@ -136,9 +145,17 @@ fn args() -> Result { "--json" => parsed.json = Some(value()?), "--label" => parsed.label = value()?, "--llm" => parsed.llm = true, + "--host" => parsed.host = value()?, + "--loop-guard" => parsed.loop_guard = true, other => return Err(format!("unknown flag {other}").into()), } } + if !matches!(parsed.host.as_str(), "scripted" | "openhuman") { + return Err(format!("unknown host {}", parsed.host).into()); + } + if parsed.loop_guard && parsed.host != "openhuman" { + return Err("--loop-guard requires --host openhuman".into()); + } if parsed.label.is_empty() { parsed.label = parsed.engine.clone(); } @@ -176,6 +193,7 @@ struct ScenarioReport { about: &'static str, writes: usize, tool_calls: usize, + pre_turn_timeouts: usize, settle_ms: f64, synthesis: Synthesis, probes: Vec, @@ -263,6 +281,17 @@ async fn main() -> Result<(), Error> { llm.as_ref().map_or("none", |llm| llm.model.as_str()), ); + let policy = if args.host == "openhuman" { + RecallPolicy { + team_limit: 0, + ..RecallPolicy::default() + } + } else { + RecallPolicy { + build_beliefs_every: Some(4), + ..RecallPolicy::default() + } + }; let eval = Eval { engine: engine.clone(), inspector, @@ -270,10 +299,8 @@ async fn main() -> Result<(), Error> { llm, run, enrich_wait, - policy: RecallPolicy { - build_beliefs_every: Some(4), - ..RecallPolicy::default() - }, + policy, + openhuman: args.host == "openhuman", }; let usage_before = match &eval.inspector { Some(inspector) => Some(inspector.usage().await?), @@ -303,6 +330,16 @@ async fn main() -> Result<(), Error> { reports.push(report); } + let loop_guard = if args.loop_guard { + let report = loop_guard::run(engine.clone(), run, &eval.policy).await?; + println!( + "\n## Loop guard\n\n```json\n{}\n```", + serde_json::to_string_pretty(&report)? + ); + Some(report) + } else { + None + }; print_summary(&args.label, &reports, &timings); let usage = match (&eval.inspector, usage_before) { (Some(inspector), Some(before)) => Some(inspector.usage().await?.since(&before)), @@ -341,8 +378,10 @@ async fn main() -> Result<(), Error> { "flags": flags, "server": server, "engine": engine.descriptor().id, + "host": args.host, "run": run, "scenarios": reports, + "loop_guard": loop_guard, "timings": timings, "usage": usage, "kpis": kpis, @@ -350,6 +389,12 @@ async fn main() -> Result<(), Error> { std::fs::write(path, serde_json::to_string_pretty(&out)?)?; println!("\nwrote {path}"); } + if loop_guard + .as_ref() + .is_some_and(|report| report.echoed_items > 0) + { + return Err("loop guard found injected pack text in stored items".into()); + } Ok(()) } @@ -387,6 +432,7 @@ struct Eval { run: u64, enrich_wait: u64, policy: RecallPolicy, + openhuman: bool, } impl Eval { @@ -416,6 +462,7 @@ impl Eval { let mut jobs: Vec = Vec::new(); let mut writes: BTreeMap<&'static str, usize> = BTreeMap::new(); let mut tool_calls = 0; + let mut pre_turn_timeouts = 0; for step in &scenario.steps { match step { Step::Doc { @@ -457,17 +504,24 @@ impl Eval { } => { let mut scripted = ScriptedAgent::new(memory(tenant, agent)?, thread, WINDOW) .at(epoch + chrono::Duration::days(*day)); + if self.openhuman { + scripted = scripted.openhuman(); + } for (text, tools) in turns { let record = scripted.user(text, tools).await?; timings.add("pre_turn (log + recall)", record.pre_ms); timings.add("post_turn (log)", record.post_ms); - if !record.logged { + pre_turn_timeouts += usize::from(record.timed_out); + if record.timed_out { + println!(" ! pre_turn timed out on {thread}"); + } else if !record.logged { println!(" ! a turn of {thread} was not logged"); } tool_calls += record.tool_calls; jobs.extend(record.jobs); - *writes.entry(tenant).or_default() += 2; + *writes.entry(tenant).or_default() += 1 + usize::from(record.logged); } + *writes.entry(tenant).or_default() += scripted.flush().await?; } } } @@ -610,6 +664,7 @@ impl Eval { about: scenario.about, writes: writes.values().sum(), tool_calls, + pre_turn_timeouts, settle_ms, synthesis, probes, @@ -662,15 +717,20 @@ impl Eval { )? .with_policy(policy.clone()); let started = Instant::now(); + let mut timed_out = false; + let mut host_elapsed = None; let (markdown, tokens) = match &probe.via { Via::Ask => { let thread = format!("probe-{}", probe.id); - pack( - memory - .pre_turn(PreTurn::new(thread, 0, probe.question)) - .await? - .pack, - ) + let pre = PreTurn::new(thread, 0, probe.question); + if self.openhuman { + let (pack, timeout, elapsed) = host_probe_pack(memory.clone(), pre).await?; + timed_out = timeout; + host_elapsed = Some(elapsed); + pack + } else { + pack(memory.pre_turn(pre).await?.pack) + } } Via::Resume { thread, focus } => pack( memory @@ -699,7 +759,14 @@ impl Eval { } => { let mut pre = PreTurn::new(*thread, *turn_index, probe.question); pre.in_prompt_from = *in_prompt_from; - pack(memory.pre_turn(pre).await?.pack) + if self.openhuman { + let (pack, timeout, elapsed) = host_probe_pack(memory.clone(), pre).await?; + timed_out = timeout; + host_elapsed = Some(elapsed); + pack + } else { + pack(memory.pre_turn(pre).await?.pack) + } } Via::ContextDoc { heading } => { let root = layout(run, scenario.name, probe.tenant)?.root().clone(); @@ -712,8 +779,9 @@ impl Eval { (doc.markdown, doc.tokens) } }; - let elapsed = ms(started); + let elapsed = host_elapsed.unwrap_or_else(|| ms(started)); let mut result = score(scenario.name, phase, probe, &markdown, tokens, elapsed); + result.timed_out = timed_out; if let Some(llm) = llm { let answer = llm.answer(&markdown, probe.question).await?; result.llm_ok = grade(probe, Some(&answer.text)); @@ -731,6 +799,25 @@ fn pack(pack: ContextPack) -> (String, usize) { (pack.markdown, pack.tokens) } +/// Run a probe's pre-turn under the host deadline. A timed-out task still +/// finishes its write, but its pack is absent from the simulated prompt. +async fn host_probe_pack( + memory: AgentMemory, + pre: PreTurn, +) -> Result<((String, usize), bool, f64), Error> { + let started = Instant::now(); + let mut task = + tokio::spawn(async move { memory.pre_turn_dated(pre, false, async { None }).await }); + match tokio::time::timeout(PRE_TURN_TIMEOUT, &mut task).await { + Ok(context) => Ok((pack(context??.pack), false, ms(started))), + Err(_) => { + let elapsed = ms(started); + task.await??; + Ok(((String::new(), 0), true, elapsed)) + } + } +} + /// One accuracy row. fn row(name: &str, phase: &str, t: &Totals) -> String { format!( @@ -785,13 +872,13 @@ fn print_summary(label: &str, reports: &[ScenarioReport], timings: &Timings) { } println!("\n## Latency (ms)\n"); - println!("| Step | n | p50 | p95 | max |"); - println!("| --- | --- | --- | --- | --- |"); + println!("| Step | n | p50 | p95 | p99 | max |"); + println!("| --- | --- | --- | --- | --- | --- |"); for (step, samples) in &timings.0 { let l = Latency::of(samples); println!( - "| {step} | {} | {:.1} | {:.1} | {:.1} |", - l.n, l.p50, l.p95, l.max + "| {step} | {} | {:.1} | {:.1} | {:.1} | {:.1} |", + l.n, l.p50, l.p95, l.p99, l.max ); } @@ -802,9 +889,10 @@ fn print_summary(label: &str, reports: &[ScenarioReport], timings: &Timings) { || p.llm_ok == Some(false) || p.leak || p.stale_first + || p.timed_out }) { println!( - "- {} {}/{} ({}, {}): hit {:?}, rank {:?}, stale first {}, leak {}; extractive {:?}; model {:?}", + "- {} {}/{} ({}, {}): hit {:?}, rank {:?}, stale first {}, leak {}, timeout {}; extractive {:?}; model {:?}", result.phase, result.scenario, result.id, @@ -814,6 +902,7 @@ fn print_summary(label: &str, reports: &[ScenarioReport], timings: &Timings) { result.rank, result.stale_first, result.leak, + result.timed_out, result .answer .as_deref() diff --git a/crates/tinymemory-integrations/examples/memory_eval/score.rs b/crates/tinymemory-integrations/examples/memory_eval/score.rs index 0073ef5d..c858b6b0 100644 --- a/crates/tinymemory-integrations/examples/memory_eval/score.rs +++ b/crates/tinymemory-integrations/examples/memory_eval/score.rs @@ -29,6 +29,8 @@ pub(crate) struct ProbeResult { pub(crate) id: &'static str, pub(crate) via: &'static str, pub(crate) style: &'static str, + /// Whether the host deadline expired before this probe received a pack. + pub(crate) timed_out: bool, /// `None` when the probe expects nothing (a pure leak check). pub(crate) hit: Option, pub(crate) rank: Option, @@ -122,6 +124,7 @@ pub(crate) fn score( Style::Lexical => "lexical", Style::Paraphrase => "paraphrase", }, + timed_out: false, hit: (!probe.expect.is_empty()).then(|| probe.expect.iter().all(has)), rank: expected.map(|at| at + 1), section: expected.map(|at| units[at].0.clone()), @@ -232,6 +235,7 @@ pub(crate) struct Latency { pub(crate) n: usize, pub(crate) p50: f64, pub(crate) p95: f64, + pub(crate) p99: f64, pub(crate) max: f64, } @@ -248,6 +252,7 @@ impl Latency { n: sorted.len(), p50: at(0.5), p95: at(0.95), + p99: at(0.99), max: sorted[sorted.len() - 1], } } diff --git a/docs/evals/README.md b/docs/evals/README.md index 661ae3fa..fae455c8 100644 --- a/docs/evals/README.md +++ b/docs/evals/README.md @@ -8,6 +8,7 @@ measure how well that behaviour serves an agent. | --- | --- | --- | | [Agent memory](agent-memory.md) | The lifecycle (`pre_turn`, `post_turn`, `start_session`, compaction, belief builds) across twelve scenarios | 2026-10-04, CortexDB v0.10.4 | | [CortexDB flags](cortex-flags.md) | Whether CortexDB's server flags move accuracy, learning, surprise, conflicts, cost and latency | 2026-10-04, CortexDB v0.10.4 | +| [OpenHuman host profile](openhuman-host.md) | OpenHuman turn deadline, logging parity and a 500-turn recall loop guard | 2026-10-10, reference and CortexDB | ## The agent memory eval @@ -24,6 +25,7 @@ The harness is the `memory_eval` example in | `compare.rs` | `memory_eval compare`: the KPIs of several runs against a baseline | | `inspect.rs` | Reads CortexDB's derived layers (facts, beliefs, conflicts) and model usage straight off the wire | | `llm.rs` | The optional model that answers each probe from its pack (`--llm`) | +| `loop_guard.rs` | Replays the JSON-scripted 500-turn pack echo probe (`--loop-guard`) | ### Running it @@ -64,10 +66,13 @@ and the run stops at the settle step with "only 0 of N writes visible". The script prints the report and writes `target/memory-eval/