From 5defb6766df394d67593e9facc1b8fbf285480bd Mon Sep 17 00:00:00 2001 From: acamilo Date: Tue, 22 Sep 2026 11:46:57 +0000 Subject: [PATCH] feat(session): fly-session, the SESSION-01 synthetic sequential transaction over flybus The lockstep coordinator, its phase machine and a synthetic composition, as a new workspace member. Every worker method is a Flybus RPC to an incarnation-pinned service; the domain request ids and result caches of ipc-v1 section 5 sit in front of every mutation. - coordinator: the step-v1 section 3 transaction in order -- prepare every agent concurrently, run one executor per agent in sorted agent-id order, assemble all port controls in descriptor order, send exactly one Environment.Advance, evaluate the task once, commit every agent concurrently, and move the committed boundary only when all of them succeeded. Sequential, concurrent and reversed dispatch are selectable and must agree. - phase: the section 2 machine as an explicit edge table, Paused and Failed included, with a committed-boundary predicate that gates pauses, captures and publication. - clock: checked rational accumulation. A 60 Hz world with a 1 ms model tick runs 16, 17, 17 ticks and comes back to a remainder of exactly zero. - dedup: operation keys, the cached reply with its own artifact holds, CONFLICT, IN_PROGRESS, RESULT_EXPIRED, STALE_STEP, the lifecycle bound and Worker.Acknowledge. - worker: the dispatch shell. Admission order and the result cache live here; the endpoint mutex is the worker's simulation lock, so one mutation runs at a time while Worker.Status answers from a separate cell. - agent, environment, task: a fake model with an explicit seed and a mutation counter the tests read, a fixed readout stub, a counter arena that seals one immutable frame per boundary, a deterministic task and the identity executor. - fly_session_types: the CONTRACT-01 domain types as a local stand-in, reconciled with the shared crate in a following commit. Tests run twice, over the in-memory transport and over a Unix socket, through the same router: the SESSION-01 acceptance bullets and every section 4 failure-injection row that applies to this slice. --- services/flysim/Cargo.lock | 13 + services/flysim/Cargo.toml | 8 +- services/flysim/crates/fly-session/Cargo.toml | 25 + services/flysim/crates/fly-session/README.md | 135 ++ .../crates/fly-session/examples/session.rs | 53 + .../flysim/crates/fly-session/src/agent.rs | 687 ++++++ .../flysim/crates/fly-session/src/clock.rs | 218 ++ .../crates/fly-session/src/coordinator.rs | 2043 +++++++++++++++++ .../flysim/crates/fly-session/src/dedup.rs | 440 ++++ .../crates/fly-session/src/environment.rs | 454 ++++ .../src/fly_session_types/canonical.rs | 139 ++ .../src/fly_session_types/methods.rs | 1026 +++++++++ .../fly-session/src/fly_session_types/mod.rs | 28 + .../src/fly_session_types/scalars.rs | 433 ++++ .../src/fly_session_types/trace.rs | 121 + .../flysim/crates/fly-session/src/harness.rs | 379 +++ services/flysim/crates/fly-session/src/lib.rs | 38 + .../flysim/crates/fly-session/src/phase.rs | 223 ++ services/flysim/crates/fly-session/src/rpc.rs | 139 ++ .../flysim/crates/fly-session/src/task.rs | 385 ++++ .../flysim/crates/fly-session/src/worker.rs | 719 ++++++ .../crates/fly-session/tests/common/mod.rs | 96 + .../crates/fly-session/tests/failures.rs | 377 +++ .../crates/fly-session/tests/session.rs | 401 ++++ 24 files changed, 8579 insertions(+), 1 deletion(-) create mode 100644 services/flysim/crates/fly-session/Cargo.toml create mode 100644 services/flysim/crates/fly-session/README.md create mode 100644 services/flysim/crates/fly-session/examples/session.rs create mode 100644 services/flysim/crates/fly-session/src/agent.rs create mode 100644 services/flysim/crates/fly-session/src/clock.rs create mode 100644 services/flysim/crates/fly-session/src/coordinator.rs create mode 100644 services/flysim/crates/fly-session/src/dedup.rs create mode 100644 services/flysim/crates/fly-session/src/environment.rs create mode 100644 services/flysim/crates/fly-session/src/fly_session_types/canonical.rs create mode 100644 services/flysim/crates/fly-session/src/fly_session_types/methods.rs create mode 100644 services/flysim/crates/fly-session/src/fly_session_types/mod.rs create mode 100644 services/flysim/crates/fly-session/src/fly_session_types/scalars.rs create mode 100644 services/flysim/crates/fly-session/src/fly_session_types/trace.rs create mode 100644 services/flysim/crates/fly-session/src/harness.rs create mode 100644 services/flysim/crates/fly-session/src/lib.rs create mode 100644 services/flysim/crates/fly-session/src/phase.rs create mode 100644 services/flysim/crates/fly-session/src/rpc.rs create mode 100644 services/flysim/crates/fly-session/src/task.rs create mode 100644 services/flysim/crates/fly-session/src/worker.rs create mode 100644 services/flysim/crates/fly-session/tests/common/mod.rs create mode 100644 services/flysim/crates/fly-session/tests/failures.rs create mode 100644 services/flysim/crates/fly-session/tests/session.rs diff --git a/services/flysim/Cargo.lock b/services/flysim/Cargo.lock index b2a05e3..1d0a7cc 100644 --- a/services/flysim/Cargo.lock +++ b/services/flysim/Cargo.lock @@ -416,6 +416,19 @@ dependencies = [ "serde", ] +[[package]] +name = "fly-session" +version = "0.1.1" +dependencies = [ + "flybus", + "ryu-js", + "serde", + "serde_json", + "sha2", + "tempfile", + "tokio", +] + [[package]] name = "flybrain-core" version = "0.1.1" diff --git a/services/flysim/Cargo.toml b/services/flysim/Cargo.toml index 79b775e..92956b3 100644 --- a/services/flysim/Cargo.toml +++ b/services/flysim/Cargo.toml @@ -1,6 +1,12 @@ [workspace] resolver = "3" -members = ["crates/flybrain-core", "crates/flybrain-gb", "crates/flybus", "crates/flysim"] +members = [ + "crates/flybrain-core", + "crates/flybrain-gb", + "crates/flybus", + "crates/fly-session", + "crates/flysim", +] [workspace.package] version = "0.1.1" diff --git a/services/flysim/crates/fly-session/Cargo.toml b/services/flysim/crates/fly-session/Cargo.toml new file mode 100644 index 0000000..00d0f8b --- /dev/null +++ b/services/flysim/crates/fly-session/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "fly-session" +version.workspace = true +edition = "2024" +rust-version.workspace = true +license.workspace = true +publish = false +description = "The lockstep session coordinator, its phase machine and a synthetic composition over flybus." + +[lib] +name = "fly_session" +path = "src/lib.rs" + +[dependencies] +flybus = { path = "../flybus" } + +ryu-js = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +tokio = { version = "1", features = ["rt", "sync", "time", "macros"] } + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } diff --git a/services/flysim/crates/fly-session/README.md b/services/flysim/crates/fly-session/README.md new file mode 100644 index 0000000..5aa4b70 --- /dev/null +++ b/services/flysim/crates/fly-session/README.md @@ -0,0 +1,135 @@ +# fly-session + +The lockstep session coordinator, its phase machine and a synthetic composition over +[`flybus`](../flybus). + +This crate is the SESSION-01 slice of the session-framework implementation guide: the +sequential transaction of `step-v1`, driven over the Flybus router, with small fake workers +standing in for a brain and an emulator. It contains no public controller API, no implicit +best-effort retry, no real emulator and no real brain. + +```text +Ready(k) ─ Prepare all agents concurrently ────────────> every agent Prepared(k) + ─ one executor per agent, sorted agent-id order + ─ one complete port batch, descriptor port order + ─ exactly one Environment.Advance(k, batch) ──> boundary k+1 + ─ task.evaluate_transition, once + ─ Commit all agents concurrently ─────────────> every agent Ready(k+1) + ─ committed boundary k+1, publish, next Prepare allowed +``` + +## Layout + +| Module | Contents | +| --- | --- | +| `fly_session_types` | The CONTRACT-01 domain types, as a local stand-in until that crate exists | +| `clock` | The `step-v1` section 5 rational tick accumulator and the coordinator's pacing | +| `phase` | The `step-v1` section 2 state machine as an explicit edge table | +| `dedup` | The `ipc-v1` section 5 operation keys, result caches and retention | +| `worker` | The worker dispatch shell: one service, the common `Worker.*` methods, admission | +| `agent` | A fake agent worker: seeded model, mutation counter, fixed readout stub | +| `environment` | The counter arena: one complete batch per advance, one native frame | +| `task` | The task and executor traits, the deterministic counter task, the identity executor | +| `rpc` | Domain calls: `req-` serials, incarnation pinning, the retry rule | +| `coordinator` | The transaction, the trace, the failure rules and the publication boundary | +| `harness` | The runnable composition: router, two agents, one arena, one coordinator | + +## What it implements + +- **The transaction, in order.** Prepare all agents concurrently; run each task-local executor + once in sorted agent-id order; assemble all configured port controls in descriptor port + order; send exactly one `Environment.Advance`; evaluate the task once; commit all agents + concurrently. The committed boundary moves only when every commit has succeeded. +- **The state machine**, including `Paused` and `Failed`, with every transition recorded. A + transition the `step-v1` section 2 table does not list returns `INVALID_PHASE`. +- **The committed boundary rule.** Only `Ready(k)` or `Paused(k)` is a committed boundary; a + snapshot publishes one of those and never an in-progress mix of new agent state and an old + world. +- **Time and pacing** with checked rational accumulation. A 60 Hz world with a 1 ms model tick + produces 16, 17, 17 ticks over three steps, totalling 50, with a remainder of exactly zero. + Wall time is only pacing: when behind, the coordinator omits the sleep and reports the lag. +- **Initialization, pause and episodes.** The environment initializes first, while stopped; + the task bootstraps; then the agents warm up with learning disabled. Nothing in bootstrap + advances the world or produces a gameplay reward. A pause arriving mid-step completes the + transition and pauses at its committed boundary. A terminal task event commits its final + rewards, then the session pauses; no worker resets itself. +- **The failure rules.** A partial commit fails the epoch; an uncertain Advance is resolved + against its original domain request id and never becomes a second batch; a worker + incarnation change invalidates the epoch. +- **Domain deduplication over bus calls.** Same key, request and body replays its cached + reply with fresh delivery ownership over retained artifacts; a changed body is `CONFLICT`; a + duplicate of a running operation is `IN_PROGRESS` for that bus call while the original + completes; an evicted record is `RESULT_EXPIRED`; a newly issued request naming an old step + is `STALE_STEP`. `Worker.Acknowledge` releases a domain result cache, which is not a bus + `delivery.consumed`. + +## API + +```rust +let harness = SessionHarness::start(Via::Unix, dir.path(), HarnessConfig::default()).await?; +harness.coordinator.bootstrap().await?; // Ready(0), world stopped at boundary 0 +let reports = harness.coordinator.run(3).await?; // three transitions +harness.coordinator.pause_handle().request(); // finish this transition, then pause +harness.coordinator.trace.behavior(); // the step-v1 section 8 behaviour trace +harness.shutdown().await; +``` + +- `Coordinator::dispatch` selects `Sequential`, `Concurrent` or `Reversed` per-agent dispatch. + All three must produce the same behaviour trace; that is a test. +- `Coordinator::injections` asks for one deliberate message fault at one step: a duplicate + Prepare or Commit, an abandoned Advance result, an altered control batch, or a consumed + result artifact followed by a replay. `injection_log` reports what came back. +- `Coordinator::probe_raw` sends one domain request as it stands and returns the worker's own + terminal outcome, without letting the answer change session state. +- `AgentFaults` and `EnvironmentFaults` ask a worker for a deliberate delay or failure. + +## The synthetic composition + +- **Agents.** A fake model is an LCG with an explicit seed and one counter of everything that + mutated it: ticks, stimulations, reinforcements and input installs. The worker reports that + counter as its `progressCounter`, which is how a test proves a duplicate repeated nothing. + The readout is a fixed stub: it reads bits of the current state, masked by the declared + available actions, and never changes its own weights or invents a default winner. +- **Environment.** A signed counter. `inc` adds one, `dec` subtracts one, and one bipolar + `bias` axis is carried and validated but does not move the world. Each observation seals one + immutable 4x4 RGBA frame whose bytes carry the counter, so an agent reading its sensory view + reads the world rather than a constant. +- **Task.** Rewards are the counter delta of each agent's own port control, with deterministic + event ids derived from epoch, source step, rule and ordinal. +- **Executors.** The stateless identity executor only, as v1 specifies. + +## Limitations + +- **Fake workers.** There is no neural model and no emulator. What is modelled exactly is the + ordering, the identity rules and the retry rules, not any numerical behaviour. +- **No state methods.** `State.Capture`, `State.StageRestore` and `State.ActivateRestore` are + STATE-01. The phase machine has their edges (`Capturing`, `Restoring`) and the workers do not + advertise them as implemented methods. +- **One process.** SESSION-01 runs every participant in one process over the same router. + SESSION-02 is the per-fly process split. +- **No audience input.** The admitted pre-step stimulation list exists and is always empty. +- **Pacing is coarse.** The pacing deadline rounds one step to whole nanoseconds for sleeping + only; simulation time stays rational and that rounding never re-enters the accumulator. + +## Tests + +```text +cargo test -p fly-session # unit + both integration suites +cargo run -p fly-session --example session # the runnable synthetic session +``` + +Every integration test runs twice, once over the in-memory transport and once over a Unix +socket, through the same router code: + +- `tests/session.rs`: one world advance per complete batch; every agent Prepared before the + advance; one task evaluation per transition; every agent committed before the next Prepare or + any committed publication; the 16/17/17 tick profile with a zero remainder; a mid-step pause + completing its transition; bootstrap advancing nothing; the committed snapshot naming the + transition that just ended; a terminal episode pausing at its own boundary; `Worker.Status` + during a session; and sequential, concurrent and reversed dispatch producing one behaviour + trace. +- `tests/failures.rs`: a duplicate Prepare after a lost reply; a duplicate Commit; the same + batch with altered controls; a lost Advance result; a cached artifact consumed by its first + caller; one Commit failing after another succeeded; a replaced registration; a reply from + another incarnation; a world that advanced without sensory data; an exact duplicate of a + running operation; and an old-epoch operation. diff --git a/services/flysim/crates/fly-session/examples/session.rs b/services/flysim/crates/fly-session/examples/session.rs new file mode 100644 index 0000000..9178a00 --- /dev/null +++ b/services/flysim/crates/fly-session/examples/session.rs @@ -0,0 +1,53 @@ +//! The synthetic sequential transaction, run over both transports and printed. +//! +//! ```text +//! cargo run -p fly-session --example session +//! ``` +//! +//! Two fake agents, one counter arena, one coordinator, one router. Nothing here needs a ROM, +//! a dataset, a GPU or a network. + +use fly_session::harness::{HarnessConfig, SessionHarness, Via}; + +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +async fn main() { + for via in [Via::Memory, Via::Unix] { + let dir = tempfile::tempdir().expect("a temporary directory"); + let mut harness = SessionHarness::start(via, dir.path(), HarnessConfig::default()) + .await + .expect("the session starts"); + harness.coordinator.bootstrap().await.expect("bootstrap"); + println!("--- {via:?}: Ready(0) with the world stopped at boundary 0"); + let reports = harness.coordinator.run(3).await.expect("three transitions"); + + for (k, transition) in harness.coordinator.trace.transitions.iter().enumerate() { + let ticks: Vec = transition + .agents + .iter() + .map(|a| format!("{}={} ticks", a.agent_id, a.ticks_advanced)) + .collect(); + println!( + "step {k}: {} batch={} boundary={} events={}", + ticks.join(" "), + transition.batch_id, + transition.acknowledged_boundary, + transition.task_event_ids.len() + ); + } + let progress = harness.coordinator.task_progress(); + println!( + "{via:?}: {} advances, counter {}, reward {}, {} publications, last boundary {}", + harness.coordinator.stats().advances, + progress.integer("counter").unwrap_or_default(), + progress.number("totalReward").unwrap_or_default(), + harness.coordinator.stats().publications, + reports.last().map(|r| r.boundary).unwrap_or_default(), + ); + // The behaviour trace is what two runs in different dispatch orders must agree on. + for line in harness.coordinator.trace.behavior() { + println!(" behaviour: {line}"); + } + harness.shutdown().await; + drop(dir); + } +} diff --git a/services/flysim/crates/fly-session/src/agent.rs b/services/flysim/crates/fly-session/src/agent.rs new file mode 100644 index 0000000..a767079 --- /dev/null +++ b/services/flysim/crates/fly-session/src/agent.rs @@ -0,0 +1,687 @@ +//! A small fake agent worker: an explicit seed, a mutation counter the tests read, and a fixed +//! readout stub. +//! +//! There is no neural model here and no attempt to imitate one. What it does model exactly is +//! the *ordering* `workers-v1` section 2 requires: Prepare applies pre-step stimulation, then +//! advances whole ticks, then decodes; Commit installs the next input, then applies task +//! stimulation, then reinforces once, and executes no tick at all. Every mutating step bumps +//! one counter, which is how a test proves a duplicate request changed nothing. + +use std::collections::BTreeMap; + +use serde_json::Value; + +use crate::clock::TickAccumulator; +use crate::dedup::OpClass; +use crate::task::{context_schema, decision_schema}; +use crate::types::{ + AgentCommitResult, AgentInitializeParams, AgentInitializeResult, AgentTelemetry, AssetRef, + AxisValue, ButtonState, CommitParams, Digest, DomainError, DomainResult, ErrorCode, Id, + LearningTelemetry, MAX_EVENT_ARRAY, Mutation, PrepareParams, PreparedDecision, RateSample, + RationalNs, Role, Scope, SensoryInput, Stimulus, TypedValue, U64, WorkerState, +}; +use crate::worker::{BoxFuture, HandlerCtx, HandlerReply, StatusCell, WorkerEndpoint}; + +/// The fake numerical model: a seeded stream and a count of everything that mutated it. +#[derive(Clone, Debug)] +pub struct FakeModel { + seed: i32, + state: u64, + mutations: u64, + ticks: u64, + stimulations: u64, + reinforcements: u64, + learning_enabled: bool, + learning_updates: u64, + learning_changed: u64, + last_signal: f64, + input_value: i64, + input_installs: u64, +} + +impl FakeModel { + /// A model at its initial state for `seed`. The seed is run configuration and state, and + /// the same seed always produces the same stream. + pub fn new(seed: i32) -> FakeModel { + FakeModel { + seed, + // Sign-extend so a negative seed is a distinct stream rather than a truncation. + state: (seed as i64 as u64) ^ 0x9e37_79b9_7f4a_7c15, + mutations: 0, + ticks: 0, + stimulations: 0, + reinforcements: 0, + learning_enabled: false, + learning_updates: 0, + learning_changed: 0, + last_signal: 0.0, + input_value: 0, + input_installs: 0, + } + } + + pub fn seed(&self) -> i32 { + self.seed + } + + /// Every mutation this model has taken: ticks, stimulations, reinforcements and installs. + /// + /// Tests read this to prove a duplicate request repeated nothing. + pub fn mutations(&self) -> u64 { + self.mutations + } + + pub fn ticks(&self) -> u64 { + self.ticks + } + + pub fn reinforcements(&self) -> u64 { + self.reinforcements + } + + pub fn stimulations(&self) -> u64 { + self.stimulations + } + + /// How many times the next sensory input was installed: once per Initialize and Commit. + pub fn input_installs(&self) -> u64 { + self.input_installs + } + + /// The scalar the encoder last installed. + pub fn input_value(&self) -> i64 { + self.input_value + } + + fn draw(&mut self) -> u64 { + self.state = self + .state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + self.mutations += 1; + self.state + } + + /// Advances `ticks` whole model ticks. This is the only place ticks are executed. + fn advance(&mut self, ticks: u64) { + for _ in 0..ticks { + self.draw(); + self.ticks += 1; + } + } + + /// Applies one declared stimulus. The kind resolves through the profile; a caller never + /// names a neuron or a drive value. + fn stimulate(&mut self, stimulus: &Stimulus) { + let kind = u64::from_le_bytes({ + let d = Digest::of(stimulus.kind_id.as_str().as_bytes()); + let bytes = d.as_str().as_bytes(); + let mut out = [0u8; 8]; + out.copy_from_slice(&bytes[..8]); + out + }); + self.state ^= kind ^ (stimulus.duration_ms.to_bits()); + self.mutations += 1; + self.stimulations += 1; + } + + /// Installs the next encoded sensory input for the following Prepare. + fn install_input(&mut self, value: i64) { + self.input_value = value; + self.state ^= value as u64; + self.mutations += 1; + self.input_installs += 1; + } + + /// Reinforces once at the current brain time. A zero sum still reinforces: the profile's + /// legacy-equivalent behaviour is not optimized away without evidence. + fn reinforce(&mut self, signal: f64) { + self.last_signal = signal; + self.reinforcements += 1; + self.mutations += 1; + if self.learning_enabled { + self.learning_updates += 1; + if signal != 0.0 { + self.learning_changed += 1; + self.state ^= signal.to_bits(); + } + } + } + + /// The fixed readout: a deterministic decode of the current state, masked by the declared + /// available actions. It never invents a default winner or changes its own weights. + fn readout(&self, available: &[String]) -> (bool, bool, f64) { + // Separate bits of one rotated word, because an LCG's low bits are too regular to + // read a decision off directly. + let draw = self.state.rotate_right(29); + let mut inc = draw & 1 == 1; + let mut dec = !inc && (draw >> 7) & 1 == 1; + if !available.iter().any(|a| a == "inc") { + inc = false; + } + if !available.iter().any(|a| a == "dec") { + dec = false; + } + // Exactly representable in f64, so a digest over the decision is stable. + let bias = (((draw >> 13) % 5) as f64 - 2.0) / 4.0; + (inc, dec, bias) + } + + fn telemetry(&self) -> AgentTelemetry { + let draw = self.state >> 29; + AgentTelemetry { + brain_ticks: U64(self.ticks), + population_rate_hz: (draw % 1000) as f64 / 10.0, + rates: vec![ + RateSample { role_id: Id::lit("kc"), hz: (draw % 700) as f64 / 10.0 }, + RateSample { role_id: Id::lit("mbon"), hz: (draw % 310) as f64 / 10.0 }, + ], + learning: LearningTelemetry { + enabled: self.learning_enabled, + updates: U64(self.learning_updates), + changed: U64(self.learning_changed), + signal: self.last_signal, + }, + } + } +} + +/// Deliberate faults a test can ask this worker to produce. +#[derive(Clone, Debug, Default)] +pub struct AgentFaults { + /// Fail `Agent.Commit` at this step, after the next input was installed, so the coordinator + /// meets a partially applied mutation rather than a clean refusal. + pub fail_commit_at_step: Option, + /// Hold `Agent.Prepare` open for this long, to reorder completions. + pub prepare_delay_ms: u64, + /// Hold `Agent.Commit` open for this long. + pub commit_delay_ms: u64, +} + +/// One fake agent worker's configuration. +#[derive(Clone, Debug)] +pub struct AgentConfig { + pub session_id: Id, + pub agent_id: Id, + pub incarnation_id: Id, + pub tick_duration: RationalNs, + pub warmup_ticks: u64, + pub faults: AgentFaults, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum AgentPhase { + Uninitialized, + Ready(u64), + Prepared(u64), + Failed, +} + +/// The agent worker endpoint: model, sensor encoder and readout in one bundle. +pub struct FakeAgentWorker { + config: AgentConfig, + status: StatusCell, + phase: AgentPhase, + epoch: Option, + profile: Option, + accumulator: Option, + model: FakeModel, + context: Option, + context_digest: Option, + prepared: Option<(Id, PreparedDecision)>, +} + +impl FakeAgentWorker { + pub fn new(config: AgentConfig) -> FakeAgentWorker { + FakeAgentWorker { + status: StatusCell::new(), + phase: AgentPhase::Uninitialized, + epoch: None, + profile: None, + accumulator: None, + model: FakeModel::new(0), + context: None, + context_digest: None, + prepared: None, + config, + } + } + + pub fn status(&self) -> StatusCell { + self.status.clone() + } + + fn check_epoch(&self, scope: &Scope) -> DomainResult<()> { + if scope.session_id != self.config.session_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "this worker belongs to another session", + )); + } + match &self.epoch { + Some(epoch) if *epoch == scope.epoch => Ok(()), + Some(_) => Err(DomainError::before( + ErrorCode::StaleEpoch, + "this scope names an epoch this worker has left", + )), + None => Err(DomainError::before( + ErrorCode::InvalidPhase, + "this worker is uninitialized", + )), + } + } + + /// Encodes a sensory input into the one scalar the fake model consumes. + /// + /// Reading the pixels is what proves the attachment was a live owned handle rather than a + /// bare reference; a missing required view is an error, never zero input. + async fn encode(&self, ctx: &HandlerCtx<'_>, input: &SensoryInput) -> DomainResult { + input.validate().map_err(DomainError::invalid)?; + let mut total: i64 = 0; + for view in &input.views { + let name = format!("view.{}", view.view_id); + let artifact = ctx.artifact(&name)?; + if artifact.reference() != &view.pixels.0 { + return Err(DomainError::before( + ErrorCode::BufferInvalid, + format!("attachment {name} is not the artifact the payload names"), + )); + } + let bytes = artifact.read_all().await.map_err(|e| { + DomainError::before( + ErrorCode::BufferInvalid, + format!("view {} could not be read: {}", view.view_id, e.message), + ) + })?; + if bytes.len() as u64 != view.pixels.0.byte_length { + return Err(DomainError::before( + ErrorCode::BufferInvalid, + format!("view {} is the wrong length", view.view_id), + )); + } + total += i64::from(bytes.first().copied().unwrap_or_default()); + } + if let Some(structured) = &input.structured { + total += structured.integer("counter").map_err(DomainError::invalid)?; + } + Ok(total) + } + + fn available_actions(context: &TypedValue) -> DomainResult> { + if context.schema != context_schema() { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "the decision context does not carry the schema this profile allows", + )); + } + let list = context + .value + .get("available") + .and_then(Value::as_array) + .ok_or_else(|| DomainError::invalid("the decision context declares no actions"))?; + list.iter() + .map(|v| { + v.as_str() + .map(str::to_owned) + .ok_or_else(|| DomainError::invalid("an available action is not a string")) + }) + .collect() + } + + fn decision(&self, available: &[String]) -> TypedValue { + let (inc, dec, bias) = self.model.readout(available); + let intent = crate::types::ControllerIntent { + buttons: vec![ + ButtonState { id: Id::lit("inc"), down: inc }, + ButtonState { id: Id::lit("dec"), down: dec }, + ], + axes: vec![AxisValue { id: Id::lit("bias"), value: bias }], + }; + let value = match serde_json::to_value(&intent).expect("an intent serializes") { + Value::Object(m) => m, + _ => unreachable!(), + }; + TypedValue::new(decision_schema(), value) + } + + async fn initialize(&mut self, ctx: &HandlerCtx<'_>) -> DomainResult { + let scope = ctx.scope()?.clone(); + if self.phase != AgentPhase::Uninitialized { + return Err(DomainError::before( + ErrorCode::InvalidPhase, + "Agent.Initialize is only allowed on an uninitialized agent; restore uses the \ + state interface", + )); + } + if scope.step.0 != 0 { + return Err(DomainError::before( + ErrorCode::FutureStep, + "Agent.Initialize uses the new epoch at step 0", + )); + } + if scope.session_id != self.config.session_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "this worker belongs to another session", + )); + } + let params: AgentInitializeParams = ctx.params()?; + if params.agent_id != self.config.agent_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "Agent.Initialize names another agent", + )); + } + if params.worker_threads == 0 { + return Err(DomainError::invalid("workerThreads must be >= 1")); + } + params.initial_decision_context.validate().map_err(DomainError::invalid)?; + let available = FakeAgentWorker::available_actions(¶ms.initial_decision_context)?; + // Everything is validated before the model is constructed. + let encoded = self.encode(ctx, ¶ms.initial_input).await?; + if params.initial_input.boundary.0 != 0 { + return Err(DomainError::invalid("the initial input must observe boundary 0")); + } + + let mut accumulator = + TickAccumulator::new(self.config.tick_duration).map_err(DomainError::invalid)?; + let mut model = FakeModel::new(params.seed); + model.install_input(encoded); + // Warm-up runs with learning disabled and produces no gameplay reward or control. + model.advance(self.config.warmup_ticks); + accumulator.warm_up(self.config.warmup_ticks).map_err(|e| { + DomainError::new(ErrorCode::Internal, e, Mutation::Applied) + })?; + // Calibration happens on settled rates, after warm-up. The readout is a pure read of + // the model, so calibrating it mutates nothing. + let _calibration = model.readout(&available); + model.learning_enabled = true; + + self.model = model; + self.accumulator = Some(accumulator); + self.epoch = Some(scope.epoch.clone()); + self.profile = Some(params.profile.clone()); + self.context_digest = Some(params.initial_decision_context.digest()); + self.context = Some(params.initial_decision_context); + self.phase = AgentPhase::Ready(0); + self.status.set_state(WorkerState::Ready); + self.status.set_scope(Some(scope.clone())); + self.status.advance_to(self.model.mutations()); + + let result = AgentInitializeResult { + agent_id: self.config.agent_id.clone(), + profile_digest: params.profile.digest.clone(), + tick_duration: self.config.tick_duration, + warmup_ticks: U64(self.config.warmup_ticks), + committed_step: U64(0), + decision_context_digest: self.context_digest.clone().expect("just set"), + telemetry: self.model.telemetry(), + }; + Ok(HandlerReply::from(&result)) + } + + async fn prepare(&mut self, ctx: &HandlerCtx<'_>) -> DomainResult { + let scope = ctx.scope()?.clone(); + self.check_epoch(&scope)?; + let AgentPhase::Ready(k) = self.phase.clone() else { + return Err(DomainError::before( + ErrorCode::InvalidPhase, + format!("Agent.Prepare needs Ready(k); this worker is {:?}", self.phase), + )); + }; + if scope.step.0 < k { + return Err(DomainError::before( + ErrorCode::StaleStep, + "Agent.Prepare names a step this worker has left", + )); + } + if scope.step.0 > k { + return Err(DomainError::before( + ErrorCode::FutureStep, + "Agent.Prepare names a step beyond this worker's committed boundary", + )); + } + let params: PrepareParams = ctx.params()?; + if params.agent_id != self.config.agent_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "Agent.Prepare names another agent", + )); + } + let profile = self.profile.as_ref().expect("initialized"); + if params.profile_digest != profile.digest { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "Agent.Prepare names another profile", + )); + } + if Some(¶ms.decision_context_digest) != self.context_digest.as_ref() { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "the cached decision context digest does not match", + )); + } + params.interval.validate().map_err(DomainError::invalid)?; + if params.pre_step_stimulations.len() > MAX_EVENT_ARRAY { + return Err(DomainError::invalid("at most 64 pre-step stimulations")); + } + for stimulus in ¶ms.pre_step_stimulations { + stimulus.validate().map_err(DomainError::invalid)?; + } + let available = + FakeAgentWorker::available_actions(self.context.as_ref().expect("initialized"))?; + + self.status.set_state(WorkerState::Preparing); + if self.config.faults.prepare_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis( + self.config.faults.prepare_delay_ms, + )) + .await; + } + + // 1. admitted pre-step stimulation, in deterministic command sequence order + for stimulus in ¶ms.pre_step_stimulations { + self.model.stimulate(stimulus); + } + // 2. advance the numerical model for the environment interval + let ticks = { + let accumulator = self.accumulator.as_mut().expect("initialized"); + accumulator.advance(¶ms.interval).map_err(|e| { + DomainError::new(ErrorCode::InvalidArgument, e, Mutation::Applied) + })? + }; + self.model.advance(ticks); + // 3. read rates and perform the fixed readout with the declared decision context + let decision = self.decision(&available); + let (brain_ticks, remainder) = { + let accumulator = self.accumulator.as_ref().expect("initialized"); + (accumulator.brain_ticks(), accumulator.remainder()) + }; + let prepared = PreparedDecision { + agent_id: self.config.agent_id.clone(), + ticks_advanced: U64(ticks), + brain_ticks, + remainder, + decision, + }; + self.prepared = Some((ctx.request.request_id.clone(), prepared.clone())); + self.phase = AgentPhase::Prepared(k); + self.status.set_state(WorkerState::Prepared); + self.status.advance_to(self.model.mutations()); + Ok(HandlerReply::from(&prepared)) + } + + async fn commit(&mut self, ctx: &HandlerCtx<'_>) -> DomainResult { + let scope = ctx.scope()?.clone(); + self.check_epoch(&scope)?; + let AgentPhase::Prepared(k) = self.phase.clone() else { + return Err(DomainError::before( + ErrorCode::InvalidPhase, + format!("Agent.Commit needs Prepared(k); this worker is {:?}", self.phase), + )); + }; + if scope.step.0 != k { + return Err(DomainError::before( + if scope.step.0 < k { ErrorCode::StaleStep } else { ErrorCode::FutureStep }, + "Agent.Commit must carry the step of its transition, not the new boundary", + )); + } + let params: CommitParams = ctx.params()?; + if params.agent_id != self.config.agent_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "Agent.Commit names another agent", + )); + } + let (prepared_request, _) = self.prepared.as_ref().expect("prepared"); + if params.prepared_request_id != *prepared_request { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "Agent.Commit does not match this worker's Prepare request", + )); + } + if params.next_input.boundary.0 != k + 1 { + return Err(DomainError::invalid( + "the next sensory input must observe boundary k+1", + )); + } + if params.rewards.len() > MAX_EVENT_ARRAY || params.task_stimulations.len() > MAX_EVENT_ARRAY + { + return Err(DomainError::invalid("at most 64 rewards and 64 stimulations")); + } + for reward in ¶ms.rewards { + reward.validate().map_err(DomainError::invalid)?; + } + for stimulus in ¶ms.task_stimulations { + stimulus.validate().map_err(DomainError::invalid)?; + } + params.next_decision_context.validate().map_err(DomainError::invalid)?; + FakeAgentWorker::available_actions(¶ms.next_decision_context)?; + // The complete request and every required owned artifact are validated before anything + // is applied. + let encoded = self.encode(ctx, ¶ms.next_input).await?; + + self.status.set_state(WorkerState::Committing); + if self.config.faults.commit_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis( + self.config.faults.commit_delay_ms, + )) + .await; + } + + // 1. encode and install the next sensory input for the following Prepare + self.model.install_input(encoded); + if self.config.faults.fail_commit_at_step == Some(k) { + // A deliberate fault after the input was installed: the brain is already mutated, + // so the coordinator has to recover the group rather than retry this agent. + self.phase = AgentPhase::Failed; + self.status.set_state(WorkerState::Failed); + return Err(DomainError::new( + ErrorCode::BackendFailure, + "injected commit failure after the next input was installed", + Mutation::Applied, + )); + } + // 2. apply task-derived stimulation in returned event order + for stimulus in ¶ms.task_stimulations { + self.model.stimulate(stimulus); + } + // 3. sum this agent's rewards in returned event order and reinforce once + let mut signal = 0.0f64; + for reward in ¶ms.rewards { + signal += reward.value; + } + self.model.reinforce(signal); + // 4. retain the next decision context and acknowledge boundary k+1 + self.context_digest = Some(params.next_decision_context.digest()); + self.context = Some(params.next_decision_context); + self.prepared = None; + self.phase = AgentPhase::Ready(k + 1); + self.status.set_state(WorkerState::Ready); + self.status.set_scope(Some(Scope::new(&scope.session_id, &scope.epoch, k + 1))); + self.status.advance_to(self.model.mutations()); + + let result = AgentCommitResult { + agent_id: self.config.agent_id.clone(), + committed_step: U64(k + 1), + decision_context_digest: self.context_digest.clone().expect("just set"), + telemetry: self.model.telemetry(), + }; + Ok(HandlerReply::from(&result)) + } + + /// The model, for a test that wants its mutation counter directly. + pub fn model(&self) -> &FakeModel { + &self.model + } +} + +impl WorkerEndpoint for FakeAgentWorker { + fn worker_id(&self) -> Id { + self.config.agent_id.clone() + } + + fn incarnation_id(&self) -> Id { + self.config.incarnation_id.clone() + } + + fn session_id(&self) -> Id { + self.config.session_id.clone() + } + + fn role(&self) -> Role { + Role::Agent + } + + fn capabilities(&self) -> Vec { + vec![Id::lit("agent-step-v1"), Id::lit("pixel-observation-v1")] + } + + fn status_cell(&self) -> StatusCell { + self.status.clone() + } + + fn methods(&self) -> Vec<&'static str> { + vec!["Agent.Initialize", "Agent.Prepare", "Agent.Commit"] + } + + fn handle<'a>(&'a mut self, ctx: HandlerCtx<'a>) -> BoxFuture<'a, DomainResult> { + Box::pin(async move { + match ctx.method { + "Agent.Initialize" => self.initialize(&ctx).await, + "Agent.Prepare" => self.prepare(&ctx).await, + "Agent.Commit" => self.commit(&ctx).await, + other => Err(DomainError::before( + ErrorCode::Unsupported, + format!("{other} is not an agent method"), + )), + } + }) + } +} + +/// The retention class table an agent endpoint follows, for a caller that wants it. +pub fn agent_op_class(method: &str) -> Option { + match method { + "Agent.Initialize" => Some(OpClass::Lifecycle), + "Agent.Prepare" | "Agent.Commit" => Some(OpClass::StepMutation), + _ => None, + } +} + +/// A synthetic profile asset for one agent. The digest covers its effective identities. +pub fn synthetic_profile(agent_id: &Id, tick_duration: &RationalNs, warmup_ticks: u64) -> AssetRef { + let text = format!( + "arena-direct-v1\nagent={agent_id}\ntick={}/{}\nwarmup={warmup_ticks}\n", + tick_duration.numerator, tick_duration.denominator + ); + AssetRef { + id: Id::lit("arena-direct-v1"), + digest: Digest::of(text.as_bytes()), + byte_length: U64(text.len() as u64), + format: Id::lit("fly-profile-v1"), + } +} + +/// The per-agent contexts a bootstrap produced, keyed by agent id. +pub type Contexts = BTreeMap; diff --git a/services/flysim/crates/fly-session/src/clock.rs b/services/flysim/crates/fly-session/src/clock.rs new file mode 100644 index 0000000..a7a3252 --- /dev/null +++ b/services/flysim/crates/fly-session/src/clock.rs @@ -0,0 +1,218 @@ +//! Time and pacing: `step-v1` section 5. +//! +//! The accumulator is exact rational arithmetic, never rounded nanoseconds. A 60 Hz world with +//! a 1 ms model tick advances 16, 17, 17 ticks over its first three steps and comes back to a +//! remainder of exactly zero; rounding to microseconds does not. + +use crate::types::{DomainError, ErrorCode, RationalNs, U64}; + +/// One agent's tick accumulator: its tick duration, its remainder and its executed count. +#[derive(Clone, Debug)] +pub struct TickAccumulator { + tick_duration: RationalNs, + remainder: RationalNs, + executed_ticks: u64, + warmup_offset: u64, +} + +impl TickAccumulator { + /// A fresh accumulator. The tick duration must be positive. + pub fn new(tick_duration: RationalNs) -> Result { + tick_duration.validate()?; + if !tick_duration.is_positive() { + return Err("tick duration must be positive".to_owned()); + } + Ok(TickAccumulator { + tick_duration, + remainder: RationalNs::zero(), + executed_ticks: 0, + warmup_offset: 0, + }) + } + + pub fn tick_duration(&self) -> RationalNs { + self.tick_duration + } + + /// The persisted remainder: always >= 0 and < one model tick. + pub fn remainder(&self) -> RationalNs { + self.remainder + } + + /// Every tick this accumulator has executed, warm-up included. + pub fn executed_ticks(&self) -> u64 { + self.executed_ticks + } + + /// The warm-up ticks executed before the first gameplay transition. + pub fn warmup_offset(&self) -> u64 { + self.warmup_offset + } + + /// Accounts for `ticks` of warm-up. Warm-up does not consume an environment interval, so + /// it never touches the remainder. + pub fn warm_up(&mut self, ticks: u64) -> Result<(), String> { + self.warmup_offset = self + .warmup_offset + .checked_add(ticks) + .ok_or_else(|| "warm-up tick count overflows".to_owned())?; + self.executed_ticks = self + .executed_ticks + .checked_add(ticks) + .ok_or_else(|| "executed tick count overflows".to_owned())?; + Ok(()) + } + + /// Adds one environment interval and returns the whole ticks it covers. + /// + /// ```text + /// accumulator += environment step duration + /// ticks = floor(accumulator / model tick duration) + /// accumulator -= ticks * model tick duration + /// ``` + pub fn advance(&mut self, interval: &RationalNs) -> Result { + interval.validate()?; + if !interval.is_positive() { + return Err("environment interval must be positive".to_owned()); + } + let accumulated = self.remainder.checked_add(interval)?; + let ticks = accumulated.checked_div_floor(&self.tick_duration)?; + let consumed = self.tick_duration.checked_mul_u64(ticks)?; + let remainder = accumulated.checked_sub(&consumed)?; + debug_assert_eq!( + remainder.cmp_value(&self.tick_duration), + std::cmp::Ordering::Less, + "the remainder must stay below one model tick" + ); + self.remainder = remainder; + self.executed_ticks = self + .executed_ticks + .checked_add(ticks) + .ok_or_else(|| "executed tick count overflows".to_owned())?; + Ok(ticks) + } + + pub fn brain_ticks(&self) -> U64 { + U64(self.executed_ticks) + } +} + +/// The coordinator's pacing authority: absolute deadlines after committed boundaries. +/// +/// Only one pacing authority may be active, so this is the coordinator's and the backend does +/// not throttle as well. When behind, it omits the sleep and reports the lag; it never skips a +/// world step or drops a neural tick. +#[derive(Clone, Debug)] +pub struct Pacing { + step_duration: RationalNs, + next_deadline: Option, + lag: std::time::Duration, + lagged_steps: u64, +} + +impl Pacing { + pub fn new(step_duration: RationalNs) -> Pacing { + Pacing { + step_duration, + next_deadline: None, + lag: std::time::Duration::ZERO, + lagged_steps: 0, + } + } + + /// The wall-clock period of one step, rounded for sleeping only. Simulation time stays + /// rational; this value is never fed back into the accumulator. + fn period(&self) -> std::time::Duration { + let ns = u128::from(self.step_duration.numerator.0) / u128::from(self.step_duration.denominator.0).max(1); + std::time::Duration::from_nanos(u64::try_from(ns).unwrap_or(u64::MAX)) + } + + /// Waits until this step's deadline. Returns the lag if the deadline had already passed. + pub async fn wait(&mut self) -> Option { + let now = std::time::Instant::now(); + let deadline = self.next_deadline.unwrap_or(now); + let outcome = if deadline > now { + tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)).await; + None + } else { + let behind = now.duration_since(deadline); + if !behind.is_zero() { + self.lag += behind; + self.lagged_steps += 1; + } + Some(behind) + }; + self.next_deadline = Some(deadline.max(now) + self.period()); + outcome + } + + pub fn total_lag(&self) -> std::time::Duration { + self.lag + } + + pub fn lagged_steps(&self) -> u64 { + self.lagged_steps + } +} + +/// Converts a whole tick count to the legacy f64 millisecond clock, refusing a run that has +/// left the exactly representable range. +pub fn ticks_to_legacy_millis(ticks: u64, tick_duration: &RationalNs) -> Result { + // 2^53 is the last integer f64 represents exactly; beyond it a millisecond clock starts + // skipping representable ticks, so the run is refused rather than silently rounded. + const EXACT_F64_INTEGERS: u64 = 1 << 53; + if ticks >= EXACT_F64_INTEGERS { + return Err(DomainError::before( + ErrorCode::InvalidArgument, + "tick count exceeds the range the legacy millisecond clock represents exactly", + )); + } + let per_tick_ms = tick_duration.numerator.0 as f64 + / (tick_duration.denominator.0 as f64 * 1_000_000.0); + Ok(ticks as f64 * per_tick_ms) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_60_hz_world_with_a_1_ms_tick_runs_16_17_17() { + let step = RationalNs::from_hz(60).unwrap(); + let mut acc = TickAccumulator::new(RationalNs::from_millis(1).unwrap()).unwrap(); + let ticks: Vec = (0..3).map(|_| acc.advance(&step).unwrap()).collect(); + assert_eq!(ticks, vec![16, 17, 17]); + assert_eq!(ticks.iter().sum::(), 50); + assert!(acc.remainder().is_zero()); + } + + #[test] + fn the_remainder_stays_below_one_tick_and_never_goes_negative() { + let step = RationalNs::from_hz(60).unwrap(); + let tick = RationalNs::from_millis(1).unwrap(); + let mut acc = TickAccumulator::new(tick).unwrap(); + for _ in 0..600 { + acc.advance(&step).unwrap(); + assert_eq!(acc.remainder().cmp_value(&tick), std::cmp::Ordering::Less); + } + // 600 steps of 1/60 s is exactly 10 s, which is 10,000 whole milliseconds. + assert_eq!(acc.executed_ticks(), 10_000); + assert!(acc.remainder().is_zero()); + } + + #[test] + fn warm_up_ticks_do_not_touch_the_remainder() { + let mut acc = TickAccumulator::new(RationalNs::from_millis(1).unwrap()).unwrap(); + acc.warm_up(25).unwrap(); + assert_eq!(acc.executed_ticks(), 25); + assert_eq!(acc.warmup_offset(), 25); + assert!(acc.remainder().is_zero()); + assert_eq!(acc.advance(&RationalNs::from_hz(60).unwrap()).unwrap(), 16); + } + + #[test] + fn a_zero_interval_is_refused_rather_than_silently_producing_no_ticks() { + let mut acc = TickAccumulator::new(RationalNs::from_millis(1).unwrap()).unwrap(); + assert!(acc.advance(&RationalNs::zero()).is_err()); + } +} diff --git a/services/flysim/crates/fly-session/src/coordinator.rs b/services/flysim/crates/fly-session/src/coordinator.rs new file mode 100644 index 0000000..34be799 --- /dev/null +++ b/services/flysim/crates/fly-session/src/coordinator.rs @@ -0,0 +1,2043 @@ +//! The session coordinator: the transaction of `step-v1` section 3, exactly in order. +//! +//! Phase A prepares every agent concurrently. Phase B runs each executor once in sorted +//! agent-id order, assembles one complete port batch in descriptor order, and sends exactly +//! one `Environment.Advance`. Phase C evaluates the task once. Phase D commits every agent +//! concurrently, and only when all of them have succeeded does the committed boundary move, +//! the snapshot publish and the next Prepare become allowed. +//! +//! There is no implicit best-effort retry here. An uncertain call is resolved against the same +//! domain request id, and anything that cannot be resolved fails the epoch. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Map, Value, json}; + +use crate::clock::Pacing; +use crate::phase::{Phase, PhaseMachine}; +use crate::rpc::{self, DomainReply, Serials, WorkerRef}; +use crate::task::{ActionExecutor, Task}; +use crate::types::{ + AgentCommitResult, AgentInitializeParams, AgentInitializeResult, AgentOutcome, AssetRef, + CommitParams, Digest, DomainError, EnvironmentDescriptor, EnvironmentInitializeParams, + EnvironmentInitializeResult, EpisodeRequest, ErrorCode, HelloParams, HelloResult, Id, Mutation, + PortBinding, PortControl, PrepareParams, PreparedDecision, RationalNs, RequestId, Role, Scope, + SensoryInput, StatusResult, StepResult, Stimulus, TypedValue, U64, WorldObservation, + contract_digest, controls_digest, +}; +use crate::types::{AgentTransitionTrace, TraceLog, TransitionTrace, ViewProvenance}; + +/// The order the coordinator dispatches and awaits its per-agent phases in. +/// +/// Completion order never affects port or action order, so all three must produce the same +/// behaviour trace. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum DispatchOrder { + /// Dispatch every agent, then await them all. + #[default] + Concurrent, + /// Dispatch and await one agent at a time, in sorted agent-id order. + Sequential, + /// Dispatch and await one agent at a time, in reverse sorted agent-id order. + Reversed, +} + +/// Deliberate message-level faults, injected at one step. +#[derive(Clone, Debug, Default)] +pub struct Injections { + /// The step these injections apply to. + pub at_step: u64, + /// Re-send this agent's Prepare on a new bus call with the original request id and body. + pub duplicate_prepare: Option, + /// Re-send this agent's Commit the same way. + pub duplicate_commit: Option, + /// Abandon the first Advance result and resolve the same operation afterwards. + pub lose_advance_result: bool, + /// Re-send the Advance with the original request id and an altered control batch. + pub altered_advance_controls: bool, + /// Read and release the Advance result's frame, then replay the same operation. + pub consume_advance_artifact_then_retry: bool, +} + +/// What an injection produced, for a test to assert on. +#[derive(Clone, Debug, PartialEq)] +pub struct InjectionOutcome { + pub what: String, + pub code: Option, + pub identical: bool, +} + +/// A pause request that can be set from outside the transaction. +/// +/// A request arriving mid-step means "finish this transition, then pause"; it is read once +/// the committed boundary has moved, never in the middle of one. +#[derive(Clone, Debug)] +pub struct PauseRequest(std::sync::Arc); + +impl PauseRequest { + pub fn request(&self) { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } + + pub fn is_requested(&self) -> bool { + self.0.load(std::sync::atomic::Ordering::SeqCst) + } +} + +/// Why the epoch failed. +#[derive(Clone, Debug)] +pub struct SessionFailure { + pub error: DomainError, + pub phase: String, + pub detail: String, +} + +impl std::fmt::Display for SessionFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} at {}: {}", self.detail, self.phase, self.error) + } +} + +impl std::error::Error for SessionFailure {} + +type Outcome = Result; + +/// The bus addresses this session publishes on. Chosen by the composition, not the router. +#[derive(Clone, Debug)] +pub struct Topics { + pub descriptor: String, + pub snapshots: String, + pub events: String, +} + +impl Topics { + pub fn for_session(session_id: &Id) -> Topics { + Topics { + descriptor: format!("session.{session_id}.descriptor"), + snapshots: format!("session.{session_id}.snapshots"), + events: format!("session.{session_id}.events"), + } + } +} + +/// One agent's session-side record: its port, its profile, its retained context. +pub struct AgentSlot { + pub worker: WorkerRef, + pub agent_id: Id, + pub port_id: Id, + pub profile: AssetRef, + pub seed: i32, + pub tick_duration: RationalNs, + pub warmup_ticks: U64, + pub committed_step: u64, + context: TypedValue, + context_digest: Digest, + prepared: Option, + prepare_request: Option, +} + +impl AgentSlot { + pub fn new( + worker: WorkerRef, + agent_id: Id, + port_id: Id, + profile: AssetRef, + seed: i32, + ) -> AgentSlot { + AgentSlot { + worker, + agent_id, + port_id, + profile, + seed, + tick_duration: RationalNs::zero(), + warmup_ticks: U64(0), + committed_step: 0, + context: TypedValue::new(crate::task::context_schema(), Map::new()), + context_digest: Digest::of(b""), + prepared: None, + prepare_request: None, + } + } +} + +/// The session's counters, for the acceptance assertions. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Stats { + pub advances: u64, + pub publications: u64, + pub prepares: u64, + pub commits: u64, +} + +pub struct Coordinator { + bus: flybus::Client, + session_id: Id, + epoch: Id, + episode_id: Id, + phases: PhaseMachine, + agents: Vec, + environment: WorkerRef, + descriptor: Option, + task: Box, + executors: BTreeMap>, + observation: Option, + /// Explicit holds on the current boundary's views, forwarded to every Commit and to + /// publication and dropped once the boundary is committed. + views: BTreeMap, + /// Holds on the boundary the world has just reached, before it is committed. + pending_views: BTreeMap, + serials: Serials, + topics: Topics, + pacing: Option, + /// Set by whoever asks for a normal pause, possibly while a transition is in flight. + pause: std::sync::Arc, + episode: Option, + lifecycle_acks: Vec<(WorkerRef, Id)>, + stats: Stats, + /// The ordered actions this session took, for the ordering assertions. + pub audit: Vec, + pub trace: TraceLog, + pub dispatch: DispatchOrder, + pub injections: Injections, + pub injection_log: Vec, + /// How many times an exact duplicate met IN_PROGRESS while resolving an uncertain call. + pub in_progress_replies: u64, +} + +impl Coordinator { + #[allow(clippy::too_many_arguments)] + pub fn new( + bus: flybus::Client, + session_id: Id, + epoch: Id, + episode_id: Id, + environment: WorkerRef, + agents: Vec, + task: Box, + executors: BTreeMap>, + ) -> Coordinator { + let mut agents = agents; + // Sorted agent-id order is the executor and control order, so it is fixed here once. + agents.sort_by(|a, b| a.agent_id.cmp(&b.agent_id)); + let topics = Topics::for_session(&session_id); + Coordinator { + bus, + session_id, + epoch, + episode_id, + phases: PhaseMachine::new(), + agents, + environment, + descriptor: None, + task, + executors, + observation: None, + views: BTreeMap::new(), + pending_views: BTreeMap::new(), + serials: Serials::default(), + topics, + pacing: None, + pause: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + episode: None, + lifecycle_acks: Vec::new(), + stats: Stats::default(), + audit: Vec::new(), + trace: TraceLog::default(), + dispatch: DispatchOrder::default(), + injections: Injections::default(), + injection_log: Vec::new(), + in_progress_replies: 0, + } + } + + pub fn phase(&self) -> Phase { + self.phases.phase() + } + + pub fn stats(&self) -> Stats { + self.stats + } + + pub fn topics(&self) -> &Topics { + &self.topics + } + + pub fn epoch(&self) -> &Id { + &self.epoch + } + + pub fn descriptor(&self) -> Option<&EnvironmentDescriptor> { + self.descriptor.as_ref() + } + + pub fn observation(&self) -> Option<&WorldObservation> { + self.observation.as_ref() + } + + pub fn episode_request(&self) -> Option<&EpisodeRequest> { + self.episode.as_ref() + } + + /// The committed boundary, when the session is at one. + pub fn committed_boundary(&self) -> Option { + self.phases.phase().committed_boundary() + } + + pub fn agent_ids(&self) -> Vec { + self.agents.iter().map(|a| a.agent_id.clone()).collect() + } + + /// How many times the task evaluated a transition. + pub fn evaluations(&self) -> u64 { + self.task.evaluations() + } + + pub fn task_progress(&self) -> TypedValue { + self.task.progress() + } + + /// "Finish this transition, then pause." It never truncates a transition. + pub fn request_pause(&mut self) { + self.pause.store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// A handle a supervisor can use to request a pause while a transition is in flight. + pub fn pause_handle(&self) -> PauseRequest { + PauseRequest(self.pause.clone()) + } + + pub fn pause_requested(&self) -> bool { + self.pause.load(std::sync::atomic::Ordering::SeqCst) + } + + /// Leaves a normal pause at its committed boundary. + pub fn resume(&mut self) -> Outcome<()> { + let Phase::Paused(k) = self.phases.phase() else { + return Err(self.fail_now( + DomainError::before(ErrorCode::InvalidPhase, "the session is not paused"), + "resume", + )); + }; + self.transition(Phase::Ready(k))?; + Ok(()) + } + + fn scope(&self, step: u64) -> Scope { + Scope::new(&self.session_id, &self.epoch, step) + } + + fn transition(&mut self, next: Phase) -> Outcome<()> { + match self.phases.to(next) { + Ok((from, to)) => { + self.trace.phase(from, to); + Ok(()) + } + Err(error) => Err(self.fail_now(error, "phase")), + } + } + + /// Fails the epoch and records the transition to Failed. + fn fail_now(&mut self, error: DomainError, detail: &str) -> SessionFailure { + let phase = self.phases.phase().label(); + let (from, to) = self.phases.fail(); + self.trace.phase(from, to); + self.audit.push(format!("fail:{detail}")); + SessionFailure { error, phase, detail: detail.to_owned() } + } + + fn agent(&self, agent_id: &Id) -> Option<&AgentSlot> { + self.agents.iter().find(|a| a.agent_id == *agent_id) + } + + // --------------------------------------------------------------------------------------- + // Initialization + + /// Negotiates with every worker, initializes the environment and then the agents, and + /// establishes Ready(0). + /// + /// Nothing here advances the environment or produces a gameplay reward. + pub async fn bootstrap(&mut self) -> Outcome<()> { + self.hello_environment().await?; + for index in 0..self.agents.len() { + self.hello_agent(index).await?; + } + self.declare_topics().await?; + self.initialize_environment().await?; + self.bootstrap_task()?; + for index in 0..self.agents.len() { + self.initialize_agent(index).await?; + } + self.acknowledge_lifecycle().await?; + self.transition(Phase::Ready(0))?; + let pacing = self + .descriptor + .as_ref() + .map(|descriptor| Pacing::new(descriptor.step_duration)); + self.pacing = pacing; + self.publish_descriptor().await?; + self.publish_snapshot(0, &BTreeMap::new(), &[], &[]).await?; + Ok(()) + } + + async fn hello(&mut self, worker: WorkerRef, role: Role, required: &str) -> Outcome { + let params = HelloParams { + session_id: self.session_id.clone(), + expected_worker_id: worker.worker_id.clone(), + role, + supported_majors: vec![1], + }; + let reply = self + .call(&worker, "Worker.Hello", None, ¶ms, &[], &[]) + .await?; + let result: HelloResult = reply.parse().map_err(|e| self.fail_now(e, "hello"))?; + if result.contract_digest != contract_digest() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the worker implements another contract revision", + ), + "hello", + )); + } + if result.role != role || result.worker_id != worker.worker_id { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the worker is not the role or the worker the composition expected", + ), + "hello", + )); + } + let required = Id::lit(required); + if !result.capabilities.contains(&required) { + return Err(self.fail_now( + DomainError::before( + ErrorCode::Unsupported, + format!("the worker does not advertise {required}"), + ), + "hello", + )); + } + self.audit.push(format!("hello:{}", worker.worker_id)); + Ok(result.incarnation_id) + } + + async fn hello_environment(&mut self) -> Outcome<()> { + let worker = self.environment.clone(); + let incarnation = self.hello(worker, Role::Environment, "world-step-v1").await?; + self.environment.domain_incarnation = Some(incarnation); + Ok(()) + } + + async fn hello_agent(&mut self, index: usize) -> Outcome<()> { + let worker = self.agents[index].worker.clone(); + let incarnation = self.hello(worker, Role::Agent, "agent-step-v1").await?; + self.agents[index].worker.domain_incarnation = Some(incarnation); + Ok(()) + } + + async fn declare_topics(&mut self) -> Outcome<()> { + for (name, retained) in [ + (self.topics.descriptor.clone(), flybus::Retained::Latest), + (self.topics.snapshots.clone(), flybus::Retained::Latest), + (self.topics.events.clone(), flybus::Retained::None), + ] { + self.bus.declare_topic(&name, retained).await.map_err(|e| { + let error = DomainError::new( + ErrorCode::BackendFailure, + format!("declaring {name}: {}", e.message), + Mutation::None, + ); + self.fail_now(error, "declare-topic") + })?; + } + Ok(()) + } + + async fn initialize_environment(&mut self) -> Outcome<()> { + let bindings: Vec = self + .agents + .iter() + .map(|a| PortBinding { port_id: a.port_id.clone(), agent_id: a.agent_id.clone() }) + .collect(); + let params = EnvironmentInitializeParams { + backend_config: crate::environment::synthetic_asset( + "counter-arena-backend", + "counter-arena-backend-v1", + ), + task_config: crate::environment::synthetic_asset( + "counter-arena-setup", + "counter-arena-setup-v1", + ), + episode_id: self.episode_id.clone(), + port_bindings: bindings, + }; + let worker = self.environment.clone(); + let scope = self.scope(0); + let reply = self + .call( + &worker, + "Environment.Initialize", + Some(scope), + ¶ms, + &[], + &["view.arena".to_owned()], + ) + .await?; + let result: EnvironmentInitializeResult = + reply.parse().map_err(|e| self.fail_now(e, "environment-initialize"))?; + result + .descriptor + .validate() + .map_err(|e| self.fail_now(DomainError::invalid(e), "environment-descriptor"))?; + result + .observation + .validate(&result.descriptor) + .map_err(|e| self.fail_now(DomainError::invalid(e), "observation-0"))?; + if result.observation.boundary.0 != 0 || !result.observation.world_time.is_zero() { + return Err(self.fail_now( + DomainError::invalid("boundary 0 must have world time zero"), + "observation-0", + )); + } + // Every port the descriptor declares must be bound to a configured agent, and every + // agent's port must exist. + let declared: BTreeSet = + result.descriptor.ports.iter().map(|p| p.port_id.clone()).collect(); + let assigned: BTreeSet = self.agents.iter().map(|a| a.port_id.clone()).collect(); + if declared != assigned { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the descriptor's ports and the composition's port assignment disagree", + ), + "port-assignment", + )); + } + self.views = reply.artifacts; + self.descriptor = Some(result.descriptor); + self.observation = Some(result.observation); + self.lifecycle_acks.push((worker, reply.request_id.id())); + self.audit.push("environment.initialize".to_owned()); + Ok(()) + } + + fn bootstrap_task(&mut self) -> Outcome<()> { + let observation = self.observation.clone().expect("initialized"); + let bindings: Vec = self + .agents + .iter() + .map(|a| PortBinding { port_id: a.port_id.clone(), agent_id: a.agent_id.clone() }) + .collect(); + let bootstrap = self + .task + .bootstrap(&observation.inspection, &bindings) + .map_err(|e| self.fail_now(e, "task-bootstrap"))?; + for event in &bootstrap.events { + if event.source_step.0 != 0 { + return Err(self.fail_now( + DomainError::invalid("a bootstrap event has source step 0"), + "task-bootstrap", + )); + } + } + for index in 0..self.agents.len() { + let agent_id = self.agents[index].agent_id.clone(); + let context = bootstrap.contexts.get(&agent_id).cloned().ok_or_else(|| { + DomainError::before( + ErrorCode::IdentityMismatch, + format!("the task bootstrapped no context for {agent_id}"), + ) + }); + let context = match context { + Ok(context) => context, + Err(e) => return Err(self.fail_now(e, "task-bootstrap")), + }; + self.agents[index].context_digest = context.digest(); + self.agents[index].context = context; + } + self.audit.push("task.bootstrap".to_owned()); + Ok(()) + } + + async fn initialize_agent(&mut self, index: usize) -> Outcome<()> { + let observation = self.observation.clone().expect("initialized"); + let slot_worker = self.agents[index].worker.clone(); + let params = AgentInitializeParams { + agent_id: self.agents[index].agent_id.clone(), + profile: self.agents[index].profile.clone(), + seed: self.agents[index].seed, + initial_input: self.sensory_input(&observation, 0), + initial_decision_context: self.agents[index].context.clone(), + worker_threads: 1, + }; + let attachments = self.view_attachments(); + let scope = self.scope(0); + let reply = { + let refs: Vec<(&str, &flybus::Artifact)> = + attachments.iter().map(|(n, a)| (n.as_str(), a)).collect(); + self.call(&slot_worker, "Agent.Initialize", Some(scope), ¶ms, &refs, &[]) + .await? + }; + let result: AgentInitializeResult = + reply.parse().map_err(|e| self.fail_now(e, "agent-initialize"))?; + if result.committed_step.0 != 0 { + return Err(self.fail_now( + DomainError::invalid("Agent.Initialize must establish committed step 0"), + "agent-initialize", + )); + } + if result.profile_digest != self.agents[index].profile.digest { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the agent initialized another profile", + ), + "agent-initialize", + )); + } + if result.decision_context_digest != self.agents[index].context_digest { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the agent retained another decision context", + ), + "agent-initialize", + )); + } + result + .telemetry + .validate() + .map_err(|e| self.fail_now(DomainError::invalid(e), "agent-initialize"))?; + self.agents[index].tick_duration = result.tick_duration; + self.agents[index].warmup_ticks = result.warmup_ticks; + self.agents[index].committed_step = 0; + self.lifecycle_acks.push((slot_worker, reply.request_id.id())); + let agent_id = self.agents[index].agent_id.clone(); + self.audit.push(format!("agent.initialize:{agent_id}")); + Ok(()) + } + + /// Releases every lifecycle reply the workers are holding for us. + /// + /// This is the domain acknowledgment of `ipc-v1` section 5, which drops a worker's result + /// cache. It is not a bus `delivery.consumed`, which the SDK did when we dropped the + /// replies. + async fn acknowledge_lifecycle(&mut self) -> Outcome<()> { + let mut by_worker: BTreeMap)> = BTreeMap::new(); + for (worker, request_id) in std::mem::take(&mut self.lifecycle_acks) { + by_worker + .entry(worker.service.clone()) + .or_insert_with(|| (worker.clone(), Vec::new())) + .1 + .push(request_id); + } + for (worker, ids) in by_worker.into_values() { + let params = crate::types::AcknowledgeParams { request_ids: ids.clone() }; + let reply = self + .call(&worker, "Worker.Acknowledge", None, ¶ms, &[], &[]) + .await?; + let result: crate::types::AcknowledgeResult = + reply.parse().map_err(|e| self.fail_now(e, "acknowledge"))?; + if result.acknowledged.len() != ids.len() { + return Err(self.fail_now( + DomainError::invalid("a worker did not acknowledge every lifecycle reply"), + "acknowledge", + )); + } + } + self.audit.push("acknowledge.lifecycle".to_owned()); + Ok(()) + } + + /// Queries one worker's status without waiting for its current mutation. + pub async fn status(&mut self, worker: &WorkerRef) -> Outcome { + let reply = self + .call(worker, "Worker.Status", None, &json!({}), &[], &[]) + .await?; + reply.parse().map_err(|e| self.fail_now(e, "status")) + } + + /// Asks one worker to stop. Only the configured supervisor may do this. + pub async fn shutdown(&mut self, worker: &WorkerRef, reason: &str) -> Outcome<()> { + let params = crate::types::ShutdownParams { reason: Id::lit(reason) }; + let reply = self + .call(worker, "Worker.Shutdown", None, ¶ms, &[], &[]) + .await?; + let result: crate::types::ShutdownResult = + reply.parse().map_err(|e| self.fail_now(e, "shutdown"))?; + if !result.stopping { + return Err(self.fail_now( + DomainError::invalid("a responsive worker reports stopping:true"), + "shutdown", + )); + } + Ok(()) + } + + pub fn environment_ref(&self) -> &WorkerRef { + &self.environment + } + + pub fn agent_ref(&self, agent_id: &Id) -> Option<&WorkerRef> { + self.agent(agent_id).map(|slot| &slot.worker) + } +} + +// ------------------------------------------------------------------------------------------- +// One transaction + +/// What one completed transition reports back. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StepReport { + /// The new committed boundary. + pub boundary: u64, + /// True when the session is now paused at that boundary. + pub paused: bool, + /// True when the task asked for a terminal episode transition. + pub terminal: bool, +} + +/// One per-agent bus call, ready to dispatch. +struct Job { + agent_id: Id, + worker: WorkerRef, + method: &'static str, + scope: Option, + params: Map, + attachments: Vec<(String, flybus::Artifact)>, + request_id: RequestId, +} + +/// Issues one domain call with owned arguments, so it can run in its own task. +#[allow(clippy::too_many_arguments)] +async fn call_owned( + bus: flybus::Client, + worker: WorkerRef, + method: &'static str, + scope: Option, + params: Map, + attachments: Vec<(String, flybus::Artifact)>, + request_id: RequestId, + want: Vec, +) -> Result { + let refs: Vec<(&str, &flybus::Artifact)> = + attachments.iter().map(|(n, a)| (n.as_str(), a)).collect(); + rpc::call(&bus, &worker, method, scope, params, &refs, request_id, &want).await +} + +impl Coordinator { + /// Serializes `params`, takes the next request serial for this worker, calls, and checks + /// the reply's identity and echoed scope. + async fn call( + &mut self, + worker: &WorkerRef, + method: &'static str, + scope: Option, + params: &P, + attachments: &[(&str, &flybus::Artifact)], + want: &[String], + ) -> Outcome { + let params = match serde_json::to_value(params).expect("params serialize") { + Value::Object(m) => m, + _ => Map::new(), + }; + let request_id = self.serials.next(&worker.service); + let owned: Vec<(String, flybus::Artifact)> = attachments + .iter() + .map(|(n, a)| ((*n).to_owned(), (*a).clone())) + .collect(); + let reply = call_owned( + self.bus.clone(), + worker.clone(), + method, + scope.clone(), + params, + owned, + request_id, + want.to_vec(), + ) + .await; + let reply = match reply { + Ok(reply) => reply, + Err(e) => return Err(self.fail_now(e, method)), + }; + self.check_reply(worker, &reply, &scope, method)?; + match reply.result() { + Ok(_) => Ok(reply), + Err(e) => Err(self.fail_now(e, method)), + } + } + + /// The worker's identity and the echoed scope must both match, whatever the outcome was. + /// + /// A reply from another incarnation means every live participant belongs to an invalid + /// epoch, so it fails the epoch instead of being read as this session's answer. + fn check_reply( + &mut self, + worker: &WorkerRef, + reply: &DomainReply, + scope: &Option, + method: &'static str, + ) -> Outcome<()> { + let (worker_id, incarnation, echoed) = match &reply.outcome { + crate::types::SessionRpcOutcome::Success(s) => { + (&s.worker_id, &s.incarnation_id, &s.scope) + } + crate::types::SessionRpcOutcome::Failure(f) => { + (&f.worker_id, &f.incarnation_id, &f.scope) + } + }; + if *worker_id != worker.worker_id { + return Err(self.fail_now( + DomainError::before(ErrorCode::IdentityMismatch, "another worker answered"), + method, + )); + } + if let Some(expected) = &worker.domain_incarnation + && incarnation != expected + { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the reply comes from another worker incarnation", + ), + method, + )); + } + if echoed != scope { + return Err(self.fail_now( + DomainError::before(ErrorCode::IdentityMismatch, "the reply echoes another scope"), + method, + )); + } + if *reply.outcome.request_id() != reply.request_id.id() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the reply echoes another request id", + ), + method, + )); + } + Ok(()) + } + + /// Resolves an uncertain operation: a fresh bus call with the original domain request id + /// and body, pinned to the same service incarnation. It never issues a new request id and + /// never recomputes a decision. + #[allow(clippy::too_many_arguments)] + async fn resolve( + &mut self, + worker: &WorkerRef, + method: &'static str, + scope: Option, + params: Map, + attachments: Vec<(String, flybus::Artifact)>, + request_id: RequestId, + want: &[String], + ) -> Outcome { + // The original may still be running, which answers IN_PROGRESS for this bus call and + // starts no second mutation. Waiting and asking again is the resolution, not a retry + // of the operation. + for attempt in 0..200u32 { + let reply = call_owned( + self.bus.clone(), + worker.clone(), + method, + scope.clone(), + params.clone(), + attachments.clone(), + request_id, + want.to_vec(), + ) + .await; + let reply = match reply { + Ok(reply) => reply, + Err(e) => return Err(self.fail_now(e, method)), + }; + self.check_reply(worker, &reply, &scope, method)?; + match reply.result() { + Ok(_) => return Ok(reply), + Err(e) if e.code == ErrorCode::InProgress => { + let _ = attempt; + self.in_progress_replies += 1; + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + Err(e) => return Err(self.fail_now(e, method)), + } + } + Err(self.fail_now( + DomainError::new( + ErrorCode::BackendFailure, + "the uncertain operation never resolved", + Mutation::Unknown, + ), + method, + )) + } + + /// Sends the same domain request again on a fresh bus call and reports what came back, + /// without letting the answer change session state. + #[allow(clippy::too_many_arguments)] + async fn probe_duplicate( + &mut self, + worker: &WorkerRef, + method: &'static str, + scope: Option, + params: Map, + attachments: Vec<(String, flybus::Artifact)>, + request_id: RequestId, + expected: Option<&Map>, + what: &str, + ) { + let reply = call_owned( + self.bus.clone(), + worker.clone(), + method, + scope, + params, + attachments, + request_id, + Vec::new(), + ) + .await; + let outcome = match reply { + Ok(reply) => match reply.result() { + Ok(result) => InjectionOutcome { + what: what.to_owned(), + code: None, + identical: expected == Some(result), + }, + Err(e) => InjectionOutcome { + what: what.to_owned(), + code: Some(e.code), + identical: false, + }, + }, + Err(e) => InjectionOutcome { + what: what.to_owned(), + code: Some(e.code), + identical: false, + }, + }; + self.injection_log.push(outcome); + } + + /// Sends one domain request exactly as given and returns the worker's own terminal + /// outcome, without letting the answer change session state. + /// + /// This is how a supervisor or a test observes a worker's refusal -- a stale epoch, a + /// wrong phase -- rather than inferring it from a failed session. + pub async fn probe_raw( + &mut self, + worker: &WorkerRef, + method: &'static str, + scope: Option, + params: Value, + ) -> Result, DomainError> { + let params = match params { + Value::Object(m) => m, + _ => Map::new(), + }; + let request_id = self.serials.next(&worker.service); + let reply = call_owned( + self.bus.clone(), + worker.clone(), + method, + scope, + params, + Vec::new(), + request_id, + Vec::new(), + ) + .await?; + reply.result().cloned() + } + + /// The sensory input one agent is permitted to consume at `boundary`. + fn sensory_input(&self, observation: &WorldObservation, boundary: u64) -> SensoryInput { + SensoryInput { + boundary: U64(boundary), + views: observation.sensory_views.clone(), + // This profile senses pixels only, so structured input stays null rather than + // smuggling inspection data into the neural path. + structured: None, + } + } + + fn view_attachments(&self) -> Vec<(String, flybus::Artifact)> { + self.views.iter().map(|(n, a)| (n.clone(), a.clone())).collect() + } + + /// Runs one set of per-agent jobs in the configured dispatch order. + async fn run_jobs( + &mut self, + jobs: Vec, + order: DispatchOrder, + ) -> Vec<(Id, Result, Option, WorkerRef, &'static str)> { + let mut out = Vec::new(); + match order { + DispatchOrder::Sequential | DispatchOrder::Reversed => { + let mut jobs = jobs; + if order == DispatchOrder::Reversed { + jobs.reverse(); + } + for job in jobs { + let reply = call_owned( + self.bus.clone(), + job.worker.clone(), + job.method, + job.scope.clone(), + job.params, + job.attachments, + job.request_id, + Vec::new(), + ) + .await; + out.push((job.agent_id, reply, job.scope, job.worker, job.method)); + } + } + DispatchOrder::Concurrent => { + let mut tasks = Vec::new(); + for job in jobs { + let bus = self.bus.clone(); + let agent_id = job.agent_id.clone(); + let worker = job.worker.clone(); + let scope = job.scope.clone(); + let method = job.method; + tasks.push(tokio::spawn(async move { + let reply = call_owned( + bus, + job.worker, + job.method, + job.scope, + job.params, + job.attachments, + job.request_id, + Vec::new(), + ) + .await; + (agent_id, reply, scope, worker, method) + })); + } + for task in tasks { + match task.await { + Ok(result) => out.push(result), + Err(e) => panic!("a dispatch task panicked: {e}"), + } + } + } + } + // Completion order never affects anything downstream, so the results are put back in + // sorted agent-id order here and nowhere else. + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } +} + +impl Coordinator { + /// One complete transition `k -> k+1`. + pub async fn step(&mut self) -> Outcome { + let Phase::Ready(k) = self.phases.phase() else { + return Err(self.fail_now( + DomainError::before( + ErrorCode::InvalidPhase, + "a transition starts only from a committed Ready boundary", + ), + "step", + )); + }; + if self.episode.is_some() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::InvalidPhase, + "the task asked for a terminal episode; the episode policy runs first", + ), + "step", + )); + } + let descriptor = self.descriptor.clone().ok_or_else(|| { + DomainError::before(ErrorCode::InvalidPhase, "the session never bootstrapped") + }); + let descriptor = match descriptor { + Ok(descriptor) => descriptor, + Err(e) => return Err(self.fail_now(e, "step")), + }; + let old_observation = self.observation.clone().expect("bootstrapped"); + if let Some(pacing) = self.pacing.as_mut() { + // Wall time is only pacing. Being late omits the sleep and is reported; it never + // skips a world step or a neural tick. + pacing.wait().await; + } + + // ---- Phase A: prepare all agents concurrently + self.transition(Phase::Preparing(k))?; + let prepared = self.prepare_all(k, descriptor.step_duration).await?; + + // ---- Phase B: build and apply one complete batch + self.transition(Phase::Applying(k))?; + let controls = self.build_batch(k, &descriptor, &old_observation)?; + let batch_id = self.batch_id(k); + let step_result = self.advance(k, &batch_id, &controls).await?; + + // ---- Phase C: observe and evaluate the task once + self.transition(Phase::Observing(k + 1))?; + self.verify_step_result(k, &descriptor, &batch_id, &controls, &old_observation, &step_result)?; + let scope = self.scope(k); + let evaluation = self + .task + .evaluate_transition( + &scope, + &old_observation.inspection, + &step_result.observation.inspection, + &controls, + ) + .map_err(|e| { + // Task interpretation failed after prepared brains and the world already + // changed, so the epoch is failed rather than re-evaluated. + self.fail_now(e, "task-evaluate") + })?; + self.audit.push(format!("evaluate:{k}")); + let mut outcomes = evaluation.outcomes.clone(); + let mut next_contexts = evaluation.next_contexts.clone(); + for agent in self.agent_ids() { + if !outcomes.contains_key(&agent) || !next_contexts.contains_key(&agent) { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + format!("the task produced no outcome or context for {agent}"), + ), + "task-evaluate", + )); + } + } + + // ---- Phase D: commit all agent outcomes concurrently + self.transition(Phase::Committing(k))?; + let new_views: BTreeMap = step_result + .observation + .sensory_views + .iter() + .filter_map(|view| { + let name = format!("view.{}", view.view_id); + self.pending_views.get(&name).map(|a| (name, a.clone())) + }) + .collect(); + let commits = self + .commit_all(k, &step_result.observation, &mut outcomes, &mut next_contexts, &new_views) + .await?; + + // Only once every commit succeeded does the committed boundary move. + self.transition(Phase::Ready(k + 1))?; + for index in 0..self.agents.len() { + let agent_id = self.agents[index].agent_id.clone(); + let context = next_contexts.remove(&agent_id).expect("checked above"); + self.agents[index].context_digest = context.digest(); + self.agents[index].context = context; + self.agents[index].committed_step = k + 1; + self.agents[index].prepared = None; + self.agents[index].prepare_request = None; + } + // The previous boundary's handles are no longer needed; the new ones take over. + self.views = new_views; + self.pending_views.clear(); + self.observation = Some(step_result.observation.clone()); + self.stats.advances += 1; + + let event_ids: Vec = evaluation.events.iter().map(|e| e.id.clone()).collect(); + let decisions: BTreeMap = prepared + .iter() + .map(|(agent, decision)| (agent.clone(), decision.decision.clone())) + .collect(); + self.record_trace( + k, + &descriptor, + &prepared, + &commits, + &controls, + &batch_id, + &step_result, + &outcomes, + &event_ids, + ); + self.publish_events(k + 1, &evaluation.events).await?; + self.publish_snapshot(k + 1, &decisions, &controls, &event_ids).await?; + + let terminal = evaluation.episode.is_some(); + if terminal { + // A terminal event's final rewards are committed; then the session pauses at this + // boundary and the declared episode policy runs. No worker resets itself. + self.episode = evaluation.episode.clone(); + } + let paused = self.pause_requested() || terminal; + if paused { + self.transition(Phase::Paused(k + 1))?; + self.pause.store(false, std::sync::atomic::Ordering::SeqCst); + self.audit.push(format!("pause:{}", k + 1)); + } + Ok(StepReport { boundary: k + 1, paused, terminal }) + } + + /// Runs `steps` transitions, stopping early at a pause or a terminal episode. + pub async fn run(&mut self, steps: u64) -> Outcome> { + let mut reports = Vec::new(); + for _ in 0..steps { + let report = self.step().await?; + let stop = report.paused; + reports.push(report); + if stop { + break; + } + } + Ok(reports) + } + + fn batch_id(&self, k: u64) -> Id { + Id::parse(&format!("batch-{}-{k}", self.epoch)).expect("epoch and step make an Id") + } + + /// Phase A. Every agent sees the same environment interval and the same world boundary. + async fn prepare_all( + &mut self, + k: u64, + interval: RationalNs, + ) -> Outcome> { + let scope = self.scope(k); + // The task's per-agent decision contexts and the admitted pre-step stimulation list + // are frozen here; anything accepted later waits for the next boundary. + let mut jobs = Vec::new(); + let mut bodies = Vec::new(); + for index in 0..self.agents.len() { + let slot = &self.agents[index]; + let params = PrepareParams { + agent_id: slot.agent_id.clone(), + profile_digest: slot.profile.digest.clone(), + interval, + decision_context_digest: slot.context_digest.clone(), + // No audience input exists in the first synthetic composition. + pre_step_stimulations: Vec::::new(), + }; + let params = match serde_json::to_value(¶ms).expect("params serialize") { + Value::Object(m) => m, + _ => Map::new(), + }; + let request_id = self.serials.next(&slot.worker.service); + self.agents[index].prepare_request = Some(request_id); + let slot = &self.agents[index]; + bodies.push((slot.agent_id.clone(), slot.worker.clone(), params.clone(), request_id)); + jobs.push(Job { + agent_id: slot.agent_id.clone(), + worker: slot.worker.clone(), + method: "Agent.Prepare", + scope: Some(scope.clone()), + params, + attachments: Vec::new(), + request_id, + }); + } + let results = self.run_jobs(jobs, self.dispatch).await; + let mut prepared = Vec::new(); + for (agent_id, reply, scope, worker, method) in results { + let reply = match reply { + Ok(reply) => reply, + // Some agents are already Prepared. Dispatch stops and the epoch fails; a + // prepared agent is never asked to prepare again. + Err(e) => return Err(self.fail_now(e, method)), + }; + self.check_reply(&worker, &reply, &scope, method)?; + let decision: PreparedDecision = match reply.parse() { + Ok(decision) => decision, + Err(e) => return Err(self.fail_now(e, method)), + }; + if decision.agent_id != agent_id { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "a PreparedDecision names another agent", + ), + method, + )); + } + if decision.decision.schema != crate::task::decision_schema() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the decision does not carry the profile's registered intent schema", + ), + method, + )); + } + self.audit.push(format!("prepared:{agent_id}@{k}")); + self.stats.prepares += 1; + prepared.push((agent_id, decision)); + } + prepared.sort_by(|a, b| a.0.cmp(&b.0)); + for index in 0..self.agents.len() { + let agent_id = self.agents[index].agent_id.clone(); + let decision = prepared + .iter() + .find(|(id, _)| *id == agent_id) + .map(|(_, decision)| decision.clone()); + self.agents[index].prepared = decision; + } + if self.agents.iter().any(|slot| slot.prepared.is_none()) { + return Err(self.fail_now( + DomainError::before( + ErrorCode::InvalidPhase, + "the batch is built only after every PreparedDecision has arrived", + ), + "prepare", + )); + } + + if self.injections.at_step == k + && let Some(target) = self.injections.duplicate_prepare.clone() + { + let expected = prepared + .iter() + .find(|(id, _)| *id == target) + .map(|(_, decision)| match serde_json::to_value(decision).expect("serializes") { + Value::Object(m) => m, + _ => Map::new(), + }); + if let Some((agent_id, worker, params, request_id)) = + bodies.into_iter().find(|(id, _, _, _)| *id == target) + { + let _ = agent_id; + self.probe_duplicate( + &worker, + "Agent.Prepare", + Some(self.scope(k)), + params, + Vec::new(), + request_id, + expected.as_ref(), + "duplicate-prepare", + ) + .await; + } + } + Ok(prepared) + } + + /// Phase B steps 1 to 3: validate, run each executor once, assemble the complete batch. + fn build_batch( + &mut self, + k: u64, + descriptor: &EnvironmentDescriptor, + observation: &WorldObservation, + ) -> Outcome> { + let scope = self.scope(k); + let progress = self.task.progress(); + let clock = observation.world_time; + let mut intents: BTreeMap = BTreeMap::new(); + // Sorted agent-id order, never completion order. + for index in 0..self.agents.len() { + let agent_id = self.agents[index].agent_id.clone(); + let port_id = self.agents[index].port_id.clone(); + let decision = self.agents[index] + .prepared + .clone() + .expect("every agent is Prepared before the batch is built"); + let executor = self.executors.get_mut(&agent_id).ok_or_else(|| { + DomainError::before( + ErrorCode::IdentityMismatch, + format!("{agent_id} has no configured action executor"), + ) + }); + let executor = match executor { + Ok(executor) => executor, + Err(e) => return Err(self.fail_now(e, "executor")), + }; + let applied = executor.apply( + &scope, + &decision.decision, + &observation.inspection, + &progress, + &clock, + ); + let (intent, _events) = match applied { + Ok(applied) => applied, + Err(e) => return Err(self.fail_now(e, "executor")), + }; + // Only the coordinator assigns a port. + if intents.insert(port_id.clone(), intent.at_port(&port_id)).is_some() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + format!("port {port_id} is claimed by two agents"), + ), + "executor", + )); + } + } + // All configured port controls, in descriptor port order; duplicates and omissions are + // refused rather than filled in. + let mut controls = Vec::with_capacity(descriptor.ports.len()); + for port in &descriptor.ports { + let control = intents.remove(&port.port_id).ok_or_else(|| { + DomainError::before( + ErrorCode::IdentityMismatch, + format!("port {} has no control in this batch", port.port_id), + ) + }); + let control = match control { + Ok(control) => control, + Err(e) => return Err(self.fail_now(e, "batch")), + }; + if let Err(e) = port.controls.check(&control) { + return Err(self.fail_now(DomainError::invalid(e), "batch")); + } + controls.push(control); + } + if !intents.is_empty() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the batch names a port the descriptor does not declare", + ), + "batch", + )); + } + Ok(controls) + } + + /// Phase B step 4: exactly one `Environment.Advance` per complete batch. + async fn advance( + &mut self, + k: u64, + batch_id: &Id, + controls: &[PortControl], + ) -> Outcome { + let scope = self.scope(k); + let params = crate::types::AdvanceParams { + batch_id: batch_id.clone(), + controls: controls.to_vec(), + }; + let params = match serde_json::to_value(¶ms).expect("params serialize") { + Value::Object(m) => m, + _ => Map::new(), + }; + let worker = self.environment.clone(); + let request_id = self.serials.next(&worker.service); + let want = vec!["view.arena".to_owned()]; + self.audit.push(format!("advance:{k}")); + + let injected = self.injections.at_step == k; + let reply = if injected && self.injections.lose_advance_result { + // The result is lost after the world already stepped: the call is abandoned + // mid-flight, so its execution outcome is unknown. Further dispatch stops and the + // same operation is resolved; a new batch is never sent. + let pending = self + .bus + .call( + &worker.service, + Some(&worker.bus_incarnation), + "Environment.Advance", + crate::types::SessionRpcRequest::new( + &request_id, + Some(scope.clone()), + params.clone(), + ) + .to_payload(), + &[], + ) + .await + .map_err(|e| { + let error = DomainError::new( + ErrorCode::BackendFailure, + format!("Environment.Advance: bus {:?}", e.code), + Mutation::Unknown, + ); + self.fail_now(error, "advance") + })?; + let state = pending.cancel().await.ok(); + drop(pending); + self.injection_log.push(InjectionOutcome { + what: "lost-advance-result".to_owned(), + code: None, + identical: state == Some(flybus::CancelState::ExecutionUnknown) + || state == Some(flybus::CancelState::Completed), + }); + // A status probe first, exactly as the uncertain-call procedure says. + let status = self.status(&worker).await?; + self.injection_log.push(InjectionOutcome { + what: "status-after-loss".to_owned(), + code: None, + identical: status.last_batch_id.as_ref() == Some(batch_id) + || status.active_request_id.as_ref() == Some(&request_id.id()), + }); + self.resolve( + &worker, + "Environment.Advance", + Some(scope.clone()), + params.clone(), + Vec::new(), + request_id, + &want, + ) + .await? + } else { + let reply = call_owned( + self.bus.clone(), + worker.clone(), + "Environment.Advance", + Some(scope.clone()), + params.clone(), + Vec::new(), + request_id, + want.clone(), + ) + .await; + let reply = match reply { + Ok(reply) => reply, + Err(e) => return Err(self.fail_now(e, "advance")), + }; + self.check_reply(&worker, &reply, &Some(scope.clone()), "advance")?; + match reply.result() { + Ok(_) => reply, + Err(e) => return Err(self.fail_now(e, "advance")), + } + }; + + let result: StepResult = match reply.parse() { + Ok(result) => result, + Err(e) => return Err(self.fail_now(e, "advance")), + }; + self.pending_views = reply.artifacts; + + if injected && self.injections.altered_advance_controls { + // The same request id with a different body: a conflict, never a second world + // mutation. + let mut altered = controls.to_vec(); + if let Some(control) = altered.first_mut() + && let Some(button) = control.buttons.first_mut() + { + button.down = !button.down; + } + let altered_params = match serde_json::to_value(&crate::types::AdvanceParams { + batch_id: batch_id.clone(), + controls: altered, + }) + .expect("params serialize") + { + Value::Object(m) => m, + _ => Map::new(), + }; + self.probe_duplicate( + &worker, + "Environment.Advance", + Some(scope.clone()), + altered_params, + Vec::new(), + request_id, + None, + "altered-advance-controls", + ) + .await; + } + + if injected && self.injections.consume_advance_artifact_then_retry { + // The first caller consumes the result's frame, then replays the operation. The + // endpoint's cache owns its own hold, so the replay still has valid bytes. + let first = match self.pending_views.get("view.arena") { + Some(artifact) => artifact.read_all().await.ok(), + None => None, + }; + self.pending_views.clear(); + let replay = self + .resolve( + &worker, + "Environment.Advance", + Some(scope.clone()), + params.clone(), + Vec::new(), + request_id, + &want, + ) + .await?; + let again = match replay.artifacts.get("view.arena") { + Some(artifact) => artifact.read_all().await.ok(), + None => None, + }; + self.injection_log.push(InjectionOutcome { + what: "cached-artifact-after-consumption".to_owned(), + code: None, + identical: first.is_some() && first == again, + }); + self.pending_views = replay.artifacts; + } + Ok(result) + } +} + +impl Coordinator { + /// Phase C's verification: batch identity, boundary, cadence, schema and required views. + #[allow(clippy::too_many_arguments)] + fn verify_step_result( + &mut self, + k: u64, + descriptor: &EnvironmentDescriptor, + batch_id: &Id, + controls: &[PortControl], + old: &WorldObservation, + result: &StepResult, + ) -> Outcome<()> { + if result.batch_id != *batch_id { + return Err(self.fail_now( + DomainError::before(ErrorCode::IdentityMismatch, "the result names another batch"), + "step-result", + )); + } + if result.applied_from_step.0 != k || result.next_step.0 != k + 1 { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the result does not describe exactly this transition", + ), + "step-result", + )); + } + if result.applied_controls_digest != controls_digest(controls) { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the applied control digest is not the batch that was requested", + ), + "step-result", + )); + } + if result.observation.boundary.0 != k + 1 { + return Err(self.fail_now( + DomainError::before(ErrorCode::IdentityMismatch, "the observation is not k+1"), + "step-result", + )); + } + // Fixed cadence within an epoch: exactly one step duration of world time. + let expected_time = old + .world_time + .checked_add(&descriptor.step_duration) + .map_err(DomainError::invalid); + let expected_time = match expected_time { + Ok(time) => time, + Err(e) => return Err(self.fail_now(e, "step-result")), + }; + if result.observation.world_time != expected_time { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "world time did not advance by exactly one step duration", + ), + "step-result", + )); + } + if result.observation.inspection.schema != descriptor.inspection_schema { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "the inspection payload is not the declared schema", + ), + "step-result", + )); + } + // A missing required sensory input is never silently replaced by an older frame. + if let Err(e) = result.observation.validate(descriptor) { + return Err(self.fail_now( + DomainError::new(ErrorCode::BufferInvalid, e, Mutation::Unknown), + "step-result", + )); + } + for view in &result.observation.sensory_views { + let name = format!("view.{}", view.view_id); + match self.pending_views.get(&name) { + Some(artifact) if artifact.reference() == &view.pixels.0 => {} + _ => { + return Err(self.fail_now( + DomainError::new( + ErrorCode::BufferInvalid, + format!("required view {} arrived without a live owned handle", view.view_id), + Mutation::Unknown, + ), + "step-result", + )); + } + } + } + Ok(()) + } + + /// Phase D. Every agent commits before the next Prepare or any committed publication. + async fn commit_all( + &mut self, + k: u64, + observation: &WorldObservation, + outcomes: &mut BTreeMap, + next_contexts: &mut BTreeMap, + views: &BTreeMap, + ) -> Outcome> { + let scope = self.scope(k); + let attachments: Vec<(String, flybus::Artifact)> = + views.iter().map(|(n, a)| (n.clone(), a.clone())).collect(); + let mut jobs = Vec::new(); + let mut bodies = Vec::new(); + for index in 0..self.agents.len() { + let agent_id = self.agents[index].agent_id.clone(); + let prepared_request = self.agents[index] + .prepare_request + .expect("every agent prepared") + .id(); + let outcome = outcomes.get(&agent_id).cloned().unwrap_or_default(); + let next_context = next_contexts.get(&agent_id).cloned().expect("checked"); + let params = CommitParams { + agent_id: agent_id.clone(), + prepared_request_id: prepared_request, + next_input: self.sensory_input(observation, k + 1), + next_decision_context: next_context, + rewards: outcome.rewards.clone(), + task_stimulations: outcome.stimulations.clone(), + }; + let params = match serde_json::to_value(¶ms).expect("params serialize") { + Value::Object(m) => m, + _ => Map::new(), + }; + let request_id = self.serials.next(&self.agents[index].worker.service); + let worker = self.agents[index].worker.clone(); + bodies.push((agent_id.clone(), worker.clone(), params.clone(), request_id)); + jobs.push(Job { + agent_id, + worker, + method: "Agent.Commit", + scope: Some(scope.clone()), + params, + attachments: attachments.clone(), + request_id, + }); + } + let results = self.run_jobs(jobs, self.dispatch).await; + let mut commits = Vec::new(); + let mut first_failure = None; + for (agent_id, reply, scope, worker, method) in results { + match reply { + Ok(reply) => { + self.check_reply(&worker, &reply, &scope, method)?; + match reply.result() { + Ok(_) => {} + Err(e) => { + first_failure = Some(first_failure.unwrap_or(e)); + continue; + } + } + let result: AgentCommitResult = match reply.parse() { + Ok(result) => result, + Err(e) => return Err(self.fail_now(e, method)), + }; + if result.committed_step.0 != k + 1 { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "an agent acknowledged another boundary", + ), + method, + )); + } + let expected = next_contexts.get(&agent_id).expect("checked").digest(); + if result.decision_context_digest != expected { + return Err(self.fail_now( + DomainError::before( + ErrorCode::IdentityMismatch, + "an agent retained another decision context", + ), + method, + )); + } + if let Err(e) = result.telemetry.validate() { + return Err(self.fail_now(DomainError::invalid(e), method)); + } + self.audit.push(format!("committed:{agent_id}@{k}")); + self.stats.commits += 1; + commits.push((agent_id, result)); + } + Err(e) => { + first_failure = Some(first_failure.unwrap_or(e)); + } + } + } + if let Some(error) = first_failure { + // One Commit failed after others succeeded. There is no partial-match + // continuation: the epoch is failed and the group recovers together. + let _ = bodies; + return Err(self.fail_now(error, "commit")); + } + if commits.len() != self.agents.len() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::InvalidPhase, + "no next world step until every agent has committed", + ), + "commit", + )); + } + + if self.injections.at_step == k + && let Some(target) = self.injections.duplicate_commit.clone() + { + let expected = commits + .iter() + .find(|(id, _)| *id == target) + .map(|(_, result)| match serde_json::to_value(result).expect("serializes") { + Value::Object(m) => m, + _ => Map::new(), + }); + if let Some((_, worker, params, request_id)) = + bodies.into_iter().find(|(id, _, _, _)| *id == target) + { + self.probe_duplicate( + &worker, + "Agent.Commit", + Some(self.scope(k)), + params, + attachments.clone(), + request_id, + expected.as_ref(), + "duplicate-commit", + ) + .await; + } + } + commits.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(commits) + } + + /// Records the `step-v1` section 8 trace for this transition. + #[allow(clippy::too_many_arguments)] + fn record_trace( + &mut self, + k: u64, + _descriptor: &EnvironmentDescriptor, + prepared: &[(Id, PreparedDecision)], + commits: &[(Id, AgentCommitResult)], + controls: &[PortControl], + batch_id: &Id, + result: &StepResult, + outcomes: &BTreeMap, + event_ids: &[Id], + ) { + let mut agents = Vec::new(); + for (agent_id, decision) in prepared { + let slot = self.agent(agent_id).expect("configured"); + let committed = commits + .iter() + .find(|(id, _)| id == agent_id) + .map(|(_, result)| result.committed_step) + .unwrap_or(U64(0)); + let outcome = outcomes.get(agent_id).cloned().unwrap_or_default(); + agents.push(AgentTransitionTrace { + agent_id: agent_id.clone(), + profile_digest: slot.profile.digest.clone(), + ticks_advanced: decision.ticks_advanced, + brain_ticks: decision.brain_ticks, + remainder: decision.remainder, + decision_digest: decision.decision.digest(), + committed_step: committed, + rewards: outcome.rewards, + stimulations: outcome.stimulations, + prepare_request_id: slot + .prepare_request + .map(|r| r.id()) + .unwrap_or_else(|| Id::lit("req-0")), + commit_request_id: Id::parse(&format!( + "req-{}", + self.serials.highest(&slot.worker.service) + )) + .expect("a serial is an Id"), + }); + } + let observation_boundaries = result + .observation + .sensory_views + .iter() + .map(|view| ViewProvenance { + view_id: view.view_id.clone(), + produced_step: view.produced_step, + }) + .collect(); + self.trace.transition(TransitionTrace { + scope: self.scope(k), + agents, + controls_digest: controls_digest(controls), + acknowledged_boundary: result.next_step, + observation_boundaries, + task_event_ids: event_ids.to_vec(), + published_boundary: U64(k + 1), + batch_id: batch_id.clone(), + advance_request_id: Id::parse(&format!( + "req-{}", + self.serials.highest(&self.environment.service) + )) + .expect("a serial is an Id"), + }); + } + + // ----------------------------------------------------------------------------------- + // Publication + + async fn publish( + &mut self, + topic: &str, + payload: Map, + attachments: Vec<(String, flybus::Artifact)>, + ) -> Outcome<()> { + let refs: Vec<(&str, &flybus::Artifact)> = + attachments.iter().map(|(n, a)| (n.as_str(), a)).collect(); + match self.bus.publish(topic, payload, &refs).await { + Ok(_) => Ok(()), + Err(e) => { + // A disconnected or backpressured observer never stalls the world; only a + // real resource fault reaches here, and it fails the epoch honestly. + let error = DomainError::new( + ErrorCode::BackendFailure, + format!("publishing {topic}: {}", e.message), + Mutation::None, + ); + Err(self.fail_now(error, "publish")) + } + } + } + + async fn publish_descriptor(&mut self) -> Outcome<()> { + let descriptor = self.descriptor.clone().expect("bootstrapped"); + let agents: Vec = self + .agents + .iter() + .map(|slot| { + json!({ + "agentId": slot.agent_id.as_str(), + "portId": slot.port_id.as_str(), + "profileDigest": slot.profile.digest.as_str(), + "tickDuration": serde_json::to_value(slot.tick_duration).expect("rational"), + "warmupTicks": slot.warmup_ticks.to_string(), + }) + }) + .collect(); + let payload = json!({ + "sessionId": self.session_id.as_str(), + "revision": "1", + "compositionDigest": self.composition_digest().as_str(), + "schedulerId": "lockstep-v1", + "environment": serde_json::to_value(&descriptor).expect("descriptor"), + "taskSchema": serde_json::to_value(self.task.schema()).expect("schema"), + "agents": agents, + }); + let topic = self.topics.descriptor.clone(); + self.publish(&topic, match payload { + Value::Object(m) => m, + _ => Map::new(), + }, Vec::new()) + .await + } + + /// The composition identity: session, epoch, agents, ports and the contract revision. + pub fn composition_digest(&self) -> Digest { + let mut text = format!( + "fly-session/composition-v1\nsession={}\nepoch={}\ncontract={}\n", + self.session_id, + self.epoch, + contract_digest() + ); + for slot in &self.agents { + text.push_str(&format!( + "agent={} port={} profile={}\n", + slot.agent_id, slot.port_id, slot.profile.digest + )); + } + Digest::of(text.as_bytes()) + } + + async fn publish_events(&mut self, source_step: u64, events: &[crate::types::TaskEvent]) -> Outcome<()> { + if events.is_empty() { + return Ok(()); + } + let payload = json!({ + "sessionId": self.session_id.as_str(), + "epoch": self.epoch.as_str(), + "sourceStep": source_step.to_string(), + "events": serde_json::to_value(events).expect("events"), + }); + let topic = self.topics.events.clone(); + self.publish(&topic, match payload { + Value::Object(m) => m, + _ => Map::new(), + }, Vec::new()) + .await + } + + /// Publishes the committed boundary. Never an in-progress mix of new agent state and an + /// old world, and never before every agent has committed. + async fn publish_snapshot( + &mut self, + boundary: u64, + decisions: &BTreeMap, + controls: &[PortControl], + event_ids: &[Id], + ) -> Outcome<()> { + if !self.phases.phase().is_committed_boundary() { + return Err(self.fail_now( + DomainError::before( + ErrorCode::InvalidPhase, + "a snapshot represents a committed boundary only", + ), + "publish", + )); + } + let observation = self.observation.clone().expect("bootstrapped"); + let agents: Vec = self + .agents + .iter() + .map(|slot| { + let control = controls + .iter() + .find(|c| c.port_id == slot.port_id) + .map(|c| serde_json::to_value(c).expect("control")); + json!({ + "agentId": slot.agent_id.as_str(), + "selectedDecision": decisions + .get(&slot.agent_id) + .map(|d| serde_json::to_value(d).expect("decision")), + "appliedControls": control, + "committedStep": slot.committed_step.to_string(), + }) + }) + .collect(); + let payload = json!({ + "descriptorRevision": "1", + "publisherIncarnation": self.bus.info().connection_id.clone(), + "scope": serde_json::to_value(self.scope(boundary)).expect("scope"), + "episodeId": self.episode_id.as_str(), + "sequence": self.stats.publications.to_string(), + "worldTime": serde_json::to_value(observation.world_time).expect("rational"), + "agents": agents, + "progress": serde_json::to_value(self.task.progress()).expect("progress"), + "media": json!({ + "views": serde_json::to_value(&observation.broadcast_views).expect("views"), + "audio": [], + }), + "eventIds": event_ids.iter().map(Id::as_str).collect::>(), + }); + let attachments = self.view_attachments(); + let topic = self.topics.snapshots.clone(); + self.publish( + &topic, + match payload { + Value::Object(m) => m, + _ => Map::new(), + }, + attachments, + ) + .await?; + self.stats.publications += 1; + self.audit.push(format!("publish:{boundary}")); + Ok(()) + } +} diff --git a/services/flysim/crates/fly-session/src/dedup.rs b/services/flysim/crates/fly-session/src/dedup.rs new file mode 100644 index 0000000..0a7cc8e --- /dev/null +++ b/services/flysim/crates/fly-session/src/dedup.rs @@ -0,0 +1,440 @@ +//! Domain deduplication and the result cache of `ipc-v1` section 5. +//! +//! The operation key for a step mutation is `(sessionId, epoch, step, method, workerId)`, and +//! there is at most one Prepare, Commit or Advance for it. The cache decides, *before* any +//! phase check or artifact dereference, whether an arriving request is the original, a safe +//! replay, a conflicting change, a duplicate of something still running, or a retry whose +//! result is gone. +//! +//! A cached reply owns its artifacts through explicit holds, so a replay is still valid after +//! the first caller consumed its delivery. Dropping the record drops those holds. + +use std::collections::{BTreeMap, VecDeque}; + +use crate::types::{Digest, DomainError, ErrorCode, Id, RequestId, SessionRpcOutcome}; + +/// `ipc-v1` section 5: unacknowledged lifecycle replies are bounded at 16, then BUSY. +pub const MAX_UNACKNOWLEDGED: usize = 16; +/// `ipc-v1` section 5: Status and Acknowledge keep a cache of their last 16 replies. +pub const MAX_READONLY_REPLIES: usize = 16; +/// `ipc-v1` section 5: keep the current and the immediately previous step's records. +pub const RETAINED_STEPS: u64 = 2; + +/// Which retention rule an operation follows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OpClass { + /// Prepare, Commit, Advance: keyed by scope and retained for two steps. + StepMutation, + /// Initialize and capture: retained until `Worker.Acknowledge`. + Lifecycle, + /// Status and Acknowledge: a small last-16 reply cache, no mutation key. + ReadOnly, +} + +/// A step mutation's identity. Not a bus call id and not a batch id. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct OperationKey { + pub session_id: Id, + pub epoch: Id, + pub step: u64, + pub method: String, + pub worker_id: Id, +} + +/// A terminal domain reply plus the artifact holds that keep its attachments readable. +#[derive(Clone)] +pub struct CachedReply { + pub outcome: SessionRpcOutcome, + pub artifacts: Vec<(String, flybus::Artifact)>, +} + +impl CachedReply { + pub fn new(outcome: SessionRpcOutcome) -> CachedReply { + CachedReply { outcome, artifacts: Vec::new() } + } + + pub fn with_artifacts( + outcome: SessionRpcOutcome, + artifacts: Vec<(String, flybus::Artifact)>, + ) -> CachedReply { + CachedReply { outcome, artifacts } + } + + /// The attachment list for a fresh `rpc.reply`, over the same immutable bytes. + pub fn attachments(&self) -> Vec<(&str, &flybus::Artifact)> { + self.artifacts.iter().map(|(n, a)| (n.as_str(), a)).collect() + } +} + +impl std::fmt::Debug for CachedReply { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CachedReply") + .field("outcome", &self.outcome) + .field("artifacts", &self.artifacts.len()) + .finish() + } +} + +/// What the cache decided about an arriving request. +#[derive(Debug)] +pub enum Admission { + /// The original: run it, then `record` its reply. + Execute, + /// A safe replay: return this reply again, with fresh delivery ownership. + Replay(CachedReply), + /// A terminal domain error for this bus call, with no second mutation started. + Refuse(DomainError), +} + +#[derive(Clone, Debug)] +struct Record { + request_id: RequestId, + body: Digest, + reply: CachedReply, +} + +/// One worker's domain request cache. +pub struct ResultCache { + steps: BTreeMap, + active: BTreeMap, + lifecycle: BTreeMap, + lifecycle_order: VecDeque, + readonly: VecDeque<(Id, CachedReply)>, + highest_serial: Option, + step_watermark: Option, + /// Serials retired by `Worker.Acknowledge`; reuse below this is refused without keeping a + /// tombstone per request. + acknowledged_watermark: Option, +} + +impl Default for ResultCache { + fn default() -> ResultCache { + ResultCache::new() + } +} + +impl ResultCache { + pub fn new() -> ResultCache { + ResultCache { + steps: BTreeMap::new(), + active: BTreeMap::new(), + lifecycle: BTreeMap::new(), + lifecycle_order: VecDeque::new(), + readonly: VecDeque::new(), + highest_serial: None, + step_watermark: None, + acknowledged_watermark: None, + } + } + + /// Decides what to do with an arriving request. Identity is checked before phase. + pub fn admit( + &mut self, + class: OpClass, + key: &OperationKey, + request: RequestId, + body: &Digest, + ) -> Admission { + match class { + OpClass::StepMutation => self.admit_step(key, request, body), + OpClass::Lifecycle => self.admit_lifecycle(request, body), + OpClass::ReadOnly => Admission::Execute, + } + } + + fn admit_step(&mut self, key: &OperationKey, request: RequestId, body: &Digest) -> Admission { + if let Some(record) = self.steps.get(key) { + if record.request_id == request && record.body == *body { + return Admission::Replay(record.reply.clone()); + } + return Admission::Refuse(DomainError::new( + ErrorCode::Conflict, + "this operation key already holds a different request or body", + // The earlier result stands; refusing the duplicate undoes nothing. + crate::types::Mutation::None, + )); + } + if !self.active.is_empty() && !self.active.contains_key(key) { + // One mutation executes at a time. A second, different one is refused before + // admission rather than queued behind the first. + return Admission::Refuse(DomainError::before( + ErrorCode::Busy, + "another step mutation is already executing on this worker", + )); + } + if let Some((active_request, active_body)) = self.active.get(key) { + if *active_request == request && *active_body == *body { + return Admission::Refuse(DomainError::new( + ErrorCode::InProgress, + "the original operation is still executing; this duplicate started no work", + crate::types::Mutation::Unknown, + )); + } + return Admission::Refuse(DomainError::new( + ErrorCode::Conflict, + "an operation with a different body is active for this key", + crate::types::Mutation::None, + )); + } + // No record and nothing active. Eviction must never re-enable execution, so a step + // below the retention window is refused on the serial and step watermarks. + if let Some(watermark) = self.step_watermark + && key.step + RETAINED_STEPS <= watermark + { + let issued = self.highest_serial.is_some_and(|h| request.0 <= h); + return Admission::Refuse(if issued { + DomainError::new( + ErrorCode::ResultExpired, + "the retained result for this request is gone; it is never recomputed", + crate::types::Mutation::Unknown, + ) + } else { + DomainError::before( + ErrorCode::StaleStep, + "a newly issued request cannot name a step the worker has left behind", + ) + }); + } + if let Some(watermark) = self.acknowledged_watermark + && request.0 <= watermark + { + return Admission::Refuse(DomainError::new( + ErrorCode::ResultExpired, + "this request serial was acknowledged and cannot be reused", + crate::types::Mutation::Unknown, + )); + } + self.begin(key.clone(), request, body.clone()); + Admission::Execute + } + + fn admit_lifecycle(&mut self, request: RequestId, body: &Digest) -> Admission { + let id = request.id(); + if let Some(record) = self.lifecycle.get(&id) { + if record.body == *body { + return Admission::Replay(record.reply.clone()); + } + return Admission::Refuse(DomainError::before( + ErrorCode::Conflict, + "this lifecycle request id already holds a different body", + )); + } + if let Some(watermark) = self.acknowledged_watermark + && request.0 <= watermark + { + return Admission::Refuse(DomainError::new( + ErrorCode::ResultExpired, + "this request serial was acknowledged and cannot be reused", + crate::types::Mutation::Unknown, + )); + } + if self.lifecycle.len() >= MAX_UNACKNOWLEDGED { + return Admission::Refuse(DomainError::before( + ErrorCode::Busy, + "16 lifecycle replies are unacknowledged; acknowledge some before sending more", + )); + } + Admission::Execute + } + + fn begin(&mut self, key: OperationKey, request: RequestId, body: Digest) { + self.highest_serial = Some(self.highest_serial.map_or(request.0, |h| h.max(request.0))); + self.step_watermark = Some(self.step_watermark.map_or(key.step, |w| w.max(key.step))); + self.active.insert(key, (request, body)); + } + + /// Stores a terminal reply for a step mutation and prunes what has aged out. + pub fn record( + &mut self, + key: OperationKey, + request: RequestId, + body: Digest, + reply: CachedReply, + ) { + self.active.remove(&key); + let step = key.step; + self.steps.insert(key, Record { request_id: request, body, reply }); + self.step_watermark = Some(self.step_watermark.map_or(step, |w| w.max(step))); + self.prune(); + } + + /// Stores a lifecycle reply, retained until `Worker.Acknowledge`. + pub fn record_lifecycle(&mut self, request: RequestId, body: Digest, reply: CachedReply) { + let id = request.id(); + self.highest_serial = Some(self.highest_serial.map_or(request.0, |h| h.max(request.0))); + if self.lifecycle.insert(id.clone(), Record { request_id: request, body, reply }).is_none() + { + self.lifecycle_order.push_back(id); + } + } + + /// Stores a read-only reply in the last-16 cache. + pub fn record_readonly(&mut self, request: RequestId, reply: CachedReply) { + let id = request.id(); + self.readonly.retain(|(existing, _)| *existing != id); + self.readonly.push_back((id, reply)); + while self.readonly.len() > MAX_READONLY_REPLIES { + self.readonly.pop_front(); + } + } + + /// Releases an operation that did not mutate anything, so the key stays free. + pub fn abandon(&mut self, key: &OperationKey) { + self.active.remove(key); + } + + /// `Worker.Acknowledge`: drops those lifecycle records, ignoring unknown ids, and raises + /// the serial watermark so an acknowledged id cannot be reused. + pub fn acknowledge(&mut self, ids: &[Id]) -> Vec { + let mut out = Vec::new(); + for id in ids { + if let Some(record) = self.lifecycle.remove(id) { + self.lifecycle_order.retain(|existing| existing != id); + self.acknowledged_watermark = Some( + self.acknowledged_watermark + .map_or(record.request_id.0, |w| w.max(record.request_id.0)), + ); + out.push(id.clone()); + } + } + out + } + + pub fn unacknowledged(&self) -> usize { + self.lifecycle.len() + } + + pub fn retained_steps(&self) -> usize { + self.steps.len() + } + + pub fn has_active(&self) -> bool { + !self.active.is_empty() + } + + /// Deliberately drops a step record, so a retry meets RESULT_EXPIRED instead of a replay. + pub fn expire_step(&mut self, key: &OperationKey) -> bool { + self.steps.remove(key).is_some() + } + + fn prune(&mut self) { + let Some(watermark) = self.step_watermark else { return }; + let floor = watermark.saturating_sub(RETAINED_STEPS - 1); + self.steps.retain(|key, _| key.step >= floor); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{Scope, SessionRpcSuccess, SuccessTag}; + + fn key(step: u64, method: &str) -> OperationKey { + OperationKey { + session_id: Id::lit("demo"), + epoch: Id::lit("e1"), + step, + method: method.to_owned(), + worker_id: Id::lit("fly-a"), + } + } + + fn reply(tag: &str) -> CachedReply { + let mut result = serde_json::Map::new(); + result.insert("tag".into(), tag.into()); + CachedReply::new(SessionRpcOutcome::Success(SessionRpcSuccess { + kind: SuccessTag::Result, + request_id: Id::lit("req-1"), + worker_id: Id::lit("fly-a"), + incarnation_id: Id::lit("inc-1"), + scope: Some(Scope::new(&Id::lit("demo"), &Id::lit("e1"), 0)), + result, + })) + } + + fn body(s: &str) -> Digest { + Digest::of(s.as_bytes()) + } + + #[test] + fn the_same_key_request_and_body_replays() { + let mut c = ResultCache::new(); + let k = key(0, "Agent.Prepare"); + assert!(matches!(c.admit(OpClass::StepMutation, &k, RequestId(1), &body("a")), Admission::Execute)); + c.record(k.clone(), RequestId(1), body("a"), reply("first")); + match c.admit(OpClass::StepMutation, &k, RequestId(1), &body("a")) { + Admission::Replay(r) => { + assert_eq!(r.outcome.result().unwrap()["tag"], "first"); + } + other => panic!("wanted a replay, got {other:?}"), + } + } + + #[test] + fn a_changed_body_for_a_recorded_key_is_a_conflict() { + let mut c = ResultCache::new(); + let k = key(0, "Environment.Advance"); + c.admit(OpClass::StepMutation, &k, RequestId(1), &body("a")); + c.record(k.clone(), RequestId(1), body("a"), reply("first")); + match c.admit(OpClass::StepMutation, &k, RequestId(1), &body("b")) { + Admission::Refuse(e) => assert_eq!(e.code, ErrorCode::Conflict), + other => panic!("wanted CONFLICT, got {other:?}"), + } + } + + #[test] + fn a_duplicate_while_the_original_runs_is_in_progress() { + let mut c = ResultCache::new(); + let k = key(0, "Agent.Prepare"); + c.admit(OpClass::StepMutation, &k, RequestId(1), &body("a")); + match c.admit(OpClass::StepMutation, &k, RequestId(1), &body("a")) { + Admission::Refuse(e) => assert_eq!(e.code, ErrorCode::InProgress), + other => panic!("wanted IN_PROGRESS, got {other:?}"), + } + } + + #[test] + fn an_evicted_step_gives_result_expired_and_a_fresh_serial_gives_stale_step() { + let mut c = ResultCache::new(); + for step in 0..4u64 { + let k = key(step, "Agent.Prepare"); + c.admit(OpClass::StepMutation, &k, RequestId(step + 1), &body("a")); + c.record(k, RequestId(step + 1), body("a"), reply("x")); + } + assert_eq!(c.retained_steps(), 2); + // req-1 named step 0, whose record is long gone. + match c.admit(OpClass::StepMutation, &key(0, "Agent.Prepare"), RequestId(1), &body("a")) { + Admission::Refuse(e) => assert_eq!(e.code, ErrorCode::ResultExpired), + other => panic!("wanted RESULT_EXPIRED, got {other:?}"), + } + // A serial above the highest issued is a new operation naming an old step. + match c.admit(OpClass::StepMutation, &key(0, "Agent.Prepare"), RequestId(99), &body("a")) { + Admission::Refuse(e) => assert_eq!(e.code, ErrorCode::StaleStep), + other => panic!("wanted STALE_STEP, got {other:?}"), + } + } + + #[test] + fn lifecycle_replies_are_bounded_and_released_by_acknowledge() { + let mut c = ResultCache::new(); + for serial in 1..=MAX_UNACKNOWLEDGED as u64 { + assert!(matches!( + c.admit(OpClass::Lifecycle, &key(0, "Agent.Initialize"), RequestId(serial), &body("a")), + Admission::Execute + )); + c.record_lifecycle(RequestId(serial), body("a"), reply("init")); + } + match c.admit(OpClass::Lifecycle, &key(0, "Agent.Initialize"), RequestId(99), &body("a")) { + Admission::Refuse(e) => assert_eq!(e.code, ErrorCode::Busy), + other => panic!("wanted BUSY, got {other:?}"), + } + let dropped = c.acknowledge(&[Id::lit("req-1"), Id::lit("req-404")]); + assert_eq!(dropped, vec![Id::lit("req-1")]); + assert_eq!(c.unacknowledged(), MAX_UNACKNOWLEDGED - 1); + // The acknowledged serial cannot come back. + match c.admit(OpClass::Lifecycle, &key(0, "Agent.Initialize"), RequestId(1), &body("a")) { + Admission::Refuse(e) => assert_eq!(e.code, ErrorCode::ResultExpired), + other => panic!("wanted RESULT_EXPIRED, got {other:?}"), + } + } +} diff --git a/services/flysim/crates/fly-session/src/environment.rs b/services/flysim/crates/fly-session/src/environment.rs new file mode 100644 index 0000000..bbf3354 --- /dev/null +++ b/services/flysim/crates/fly-session/src/environment.rs @@ -0,0 +1,454 @@ +//! The counter arena: one environment worker with no emulator behind it. +//! +//! It holds a signed counter, applies one complete control batch per Advance, advances exactly +//! one interval, and returns boundary `k+1` with its world time advanced by `stepDuration`. It +//! never advances while waiting for the next request, and it does not free-run during agent +//! initialization. + +use std::collections::BTreeSet; +use std::io::Write; + +use crate::task::{controller_schema_ref, inspection, inspection_schema}; +use crate::types::{ + AdvanceParams, AxisRange, AxisSchema, ControllerSchema, Determinism, Digest, DomainError, + DomainResult, EnvironmentDescriptor, EnvironmentInitializeParams, EnvironmentInitializeResult, + ErrorCode, Id, MAX_PORTS, Mutation, PixelAspect, PortBinding, PortControl, PortDescriptor, + RationalNs, Recovery, Role, Scope, StepResult, U64, ViewDescriptor, ViewFormat, ViewRef, + WorkerState, WorldObservation, controls_digest, +}; +use crate::worker::{BoxFuture, HandlerCtx, HandlerReply, StatusCell, WorkerEndpoint}; + +/// The arena's view: a 4x4 RGBA8 tile whose bytes carry the counter. +pub const VIEW_WIDTH: u32 = 4; +pub const VIEW_HEIGHT: u32 = 4; + +/// Deliberate faults a test can ask the environment for. +#[derive(Clone, Debug, Default)] +pub struct EnvironmentFaults { + /// Hold `Environment.Advance` open for this long, after the world already moved. + pub advance_delay_ms: u64, + /// Drop the required sensory view from the result at this boundary, so the coordinator + /// meets a world that advanced with no usable sensory data. + pub omit_view_at_boundary: Option, +} + +#[derive(Clone, Debug)] +pub struct EnvironmentConfig { + pub session_id: Id, + pub worker_id: Id, + pub incarnation_id: Id, + /// The world's fixed reduced step duration. 60 Hz is `1/60` s. + pub step_duration: RationalNs, + pub ports: Vec, + pub faults: EnvironmentFaults, +} + +/// The counter environment endpoint. +pub struct CounterEnvironment { + config: EnvironmentConfig, + status: StatusCell, + epoch: Option, + episode_id: Option, + descriptor: Option, + bindings: Vec, + boundary: u64, + counter: i64, + world_time: RationalNs, + advances: u64, + batches: BTreeSet, +} + +impl CounterEnvironment { + pub fn new(config: EnvironmentConfig) -> CounterEnvironment { + CounterEnvironment { + status: StatusCell::new(), + epoch: None, + episode_id: None, + descriptor: None, + bindings: Vec::new(), + boundary: 0, + counter: 0, + world_time: RationalNs::zero(), + advances: 0, + batches: BTreeSet::new(), + config, + } + } + + pub fn status(&self) -> StatusCell { + self.status.clone() + } + + /// How many intervals this world has advanced. One complete batch advances it once. + pub fn advances(&self) -> u64 { + self.advances + } + + pub fn counter(&self) -> i64 { + self.counter + } + + pub fn boundary(&self) -> u64 { + self.boundary + } + + /// The controller every port of this arena declares: two buttons and one bipolar axis. + pub fn controller_schema() -> ControllerSchema { + ControllerSchema { + schema: controller_schema_ref(), + buttons: vec![Id::lit("inc"), Id::lit("dec")], + axes: vec![AxisSchema { + id: Id::lit("bias"), + range: AxisRange::Bipolar, + neutral: 0.0, + }], + } + } + + pub fn view_descriptor() -> ViewDescriptor { + ViewDescriptor { + view_id: Id::lit("arena"), + width: VIEW_WIDTH, + height: VIEW_HEIGHT, + format: ViewFormat::Rgba8, + row_stride: VIEW_WIDTH * 4, + pixel_aspect: PixelAspect { numerator: 1, denominator: 1 }, + observation_delay_steps: 0, + } + } + + fn build_descriptor(&self) -> DomainResult { + let descriptor = EnvironmentDescriptor { + backend_digest: Digest::of(b"counter-arena-backend-v1"), + content_digest: Digest::of(b"counter-arena-content-v1"), + configuration_digest: Digest::of( + format!( + "counter-arena-config-v1\nstep={}/{}\nports={}\n", + self.config.step_duration.numerator, + self.config.step_duration.denominator, + self.config.ports.len() + ) + .as_bytes(), + ), + step_duration: self.config.step_duration, + ports: self + .config + .ports + .iter() + .map(|port_id| PortDescriptor { + port_id: port_id.clone(), + controls: CounterEnvironment::controller_schema(), + }) + .collect(), + inspection_schema: inspection_schema(), + views: vec![CounterEnvironment::view_descriptor()], + audio: Vec::new(), + recovery: Recovery::ExactCheckpoint, + determinism: Determinism::FixedBuild, + }; + descriptor.validate().map_err(DomainError::invalid)?; + Ok(descriptor) + } + + /// Seals one immutable native frame for the current counter and returns the handle. + async fn render( + &self, + ctx: &HandlerCtx<'_>, + ) -> DomainResult<(ViewRef, flybus::Artifact)> { + let descriptor = CounterEnvironment::view_descriptor(); + let len = descriptor.byte_length(); + let mut writer = ctx + .client + .artifacts() + .allocate(len, "image/x-rgba") + .await + .map_err(|e| { + DomainError::new( + ErrorCode::BackendFailure, + format!("frame allocation failed: {}", e.message), + Mutation::Applied, + ) + })?; + // Every pixel carries the counter's low byte, so an agent reading the frame reads the + // world rather than a constant. + let byte = (self.counter & 0xff) as u8; + writer + .write_all(&vec![byte; len as usize]) + .map_err(|e| { + DomainError::new( + ErrorCode::BackendFailure, + format!("frame write failed: {e}"), + Mutation::Applied, + ) + })?; + let artifact = writer.seal().await.map_err(|e| { + DomainError::new( + ErrorCode::BackendFailure, + format!("frame seal failed: {}", e.message), + Mutation::Applied, + ) + })?; + let view = ViewRef { + view_id: descriptor.view_id.clone(), + produced_step: U64(descriptor.required_produced_step(self.boundary)), + pixels: crate::types::ArtifactRef(artifact.reference().clone()), + }; + Ok((view, artifact)) + } + + async fn observation( + &self, + ctx: &HandlerCtx<'_>, + ) -> DomainResult<(WorldObservation, Vec<(String, flybus::Artifact)>)> { + let omit = self.config.faults.omit_view_at_boundary == Some(self.boundary); + let (views, attachments) = if omit { + (Vec::new(), Vec::new()) + } else { + let (view, artifact) = self.render(ctx).await?; + let name = format!("view.{}", view.view_id); + (vec![view], vec![(name, artifact)]) + }; + let observation = WorldObservation { + boundary: U64(self.boundary), + world_time: self.world_time, + engine_frame: Some(self.boundary.to_string()), + sensory_views: views.clone(), + inspection: inspection(self.counter, self.boundary), + // The same immutable object serves the broadcast view; nothing is rendered twice. + broadcast_views: views, + audio: Vec::new(), + }; + Ok((observation, attachments)) + } + + async fn initialize(&mut self, ctx: &HandlerCtx<'_>) -> DomainResult { + let scope = ctx.scope()?.clone(); + if self.descriptor.is_some() { + return Err(DomainError::before( + ErrorCode::InvalidPhase, + "this environment is already initialized", + )); + } + if scope.session_id != self.config.session_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "this environment belongs to another session", + )); + } + if scope.step.0 != 0 { + return Err(DomainError::before( + ErrorCode::FutureStep, + "Environment.Initialize uses the new epoch at step 0", + )); + } + let params: EnvironmentInitializeParams = ctx.params()?; + if params.port_bindings.len() > MAX_PORTS as usize { + return Err(DomainError::invalid("at most 4 ports in the first composition")); + } + let mut seen = BTreeSet::new(); + for binding in ¶ms.port_bindings { + if !self.config.ports.contains(&binding.port_id) { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + format!("port {} is not a port of this arena", binding.port_id), + )); + } + if !seen.insert(binding.port_id.clone()) { + return Err(DomainError::invalid(format!( + "port {} is bound twice", + binding.port_id + ))); + } + } + let descriptor = self.build_descriptor()?; + self.epoch = Some(scope.epoch.clone()); + self.episode_id = Some(params.episode_id.clone()); + self.bindings = params.port_bindings.clone(); + self.boundary = 0; + self.counter = 0; + self.world_time = RationalNs::zero(); + self.batches.clear(); + self.descriptor = Some(descriptor.clone()); + // The world is stopped when O[0] goes out and cannot free-run while the brains boot. + self.status.set_state(WorkerState::Ready); + self.status.set_scope(Some(scope.clone())); + self.status.progress(1); + + let (observation, attachments) = self.observation(ctx).await?; + let result = EnvironmentInitializeResult { descriptor, observation }; + let mut reply = HandlerReply::from(&result); + reply.artifacts = attachments; + Ok(reply) + } + + async fn advance(&mut self, ctx: &HandlerCtx<'_>) -> DomainResult { + let scope = ctx.scope()?.clone(); + let Some(descriptor) = self.descriptor.clone() else { + return Err(DomainError::before( + ErrorCode::InvalidPhase, + "this environment is uninitialized", + )); + }; + if scope.session_id != self.config.session_id { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "this environment belongs to another session", + )); + } + match &self.epoch { + Some(epoch) if *epoch == scope.epoch => {} + _ => { + return Err(DomainError::before( + ErrorCode::StaleEpoch, + "this scope names an epoch this environment has left", + )); + } + } + if scope.step.0 != self.boundary { + return Err(DomainError::before( + if scope.step.0 < self.boundary { + ErrorCode::StaleStep + } else { + ErrorCode::FutureStep + }, + "Environment.Advance must name the boundary the world is at", + )); + } + let params: AdvanceParams = ctx.params()?; + // Batch ids are unique within an epoch; reusing one for a different request or step is + // a conflict, not a second world mutation. + if self.batches.contains(¶ms.batch_id) { + return Err(DomainError::before( + ErrorCode::Conflict, + format!("batch {} was already applied in this epoch", params.batch_id), + )); + } + self.check_batch(&descriptor, ¶ms.controls)?; + let digest = controls_digest(¶ms.controls); + + let applied_from = self.boundary; + // Apply the complete batch to its interval and advance exactly one framework step. + let mut delta = 0i64; + for control in ¶ms.controls { + delta += crate::task::CounterTask::delta_of(control); + } + self.counter += delta; + self.boundary += 1; + self.world_time = self + .world_time + .checked_add(&descriptor.step_duration) + .map_err(|e| DomainError::new(ErrorCode::Internal, e, Mutation::Applied))?; + self.advances += 1; + self.batches.insert(params.batch_id.clone()); + self.status.set_batch(params.batch_id.clone()); + self.status.set_scope(Some(Scope::new( + &scope.session_id, + &scope.epoch, + self.boundary, + ))); + self.status.progress(1); + + if self.config.faults.advance_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis( + self.config.faults.advance_delay_ms, + )) + .await; + } + + let (observation, attachments) = self.observation(ctx).await?; + // The record of batch id, result and next boundary exists before the acknowledgment. + let result = StepResult { + batch_id: params.batch_id, + applied_from_step: U64(applied_from), + next_step: U64(self.boundary), + applied_controls_digest: digest, + observation, + }; + let mut reply = HandlerReply::from(&result); + reply.artifacts = attachments; + Ok(reply) + } + + /// Every active port must appear exactly once, with every declared button and axis in + /// descriptor order. Uncontrolled ports were configured neutral before the epoch. + fn check_batch( + &self, + descriptor: &EnvironmentDescriptor, + controls: &[PortControl], + ) -> DomainResult<()> { + if controls.len() != descriptor.ports.len() { + return Err(DomainError::invalid(format!( + "the batch has {} port controls; the descriptor declares {}", + controls.len(), + descriptor.ports.len() + ))); + } + for (declared, given) in descriptor.ports.iter().zip(controls) { + if declared.port_id != given.port_id { + return Err(DomainError::invalid(format!( + "port {} is out of descriptor order; expected {}", + given.port_id, declared.port_id + ))); + } + declared.controls.check(given).map_err(DomainError::invalid)?; + } + Ok(()) + } +} + +impl WorkerEndpoint for CounterEnvironment { + fn worker_id(&self) -> Id { + self.config.worker_id.clone() + } + + fn incarnation_id(&self) -> Id { + self.config.incarnation_id.clone() + } + + fn session_id(&self) -> Id { + self.config.session_id.clone() + } + + fn role(&self) -> Role { + Role::Environment + } + + fn capabilities(&self) -> Vec { + vec![ + Id::lit("world-step-v1"), + Id::lit("pixel-observation-v1"), + Id::lit("checkpoint-v1"), + ] + } + + fn status_cell(&self) -> StatusCell { + self.status.clone() + } + + fn methods(&self) -> Vec<&'static str> { + vec!["Environment.Initialize", "Environment.Advance"] + } + + fn handle<'a>(&'a mut self, ctx: HandlerCtx<'a>) -> BoxFuture<'a, DomainResult> { + Box::pin(async move { + match ctx.method { + "Environment.Initialize" => self.initialize(&ctx).await, + "Environment.Advance" => self.advance(&ctx).await, + other => Err(DomainError::before( + ErrorCode::Unsupported, + format!("{other} is not an environment method"), + )), + } + }) + } +} + +/// A synthetic backend/task configuration asset for the arena. +pub fn synthetic_asset(id: &str, body: &str) -> crate::types::AssetRef { + crate::types::AssetRef { + id: Id::lit(id), + digest: Digest::of(body.as_bytes()), + byte_length: U64(body.len() as u64), + format: Id::lit("fly-config-v1"), + } +} diff --git a/services/flysim/crates/fly-session/src/fly_session_types/canonical.rs b/services/flysim/crates/fly-session/src/fly_session_types/canonical.rs new file mode 100644 index 0000000..a44db56 --- /dev/null +++ b/services/flysim/crates/fly-session/src/fly_session_types/canonical.rs @@ -0,0 +1,139 @@ +//! Canonical JSON (RFC 8785) and the digests built on it. +//! +//! Two payloads mean the same domain operation when their canonical encodings are equal, so a +//! duplicate `Agent.Prepare` can be told from a changed one without depending on key order, +//! whitespace or float formatting. Object keys sort by UTF-16 code unit; numbers use the +//! ECMAScript `Number::toString` shortest form (`ryu_js`), so `1.0` and `1` are one value. + +use serde_json::Value; + +use super::scalars::Digest; + +/// The canonical JSON encoding of `value`. +/// +/// Panics on a non-finite number, which cannot appear in a validated payload and cannot be +/// represented in JSON at all. +pub fn canonical_json(value: &Value) -> String { + let mut out = String::new(); + write_value(&mut out, value); + out +} + +/// The SHA-256 of the canonical encoding. +pub fn canonical_digest(value: &Value) -> Digest { + Digest::of(canonical_json(value).as_bytes()) +} + +fn write_value(out: &mut String, value: &Value) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(true) => out.push_str("true"), + Value::Bool(false) => out.push_str("false"), + Value::Number(n) => write_number(out, n), + Value::String(s) => write_string(out, s), + Value::Array(items) => { + out.push('['); + for (i, item) in items.iter().enumerate() { + if i > 0 { + out.push(','); + } + write_value(out, item); + } + out.push(']'); + } + Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort_by(|a, b| utf16_cmp(a, b)); + out.push('{'); + for (i, key) in keys.iter().enumerate() { + if i > 0 { + out.push(','); + } + write_string(out, key); + out.push(':'); + write_value(out, &map[*key]); + } + out.push('}'); + } + } +} + +fn write_number(out: &mut String, n: &serde_json::Number) { + if let Some(u) = n.as_u64() { + out.push_str(&u.to_string()); + return; + } + if let Some(i) = n.as_i64() { + out.push_str(&i.to_string()); + return; + } + let f = n.as_f64().expect("a JSON number is representable"); + assert!(f.is_finite(), "canonical JSON cannot encode a non-finite number"); + let mut buf = ryu_js::Buffer::new(); + out.push_str(buf.format(f)); +} + +fn write_string(out: &mut String, s: &str) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\u{08}' => out.push_str("\\b"), + '\u{0c}' => out.push_str("\\f"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => { + out.push_str(&format!("\\u{:04x}", c as u32)); + } + c => out.push(c), + } + } + out.push('"'); +} + +/// Compares two strings by UTF-16 code unit, which is what RFC 8785 sorts object keys by. +/// +/// Byte order and code-unit order disagree only above the BMP, so the surrogate expansion +/// matters for a key containing an astral character. +fn utf16_cmp(a: &str, b: &str) -> std::cmp::Ordering { + let mut ai = a.encode_utf16(); + let mut bi = b.encode_utf16(); + loop { + match (ai.next(), bi.next()) { + (None, None) => return std::cmp::Ordering::Equal, + (None, Some(_)) => return std::cmp::Ordering::Less, + (Some(_), None) => return std::cmp::Ordering::Greater, + (Some(x), Some(y)) if x == y => continue, + (Some(x), Some(y)) => return x.cmp(&y), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn keys_sort_and_floats_take_their_shortest_form() { + let v = json!({"b": 1.0, "a": [1, 2.5], "\u{00e9}": "x"}); + assert_eq!(canonical_json(&v), "{\"a\":[1,2.5],\"b\":1,\"\u{00e9}\":\"x\"}"); + } + + #[test] + fn the_same_object_written_two_ways_has_one_digest() { + let a: Value = serde_json::from_str("{\"x\":1,\"y\":{\"p\":2,\"q\":3}}").unwrap(); + let b: Value = serde_json::from_str("{\"y\":{\"q\":3,\"p\":2},\"x\":1}").unwrap(); + assert_eq!(canonical_digest(&a), canonical_digest(&b)); + } + + #[test] + fn an_astral_key_sorts_by_code_unit_not_by_byte() { + // U+10000 encodes as the surrogate pair D800 DC00, which is below U+FFFD's E000-range + // code unit in UTF-16 but above it byte-wise. + let v = json!({"\u{10000}": 1, "\u{fffd}": 2}); + assert_eq!(canonical_json(&v), "{\"\u{10000}\":1,\"\u{fffd}\":2}"); + } +} diff --git a/services/flysim/crates/fly-session/src/fly_session_types/methods.rs b/services/flysim/crates/fly-session/src/fly_session_types/methods.rs new file mode 100644 index 0000000..dadd6b5 --- /dev/null +++ b/services/flysim/crates/fly-session/src/fly_session_types/methods.rs @@ -0,0 +1,1026 @@ +//! The domain envelope, the error table and every method payload of `ipc-v1` and `workers-v1`. +//! +//! These are the bodies carried inside a Flybus `rpc.call` payload and `rpc.result` outcome. +//! The bus owns framing, routing and artifact ownership; nothing below knows about either. + +use serde::de::Error as _; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::{Map, Value}; + +use super::canonical::canonical_digest; +use super::scalars::{Digest, Id, RationalNs, SchemaRef, Scope, TypedValue, U64}; + +/// A transient artifact identity, as it appears inside a domain payload. +/// +/// Every one of these must also be listed in the surrounding bus attachments and backed by a +/// live owned handle; the reference alone is not authority to read. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ArtifactRef(pub flybus::ArtifactRef); + +impl Serialize for ArtifactRef { + fn serialize(&self, s: S) -> Result { + self.0.to_json().serialize(s) + } +} + +impl<'de> Deserialize<'de> for ArtifactRef { + fn deserialize>(d: D) -> Result { + let v = Value::deserialize(d)?; + flybus::ArtifactRef::from_json(&v).map(ArtifactRef).map_err(D::Error::custom) + } +} + +// --------------------------------------------------------------------------------------------- +// Errors + +/// `ipc-v1` section 7. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ErrorCode { + #[serde(rename = "INVALID_ARGUMENT")] + InvalidArgument, + #[serde(rename = "UNSUPPORTED")] + Unsupported, + #[serde(rename = "IDENTITY_MISMATCH")] + IdentityMismatch, + #[serde(rename = "STALE_EPOCH")] + StaleEpoch, + #[serde(rename = "STALE_STEP")] + StaleStep, + #[serde(rename = "FUTURE_STEP")] + FutureStep, + #[serde(rename = "INVALID_PHASE")] + InvalidPhase, + #[serde(rename = "CONFLICT")] + Conflict, + #[serde(rename = "IN_PROGRESS")] + InProgress, + #[serde(rename = "BUSY")] + Busy, + #[serde(rename = "BUFFER_INVALID")] + BufferInvalid, + #[serde(rename = "RESULT_EXPIRED")] + ResultExpired, + #[serde(rename = "INCOMPATIBLE_STATE")] + IncompatibleState, + #[serde(rename = "BACKEND_FAILURE")] + BackendFailure, + #[serde(rename = "INTERNAL")] + Internal, +} + +/// How much of the operation had already happened when the error was produced. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Mutation { + None, + Applied, + Unknown, +} + +/// A domain failure: the code, a bounded message and the mutation certainty. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DomainError { + pub code: ErrorCode, + pub message: String, + pub mutation: Mutation, +} + +/// `ipc-v1` section 7: messages are at most 512 code points and carry no raw memory. +pub const MAX_ERROR_MESSAGE_CHARS: usize = 512; + +impl DomainError { + pub fn new(code: ErrorCode, message: impl Into, mutation: Mutation) -> DomainError { + let message: String = message.into(); + let message = if message.chars().count() > MAX_ERROR_MESSAGE_CHARS { + message.chars().take(MAX_ERROR_MESSAGE_CHARS).collect() + } else { + message + }; + DomainError { code, message, mutation } + } + + /// An error raised before anything was mutated. + pub fn before(code: ErrorCode, message: impl Into) -> DomainError { + DomainError::new(code, message, Mutation::None) + } + + pub fn invalid(message: impl Into) -> DomainError { + DomainError::before(ErrorCode::InvalidArgument, message) + } +} + +impl std::fmt::Display for DomainError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?} ({:?}): {}", self.code, self.mutation, self.message) + } +} + +impl std::error::Error for DomainError {} + +pub type DomainResult = Result; + +// --------------------------------------------------------------------------------------------- +// Envelope + +/// The request body of every session RPC: `{requestId, scope, params}`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SessionRpcRequest { + pub request_id: Id, + pub scope: Option, + pub params: Map, +} + +impl SessionRpcRequest { + pub fn new(request_id: &RequestId, scope: Option, params: Map) -> Self { + SessionRpcRequest { request_id: request_id.id(), scope, params } + } + + /// The canonical body digest the deduplication cache compares: method, scope and params, + /// and nothing that changes between two safe retries of one operation. + pub fn body_digest(&self, method: &str) -> Digest { + let mut m = Map::new(); + m.insert("method".into(), method.into()); + m.insert( + "scope".into(), + self.scope.as_ref().map_or(Value::Null, |s| { + serde_json::to_value(s).expect("Scope serializes") + }), + ); + m.insert("params".into(), Value::Object(self.params.clone())); + canonical_digest(&Value::Object(m)) + } + + pub fn to_payload(&self) -> Map { + match serde_json::to_value(self).expect("SessionRpcRequest serializes") { + Value::Object(m) => m, + _ => unreachable!("a struct serializes to an object"), + } + } + + pub fn from_payload(payload: &Map) -> Result { + serde_json::from_value(Value::Object(payload.clone())).map_err(|e| e.to_string()) + } +} + +/// `req-`: the domain operation identity, independent of the bus `callId`. +/// +/// A safe retry keeps this and takes a fresh `callId`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RequestId(pub u64); + +impl RequestId { + pub fn id(&self) -> Id { + Id::parse(&format!("req-{}", self.0)).expect("req- plus a U64 is an Id") + } + + /// Reads the serial back out of `req-`. + pub fn parse(id: &Id) -> Result { + let s = id.as_str(); + let rest = s + .strip_prefix("req-") + .ok_or_else(|| format!("request id {s:?} is not req-"))?; + U64::parse(rest).map(|v| RequestId(v.0)) + } +} + +impl std::fmt::Display for RequestId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "req-{}", self.0) + } +} + +/// The success half of a domain reply. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SessionRpcSuccess { + #[serde(rename = "type")] + pub kind: SuccessTag, + pub request_id: Id, + pub worker_id: Id, + pub incarnation_id: Id, + pub scope: Option, + pub result: Map, +} + +/// The failure half of a domain reply. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SessionRpcFailure { + #[serde(rename = "type")] + pub kind: FailureTag, + pub request_id: Id, + pub worker_id: Id, + pub incarnation_id: Id, + pub scope: Option, + pub error: DomainError, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SuccessTag { + Result, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FailureTag { + Error, +} + +/// One terminal domain reply, success or failure. +#[derive(Clone, Debug, PartialEq)] +pub enum SessionRpcOutcome { + Success(SessionRpcSuccess), + Failure(SessionRpcFailure), +} + +impl SessionRpcOutcome { + pub fn to_outcome(&self) -> Map { + let v = match self { + SessionRpcOutcome::Success(s) => { + serde_json::to_value(s).expect("success serializes") + } + SessionRpcOutcome::Failure(f) => serde_json::to_value(f).expect("failure serializes"), + }; + match v { + Value::Object(m) => m, + _ => unreachable!("a struct serializes to an object"), + } + } + + pub fn from_outcome(outcome: &Map) -> Result { + let tag = outcome.get("type").and_then(Value::as_str).unwrap_or_default(); + let v = Value::Object(outcome.clone()); + match tag { + "result" => serde_json::from_value(v) + .map(SessionRpcOutcome::Success) + .map_err(|e| e.to_string()), + "error" => serde_json::from_value(v) + .map(SessionRpcOutcome::Failure) + .map_err(|e| e.to_string()), + other => Err(format!("domain outcome has type {other:?}")), + } + } + + pub fn request_id(&self) -> &Id { + match self { + SessionRpcOutcome::Success(s) => &s.request_id, + SessionRpcOutcome::Failure(f) => &f.request_id, + } + } + + pub fn error(&self) -> Option<&DomainError> { + match self { + SessionRpcOutcome::Success(_) => None, + SessionRpcOutcome::Failure(f) => Some(&f.error), + } + } + + /// The `result` object of a success, or the domain error. + pub fn result(&self) -> DomainResult<&Map> { + match self { + SessionRpcOutcome::Success(s) => Ok(&s.result), + SessionRpcOutcome::Failure(f) => Err(f.error.clone()), + } + } +} + +// --------------------------------------------------------------------------------------------- +// Common worker methods + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Role { + Agent, + Environment, + Coordinator, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct HelloParams { + pub session_id: Id, + pub expected_worker_id: Id, + pub role: Role, + pub supported_majors: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct HelloResult { + pub selected_major: u16, + pub selected_minor: u16, + pub worker_id: Id, + pub incarnation_id: Id, + pub role: Role, + pub build_digest: Digest, + pub contract_digest: Digest, + pub capabilities: Vec, + pub limits: HelloLimits, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct HelloLimits { + pub max_agents: u32, + pub max_ports: u32, +} + +/// The worker phase names of `ipc-v1` section 4. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WorkerState { + Uninitialized, + Ready, + Preparing, + Prepared, + Advancing, + Committing, + Capturing, + StagedRestore, + Restoring, + Failed, + Stopping, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct StatusResult { + pub state: WorkerState, + pub current_scope: Option, + pub active_request_id: Option, + pub last_completed_request_id: Option, + pub last_batch_id: Option, + pub progress_counter: U64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AcknowledgeParams { + pub request_ids: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AcknowledgeResult { + pub acknowledged: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ShutdownParams { + pub reason: Id, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ShutdownResult { + pub stopping: bool, +} + +// --------------------------------------------------------------------------------------------- +// Shared data model (workers-v1 section 1) + +/// Persistent installed release content: a profile, a dataset, a backend build. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AssetRef { + pub id: Id, + pub digest: Digest, + pub byte_length: U64, + pub format: Id, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ViewDescriptor { + pub view_id: Id, + pub width: u32, + pub height: u32, + pub format: ViewFormat, + pub row_stride: u32, + pub pixel_aspect: PixelAspect, + pub observation_delay_steps: u8, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ViewFormat { + #[serde(rename = "rgba8")] + Rgba8, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PixelAspect { + pub numerator: u16, + pub denominator: u16, +} + +impl ViewDescriptor { + pub fn validate(&self) -> Result<(), String> { + if !(1..=4096).contains(&self.width) || !(1..=4096).contains(&self.height) { + return Err("view dimensions must be 1..=4096".to_owned()); + } + if self.row_stride != self.width * 4 { + return Err("rowStride must be exactly 4 x width; v1 has no padded rows".to_owned()); + } + if self.pixel_aspect.numerator == 0 || self.pixel_aspect.denominator == 0 { + return Err("pixel aspect must be positive".to_owned()); + } + if self.observation_delay_steps > 8 { + return Err("observationDelaySteps must be 0..=8".to_owned()); + } + Ok(()) + } + + pub fn byte_length(&self) -> u64 { + u64::from(self.row_stride) * u64::from(self.height) + } + + /// `state-media-v1` section 2: the boundary a required sensory view must come from. + pub fn required_produced_step(&self, boundary: u64) -> u64 { + boundary.saturating_sub(u64::from(self.observation_delay_steps)) + } +} + +/// One view of one boundary, with the bytes behind an owned artifact handle. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ViewRef { + pub view_id: Id, + pub produced_step: U64, + pub pixels: ArtifactRef, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AudioDescriptor { + pub stream_id: Id, + pub sample_rate: u32, + pub channels: u8, + pub format: AudioFormat, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum AudioFormat { + #[serde(rename = "f32le-interleaved")] + F32LeInterleaved, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AudioRef { + pub stream_id: Id, + pub first_sample: U64, + pub sample_frames: u32, + pub samples: ArtifactRef, + pub discontinuity: bool, +} + +/// Only the views this agent may consume, plus its permitted structured input. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensoryInput { + pub boundary: U64, + pub views: Vec, + pub structured: Option, +} + +/// `workers-v1` section 1: at most 8 views per sensory input. +pub const MAX_VIEWS: usize = 8; +/// `workers-v1` section 1: at most 64 stimuli or rewards per operation. +pub const MAX_EVENT_ARRAY: usize = 64; +/// `ipc-v1` section 2: the first composition's session limits. +pub const MAX_AGENTS: u32 = 4; +/// `ipc-v1` section 2: the first composition's session limits. +pub const MAX_PORTS: u32 = 4; +/// `ipc-v1` section 2: rate roles per agent. +pub const MAX_RATE_ROLES: usize = 64; + +impl SensoryInput { + pub fn validate(&self) -> Result<(), String> { + if self.views.len() > MAX_VIEWS { + return Err(format!("{} views, over the limit of {MAX_VIEWS}", self.views.len())); + } + let mut seen = std::collections::BTreeSet::new(); + for view in &self.views { + if !seen.insert(view.view_id.clone()) { + return Err(format!("view {} appears twice", view.view_id)); + } + } + if let Some(structured) = &self.structured { + structured.validate()?; + } + Ok(()) + } +} + +/// A profile-declared stimulus kind and duration. Never a neuron index or a drive value. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct Stimulus { + pub id: Id, + pub kind_id: Id, + pub duration_ms: f64, +} + +impl Stimulus { + pub fn validate(&self) -> Result<(), String> { + if !self.duration_ms.is_finite() || self.duration_ms <= 0.0 { + return Err("stimulus durationMs must be finite and > 0".to_owned()); + } + Ok(()) + } +} + +/// One reward value, attributed to the task event and rule that produced it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct Reward { + pub event_id: Id, + pub rule_id: Id, + pub value: f64, +} + +impl Reward { + pub fn validate(&self) -> Result<(), String> { + if !self.value.is_finite() { + return Err("reward value must be finite".to_owned()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AgentTelemetry { + pub brain_ticks: U64, + pub population_rate_hz: f64, + pub rates: Vec, + pub learning: LearningTelemetry, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct RateSample { + pub role_id: Id, + pub hz: f64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct LearningTelemetry { + pub enabled: bool, + pub updates: U64, + pub changed: U64, + pub signal: f64, +} + +impl AgentTelemetry { + pub fn validate(&self) -> Result<(), String> { + if !self.population_rate_hz.is_finite() || self.population_rate_hz < 0.0 { + return Err("populationRateHz must be finite and nonnegative".to_owned()); + } + if self.rates.len() > MAX_RATE_ROLES { + return Err(format!("{} rate roles, over 64", self.rates.len())); + } + let mut seen = std::collections::BTreeSet::new(); + for rate in &self.rates { + if !rate.hz.is_finite() || rate.hz < 0.0 { + return Err("rate hz must be finite and nonnegative".to_owned()); + } + if !seen.insert(rate.role_id.clone()) { + return Err(format!("rate role {} appears twice", rate.role_id)); + } + } + if !self.learning.signal.is_finite() { + return Err("learning signal must be finite".to_owned()); + } + Ok(()) + } +} + +// --------------------------------------------------------------------------------------------- +// Agent methods (workers-v1 section 2) + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AgentInitializeParams { + pub agent_id: Id, + pub profile: AssetRef, + pub seed: i32, + pub initial_input: SensoryInput, + pub initial_decision_context: TypedValue, + pub worker_threads: u32, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AgentInitializeResult { + pub agent_id: Id, + pub profile_digest: Digest, + pub tick_duration: RationalNs, + pub warmup_ticks: U64, + pub committed_step: U64, + pub decision_context_digest: Digest, + pub telemetry: AgentTelemetry, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct PrepareParams { + pub agent_id: Id, + pub profile_digest: Digest, + pub interval: RationalNs, + pub decision_context_digest: Digest, + pub pre_step_stimulations: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct PreparedDecision { + pub agent_id: Id, + pub ticks_advanced: U64, + pub brain_ticks: U64, + pub remainder: RationalNs, + pub decision: TypedValue, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct CommitParams { + pub agent_id: Id, + pub prepared_request_id: Id, + pub next_input: SensoryInput, + pub next_decision_context: TypedValue, + pub rewards: Vec, + pub task_stimulations: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AgentCommitResult { + pub agent_id: Id, + pub committed_step: U64, + pub decision_context_digest: Digest, + pub telemetry: AgentTelemetry, +} + +// --------------------------------------------------------------------------------------------- +// Environment methods (workers-v1 section 3) + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ControllerSchema { + pub schema: SchemaRef, + pub buttons: Vec, + pub axes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AxisSchema { + pub id: Id, + pub range: AxisRange, + pub neutral: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AxisRange { + Bipolar, + Unit, +} + +impl AxisRange { + pub fn contains(&self, v: f64) -> bool { + match self { + AxisRange::Bipolar => (-1.0..=1.0).contains(&v), + AxisRange::Unit => (0.0..=1.0).contains(&v), + } + } +} + +impl ControllerSchema { + pub fn validate(&self) -> Result<(), String> { + if self.buttons.len() > 32 { + return Err("at most 32 buttons".to_owned()); + } + if self.axes.len() > 16 { + return Err("at most 16 axes".to_owned()); + } + let mut seen = std::collections::BTreeSet::new(); + for b in &self.buttons { + if !seen.insert(b.clone()) { + return Err(format!("button {b} appears twice")); + } + } + let mut seen = std::collections::BTreeSet::new(); + for a in &self.axes { + if !seen.insert(a.id.clone()) { + return Err(format!("axis {} appears twice", a.id)); + } + if !a.neutral.is_finite() || !a.range.contains(a.neutral) { + return Err(format!("axis {} neutral lies outside its range", a.id)); + } + } + Ok(()) + } + + /// Checks that `control` names every declared button and axis, in descriptor order, with + /// every value inside its range. Out-of-range values are refused, never clamped. + pub fn check(&self, control: &PortControl) -> Result<(), String> { + if control.buttons.len() != self.buttons.len() { + return Err(format!( + "port {} has {} buttons, the schema declares {}", + control.port_id, + control.buttons.len(), + self.buttons.len() + )); + } + for (declared, given) in self.buttons.iter().zip(&control.buttons) { + if *declared != given.id { + return Err(format!( + "port {} button {} is out of descriptor order; expected {declared}", + control.port_id, given.id + )); + } + } + if control.axes.len() != self.axes.len() { + return Err(format!( + "port {} has {} axes, the schema declares {}", + control.port_id, + control.axes.len(), + self.axes.len() + )); + } + for (declared, given) in self.axes.iter().zip(&control.axes) { + if declared.id != given.id { + return Err(format!( + "port {} axis {} is out of descriptor order; expected {}", + control.port_id, given.id, declared.id + )); + } + if !given.value.is_finite() || !declared.range.contains(given.value) { + return Err(format!( + "port {} axis {} value lies outside its declared range", + control.port_id, given.id + )); + } + } + Ok(()) + } + + /// Every button up and every axis at its declared neutral. + pub fn neutral(&self, port_id: &Id) -> PortControl { + PortControl { + port_id: port_id.clone(), + buttons: self + .buttons + .iter() + .map(|id| ButtonState { id: id.clone(), down: false }) + .collect(), + axes: self + .axes + .iter() + .map(|a| AxisValue { id: a.id.clone(), value: a.neutral }) + .collect(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ButtonState { + pub id: Id, + pub down: bool, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AxisValue { + pub id: Id, + pub value: f64, +} + +/// One port's complete controller state. The port is assigned by the coordinator. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct PortControl { + pub port_id: Id, + pub buttons: Vec, + pub axes: Vec, +} + +/// What an executor returns: buttons and axes, with no port assignment. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ControllerIntent { + pub buttons: Vec, + pub axes: Vec, +} + +impl ControllerIntent { + /// Binds this intent to a port, which only the coordinator may do. + pub fn at_port(self, port_id: &Id) -> PortControl { + PortControl { port_id: port_id.clone(), buttons: self.buttons, axes: self.axes } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct PortDescriptor { + pub port_id: Id, + pub controls: ControllerSchema, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Recovery { + ExactCheckpoint, + EpisodeRestart, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Determinism { + FixedBuild, + Unverified, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct EnvironmentDescriptor { + pub backend_digest: Digest, + pub content_digest: Digest, + pub configuration_digest: Digest, + pub step_duration: RationalNs, + pub ports: Vec, + pub inspection_schema: SchemaRef, + pub views: Vec, + pub audio: Vec, + pub recovery: Recovery, + pub determinism: Determinism, +} + +impl EnvironmentDescriptor { + pub fn validate(&self) -> Result<(), String> { + self.step_duration.validate()?; + if !self.step_duration.is_positive() { + return Err("stepDuration must be positive".to_owned()); + } + if self.ports.len() > MAX_PORTS as usize { + return Err(format!("{} ports, over the limit of {MAX_PORTS}", self.ports.len())); + } + let mut seen = std::collections::BTreeSet::new(); + for port in &self.ports { + if !seen.insert(port.port_id.clone()) { + return Err(format!("port {} appears twice", port.port_id)); + } + port.controls.validate()?; + } + for view in &self.views { + view.validate()?; + } + Ok(()) + } + + pub fn port(&self, port_id: &Id) -> Option<&PortDescriptor> { + self.ports.iter().find(|p| p.port_id == *port_id) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct PortBinding { + pub port_id: Id, + pub agent_id: Id, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct EnvironmentInitializeParams { + pub backend_config: AssetRef, + pub task_config: AssetRef, + pub episode_id: Id, + pub port_bindings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct EnvironmentInitializeResult { + pub descriptor: EnvironmentDescriptor, + pub observation: WorldObservation, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct WorldObservation { + pub boundary: U64, + pub world_time: RationalNs, + pub engine_frame: Option, + pub sensory_views: Vec, + pub inspection: TypedValue, + pub broadcast_views: Vec, + pub audio: Vec, +} + +impl WorldObservation { + pub fn validate(&self, descriptor: &EnvironmentDescriptor) -> Result<(), String> { + self.world_time.validate()?; + self.inspection.validate()?; + if let Some(frame) = &self.engine_frame + && frame.chars().count() > 64 + { + return Err("engineFrame is at most 64 characters".to_owned()); + } + for view in &descriptor.views { + let got = self + .sensory_views + .iter() + .find(|v| v.view_id == view.view_id) + .ok_or_else(|| format!("required sensory view {} is missing", view.view_id))?; + let want = view.required_produced_step(self.boundary.0); + if got.produced_step.0 != want { + return Err(format!( + "view {} was produced at boundary {} but the declared delay requires {want}", + view.view_id, got.produced_step + )); + } + if got.pixels.0.byte_length != view.byte_length() { + return Err(format!( + "view {} is {} bytes; its descriptor says {}", + view.view_id, + got.pixels.0.byte_length, + view.byte_length() + )); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct AdvanceParams { + pub batch_id: Id, + pub controls: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct StepResult { + pub batch_id: Id, + pub applied_from_step: U64, + pub next_step: U64, + pub applied_controls_digest: Digest, + pub observation: WorldObservation, +} + +/// The digest of a validated, canonical control batch, in descriptor port order. +pub fn controls_digest(controls: &[PortControl]) -> Digest { + canonical_digest(&serde_json::to_value(controls).expect("controls serialize")) +} + +// --------------------------------------------------------------------------------------------- +// Task outputs (workers-v1 section 4) + +/// `{id, kindId, sourceStep, agentId, payload}`, in task-defined deterministic order. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct TaskEvent { + pub id: Id, + pub kind_id: Id, + pub source_step: U64, + pub agent_id: Option, + pub payload: TypedValue, +} + +/// A deterministic event id from epoch, source step, rule and ordinal. +pub fn event_id(epoch: &Id, source_step: u64, rule: &str, ordinal: u32) -> Id { + let digest = Digest::of(format!("{epoch}\n{source_step}\n{rule}\n{ordinal}\n").as_bytes()); + Id::parse(&format!("ev-{}", &digest.as_str()[..16])).expect("ev- plus hex is an Id") +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct EpisodeRequest { + pub kind: EpisodeKind, + pub reason: Id, + pub outcome: TypedValue, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum EpisodeKind { + Terminal, +} + +/// Everything the task routes to one agent for this transition. Always explicit, even empty. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AgentOutcome { + pub rewards: Vec, + pub stimulations: Vec, +} diff --git a/services/flysim/crates/fly-session/src/fly_session_types/mod.rs b/services/flysim/crates/fly-session/src/fly_session_types/mod.rs new file mode 100644 index 0000000..d8aff37 --- /dev/null +++ b/services/flysim/crates/fly-session/src/fly_session_types/mod.rs @@ -0,0 +1,28 @@ +//! `fly-session-types` as a local module. +//! +//! CONTRACT-01 owns these definitions -- the scalar types, the method payloads, their +//! validation, the canonical digests and the trace format -- as the crate +//! `services/flysim/crates/fly-session-types`. Until that crate exists this module holds the +//! minimum this slice needs, under the same names and field shapes, so the swap is a change of +//! `use` lines in `lib.rs` and nothing else. + +pub mod canonical; +pub mod methods; +pub mod scalars; +pub mod trace; + +pub use canonical::{canonical_digest, canonical_json}; +pub use methods::*; +pub use scalars::*; +pub use trace::*; + +/// The identity of the contract revision these types implement. +/// +/// A worker reports it in `Worker.Hello`; a coordinator refuses a worker whose value differs, +/// which is what stops a session from mixing two payload revisions. +pub const CONTRACT: &str = "fly-session-types/ipc-v1+step-v1+workers-v1 draft-2 2026-09-18"; + +/// The SHA-256 of [`CONTRACT`]. +pub fn contract_digest() -> Digest { + Digest::of(CONTRACT.as_bytes()) +} diff --git a/services/flysim/crates/fly-session/src/fly_session_types/scalars.rs b/services/flysim/crates/fly-session/src/fly_session_types/scalars.rs new file mode 100644 index 0000000..30d9319 --- /dev/null +++ b/services/flysim/crates/fly-session/src/fly_session_types/scalars.rs @@ -0,0 +1,433 @@ +//! The scalar encodings of `ipc-v1` section 2: `Id`, `U64`, `Digest`, `Scope`, `RationalNs`, +//! `SchemaRef` and `TypedValue`. +//! +//! Every type parses strictly. A `U64` is a canonical decimal string, never a JSON number; a +//! `RationalNs` is reduced with a positive denominator; an `Id` matches +//! `^[a-z0-9][a-z0-9._-]{0,63}$`. Nothing here knows about a method, a phase or a bus. + +use std::fmt; + +use serde::de::Error as _; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +/// `^[a-z0-9][a-z0-9._-]{0,63}$`. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Id(String); + +impl Id { + /// Parses an `Id`, or reports why the string is not one. + pub fn parse(s: &str) -> Result { + if s.is_empty() || s.len() > 64 { + return Err(format!("id must be 1..=64 characters, got {}", s.len())); + } + let first = s.as_bytes()[0]; + if !first.is_ascii_lowercase() && !first.is_ascii_digit() { + return Err("id must start with a lowercase letter or a digit".to_owned()); + } + for b in s.bytes() { + let ok = b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'.' || b == b'_' + || b == b'-'; + if !ok { + return Err("id may only contain [a-z0-9._-]".to_owned()); + } + } + Ok(Id(s.to_owned())) + } + + /// Parses an `Id` from a trusted literal; panics on a malformed one. + pub fn lit(s: &str) -> Id { + Id::parse(s).expect("malformed Id literal") + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for Id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.0) + } +} + +impl fmt::Display for Id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Serialize for Id { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for Id { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + Id::parse(&s).map_err(D::Error::custom) + } +} + +/// `"0"` or `[1-9][0-9]*`, at most `u64::MAX`, carried as a decimal string on the wire. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct U64(pub u64); + +impl U64 { + pub fn get(self) -> u64 { + self.0 + } + + /// Parses the canonical decimal encoding. Leading zeros and signs are refused. + pub fn parse(s: &str) -> Result { + if s.is_empty() { + return Err("u64 must not be empty".to_owned()); + } + if s != "0" && s.starts_with('0') { + return Err(format!("u64 {s:?} has a leading zero")); + } + if !s.bytes().all(|b| b.is_ascii_digit()) { + return Err(format!("u64 {s:?} is not decimal digits")); + } + s.parse::().map(U64).map_err(|_| format!("u64 {s:?} overflows")) + } +} + +impl From for U64 { + fn from(v: u64) -> U64 { + U64(v) + } +} + +impl fmt::Debug for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl fmt::Display for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl Serialize for U64 { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.0.to_string()) + } +} + +impl<'de> Deserialize<'de> for U64 { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + U64::parse(&s).map_err(D::Error::custom) + } +} + +/// 64 lowercase hexadecimal digits: a SHA-256. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Digest(String); + +impl Digest { + pub fn parse(s: &str) -> Result { + if s.len() != 64 || !s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err("digest must be 64 lowercase hexadecimal digits".to_owned()); + } + Ok(Digest(s.to_owned())) + } + + /// The SHA-256 of `bytes`, hex-encoded. + pub fn of(bytes: &[u8]) -> Digest { + use sha2::{Digest as _, Sha256}; + let mut h = Sha256::new(); + h.update(bytes); + let out = h.finalize(); + let mut s = String::with_capacity(64); + for b in out { + s.push(char::from_digit((b >> 4) as u32, 16).unwrap()); + s.push(char::from_digit((b & 0x0f) as u32, 16).unwrap()); + } + Digest(s) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.0) + } +} + +impl fmt::Display for Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Serialize for Digest { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for Digest { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + Digest::parse(&s).map_err(D::Error::custom) + } +} + +/// The simulation timeline coordinate: which session, which epoch, which step. +/// +/// Epoch protects simulation order. It is not a bus route incarnation or a store id. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct Scope { + pub session_id: Id, + pub epoch: Id, + pub step: U64, +} + +impl Scope { + pub fn new(session_id: &Id, epoch: &Id, step: u64) -> Scope { + Scope { session_id: session_id.clone(), epoch: epoch.clone(), step: U64(step) } + } + + /// The same scope at `step + 1`. + pub fn next(&self) -> Scope { + Scope { step: U64(self.step.0 + 1), ..self.clone() } + } +} + +/// A reduced, positive-denominator duration in nanoseconds. Zero is `0/1`. +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct RationalNs { + pub numerator: U64, + pub denominator: U64, +} + +impl RationalNs { + /// Reduces `numerator / denominator`; the denominator must be positive. + pub fn new(numerator: u64, denominator: u64) -> Result { + if denominator == 0 { + return Err("rational denominator must be positive".to_owned()); + } + let g = gcd(numerator, denominator); + let g = if g == 0 { 1 } else { g }; + Ok(RationalNs { numerator: U64(numerator / g), denominator: U64(denominator / g) }) + } + + pub fn zero() -> RationalNs { + RationalNs { numerator: U64(0), denominator: U64(1) } + } + + /// `hz` steps per second as an exact nanosecond duration. + pub fn from_hz(hz: u64) -> Result { + RationalNs::new(1_000_000_000, hz) + } + + pub fn from_millis(ms: u64) -> Result { + ms.checked_mul(1_000_000) + .ok_or_else(|| "millisecond duration overflows".to_owned()) + .and_then(|ns| RationalNs::new(ns, 1)) + } + + pub fn is_zero(&self) -> bool { + self.numerator.0 == 0 + } + + pub fn is_positive(&self) -> bool { + self.numerator.0 > 0 + } + + /// Rejects an unreduced, zero-denominator or non-canonical value that parsed as JSON. + pub fn validate(&self) -> Result<(), String> { + if self.denominator.0 == 0 { + return Err("rational denominator must be positive".to_owned()); + } + if self.numerator.0 == 0 { + if self.denominator.0 != 1 { + return Err("rational zero must be encoded 0/1".to_owned()); + } + return Ok(()); + } + if gcd(self.numerator.0, self.denominator.0) != 1 { + return Err("rational must be reduced".to_owned()); + } + Ok(()) + } + + /// Checked addition, reduced. + pub fn checked_add(&self, other: &RationalNs) -> Result { + let l = self.numerator.0.checked_mul(other.denominator.0); + let r = other.numerator.0.checked_mul(self.denominator.0); + let d = self.denominator.0.checked_mul(other.denominator.0); + match (l, r, d) { + (Some(l), Some(r), Some(d)) => { + let n = l.checked_add(r).ok_or_else(|| "rational add overflows".to_owned())?; + RationalNs::new(n, d) + } + _ => Err("rational add overflows".to_owned()), + } + } + + /// Checked subtraction; refuses a negative result. + pub fn checked_sub(&self, other: &RationalNs) -> Result { + let l = self + .numerator + .0 + .checked_mul(other.denominator.0) + .ok_or_else(|| "rational sub overflows".to_owned())?; + let r = other + .numerator + .0 + .checked_mul(self.denominator.0) + .ok_or_else(|| "rational sub overflows".to_owned())?; + let d = self + .denominator + .0 + .checked_mul(other.denominator.0) + .ok_or_else(|| "rational sub overflows".to_owned())?; + let n = l.checked_sub(r).ok_or_else(|| "rational sub would be negative".to_owned())?; + RationalNs::new(n, d) + } + + /// Checked multiplication by a whole count. + pub fn checked_mul_u64(&self, k: u64) -> Result { + let n = self + .numerator + .0 + .checked_mul(k) + .ok_or_else(|| "rational scale overflows".to_owned())?; + RationalNs::new(n, self.denominator.0) + } + + /// `floor(self / other)`; `other` must be positive. + pub fn checked_div_floor(&self, other: &RationalNs) -> Result { + if other.numerator.0 == 0 { + return Err("cannot divide by a zero duration".to_owned()); + } + let l = self + .numerator + .0 + .checked_mul(other.denominator.0) + .ok_or_else(|| "rational divide overflows".to_owned())?; + let r = other + .numerator + .0 + .checked_mul(self.denominator.0) + .ok_or_else(|| "rational divide overflows".to_owned())?; + Ok(l / r) + } + + pub fn cmp_value(&self, other: &RationalNs) -> std::cmp::Ordering { + let l = u128::from(self.numerator.0) * u128::from(other.denominator.0); + let r = u128::from(other.numerator.0) * u128::from(self.denominator.0); + l.cmp(&r) + } +} + +impl fmt::Debug for RationalNs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}ns", self.numerator.0, self.denominator.0) + } +} + +fn gcd(mut a: u64, mut b: u64) -> u64 { + while b != 0 { + let t = a % b; + a = b; + b = t; + } + a +} + +/// Identity of a registered payload schema: what shape, which revision, which definition. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SchemaRef { + pub id: Id, + pub version: u16, + pub digest: Digest, +} + +impl SchemaRef { + /// A schema reference whose digest is derived from its own name and version, so the + /// synthetic composition has stable identities without a schema registry file. + pub fn synthetic(id: &str, version: u16) -> SchemaRef { + let id = Id::lit(id); + let digest = Digest::of(format!("fly-session-schema-v1\n{id}\n{version}\n").as_bytes()); + SchemaRef { id, version, digest } + } + + pub fn validate(&self) -> Result<(), String> { + if self.version == 0 { + return Err("schema version must be 1..=65535".to_owned()); + } + Ok(()) + } +} + +/// A schema-tagged JSON object. The canonical encoding of `value` is capped at 32 KiB. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TypedValue { + pub schema: SchemaRef, + pub value: serde_json::Map, +} + +/// `ipc-v1` section 2: a `TypedValue`'s canonical JSON is at most 32 KiB. +pub const MAX_TYPED_VALUE_BYTES: usize = 32 * 1024; + +impl TypedValue { + pub fn new(schema: SchemaRef, value: serde_json::Map) -> TypedValue { + TypedValue { schema, value } + } + + pub fn validate(&self) -> Result<(), String> { + self.schema.validate()?; + let canonical = crate::fly_session_types::canonical::canonical_json( + &serde_json::Value::Object(self.value.clone()), + ); + if canonical.len() > MAX_TYPED_VALUE_BYTES { + return Err(format!( + "typed value is {} canonical bytes, over the 32 KiB limit", + canonical.len() + )); + } + Ok(()) + } + + /// The canonical digest of the whole typed value, schema identity included. + pub fn digest(&self) -> Digest { + crate::fly_session_types::canonical::canonical_digest( + &serde_json::to_value(self).expect("TypedValue serializes"), + ) + } + + pub fn number(&self, key: &str) -> Result { + self.value + .get(key) + .and_then(serde_json::Value::as_f64) + .filter(|v| v.is_finite()) + .ok_or_else(|| format!("typed value has no finite number {key:?}")) + } + + pub fn integer(&self, key: &str) -> Result { + self.value + .get(key) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| format!("typed value has no integer {key:?}")) + } + + pub fn boolean(&self, key: &str) -> Result { + self.value + .get(key) + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| format!("typed value has no boolean {key:?}")) + } +} diff --git a/services/flysim/crates/fly-session/src/fly_session_types/trace.rs b/services/flysim/crates/fly-session/src/fly_session_types/trace.rs new file mode 100644 index 0000000..f6a38a7 --- /dev/null +++ b/services/flysim/crates/fly-session/src/fly_session_types/trace.rs @@ -0,0 +1,121 @@ +//! The trace format of `step-v1` section 8, split into behaviour and operational metadata. +//! +//! Behaviour is what two runs of the same composition must agree on whatever order their +//! messages took. Operational metadata -- request ids, batch ids, bus call ids, wall time -- +//! is recorded but excluded from that comparison, which is exactly what section 8 asks for. + +use serde::Serialize; + +use super::methods::{Reward, Stimulus}; +use super::scalars::{Digest, Id, RationalNs, Scope, U64}; + +/// One agent's contribution to one transition. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentTransitionTrace { + pub agent_id: Id, + pub profile_digest: Digest, + pub ticks_advanced: U64, + pub brain_ticks: U64, + pub remainder: RationalNs, + pub decision_digest: Digest, + pub committed_step: U64, + pub rewards: Vec, + pub stimulations: Vec, + /// Operational: the `req-` of this agent's Prepare. + #[serde(skip_serializing)] + pub prepare_request_id: Id, + /// Operational: the `req-` of this agent's Commit. + #[serde(skip_serializing)] + pub commit_request_id: Id, +} + +/// The producing boundary of one observation view. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ViewProvenance { + pub view_id: Id, + pub produced_step: U64, +} + +/// One complete transition `k -> k+1`. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TransitionTrace { + pub scope: Scope, + /// Agents in sorted agent-id order, never completion order. + pub agents: Vec, + pub controls_digest: Digest, + pub acknowledged_boundary: U64, + pub observation_boundaries: Vec, + pub task_event_ids: Vec, + pub published_boundary: U64, + /// Operational: the complete batch's id. + #[serde(skip_serializing)] + pub batch_id: Id, + /// Operational: the `req-` of the single Environment.Advance. + #[serde(skip_serializing)] + pub advance_request_id: Id, +} + +impl TransitionTrace { + /// The canonical behaviour encoding: everything a reordered run must reproduce exactly. + pub fn behavior(&self) -> String { + let value = serde_json::to_value(self).expect("a trace serializes"); + super::canonical::canonical_json(&value) + } + + /// Operational identities, for a report rather than a comparison. + pub fn operational(&self) -> Vec<(String, String)> { + let mut out = vec![ + ("batchId".to_owned(), self.batch_id.to_string()), + ("advanceRequestId".to_owned(), self.advance_request_id.to_string()), + ]; + for agent in &self.agents { + out.push(( + format!("prepareRequestId.{}", agent.agent_id), + agent.prepare_request_id.to_string(), + )); + out.push(( + format!("commitRequestId.{}", agent.agent_id), + agent.commit_request_id.to_string(), + )); + } + out + } +} + +/// One session phase transition, recorded whether or not it ends a step. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PhaseTransition { + pub from: String, + pub to: String, +} + +/// Everything a run recorded: its phase transitions and its completed transitions. +#[derive(Clone, Debug, Default)] +pub struct TraceLog { + pub phases: Vec, + pub transitions: Vec, +} + +impl TraceLog { + pub fn phase(&mut self, from: String, to: String) { + self.phases.push(PhaseTransition { from, to }); + } + + pub fn transition(&mut self, trace: TransitionTrace) { + self.transitions.push(trace); + } + + /// The behaviour of every transition, in order, with operational metadata excluded. + pub fn behavior(&self) -> Vec { + self.transitions.iter().map(TransitionTrace::behavior).collect() + } + + /// The phase path, as `from -> to` strings. + pub fn phase_path(&self) -> Vec { + self.phases.iter().map(|p| format!("{} -> {}", p.from, p.to)).collect() + } +} diff --git a/services/flysim/crates/fly-session/src/harness.rs b/services/flysim/crates/fly-session/src/harness.rs new file mode 100644 index 0000000..ade7549 --- /dev/null +++ b/services/flysim/crates/fly-session/src/harness.rs @@ -0,0 +1,379 @@ +//! The runnable synthetic composition: one router, two fake agents, one counter arena and one +//! coordinator, over either transport. +//! +//! All participants use router semantics even when colocated, so the in-memory and +//! Unix-socket runs exercise the same code. The caller owns the store root directory, which +//! keeps this module free of a temporary-directory dependency. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +use flybus::{ + Client, ClientConfig, Grants, Pattern, Policy, Router, RouterConfig, ServiceConfig, Transport, + UnixListenerHandle, +}; + +use crate::agent::{AgentConfig, AgentFaults, FakeAgentWorker, synthetic_profile}; +use crate::coordinator::{AgentSlot, Coordinator}; +use crate::environment::{CounterEnvironment, EnvironmentConfig, EnvironmentFaults}; +use crate::rpc::WorkerRef; +use crate::task::{ActionExecutor, CounterTask, IdentityExecutor, Terminal}; +use crate::types::{Id, RationalNs}; +use crate::worker::{StatusCell, WorkerHandle, serve}; + +/// Which transport the session runs over. Both must produce the same behaviour. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Via { + Memory, + Unix, +} + +/// One agent in the composition. +#[derive(Clone, Debug)] +pub struct AgentSpec { + pub agent_id: Id, + pub port_id: Id, + /// An explicit seed. The first synthetic composition supports hand-selected seeds; the + /// derivation algorithm is specified before the real agent slice. + pub seed: i32, + pub faults: AgentFaults, +} + +/// The composition the harness builds. +#[derive(Clone, Debug)] +pub struct HarnessConfig { + pub session_id: Id, + pub epoch: Id, + pub episode_id: Id, + pub agents: Vec, + /// The world's cadence. 60 Hz with a 1 ms tick is the `step-v1` section 5 example. + pub step_hz: u64, + pub tick_ms: u64, + pub warmup_ticks: u64, + pub terminal: Terminal, + pub environment_faults: EnvironmentFaults, +} + +impl Default for HarnessConfig { + fn default() -> HarnessConfig { + HarnessConfig { + session_id: Id::lit("demo"), + epoch: Id::lit("e1"), + episode_id: Id::lit("ep1"), + agents: vec![ + AgentSpec { + agent_id: Id::lit("fly-a"), + port_id: Id::lit("p1"), + seed: 7, + faults: AgentFaults::default(), + }, + AgentSpec { + agent_id: Id::lit("fly-b"), + port_id: Id::lit("p2"), + seed: 11, + faults: AgentFaults::default(), + }, + ], + step_hz: 60, + tick_ms: 1, + warmup_ticks: 10, + terminal: Terminal::Never, + environment_faults: EnvironmentFaults::default(), + } + } +} + +const ENV_SERVICE: &str = "env.arena"; +const ENV_CLIENT: &str = "environment"; +const ENV_WORKER: &str = "arena"; + +fn agent_service(agent_id: &Id) -> String { + format!("agent.{agent_id}") +} + +fn agent_client(agent_id: &Id) -> String { + format!("worker-{agent_id}") +} + +fn grants(f: impl FnOnce(&mut Grants)) -> Grants { + let mut g = Grants::default(); + f(&mut g); + g +} + +/// Makes a connection for one launcher-bound participant, over the chosen transport. +struct Connector { + router: Router, + via: Via, + store_root: PathBuf, + sockets: PathBuf, + next_socket: AtomicU64, + listeners: Mutex>, +} + +impl Connector { + async fn client(&self, id: &str) -> Result { + let transport = match self.via { + Via::Memory => self.router.connect_in_memory_as(id), + Via::Unix => { + let n = self.next_socket.fetch_add(1, Ordering::Relaxed); + let path = self.sockets.join(format!("{id}-{n}.sock")); + let listener = self.router.listen_unix_as(&path, id).await.map_err(|e| { + flybus::BusError::new(flybus::ErrorCode::RouterLost, format!("listen: {e}")) + })?; + let transport = Transport::unix(&path).await.map_err(|e| { + flybus::BusError::new(flybus::ErrorCode::RouterLost, format!("connect: {e}")) + })?; + self.listeners.lock().expect("not poisoned").push(listener); + transport + } + }; + Client::connect(transport, ClientConfig::new(id, &self.store_root)).await + } +} + +/// What a restarted worker looks like from the outside: a new registration and a new +/// incarnation, both different from the ones the coordinator pinned. +#[derive(Clone, Debug)] +pub struct Restarted { + pub service: String, + pub service_incarnation: String, + pub incarnation_id: Id, +} + +/// A running synthetic session. +pub struct SessionHarness { + pub coordinator: Coordinator, + pub environment: WorkerHandle, + pub agents: BTreeMap, + pub config: HarnessConfig, + pub via: Via, + connector: Connector, + observers: Mutex>, +} + +impl SessionHarness { + /// Builds the router, the workers and the coordinator. Nothing has stepped yet. + pub async fn start( + via: Via, + root: &Path, + config: HarnessConfig, + ) -> Result { + let store_root = root.join("store"); + let sockets = root.join("sockets"); + std::fs::create_dir_all(&sockets).expect("the caller owns a writable directory"); + + let mut policy = Policy::closed() + .client( + "coordinator", + grants(|g| { + g.call = vec![Pattern::prefix("agent."), Pattern::prefix("env.")]; + g.publish = vec![Pattern::prefix("session.")]; + g.manage_topics = vec![Pattern::prefix("session.")]; + }), + ) + .client(ENV_CLIENT, grants(|g| g.register = vec![Pattern::exact(ENV_SERVICE)])) + .client("observer", grants(|g| g.subscribe = vec![Pattern::prefix("session.")])); + for spec in &config.agents { + let service = agent_service(&spec.agent_id); + policy = policy.client( + &agent_client(&spec.agent_id), + grants(|g| g.register = vec![Pattern::exact(&service)]), + ); + // A replacement worker connects under its own client id, so a restart is visibly a + // new participant rather than a silent reattachment to the active epoch. + policy = policy.client( + &format!("{}-r2", agent_client(&spec.agent_id)), + grants(|g| g.register = vec![Pattern::exact(&service)]), + ); + } + let mut router_config = RouterConfig::new(&store_root); + router_config.policy = policy; + let router = Router::new(router_config).map_err(|e| { + flybus::BusError::new(flybus::ErrorCode::StoreFailure, format!("router: {e}")) + })?; + let connector = Connector { + router, + via, + store_root, + sockets, + next_socket: AtomicU64::new(0), + listeners: Mutex::new(Vec::new()), + }; + + let step_duration = RationalNs::from_hz(config.step_hz).expect("a positive cadence"); + let tick_duration = RationalNs::from_millis(config.tick_ms).expect("a positive tick"); + + // The environment first: it owns the world and the descriptor. + let env_client = connector.client(ENV_CLIENT).await?; + let env_service = env_client.register(ENV_SERVICE, ServiceConfig::default()).await?; + let env_incarnation = env_service.incarnation().to_owned(); + let environment = serve( + env_client, + env_service, + CounterEnvironment::new(EnvironmentConfig { + session_id: config.session_id.clone(), + worker_id: Id::lit(ENV_WORKER), + incarnation_id: Id::lit("arena-inc-1"), + step_duration, + ports: config.agents.iter().map(|a| a.port_id.clone()).collect(), + faults: config.environment_faults.clone(), + }), + ); + + let mut slots = Vec::new(); + let mut agents = BTreeMap::new(); + for spec in &config.agents { + let service_name = agent_service(&spec.agent_id); + let client = connector.client(&agent_client(&spec.agent_id)).await?; + let service = client.register(&service_name, ServiceConfig::default()).await?; + let incarnation = service.incarnation().to_owned(); + let handle = serve( + client, + service, + FakeAgentWorker::new(AgentConfig { + session_id: config.session_id.clone(), + agent_id: spec.agent_id.clone(), + incarnation_id: Id::parse(&format!("{}-inc-1", spec.agent_id)) + .expect("an agent id plus a suffix is an Id"), + tick_duration, + warmup_ticks: config.warmup_ticks, + faults: spec.faults.clone(), + }), + ); + slots.push(AgentSlot::new( + WorkerRef::new(&service_name, &incarnation, &spec.agent_id), + spec.agent_id.clone(), + spec.port_id.clone(), + synthetic_profile(&spec.agent_id, &tick_duration, config.warmup_ticks), + spec.seed, + )); + agents.insert(spec.agent_id.clone(), handle); + } + + let coordinator_client = connector.client("coordinator").await?; + let executors: BTreeMap> = config + .agents + .iter() + .map(|spec| { + (spec.agent_id.clone(), Box::new(IdentityExecutor) as Box) + }) + .collect(); + let coordinator = Coordinator::new( + coordinator_client, + config.session_id.clone(), + config.epoch.clone(), + config.episode_id.clone(), + WorkerRef::new(ENV_SERVICE, &env_incarnation, &Id::lit(ENV_WORKER)), + slots, + Box::new(CounterTask::new(&config.epoch, config.terminal)), + executors, + ); + + Ok(SessionHarness { + coordinator, + environment, + agents, + config, + via, + connector, + observers: Mutex::new(Vec::new()), + }) + } + + pub fn router(&self) -> &Router { + &self.connector.router + } + + /// A client for `id`, connected the same way every participant is. + pub async fn client(&self, id: &str) -> Result { + self.connector.client(id).await + } + + /// An extra subscriber, for a test that watches the published boundaries. + pub async fn observer(&self) -> Result { + let client = self.connector.client("observer").await?; + self.observers.lock().expect("not poisoned").push(client.clone()); + Ok(client) + } + + /// Replaces one agent's worker with a fresh incarnation, as a restore would. + /// + /// The coordinator still pins the old registration, so its next call to that agent fails + /// rather than silently reaching another brain. + pub async fn restart_agent(&mut self, agent_id: &Id) -> Result { + let tick_duration = RationalNs::from_millis(self.config.tick_ms).expect("a positive tick"); + if let Some(old) = self.agents.remove(agent_id) { + old.stop().await; + } + let service_name = agent_service(agent_id); + let client = self.connector.client(&format!("{}-r2", agent_client(agent_id))).await?; + let service = loop { + match client.register(&service_name, ServiceConfig::default()).await { + Ok(service) => break service, + Err(e) if e.code == flybus::ErrorCode::Conflict => { + // The old registration is released when its connection finishes closing. + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + Err(e) => return Err(e), + } + }; + let spec = self + .config + .agents + .iter() + .find(|spec| spec.agent_id == *agent_id) + .expect("a configured agent") + .clone(); + let incarnation_id = + Id::parse(&format!("{agent_id}-inc-2")).expect("an agent id plus a suffix is an Id"); + let restarted = Restarted { + service: service_name, + service_incarnation: service.incarnation().to_owned(), + incarnation_id: incarnation_id.clone(), + }; + let handle = serve( + client, + service, + FakeAgentWorker::new(AgentConfig { + session_id: self.config.session_id.clone(), + agent_id: agent_id.clone(), + incarnation_id, + tick_duration, + warmup_ticks: self.config.warmup_ticks, + faults: spec.faults, + }), + ); + self.agents.insert(agent_id.clone(), handle); + Ok(restarted) + } + + /// The agent worker's progress counter, which is its fake model's mutation count. + pub fn agent_mutations(&self, agent_id: &Id) -> u64 { + self.agents.get(agent_id).map(WorkerHandle::progress_counter).unwrap_or_default() + } + + pub fn environment_mutations(&self) -> u64 { + self.environment.progress_counter() + } + + pub fn agent_status(&self, agent_id: &Id) -> Option { + self.agents.get(agent_id).map(|handle| handle.status.clone()) + } + + /// Stops every worker and closes the router. + pub async fn shutdown(self) { + let SessionHarness { coordinator, environment, agents, connector, observers, .. } = self; + drop(coordinator); + environment.stop().await; + for (_, handle) in agents { + handle.stop().await; + } + for observer in observers.into_inner().expect("not poisoned") { + observer.close().await; + } + connector.router.shutdown(); + } +} diff --git a/services/flysim/crates/fly-session/src/lib.rs b/services/flysim/crates/fly-session/src/lib.rs new file mode 100644 index 0000000..8f51492 --- /dev/null +++ b/services/flysim/crates/fly-session/src/lib.rs @@ -0,0 +1,38 @@ +//! `fly-session`: the lockstep session coordinator and a synthetic composition over Flybus. +//! +//! This crate is the SESSION-01 slice of the session-framework implementation guide: the +//! sequential transaction of [`step-v1`], driven over the [`flybus`] router, with small fake +//! workers standing in for a brain and an emulator. +//! +//! ```text +//! Coordinator ── Agent.Prepare ──> agent workers (fake model + fixed readout stub) +//! ── Environment.Advance ──> environment (a counter arena, no emulator) +//! ── task.evaluate_transition (once) +//! ── Agent.Commit ──> agent workers +//! ── committed snapshot ──> session..snapshots +//! ``` +//! +//! Every arrow is a Flybus RPC to an incarnation-pinned service, with domain request ids and +//! the result caches of `ipc-v1` section 5 in front of every mutation. Nothing here contains a +//! public controller API, an implicit best-effort retry, a real emulator or a real brain. +//! +//! [`step-v1`]: https://example.invalid/step-v1 + +pub mod agent; +pub mod clock; +pub mod coordinator; +pub mod dedup; +pub mod environment; +pub mod harness; +pub mod phase; +pub mod rpc; +pub mod task; +pub mod worker; + +// CONTRACT-01 owns the domain types. Until its crate exists they live in this module under +// the same names; swapping it for the crate is a change to these two lines. +pub mod fly_session_types; +pub use fly_session_types as types; + +pub use coordinator::{Coordinator, DispatchOrder, Injections, SessionFailure, StepReport}; +pub use phase::{Phase, PhaseMachine}; diff --git a/services/flysim/crates/fly-session/src/phase.rs b/services/flysim/crates/fly-session/src/phase.rs new file mode 100644 index 0000000..f528112 --- /dev/null +++ b/services/flysim/crates/fly-session/src/phase.rs @@ -0,0 +1,223 @@ +//! The session state machine of `step-v1` section 2, as an explicit edge table. +//! +//! ```text +//! Starting -> Ready(k) -> Preparing(k) -> Applying(k) -> Observing(k+1) +//! ^ | +//! +---------------- Ready(k+1) <- Committing(k) +//! +//! Ready(k) -> Paused(k) -> Ready(k) +//! Ready(k) / Paused(k) -> Capturing(k) -> same boundary +//! any unresolved partial failure -> Failed -> Restoring(new epoch) -> Paused(k) +//! terminal episode -> Paused(k) -> Resetting(new epoch) -> Ready(0) +//! ``` +//! +//! A transition the table does not list is a bug, not a recoverable condition, so it returns +//! INVALID_PHASE rather than being silently applied. + +use crate::types::{DomainError, ErrorCode}; + +/// Where the session is. The number is the committed boundary the phase belongs to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Phase { + Starting, + Ready(u64), + Preparing(u64), + Applying(u64), + /// `Observing(k+1)`: the world has reached `k+1` but nothing is committed yet. + Observing(u64), + /// `Committing(k)`: completing the transition `k -> k+1`. + Committing(u64), + Paused(u64), + Capturing(u64), + Failed, + /// `Restoring(k)`: installing a coherent checkpoint of boundary `k` under a new epoch. + Restoring(u64), + /// `Resetting(k)`: leaving boundary `k` for a new epoch and episode at step 0. + Resetting(u64), +} + +impl Phase { + pub fn label(&self) -> String { + match self { + Phase::Starting => "Starting".to_owned(), + Phase::Ready(k) => format!("Ready({k})"), + Phase::Preparing(k) => format!("Preparing({k})"), + Phase::Applying(k) => format!("Applying({k})"), + Phase::Observing(k) => format!("Observing({k})"), + Phase::Committing(k) => format!("Committing({k})"), + Phase::Paused(k) => format!("Paused({k})"), + Phase::Capturing(k) => format!("Capturing({k})"), + Phase::Failed => "Failed".to_owned(), + Phase::Restoring(k) => format!("Restoring({k})"), + Phase::Resetting(k) => format!("Resetting({k})"), + } + } + + /// The committed boundary, where the phase has one. `Observing(k+1)` does not: the world + /// has moved but nothing is committed, so the committed boundary is still `k`. + pub fn committed_boundary(&self) -> Option { + match self { + Phase::Ready(k) | Phase::Paused(k) | Phase::Capturing(k) => Some(*k), + _ => None, + } + } + + /// Only a committed boundary is eligible for a checkpoint or a normal pause. + pub fn is_committed_boundary(&self) -> bool { + matches!(self, Phase::Ready(_) | Phase::Paused(_)) + } +} + +/// The phase, plus the edge check that guards every change to it. +#[derive(Clone, Debug)] +pub struct PhaseMachine { + phase: Phase, + /// Where a `Capturing(k)` must return to. + capture_origin: Option, +} + +impl Default for PhaseMachine { + fn default() -> PhaseMachine { + PhaseMachine::new() + } +} + +impl PhaseMachine { + pub fn new() -> PhaseMachine { + PhaseMachine { phase: Phase::Starting, capture_origin: None } + } + + pub fn phase(&self) -> Phase { + self.phase + } + + /// True when `next` is an edge of the section 2 machine. + pub fn allows(&self, next: Phase) -> bool { + use Phase::*; + // Any unresolved partial failure fails the epoch, from wherever the session was. + if next == Failed { + return self.phase != Failed; + } + match (self.phase, next) { + (Starting, Ready(0)) => true, + (Ready(k), Preparing(j)) => k == j, + (Preparing(k), Applying(j)) => k == j, + (Applying(k), Observing(j)) => j == k + 1, + (Observing(j), Committing(k)) => j == k + 1, + (Committing(k), Ready(j)) => j == k + 1, + (Ready(k), Paused(j)) => k == j, + (Paused(k), Ready(j)) => k == j, + (Ready(k), Capturing(j)) | (Paused(k), Capturing(j)) => k == j, + // Capturing returns to the boundary it came from, and only to that one. + (Capturing(k), Ready(j)) => k == j && self.capture_origin == Some(Ready(k)), + (Capturing(k), Paused(j)) => k == j && self.capture_origin == Some(Paused(k)), + (Failed, Restoring(_)) => true, + (Restoring(k), Paused(j)) => k == j, + (Paused(k), Resetting(j)) => k == j, + (Resetting(_), Ready(0)) => true, + _ => false, + } + } + + /// Applies an edge, reporting the old and new labels for the trace. + pub fn to(&mut self, next: Phase) -> Result<(String, String), DomainError> { + if !self.allows(next) { + return Err(DomainError::before( + ErrorCode::InvalidPhase, + format!("{} cannot move to {}", self.phase.label(), next.label()), + )); + } + let from = self.phase.label(); + if matches!(next, Phase::Capturing(_)) { + self.capture_origin = Some(self.phase); + } else if !matches!(self.phase, Phase::Capturing(_)) { + self.capture_origin = None; + } + self.phase = next; + Ok((from, next.label())) + } + + /// Fails the epoch from wherever the session was. + pub fn fail(&mut self) -> (String, String) { + let from = self.phase.label(); + self.phase = Phase::Failed; + self.capture_origin = None; + (from, self.phase.label()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_happy_path_walks_the_section_2_diagram() { + let mut m = PhaseMachine::new(); + m.to(Phase::Ready(0)).unwrap(); + m.to(Phase::Preparing(0)).unwrap(); + m.to(Phase::Applying(0)).unwrap(); + m.to(Phase::Observing(1)).unwrap(); + m.to(Phase::Committing(0)).unwrap(); + m.to(Phase::Ready(1)).unwrap(); + assert_eq!(m.phase(), Phase::Ready(1)); + } + + #[test] + fn a_step_cannot_be_skipped_or_rewound() { + let mut m = PhaseMachine::new(); + m.to(Phase::Ready(0)).unwrap(); + assert!(m.to(Phase::Preparing(1)).is_err()); + m.to(Phase::Preparing(0)).unwrap(); + assert!(m.to(Phase::Observing(1)).is_err()); + m.to(Phase::Applying(0)).unwrap(); + assert!(m.to(Phase::Observing(0)).is_err()); + } + + #[test] + fn only_a_committed_boundary_pauses_or_captures() { + let mut m = PhaseMachine::new(); + m.to(Phase::Ready(0)).unwrap(); + m.to(Phase::Preparing(0)).unwrap(); + assert!(m.to(Phase::Paused(0)).is_err()); + assert!(m.to(Phase::Capturing(0)).is_err()); + assert!(!Phase::Preparing(0).is_committed_boundary()); + assert!(Phase::Ready(0).is_committed_boundary()); + assert!(Phase::Paused(3).is_committed_boundary()); + } + + #[test] + fn a_capture_returns_to_the_boundary_it_came_from() { + let mut m = PhaseMachine::new(); + m.to(Phase::Ready(0)).unwrap(); + m.to(Phase::Paused(0)).unwrap(); + m.to(Phase::Capturing(0)).unwrap(); + assert!(m.to(Phase::Ready(0)).is_err()); + m.to(Phase::Paused(0)).unwrap(); + } + + #[test] + fn failure_leads_to_restore_and_then_to_a_paused_boundary() { + let mut m = PhaseMachine::new(); + m.to(Phase::Ready(0)).unwrap(); + m.to(Phase::Preparing(0)).unwrap(); + m.fail(); + assert_eq!(m.phase(), Phase::Failed); + assert!(m.to(Phase::Ready(0)).is_err()); + m.to(Phase::Restoring(0)).unwrap(); + m.to(Phase::Paused(0)).unwrap(); + } + + #[test] + fn an_episode_reset_leaves_a_pause_and_lands_on_step_zero() { + let mut m = PhaseMachine::new(); + m.to(Phase::Ready(0)).unwrap(); + m.to(Phase::Preparing(0)).unwrap(); + m.to(Phase::Applying(0)).unwrap(); + m.to(Phase::Observing(1)).unwrap(); + m.to(Phase::Committing(0)).unwrap(); + m.to(Phase::Ready(1)).unwrap(); + m.to(Phase::Paused(1)).unwrap(); + m.to(Phase::Resetting(1)).unwrap(); + m.to(Phase::Ready(0)).unwrap(); + } +} diff --git a/services/flysim/crates/fly-session/src/rpc.rs b/services/flysim/crates/fly-session/src/rpc.rs new file mode 100644 index 0000000..2f52d52 --- /dev/null +++ b/services/flysim/crates/fly-session/src/rpc.rs @@ -0,0 +1,139 @@ +//! Domain RPC over Flybus: `req-` serials, incarnation pinning and the retry rule. +//! +//! A domain retry keeps its `requestId` and body and takes a fresh bus `callId`. Nothing here +//! retries on its own: an uncertain call is resolved by the caller, which is the coordinator. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value}; + +use crate::types::{ + DomainError, ErrorCode, Id, Mutation, RequestId, SessionRpcOutcome, SessionRpcRequest, Scope, +}; + +/// A named worker endpoint, pinned to one bus registration and one domain incarnation. +#[derive(Clone, Debug)] +pub struct WorkerRef { + pub service: String, + /// The bus `serviceIncarnation` every call pins. Not a worker process id. + pub bus_incarnation: String, + pub worker_id: Id, + /// The domain `incarnationId` Hello negotiated, once it has. + pub domain_incarnation: Option, +} + +impl WorkerRef { + pub fn new(service: &str, bus_incarnation: &str, worker_id: &Id) -> WorkerRef { + WorkerRef { + service: service.to_owned(), + bus_incarnation: bus_incarnation.to_owned(), + worker_id: worker_id.clone(), + domain_incarnation: None, + } + } +} + +/// One terminal domain reply and the artifacts it brought. +pub struct DomainReply { + pub outcome: SessionRpcOutcome, + pub request_id: RequestId, + pub artifacts: BTreeMap, +} + +impl DomainReply { + /// The success `result`, or the domain error. + pub fn result(&self) -> Result<&Map, DomainError> { + self.outcome.result() + } + + pub fn parse(&self) -> Result { + let result = self.outcome.result()?; + serde_json::from_value(Value::Object(result.clone())) + .map_err(|e| DomainError::invalid(format!("unreadable result: {e}"))) + } +} + +/// Issues one domain call and waits for its terminal reply. +/// +/// `want_artifacts` names the attachments to extract before the result delivery is dropped. +/// A bus-level failure is not a domain failure: it is reported with the dispatch certainty the +/// bus gave, because a caller-side timeout must not imply that nothing was mutated. +#[allow(clippy::too_many_arguments)] +pub async fn call( + bus: &flybus::Client, + target: &WorkerRef, + method: &str, + scope: Option, + params: Map, + attachments: &[(&str, &flybus::Artifact)], + request_id: RequestId, + want_artifacts: &[String], +) -> Result { + let request = SessionRpcRequest::new(&request_id, scope, params); + let mut pending = bus + .call( + &target.service, + Some(&target.bus_incarnation), + method, + request.to_payload(), + attachments, + ) + .await + .map_err(|e| bus_error(method, &e))?; + let result = pending.result().await.map_err(|e| bus_error(method, &e))?; + let outcome = SessionRpcOutcome::from_outcome(result.outcome()) + .map_err(|e| DomainError::invalid(format!("{method}: {e}")))?; + let mut artifacts = BTreeMap::new(); + for name in want_artifacts { + if let Ok(artifact) = result.artifact(name) { + // An independent explicit hold, so the handle outlives this delivery and can be + // forwarded to several Commit calls and to publication. + match artifact.retain().await { + Ok(hold) => { + artifacts.insert(name.clone(), hold); + } + Err(_) => { + artifacts.insert(name.clone(), artifact); + } + } + } + } + drop(result); + Ok(DomainReply { outcome, request_id, artifacts }) +} + +/// Maps a bus failure onto a domain error, preserving how certain the mutation is. +fn bus_error(method: &str, e: &flybus::BusError) -> DomainError { + let mutation = match e.dispatch { + flybus::Dispatch::NotDispatched => Mutation::None, + flybus::Dispatch::Dispatched | flybus::Dispatch::Unknown => Mutation::Unknown, + }; + let code = match e.code { + flybus::ErrorCode::TargetChanged | flybus::ErrorCode::NoService => { + ErrorCode::IdentityMismatch + } + flybus::ErrorCode::Backpressure | flybus::ErrorCode::QuotaExceeded => ErrorCode::Busy, + flybus::ErrorCode::ArtifactGone + | flybus::ErrorCode::ArtifactUnsealed + | flybus::ErrorCode::OwnerInvalid + | flybus::ErrorCode::ArtifactMismatch => ErrorCode::BufferInvalid, + _ => ErrorCode::BackendFailure, + }; + DomainError::new(code, format!("{method}: bus {:?}: {}", e.code, e.message), mutation) +} + +/// Per-worker request serials. A newly issued operation takes the next one; a retry does not. +#[derive(Clone, Debug, Default)] +pub struct Serials(BTreeMap); + +impl Serials { + pub fn next(&mut self, service: &str) -> RequestId { + let slot = self.0.entry(service.to_owned()).or_insert(0); + *slot += 1; + RequestId(*slot) + } + + pub fn highest(&self, service: &str) -> u64 { + self.0.get(service).copied().unwrap_or_default() + } +} diff --git a/services/flysim/crates/fly-session/src/task.rs b/services/flysim/crates/fly-session/src/task.rs new file mode 100644 index 0000000..9e84c25 --- /dev/null +++ b/services/flysim/crates/fly-session/src/task.rs @@ -0,0 +1,385 @@ +//! Coordinator-local task and executor interfaces (`workers-v1` section 4), plus the +//! deterministic counter task and the identity executor the synthetic composition uses. +//! +//! These are library interfaces, not extra bus services. The task interprets inspection data +//! and asks for outcomes; the executor translates a selected decision using read-only current +//! game state and task progress; the coordinator orders and applies the results. Nothing here +//! writes a controller or neural state directly. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value, json}; + +use crate::types::{ + AgentOutcome, ControllerIntent, DomainError, DomainResult, EpisodeKind, EpisodeRequest, + ErrorCode, Id, PortBinding, PortControl, Reward, SchemaRef, Scope, Stimulus, TaskEvent, + TypedValue, U64, event_id, +}; + +/// The schemas the synthetic arena composition registers. +pub fn inspection_schema() -> SchemaRef { + SchemaRef::synthetic("arena.inspection.v1", 1) +} + +pub fn decision_schema() -> SchemaRef { + SchemaRef::synthetic("arena.decision.v1", 1) +} + +pub fn context_schema() -> SchemaRef { + SchemaRef::synthetic("arena.context.v1", 1) +} + +pub fn progress_schema() -> SchemaRef { + SchemaRef::synthetic("arena.progress.v1", 1) +} + +pub fn event_schema() -> SchemaRef { + SchemaRef::synthetic("arena.event.v1", 1) +} + +pub fn episode_schema() -> SchemaRef { + SchemaRef::synthetic("arena.episode.v1", 1) +} + +pub fn controller_schema_ref() -> SchemaRef { + SchemaRef::synthetic("arena.controller.v1", 1) +} + +/// What `Task.bootstrap` produced. +#[derive(Clone, Debug)] +pub struct Bootstrap { + pub contexts: BTreeMap, + pub progress: TypedValue, + pub events: Vec, +} + +/// What `Task.evaluate_transition` produced, for exactly one transition. +#[derive(Clone, Debug)] +pub struct Evaluation { + /// Every configured agent has an entry, including an empty one. + pub outcomes: BTreeMap, + pub next_contexts: BTreeMap, + pub progress: TypedValue, + pub events: Vec, + pub episode: Option, +} + +/// A checkpointable task ledger and the two evaluation entry points. +pub trait Task: Send { + fn schema(&self) -> SchemaRef; + + /// Called once, at boundary 0, before any agent is initialized. + fn bootstrap( + &mut self, + initial_inspection: &TypedValue, + bindings: &[PortBinding], + ) -> DomainResult; + + /// Called exactly once per acknowledged world step, never against a later observation. + fn evaluate_transition( + &mut self, + scope: &Scope, + old_inspection: &TypedValue, + new_inspection: &TypedValue, + applied_controls: &[PortControl], + ) -> DomainResult; + + fn progress(&self) -> TypedValue; + + /// How many times `evaluate_transition` has run. A transition must evaluate once. + fn evaluations(&self) -> u64; +} + +/// Translates one selected decision into a controller intent, with no port assignment. +pub trait ActionExecutor: Send { + fn apply( + &mut self, + scope: &Scope, + decision: &TypedValue, + current_game_state: &TypedValue, + progress: &TypedValue, + clock: &crate::types::RationalNs, + ) -> DomainResult<(ControllerIntent, Vec)>; +} + +/// The only executor v1 supports: it passes a direct-control decision through unchanged. +#[derive(Clone, Debug, Default)] +pub struct IdentityExecutor; + +impl ActionExecutor for IdentityExecutor { + fn apply( + &mut self, + _scope: &Scope, + decision: &TypedValue, + _current_game_state: &TypedValue, + _progress: &TypedValue, + _clock: &crate::types::RationalNs, + ) -> DomainResult<(ControllerIntent, Vec)> { + if decision.schema != decision_schema() { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "the decision does not carry the profile's registered intent schema", + )); + } + let intent: ControllerIntent = + serde_json::from_value(Value::Object(decision.value.clone())) + .map_err(|e| DomainError::invalid(format!("decision: {e}")))?; + Ok((intent, Vec::new())) + } +} + +/// When the counter task asks for a terminal episode transition. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Terminal { + /// The episode runs until the application stops it. + #[default] + Never, + /// The counter reached this value or higher. + Counter(i64), + /// This many transitions were evaluated. + AfterTransitions(u64), +} + +/// The deterministic counter task: rewards come from the arena counter each agent moved. +pub struct CounterTask { + epoch: Id, + agents: Vec, + bindings: Vec, + transitions: u64, + evaluations: u64, + total_reward: f64, + counter: i64, + terminal: Terminal, +} + +impl CounterTask { + pub fn new(epoch: &Id, terminal: Terminal) -> CounterTask { + CounterTask { + epoch: epoch.clone(), + agents: Vec::new(), + bindings: Vec::new(), + transitions: 0, + evaluations: 0, + total_reward: 0.0, + counter: 0, + terminal, + } + } + + fn context(&self, step: u64, boot: bool) -> TypedValue { + TypedValue::new( + context_schema(), + match json!({ + "available": ["inc", "dec"], + "boot": boot, + "step": step, + }) { + Value::Object(m) => m, + _ => unreachable!(), + }, + ) + } + + fn progress_value(&self) -> TypedValue { + TypedValue::new( + progress_schema(), + match json!({ + "counter": self.counter, + "transitions": self.transitions, + "totalReward": self.total_reward, + }) { + Value::Object(m) => m, + _ => unreachable!(), + }, + ) + } + + /// The counter delta one port control asks for: `inc` adds one, `dec` subtracts one. + /// + /// This is the task's reading of a control, kept identical to the environment's rule so a + /// reward describes the transition the world actually took. + pub fn delta_of(control: &PortControl) -> i64 { + let mut delta = 0; + for button in &control.buttons { + if button.down { + match button.id.as_str() { + "inc" => delta += 1, + "dec" => delta -= 1, + _ => {} + } + } + } + delta + } + + fn agent_of_port(&self, port_id: &Id) -> Option<&Id> { + self.bindings.iter().find(|b| b.port_id == *port_id).map(|b| &b.agent_id) + } +} + +impl Task for CounterTask { + fn schema(&self) -> SchemaRef { + progress_schema() + } + + fn bootstrap( + &mut self, + initial_inspection: &TypedValue, + bindings: &[PortBinding], + ) -> DomainResult { + if initial_inspection.schema != inspection_schema() { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + "the environment's inspection schema is not the one this task reads", + )); + } + self.counter = initial_inspection.integer("counter").map_err(DomainError::invalid)?; + self.bindings = bindings.to_vec(); + self.agents = bindings.iter().map(|b| b.agent_id.clone()).collect(); + self.agents.sort(); + let contexts = self + .agents + .iter() + .map(|agent| (agent.clone(), self.context(0, true))) + .collect(); + // A bootstrap event has sourceStep 0 and carries no reward: warm-up produces no + // gameplay outcome at all. + let events = vec![TaskEvent { + id: event_id(&self.epoch, 0, "bootstrap", 0), + kind_id: Id::lit("arena.bootstrap"), + source_step: U64(0), + agent_id: None, + payload: TypedValue::new( + event_schema(), + match json!({"counter": self.counter}) { + Value::Object(m) => m, + _ => unreachable!(), + }, + ), + }]; + Ok(Bootstrap { contexts, progress: self.progress_value(), events }) + } + + fn evaluate_transition( + &mut self, + scope: &Scope, + old_inspection: &TypedValue, + new_inspection: &TypedValue, + applied_controls: &[PortControl], + ) -> DomainResult { + let old = old_inspection.integer("counter").map_err(DomainError::invalid)?; + let new = new_inspection.integer("counter").map_err(DomainError::invalid)?; + let source_step = scope.step.0 + 1; + self.evaluations += 1; + self.transitions += 1; + self.counter = new; + + let mut outcomes: BTreeMap = self + .agents + .iter() + .map(|agent| (agent.clone(), AgentOutcome::default())) + .collect(); + let mut events = Vec::new(); + let mut ordinal = 0u32; + // Controls arrive in descriptor port order, so the reward order is deterministic. + for control in applied_controls { + let Some(agent) = self.agent_of_port(&control.port_id).cloned() else { + return Err(DomainError::before( + ErrorCode::IdentityMismatch, + format!("port {} is bound to no agent", control.port_id), + )); + }; + let delta = CounterTask::delta_of(control); + let id = event_id(&self.epoch, source_step, "counter-delta", ordinal); + ordinal += 1; + events.push(TaskEvent { + id: id.clone(), + kind_id: Id::lit("arena.counter-delta"), + source_step: U64(source_step), + agent_id: Some(agent.clone()), + payload: TypedValue::new( + event_schema(), + match json!({"delta": delta, "counter": new}) { + Value::Object(m) => m, + _ => unreachable!(), + }, + ), + }); + let outcome = outcomes.get_mut(&agent).ok_or_else(|| { + DomainError::before( + ErrorCode::IdentityMismatch, + format!("port {} names agent {agent}, which is not configured", control.port_id), + ) + })?; + // A shipped positive-only profile would reject a negative value; this task is + // signed on purpose, so the profile that consumes it declares signed rewards. + outcome.rewards.push(Reward { + event_id: id, + rule_id: Id::lit("counter-delta"), + value: delta as f64, + }); + self.total_reward += delta as f64; + // One declared stimulus when the counter moved past a multiple of five, so the + // stimulation path is exercised without depending on reward. + if delta != 0 && new.rem_euclid(5) == 0 { + outcome.stimulations.push(Stimulus { + id: event_id(&self.epoch, source_step, "milestone", ordinal), + kind_id: Id::lit("arena.milestone"), + duration_ms: 4.0, + }); + } + } + if new != old + applied_controls.iter().map(CounterTask::delta_of).sum::() { + return Err(DomainError::new( + ErrorCode::BackendFailure, + "the world did not move by the batch the task read", + crate::types::Mutation::Unknown, + )); + } + + let next_contexts = self + .agents + .iter() + .map(|agent| (agent.clone(), self.context(source_step, false))) + .collect(); + let terminal = match self.terminal { + Terminal::Never => false, + Terminal::Counter(target) => new >= target, + Terminal::AfterTransitions(n) => self.transitions >= n, + }; + let episode = terminal.then(|| EpisodeRequest { + kind: EpisodeKind::Terminal, + reason: Id::lit("counter-target"), + outcome: TypedValue::new( + episode_schema(), + match json!({"counter": new, "transitions": self.transitions}) { + Value::Object(m) => m, + _ => unreachable!(), + }, + ), + }); + Ok(Evaluation { + outcomes, + next_contexts, + progress: self.progress_value(), + events, + episode, + }) + } + + fn progress(&self) -> TypedValue { + self.progress_value() + } + + fn evaluations(&self) -> u64 { + self.evaluations + } +} + +/// The inspection value the counter environment publishes. +pub fn inspection(counter: i64, boundary: u64) -> TypedValue { + let mut value = Map::new(); + value.insert("counter".into(), counter.into()); + value.insert("boundary".into(), boundary.into()); + TypedValue::new(inspection_schema(), value) +} diff --git a/services/flysim/crates/fly-session/src/worker.rs b/services/flysim/crates/fly-session/src/worker.rs new file mode 100644 index 0000000..c8bd595 --- /dev/null +++ b/services/flysim/crates/fly-session/src/worker.rs @@ -0,0 +1,719 @@ +//! The worker dispatch shell: one Flybus service, the common `Worker.*` methods, and the +//! domain deduplication of `ipc-v1` section 5 in front of every mutation. +//! +//! The shell owns request admission order and the result cache. An endpoint owns the +//! mutation. Exactly one mutation runs at a time -- the endpoint sits behind its own mutex -- +//! while `Worker.Status` is answered from a small shared cell, so a status query never waits +//! for a numerical operation and never advances the progress counter itself. + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use serde_json::{Map, Value}; + +use crate::dedup::{Admission, CachedReply, OpClass, OperationKey, ResultCache}; +use crate::types::{ + self as types, AcknowledgeParams, AcknowledgeResult, Digest, DomainError, DomainResult, + ErrorCode, HelloLimits, HelloParams, HelloResult, Id, Mutation, RequestId, Role, + SessionRpcFailure, SessionRpcOutcome, SessionRpcRequest, SessionRpcSuccess, ShutdownParams, + ShutdownResult, StatusResult, U64, WorkerState, +}; + +/// `Worker.Acknowledge` accepts 1..=16 request ids. +pub const MAX_ACKNOWLEDGE_IDS: usize = 16; + +/// The build identity a worker reports in Hello. It is not a profile digest. +pub fn build_digest() -> Digest { + Digest::of(b"fly-session/synthetic-workers-v1") +} + +/// The status a worker reports, kept outside the endpoint mutex so `Worker.Status` stays +/// responsive while a mutation runs. +#[derive(Clone)] +pub struct StatusCell(Arc>); + +struct StatusInner { + state: WorkerState, + current_scope: Option, + active_request_id: Option, + last_completed_request_id: Option, + last_batch_id: Option, + progress_counter: u64, +} + +impl Default for StatusCell { + fn default() -> StatusCell { + StatusCell::new() + } +} + +impl StatusCell { + pub fn new() -> StatusCell { + StatusCell(Arc::new(Mutex::new(StatusInner { + state: WorkerState::Uninitialized, + current_scope: None, + active_request_id: None, + last_completed_request_id: None, + last_batch_id: None, + progress_counter: 0, + }))) + } + + fn with(&self, f: impl FnOnce(&mut StatusInner) -> T) -> T { + let mut inner = self.0.lock().expect("the status cell is never poisoned"); + f(&mut inner) + } + + pub fn set_state(&self, state: WorkerState) { + self.with(|s| s.state = state); + } + + pub fn state(&self) -> WorkerState { + self.with(|s| s.state) + } + + pub fn set_scope(&self, scope: Option) { + self.with(|s| s.current_scope = scope); + } + + pub fn set_active(&self, request_id: Option) { + self.with(|s| s.active_request_id = request_id); + } + + pub fn set_completed(&self, request_id: Id) { + self.with(|s| { + s.active_request_id = None; + s.last_completed_request_id = Some(request_id); + }); + } + + pub fn set_batch(&self, batch_id: Id) { + self.with(|s| s.last_batch_id = Some(batch_id)); + } + + /// Records computational or phase progress. A status query never calls this. + pub fn progress(&self, by: u64) { + self.with(|s| s.progress_counter = s.progress_counter.saturating_add(by)); + } + + /// Raises the progress counter to `value`, which lets a worker report its model's own + /// mutation count as its progress. It never moves backwards. + pub fn advance_to(&self, value: u64) { + self.with(|s| s.progress_counter = s.progress_counter.max(value)); + } + + pub fn progress_counter(&self) -> u64 { + self.with(|s| s.progress_counter) + } + + pub fn snapshot(&self) -> StatusResult { + self.with(|s| StatusResult { + state: s.state, + current_scope: s.current_scope.clone(), + active_request_id: s.active_request_id.clone(), + last_completed_request_id: s.last_completed_request_id.clone(), + last_batch_id: s.last_batch_id.clone(), + progress_counter: U64(s.progress_counter), + }) + } +} + +/// What a handler produced: a domain `result` object and the artifacts it attaches. +pub struct HandlerReply { + pub result: Map, + pub artifacts: Vec<(String, flybus::Artifact)>, + /// True when a failure left the endpoint mutated; the shell reports it as such. + pub mutated: bool, +} + +impl HandlerReply { + pub fn new(result: Map) -> HandlerReply { + HandlerReply { result, artifacts: Vec::new(), mutated: true } + } + + pub fn with_artifacts( + result: Map, + artifacts: Vec<(String, flybus::Artifact)>, + ) -> HandlerReply { + HandlerReply { result, artifacts, mutated: true } + } + + pub fn from(value: &T) -> HandlerReply { + let v = serde_json::to_value(value).expect("a result serializes"); + match v { + Value::Object(m) => HandlerReply::new(m), + _ => unreachable!("a struct serializes to an object"), + } + } +} + +/// Everything a handler is given: the parsed domain request and the bus request behind it. +pub struct HandlerCtx<'a> { + pub method: &'a str, + pub request: &'a SessionRpcRequest, + pub client: &'a flybus::Client, + pub incoming: &'a flybus::Request, +} + +impl HandlerCtx<'_> { + /// Deserializes `params` into a method payload, reporting INVALID_ARGUMENT. + pub fn params(&self) -> DomainResult { + serde_json::from_value(Value::Object(self.request.params.clone())) + .map_err(|e| DomainError::invalid(format!("{}: {e}", self.method))) + } + + /// The scope the request must carry. + pub fn scope(&self) -> DomainResult<&types::Scope> { + self.request + .scope + .as_ref() + .ok_or_else(|| DomainError::invalid(format!("{} requires a scope", self.method))) + } + + /// An owned handle on one of the request's declared attachments. + pub fn artifact(&self, name: &str) -> DomainResult { + self.incoming.artifact(name).map_err(|e| { + DomainError::before( + ErrorCode::BufferInvalid, + format!("attachment {name:?} is missing or unowned: {}", e.message), + ) + }) + } +} + +/// A handler's boxed future, so the endpoint trait stays dyn-compatible. +pub type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// One worker's domain behaviour. The shell owns everything else. +pub trait WorkerEndpoint: Send + 'static { + fn worker_id(&self) -> Id; + fn incarnation_id(&self) -> Id; + fn session_id(&self) -> Id; + fn role(&self) -> Role; + fn capabilities(&self) -> Vec; + fn status_cell(&self) -> StatusCell; + + /// The domain methods this endpoint implements, beyond the common `Worker.*` set. + /// Anything else returns UNSUPPORTED without entering the endpoint. + fn methods(&self) -> Vec<&'static str>; + + fn handle<'a>(&'a mut self, ctx: HandlerCtx<'a>) -> BoxFuture<'a, DomainResult>; +} + +/// A running worker: its bus client, its service and the task serving it. +pub struct WorkerHandle { + pub worker_id: Id, + pub incarnation_id: Id, + pub service_name: String, + pub service_incarnation: String, + pub status: StatusCell, + pub cache: Arc>, + task: tokio::task::JoinHandle<()>, + client: flybus::Client, +} + +impl WorkerHandle { + /// The worker's own progress counter, which is the fake model's mutation count. + pub fn progress_counter(&self) -> u64 { + self.status.progress_counter() + } + + pub fn state(&self) -> WorkerState { + self.status.state() + } + + /// Stops serving and closes the worker's bus connection, which drops its registration and + /// every owner it held. A later reply from it can attach to nothing. + pub async fn stop(self) { + self.task.abort(); + let _ = self.task.await; + self.client.close().await; + } +} + +/// Registers `service_name` and serves `endpoint` on it until the service ends or Shutdown. +pub fn serve( + client: flybus::Client, + service: flybus::Service, + endpoint: E, +) -> WorkerHandle { + let worker_id = endpoint.worker_id(); + let incarnation_id = endpoint.incarnation_id(); + let status = endpoint.status_cell(); + let service_name = service.name().to_owned(); + let service_incarnation = service.incarnation().to_owned(); + let cache = Arc::new(tokio::sync::Mutex::new(ResultCache::new())); + let task = tokio::spawn(run(client.clone(), service, Arc::new(tokio::sync::Mutex::new(endpoint)), cache.clone())); + WorkerHandle { + worker_id, + incarnation_id, + service_name, + service_incarnation, + status, + cache, + task, + client, + } +} + +async fn run( + client: flybus::Client, + mut service: flybus::Service, + endpoint: Arc>, + cache: Arc>, +) { + // Identity and capabilities are fixed for the endpoint's lifetime, so the shell reads them + // once and never takes the endpoint mutex to answer Hello or Status. + let (worker_id, incarnation_id, session_id, role, capabilities, status, methods) = { + let e = endpoint.lock().await; + ( + e.worker_id(), + e.incarnation_id(), + e.session_id(), + e.role(), + e.capabilities(), + e.status_cell(), + e.methods(), + ) + }; + let mut running: Vec> = Vec::new(); + while let Some(incoming) = service.next().await { + let method = incoming.method().to_owned(); + let responder = incoming.responder(); + let request = match SessionRpcRequest::from_payload(incoming.payload()) { + Ok(request) => request, + Err(e) => { + // A malformed envelope has no usable requestId, so the reply names req-0. + let failure = failure( + &Id::lit("req-0"), + &worker_id, + &incarnation_id, + None, + DomainError::invalid(format!("{method}: {e}")), + ); + let _ = responder.reply(failure.to_outcome(), &[]).await; + continue; + } + }; + let serial = match RequestId::parse(&request.request_id) { + Ok(serial) => serial, + Err(e) => { + let failure = failure( + &request.request_id, + &worker_id, + &incarnation_id, + request.scope.clone(), + DomainError::invalid(e), + ); + let _ = responder.reply(failure.to_outcome(), &[]).await; + continue; + } + }; + + // The common methods never enter the endpoint mutex, so they answer during a mutation. + match method.as_str() { + "Worker.Hello" => { + let outcome = hello( + &request, + &session_id, + &worker_id, + &incarnation_id, + role, + &capabilities, + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + } + "Worker.Status" => { + let result = serde_json::to_value(status.snapshot()).expect("status serializes"); + let outcome = success( + &request, + &worker_id, + &incarnation_id, + object(result), + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + } + "Worker.Acknowledge" => { + let outcome = match acknowledge(&request, &cache).await { + Ok(result) => success(&request, &worker_id, &incarnation_id, result), + Err(e) => { + failure(&request.request_id, &worker_id, &incarnation_id, request.scope.clone(), e) + } + }; + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + } + "Worker.Shutdown" => { + let outcome = match request.params.get("reason") { + Some(_) => { + match serde_json::from_value::(Value::Object( + request.params.clone(), + )) { + Ok(_) => { + status.set_state(WorkerState::Stopping); + Ok(ShutdownResult { stopping: true }) + } + Err(e) => Err(DomainError::invalid(format!("Worker.Shutdown: {e}"))), + } + } + None => Err(DomainError::invalid("Worker.Shutdown requires a reason")), + }; + match outcome { + Ok(result) => { + let value = serde_json::to_value(result).expect("shutdown serializes"); + let outcome = + success(&request, &worker_id, &incarnation_id, object(value)); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + return; + } + Err(e) => { + let outcome = failure( + &request.request_id, + &worker_id, + &incarnation_id, + request.scope.clone(), + e, + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + } + } + } + _ => {} + } + + let class = if methods.contains(&method.as_str()) { + classify_default(&method) + } else { + None + }; + let Some(class) = class else { + let outcome = failure( + &request.request_id, + &worker_id, + &incarnation_id, + request.scope.clone(), + DomainError::before(ErrorCode::Unsupported, format!("{method} is not supported")), + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + }; + + let body = request.body_digest(&method); + let key = match (class, &request.scope) { + (OpClass::StepMutation, Some(scope)) => OperationKey { + session_id: scope.session_id.clone(), + epoch: scope.epoch.clone(), + step: scope.step.0, + method: method.clone(), + worker_id: worker_id.clone(), + }, + (OpClass::StepMutation, None) => { + let outcome = failure( + &request.request_id, + &worker_id, + &incarnation_id, + None, + DomainError::invalid(format!("{method} requires a scope")), + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + } + _ => OperationKey { + session_id: session_id.clone(), + epoch: Id::lit("lifecycle"), + step: 0, + method: method.clone(), + worker_id: worker_id.clone(), + }, + }; + + // Retained request identity is checked before phase checks and before any attachment + // is dereferenced, because a duplicate may arrive after its input delivery was + // consumed and needs only the cached result. + let admission = { + let mut c = cache.lock().await; + c.admit(class, &key, serial, &body) + }; + match admission { + Admission::Replay(reply) => { + let _ = responder.reply(reply.outcome.to_outcome(), &reply.attachments()).await; + continue; + } + Admission::Refuse(e) => { + let outcome = failure( + &request.request_id, + &worker_id, + &incarnation_id, + request.scope.clone(), + e, + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + continue; + } + Admission::Execute => {} + } + + status.set_active(Some(request.request_id.clone())); + // The mutation runs in its own task so the shell keeps reading. That is what lets an + // exact duplicate arriving mid-execution be refused with IN_PROGRESS while the + // original bus call still completes normally. The endpoint mutex, not this loop, + // enforces one mutation at a time. + running.retain(|task| !task.is_finished()); + running.push(tokio::spawn(execute( + client.clone(), + endpoint.clone(), + cache.clone(), + status.clone(), + worker_id.clone(), + incarnation_id.clone(), + class, + key, + serial, + body, + method, + request, + incoming, + ))); + } + for task in running { + let _ = task.await; + } +} + +/// Runs one admitted mutation, records its reply and answers the bus call. +#[allow(clippy::too_many_arguments)] +async fn execute( + client: flybus::Client, + endpoint: Arc>, + cache: Arc>, + status: StatusCell, + worker_id: Id, + incarnation_id: Id, + class: OpClass, + key: OperationKey, + serial: RequestId, + body: Digest, + method: String, + request: SessionRpcRequest, + incoming: flybus::Request, +) { + let responder = incoming.responder(); + let outcome = { + // One mutation at a time: the endpoint mutex is the worker's simulation lock, and it is + // never held across a bus round trip taken by anything else. + let mut e = endpoint.lock().await; + let ctx = HandlerCtx { + method: &method, + request: &request, + client: &client, + incoming: &incoming, + }; + e.handle(ctx).await + }; + match outcome { + Ok(reply) => { + let outcome = success(&request, &worker_id, &incarnation_id, reply.result.clone()); + let mut holds = Vec::with_capacity(reply.artifacts.len()); + for (name, artifact) in &reply.artifacts { + // The cache owns its own hold, so a replay survives the first caller + // consuming its delivery. + match artifact.retain().await { + Ok(hold) => holds.push((name.clone(), hold)), + Err(_) => holds.push((name.clone(), artifact.clone())), + } + } + let cached = CachedReply::with_artifacts(outcome.clone(), holds); + { + let mut c = cache.lock().await; + match class { + OpClass::StepMutation => c.record(key, serial, body, cached), + OpClass::Lifecycle => c.record_lifecycle(serial, body, cached), + OpClass::ReadOnly => c.record_readonly(serial, cached), + } + } + status.set_completed(request.request_id.clone()); + let attachments: Vec<(&str, &flybus::Artifact)> = + reply.artifacts.iter().map(|(n, a)| (n.as_str(), a)).collect(); + let _ = responder.reply(outcome.to_outcome(), &attachments).await; + } + Err(e) => { + if e.mutation == Mutation::None { + // Nothing happened, so the key stays free for the corrected request. + let mut c = cache.lock().await; + c.abandon(&key); + } else { + let cached = CachedReply::new(SessionRpcOutcome::Failure(SessionRpcFailure { + kind: types::FailureTag::Error, + request_id: request.request_id.clone(), + worker_id: worker_id.clone(), + incarnation_id: incarnation_id.clone(), + scope: request.scope.clone(), + error: e.clone(), + })); + let mut c = cache.lock().await; + if class == OpClass::StepMutation { + c.record(key, serial, body, cached); + } else { + c.abandon(&key); + } + status.set_state(WorkerState::Failed); + } + status.set_active(None); + let outcome = failure( + &request.request_id, + &worker_id, + &incarnation_id, + request.scope.clone(), + e, + ); + let _ = responder.reply(outcome.to_outcome(), &[]).await; + } + } +} + +/// The retention class of every method name the contracts define. +fn classify_default(method: &str) -> Option { + match method { + "Agent.Prepare" | "Agent.Commit" | "Environment.Advance" => Some(OpClass::StepMutation), + "Agent.Initialize" | "Environment.Initialize" => Some(OpClass::Lifecycle), + "Worker.Hello" | "Worker.Status" | "Worker.Acknowledge" | "Worker.Shutdown" => { + Some(OpClass::ReadOnly) + } + _ => None, + } +} + +fn object(value: Value) -> Map { + match value { + Value::Object(m) => m, + _ => unreachable!("a struct serializes to an object"), + } +} + +fn success( + request: &SessionRpcRequest, + worker_id: &Id, + incarnation_id: &Id, + result: Map, +) -> SessionRpcOutcome { + SessionRpcOutcome::Success(SessionRpcSuccess { + kind: types::SuccessTag::Result, + request_id: request.request_id.clone(), + worker_id: worker_id.clone(), + incarnation_id: incarnation_id.clone(), + scope: request.scope.clone(), + result, + }) +} + +fn failure( + request_id: &Id, + worker_id: &Id, + incarnation_id: &Id, + scope: Option, + error: DomainError, +) -> SessionRpcOutcome { + SessionRpcOutcome::Failure(SessionRpcFailure { + kind: types::FailureTag::Error, + request_id: request_id.clone(), + worker_id: worker_id.clone(), + incarnation_id: incarnation_id.clone(), + scope, + error, + }) +} + +fn hello( + request: &SessionRpcRequest, + session_id: &Id, + worker_id: &Id, + incarnation_id: &Id, + role: Role, + capabilities: &[Id], +) -> SessionRpcOutcome { + let params: HelloParams = + match serde_json::from_value(Value::Object(request.params.clone())) { + Ok(params) => params, + Err(e) => { + return failure( + &request.request_id, + worker_id, + incarnation_id, + None, + DomainError::invalid(format!("Worker.Hello: {e}")), + ); + } + }; + if request.scope.is_some() { + return failure( + &request.request_id, + worker_id, + incarnation_id, + None, + DomainError::invalid("Worker.Hello has no scope"), + ); + } + if params.session_id != *session_id + || params.expected_worker_id != *worker_id + || params.role != role + { + return failure( + &request.request_id, + worker_id, + incarnation_id, + None, + DomainError::before( + ErrorCode::IdentityMismatch, + "this worker is not the session, worker or role the caller expected", + ), + ); + } + if !params.supported_majors.contains(&1) { + return failure( + &request.request_id, + worker_id, + incarnation_id, + None, + DomainError::before(ErrorCode::Unsupported, "no common major version"), + ); + } + let result = HelloResult { + selected_major: 1, + selected_minor: 0, + worker_id: worker_id.clone(), + incarnation_id: incarnation_id.clone(), + role, + build_digest: build_digest(), + contract_digest: types::contract_digest(), + capabilities: capabilities.to_vec(), + limits: HelloLimits { max_agents: types::MAX_AGENTS, max_ports: types::MAX_PORTS }, + }; + success( + request, + worker_id, + incarnation_id, + object(serde_json::to_value(result).expect("hello serializes")), + ) +} + +async fn acknowledge( + request: &SessionRpcRequest, + cache: &Arc>, +) -> DomainResult> { + let params: AcknowledgeParams = + serde_json::from_value(Value::Object(request.params.clone())) + .map_err(|e| DomainError::invalid(format!("Worker.Acknowledge: {e}")))?; + if params.request_ids.is_empty() || params.request_ids.len() > MAX_ACKNOWLEDGE_IDS { + return Err(DomainError::invalid("Worker.Acknowledge takes 1..=16 request ids")); + } + let acknowledged = { + let mut c = cache.lock().await; + c.acknowledge(¶ms.request_ids) + }; + let result = AcknowledgeResult { acknowledged }; + Ok(object(serde_json::to_value(result).expect("acknowledge serializes"))) +} diff --git a/services/flysim/crates/fly-session/tests/common/mod.rs b/services/flysim/crates/fly-session/tests/common/mod.rs new file mode 100644 index 0000000..39d9b63 --- /dev/null +++ b/services/flysim/crates/fly-session/tests/common/mod.rs @@ -0,0 +1,96 @@ +//! Shared test fixture: the synthetic session on a temporary store, over either transport. + +#![allow(dead_code)] + +use std::time::Duration; + +use fly_session::harness::{HarnessConfig, SessionHarness, Via}; +use fly_session::types::Id; + +pub const WAIT: Duration = Duration::from_secs(20); + +/// Generates one test per transport from an `async fn name(via: Via)`. +#[macro_export] +macro_rules! both_transports { + ($($name:ident),* $(,)?) => { + mod in_memory { + $( + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn $name() { + super::$name($crate::common::via_memory()).await + } + )* + } + mod unix_socket { + $( + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn $name() { + super::$name($crate::common::via_unix()).await + } + )* + } + }; +} + +pub fn via_memory() -> Via { + Via::Memory +} + +pub fn via_unix() -> Via { + Via::Unix +} + +/// A started session plus the temporary directory its store lives in. +pub struct Fixture { + pub dir: tempfile::TempDir, + pub harness: SessionHarness, +} + +impl Fixture { + pub async fn shutdown(self) { + let Fixture { dir, harness } = self; + harness.shutdown().await; + drop(dir); + } +} + +pub async fn fixture(via: Via, config: HarnessConfig) -> Fixture { + let dir = tempfile::tempdir().expect("a temporary directory"); + let harness = SessionHarness::start(via, dir.path(), config) + .await + .expect("the synthetic session starts"); + Fixture { dir, harness } +} + +/// The default two-agent composition: 60 Hz world, 1 ms model tick. +pub async fn default_fixture(via: Via) -> Fixture { + fixture(via, HarnessConfig::default()).await +} + +pub fn fly_a() -> Id { + Id::lit("fly-a") +} + +pub fn fly_b() -> Id { + Id::lit("fly-b") +} + +/// The index of an audit entry, or a panic naming what was missing. +pub fn at(audit: &[String], what: &str) -> usize { + audit + .iter() + .position(|entry| entry == what) + .unwrap_or_else(|| panic!("the audit has no {what:?}: {audit:?}")) +} + +pub fn count(audit: &[String], what: &str) -> usize { + audit.iter().filter(|entry| *entry == what).count() +} + +/// Fails the test rather than hanging, so a missed reply is a failure and not a stuck job. +pub async fn within(what: &str, f: impl std::future::Future) -> T { + match tokio::time::timeout(WAIT, f).await { + Ok(v) => v, + Err(_) => panic!("{what}: timed out"), + } +} diff --git a/services/flysim/crates/fly-session/tests/failures.rs b/services/flysim/crates/fly-session/tests/failures.rs new file mode 100644 index 0000000..4048760 --- /dev/null +++ b/services/flysim/crates/fly-session/tests/failures.rs @@ -0,0 +1,377 @@ +//! The failure-injection rows of the implementation guide's section 4 that apply to +//! SESSION-01, plus the `step-v1` section 7 rules they enforce. +//! +//! Most rows are proved by comparing an injected run with a clean run of the same +//! composition: same seeds, same cadence, same number of steps. If the injected run's +//! behaviour trace, model mutation counts and world counter are identical, then the injected +//! message added no tick, no RNG draw, no stimulation, no reward and no world step. + +mod common; + +use common::{Fixture, at, fly_a, fly_b, within}; +use fly_session::agent::AgentFaults; +use fly_session::coordinator::Injections; +use fly_session::environment::EnvironmentFaults; +use fly_session::harness::{HarnessConfig, SessionHarness, Via}; +use fly_session::phase::Phase; +use fly_session::types::{ErrorCode, Id}; + +both_transports!( + a_duplicate_prepare_after_a_lost_reply_repeats_nothing, + a_duplicate_commit_replays_without_a_second_reinforcement, + the_same_batch_with_altered_controls_conflicts, + a_lost_advance_result_resolves_the_same_operation, + a_cached_artifact_consumed_by_its_first_caller_survives_a_retry, + one_commit_failing_after_another_succeeds_fails_the_epoch, + a_replaced_registration_is_not_silently_reached, + a_reply_from_another_incarnation_is_rejected, + a_world_that_advanced_without_sensory_data_fails_the_transition, + an_exact_duplicate_of_a_running_operation_is_in_progress, +); + +const STEPS: u64 = 4; +const INJECT_AT: u64 = 2; + +/// What a run of the standard composition produced. +struct Run { + behaviour: Vec, + mutations: Vec<(Id, u64)>, + counter: i64, + advances: u64, + injections: Vec, + in_progress: u64, +} + +async fn run_with(via: Via, injections: Injections) -> Run { + let mut f = clean_fixture(via).await; + f.harness.coordinator.injections = injections; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(STEPS)).await.unwrap(); + let run = Run { + behaviour: f.harness.coordinator.trace.behavior(), + mutations: vec![ + (fly_a(), f.harness.agent_mutations(&fly_a())), + (fly_b(), f.harness.agent_mutations(&fly_b())), + ], + counter: f + .harness + .coordinator + .task_progress() + .integer("counter") + .unwrap(), + advances: f.harness.coordinator.stats().advances, + injections: f.harness.coordinator.injection_log.clone(), + in_progress: f.harness.coordinator.in_progress_replies, + }; + f.shutdown().await; + run +} + +async fn clean_fixture(via: Via) -> Fixture { + common::fixture(via, HarnessConfig::default()).await +} + +fn assert_same(injected: &Run, clean: &Run, what: &str) { + assert_eq!(injected.behaviour, clean.behaviour, "{what}: behaviour trace"); + assert_eq!(injected.mutations, clean.mutations, "{what}: model mutations"); + assert_eq!(injected.counter, clean.counter, "{what}: world counter"); + assert_eq!(injected.advances, clean.advances, "{what}: world advances"); + assert_eq!(injected.advances, STEPS, "{what}: one advance per batch"); + assert_eq!(clean.in_progress, 0, "{what}: a clean run never meets a duplicate"); +} + +/// Row: duplicate Prepare after a lost reply. No extra ticks, RNG draws, stimulation or +/// decode, and the cached decision comes back unchanged. +async fn a_duplicate_prepare_after_a_lost_reply_repeats_nothing(via: Via) { + let clean = run_with(via, Injections::default()).await; + let injected = run_with( + via, + Injections { + at_step: INJECT_AT, + duplicate_prepare: Some(fly_a()), + ..Injections::default() + }, + ) + .await; + let probe = injected + .injections + .iter() + .find(|o| o.what == "duplicate-prepare") + .expect("the duplicate was sent"); + assert_eq!(probe.code, None, "a safe replay is a success, not an error"); + assert!(probe.identical, "the replay returned the same decision, ticks and remainder"); + assert_same(&injected, &clean, "duplicate prepare"); +} + +/// Row: the same Commit again. It replays its cached reply and reinforces nothing twice. +async fn a_duplicate_commit_replays_without_a_second_reinforcement(via: Via) { + let clean = run_with(via, Injections::default()).await; + let injected = run_with( + via, + Injections { + at_step: INJECT_AT, + duplicate_commit: Some(fly_b()), + ..Injections::default() + }, + ) + .await; + let probe = injected + .injections + .iter() + .find(|o| o.what == "duplicate-commit") + .expect("the duplicate was sent"); + assert_eq!(probe.code, None); + assert!(probe.identical, "the replay returned the same committed step and telemetry"); + assert_same(&injected, &clean, "duplicate commit"); +} + +/// Row: the same batch with altered controls. A conflict, never a second world mutation. +async fn the_same_batch_with_altered_controls_conflicts(via: Via) { + let clean = run_with(via, Injections::default()).await; + let injected = run_with( + via, + Injections { + at_step: INJECT_AT, + altered_advance_controls: true, + ..Injections::default() + }, + ) + .await; + let probe = injected + .injections + .iter() + .find(|o| o.what == "altered-advance-controls") + .expect("the altered batch was sent"); + assert_eq!(probe.code, Some(ErrorCode::Conflict)); + assert_same(&injected, &clean, "altered controls"); +} + +/// Row: the Advance result is lost after the world stepped. The same operation is resolved +/// against its original request id; no new batch is ever sent. +async fn a_lost_advance_result_resolves_the_same_operation(via: Via) { + let clean = run_with(via, Injections::default()).await; + let injected = run_with( + via, + Injections { at_step: INJECT_AT, lose_advance_result: true, ..Injections::default() }, + ) + .await; + assert!( + injected.injections.iter().any(|o| o.what == "lost-advance-result"), + "the call was abandoned with an uncertain outcome" + ); + assert!( + injected.injections.iter().any(|o| o.what == "status-after-loss" && o.identical), + "the uncertain call was probed with Status before being resolved" + ); + assert_same(&injected, &clean, "lost advance result"); +} + +/// Row: the cached RPC artifact is consumed by its first caller. The endpoint's domain cache +/// still owns it, so the retry gets valid bytes. +async fn a_cached_artifact_consumed_by_its_first_caller_survives_a_retry(via: Via) { + let clean = run_with(via, Injections::default()).await; + let injected = run_with( + via, + Injections { + at_step: INJECT_AT, + consume_advance_artifact_then_retry: true, + ..Injections::default() + }, + ) + .await; + let probe = injected + .injections + .iter() + .find(|o| o.what == "cached-artifact-after-consumption") + .expect("the frame was read, released and replayed"); + assert!(probe.identical, "the replayed frame has the same bytes as the consumed one"); + assert_same(&injected, &clean, "cached artifact"); +} + +/// Row: one Commit fails after another succeeds. No next world step, and the epoch fails +/// rather than continuing with a partial match. +async fn one_commit_failing_after_another_succeeds_fails_the_epoch(via: Via) { + let mut config = HarnessConfig::default(); + // fly-a commits quickly and succeeds; fly-b fails after its next input was installed. + config.agents[1].faults = + AgentFaults { fail_commit_at_step: Some(1), commit_delay_ms: 15, ..AgentFaults::default() }; + let mut f = common::fixture(via, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("step", f.harness.coordinator.step()).await.unwrap(); + let err = within("failing step", f.harness.coordinator.step()) + .await + .expect_err("the epoch fails when one commit fails"); + assert_eq!(err.error.code, ErrorCode::BackendFailure); + assert_eq!(err.error.mutation, fly_session::types::Mutation::Applied); + assert_eq!(f.harness.coordinator.phase(), Phase::Failed); + + // The world took the step whose commits failed, and it takes no further step. + let advances = f.harness.coordinator.stats().advances; + assert_eq!(advances, 1, "the failed transition never reached a committed boundary"); + let env = f.harness.coordinator.environment_ref().clone(); + let status = within("status", f.harness.coordinator.status(&env)).await.unwrap(); + assert_eq!(status.current_scope.as_ref().unwrap().step.get(), 2); + let again = f.harness.coordinator.step().await.expect_err("no step from Failed"); + assert_eq!(again.error.code, ErrorCode::InvalidPhase); + let status = within("status", f.harness.coordinator.status(&env)).await.unwrap(); + assert_eq!( + status.current_scope.unwrap().step.get(), + 2, + "no world step follows a partial commit" + ); + + // The agent that succeeded is at the new boundary; the one that failed reports Failed. + let a = f.harness.coordinator.agent_ref(&fly_a()).cloned().unwrap(); + let status = within("status", f.harness.coordinator.status(&a)).await.unwrap(); + assert_eq!(status.current_scope.unwrap().step.get(), 2); + assert_eq!( + f.harness.agent_status(&fly_b()).unwrap().state(), + fly_session::types::WorkerState::Failed + ); + // Nothing was published for the boundary that failed to commit. + let audit = f.harness.coordinator.audit.clone(); + assert!(!audit.iter().any(|entry| entry == "publish:2")); + assert!(at(&audit, "publish:1") < at(&audit, "advance:1")); + f.shutdown().await; +} + +/// Row: an old worker is replaced. The coordinator pinned the old registration, so its next +/// call fails rather than silently reaching another brain. +async fn a_replaced_registration_is_not_silently_reached(via: Via) { + let mut f = clean_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("step", f.harness.coordinator.step()).await.unwrap(); + let restarted = f.harness.restart_agent(&fly_b()).await.unwrap(); + let pinned = f.harness.coordinator.agent_ref(&fly_b()).cloned().unwrap(); + assert_ne!( + restarted.service_incarnation, pinned.bus_incarnation, + "a replacement registration is a new incarnation" + ); + let err = within("step after restart", f.harness.coordinator.step()) + .await + .expect_err("the pinned incarnation is gone"); + assert_eq!(err.error.code, ErrorCode::IdentityMismatch); + assert_eq!(f.harness.coordinator.phase(), Phase::Failed); + assert_eq!(f.harness.coordinator.stats().advances, 1, "no world step under a lost pin"); + f.shutdown().await; +} + +/// Row: a reply carrying another domain incarnation is rejected, even when the bus route is +/// live and answering. +async fn a_reply_from_another_incarnation_is_rejected(via: Via) { + let mut f = clean_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + let old = f.harness.coordinator.agent_ref(&fly_b()).cloned().unwrap(); + let restarted = f.harness.restart_agent(&fly_b()).await.unwrap(); + // Follow the new registration, but keep pinning the incarnation the old worker negotiated. + let stale = fly_session::rpc::WorkerRef { + service: restarted.service.clone(), + bus_incarnation: restarted.service_incarnation.clone(), + worker_id: fly_b(), + domain_incarnation: old.domain_incarnation.clone(), + }; + assert_ne!(old.domain_incarnation, Some(restarted.incarnation_id.clone())); + let err = within("status", f.harness.coordinator.status(&stale)) + .await + .expect_err("the replacement is not the negotiated incarnation"); + assert_eq!(err.error.code, ErrorCode::IdentityMismatch); + assert_eq!(f.harness.coordinator.phase(), Phase::Failed); + f.shutdown().await; +} + +/// `step-v1` section 7: the world advanced but the required sensory data is unavailable. The +/// transition fails; nothing is rewarded or continued on guessed input. +async fn a_world_that_advanced_without_sensory_data_fails_the_transition(via: Via) { + let config = HarnessConfig { + environment_faults: EnvironmentFaults { + omit_view_at_boundary: Some(1), + ..EnvironmentFaults::default() + }, + ..HarnessConfig::default() + }; + let mut f = common::fixture(via, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + let err = within("step", f.harness.coordinator.step()) + .await + .expect_err("a missing required view is not silently replaced"); + assert_eq!(err.error.code, ErrorCode::BufferInvalid); + assert_eq!(f.harness.coordinator.phase(), Phase::Failed); + // The task never interpreted the transition, so nothing was rewarded. + assert_eq!(f.harness.coordinator.evaluations(), 0); + assert_eq!( + f.harness.coordinator.task_progress().number("totalReward").unwrap(), + 0.0 + ); + let audit = f.harness.coordinator.audit.clone(); + assert!(!audit.iter().any(|entry| entry.starts_with("committed:"))); + assert!(!audit.iter().any(|entry| entry == "publish:1")); + f.shutdown().await; +} + +/// `ipc-v1` section 5: an exact duplicate arriving while the original is still executing gets +/// IN_PROGRESS for that bus call, and the original completes normally. +async fn an_exact_duplicate_of_a_running_operation_is_in_progress(via: Via) { + let config = HarnessConfig { + environment_faults: EnvironmentFaults { + // The world moves, then the reply is held, so the resolution attempt lands while + // the original operation is still active. + advance_delay_ms: 120, + ..EnvironmentFaults::default() + }, + ..HarnessConfig::default() + }; + let mut f = common::fixture(via, config).await; + f.harness.coordinator.injections = + Injections { at_step: 0, lose_advance_result: true, ..Injections::default() }; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("step", f.harness.coordinator.step()).await.unwrap(); + assert!( + f.harness.coordinator.in_progress_replies > 0, + "the duplicate met the original still running" + ); + // And the original still completed: exactly one world step, at one boundary. + assert_eq!(f.harness.coordinator.stats().advances, 1); + assert_eq!(f.harness.coordinator.observation().unwrap().boundary.get(), 1); + f.shutdown().await; +} + +/// A restarted worker under a new epoch refuses an operation from the old one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_old_epoch_operation_is_refused_with_stale_epoch() { + let dir = tempfile::tempdir().unwrap(); + let mut harness = SessionHarness::start(Via::Memory, dir.path(), HarnessConfig::default()) + .await + .unwrap(); + within("bootstrap", harness.coordinator.bootstrap()).await.unwrap(); + within("step", harness.coordinator.step()).await.unwrap(); + + // The agent is live and initialized under epoch e1. An operation naming another epoch is + // refused as stale rather than applied to this brain. + let worker = harness.coordinator.agent_ref(&fly_a()).cloned().unwrap(); + let scope = fly_session::types::Scope::new( + &Id::lit("demo"), + &Id::lit("e0"), + 1, + ); + let params = serde_json::json!({ + "agentId": "fly-a", + "profileDigest": fly_session::types::Digest::of(b"whatever").to_string(), + "interval": {"numerator": "16666667", "denominator": "1"}, + "decisionContextDigest": fly_session::types::Digest::of(b"whatever").to_string(), + "preStepStimulations": [], + }); + let bus = harness.client("coordinator2").await; + // The launcher grants no second coordinator, so the old-epoch probe goes through the + // session's own client instead. + assert!(bus.is_err(), "an unconfigured client id is refused before it can route"); + let err = within( + "stale epoch", + harness.coordinator.probe_raw(&worker, "Agent.Prepare", Some(scope), params), + ) + .await + .expect_err("an old epoch cannot mutate this worker"); + assert_eq!(err.code, ErrorCode::StaleEpoch); + assert_eq!(harness.coordinator.stats().advances, 1); + harness.shutdown().await; + drop(dir); +} diff --git a/services/flysim/crates/fly-session/tests/session.rs b/services/flysim/crates/fly-session/tests/session.rs new file mode 100644 index 0000000..464d233 --- /dev/null +++ b/services/flysim/crates/fly-session/tests/session.rs @@ -0,0 +1,401 @@ +//! SESSION-01 acceptance: the synthetic sequential transaction, over both transports. +//! +//! Every test here is one of the acceptance bullets of the implementation guide's SESSION-01 +//! slice, or one of the initialization, pause and episode rules of `step-v1` section 6. + +mod common; + +use std::collections::BTreeMap; + +use common::{Fixture, at, count, default_fixture, fixture, fly_a, fly_b, within}; +use fly_session::coordinator::DispatchOrder; +use fly_session::harness::{AgentSpec, HarnessConfig, Via}; +use fly_session::phase::Phase; +use fly_session::types::Id; +use fly_session::agent::AgentFaults; + +both_transports!( + one_world_advance_per_complete_batch, + every_agent_is_prepared_before_the_world_advances, + the_task_evaluates_each_transition_once, + every_agent_commits_before_the_next_prepare_or_publication, + a_60_hz_world_with_a_1_ms_tick_runs_16_17_17, + a_pause_mid_step_completes_the_step_and_pauses_at_the_boundary, + bootstrap_cannot_advance_the_world_or_produce_a_reward, + the_committed_snapshot_names_the_boundary_that_just_ended, + a_terminal_episode_pauses_at_its_own_boundary, + status_answers_with_the_committed_boundary, + a_worker_refuses_a_second_initialize, +); + +const STEPS: u64 = 3; + +/// One `Environment.Advance` per complete batch, and one boundary per advance. +async fn one_world_advance_per_complete_batch(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + let before = f.harness.environment_mutations(); + let reports = within("run", f.harness.coordinator.run(STEPS)).await.unwrap(); + assert_eq!(reports.len() as u64, STEPS); + assert_eq!(f.harness.coordinator.stats().advances, STEPS); + assert_eq!(f.harness.coordinator.phase(), Phase::Ready(STEPS)); + assert_eq!( + f.harness.coordinator.observation().unwrap().boundary.get(), + STEPS, + "the world is at exactly one boundary per batch" + ); + // The environment's progress counter moves once per advance and not otherwise. + assert_eq!(f.harness.environment_mutations() - before, STEPS); + assert_eq!(count(&f.harness.coordinator.audit, "advance:0"), 1); + assert_eq!(f.harness.coordinator.trace.transitions.len() as u64, STEPS); + f.shutdown().await; +} + +/// Every agent reaches Prepared before the batch is built and the world advances. +async fn every_agent_is_prepared_before_the_world_advances(via: Via) { + // Different completion delays, so "all prepared" cannot be an accident of timing. + let mut config = HarnessConfig::default(); + config.agents[0].faults = AgentFaults { prepare_delay_ms: 15, ..AgentFaults::default() }; + config.agents[1].faults = AgentFaults { prepare_delay_ms: 1, ..AgentFaults::default() }; + let mut f = fixture(via, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(STEPS)).await.unwrap(); + let audit = f.harness.coordinator.audit.clone(); + for k in 0..STEPS { + let advance = at(&audit, &format!("advance:{k}")); + for agent in [fly_a(), fly_b()] { + let prepared = at(&audit, &format!("prepared:{agent}@{k}")); + assert!( + prepared < advance, + "{agent} must be Prepared({k}) before the world advances: {audit:?}" + ); + } + } + f.shutdown().await; +} + +/// The task's transition evaluation runs exactly once per acknowledged world step. +async fn the_task_evaluates_each_transition_once(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + assert_eq!(f.harness.coordinator.evaluations(), 0, "bootstrap evaluates no transition"); + within("run", f.harness.coordinator.run(STEPS)).await.unwrap(); + assert_eq!(f.harness.coordinator.evaluations(), STEPS); + let audit = f.harness.coordinator.audit.clone(); + for k in 0..STEPS { + assert_eq!(count(&audit, &format!("evaluate:{k}")), 1); + } + f.shutdown().await; +} + +/// No agent starts the next Prepare, and nothing is published as committed, until every agent +/// has committed this transition. +async fn every_agent_commits_before_the_next_prepare_or_publication(via: Via) { + let mut config = HarnessConfig::default(); + config.agents[0].faults = AgentFaults { commit_delay_ms: 12, ..AgentFaults::default() }; + let mut f = fixture(via, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(STEPS)).await.unwrap(); + let audit = f.harness.coordinator.audit.clone(); + for k in 0..STEPS { + let publish = at(&audit, &format!("publish:{}", k + 1)); + for agent in [fly_a(), fly_b()] { + let committed = at(&audit, &format!("committed:{agent}@{k}")); + assert!( + committed < publish, + "{agent} must commit before boundary {} is published: {audit:?}", + k + 1 + ); + if k + 1 < STEPS { + let next = at(&audit, &format!("prepared:{agent}@{}", k + 1)); + for other in [fly_a(), fly_b()] { + let other_commit = at(&audit, &format!("committed:{other}@{k}")); + assert!( + other_commit < next, + "{other} must commit step {k} before {agent} prepares {}: {audit:?}", + k + 1 + ); + } + } + } + } + assert_eq!(f.harness.coordinator.stats().publications, STEPS + 1, "one per boundary, plus 0"); + f.shutdown().await; +} + +/// `step-v1` section 5: a 60 Hz world with a 1 ms model tick runs 16, 17, 17 ticks over three +/// steps, totalling 50, with a remainder of exactly zero. +async fn a_60_hz_world_with_a_1_ms_tick_runs_16_17_17(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(3)).await.unwrap(); + let transitions = &f.harness.coordinator.trace.transitions; + assert_eq!(transitions.len(), 3); + for agent in [fly_a(), fly_b()] { + let ticks: Vec = transitions + .iter() + .map(|t| { + t.agents + .iter() + .find(|a| a.agent_id == agent) + .expect("the agent is in every transition") + .ticks_advanced + .get() + }) + .collect(); + assert_eq!(ticks, vec![16, 17, 17], "{agent} tick profile"); + assert_eq!(ticks.iter().sum::(), 50); + let last = transitions + .last() + .unwrap() + .agents + .iter() + .find(|a| a.agent_id == agent) + .unwrap(); + assert!(last.remainder.is_zero(), "{agent} remainder after three steps"); + // Warm-up ticks are counted too, so brainTicks is warm-up plus the 50 gameplay ticks. + assert_eq!(last.brain_ticks.get(), 50 + f.harness.config.warmup_ticks); + } + f.shutdown().await; +} + +/// A pause arriving mid-step means "finish this transition, then pause", and it pauses at the +/// committed boundary rather than truncating anything. +async fn a_pause_mid_step_completes_the_step_and_pauses_at_the_boundary(via: Via) { + let mut config = HarnessConfig::default(); + // Both agents hold their Commit open, so the pause request lands inside the transition. + config.agents[0].faults = AgentFaults { commit_delay_ms: 40, ..AgentFaults::default() }; + config.agents[1].faults = AgentFaults { commit_delay_ms: 60, ..AgentFaults::default() }; + let mut f = fixture(via, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("step", f.harness.coordinator.step()).await.unwrap(); + + // A supervisor asks for a pause while the second transition is still running. + let handle = f.harness.coordinator.pause_handle(); + let asked = tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + handle.request(); + }); + let report = within("paused step", f.harness.coordinator.step()).await.unwrap(); + asked.await.unwrap(); + + // The transition completed and the session paused at its committed boundary. + assert_eq!(report.boundary, 2); + assert!(report.paused); + assert_eq!(f.harness.coordinator.phase(), Phase::Paused(2)); + assert!(f.harness.coordinator.phase().is_committed_boundary()); + assert_eq!(f.harness.coordinator.stats().advances, 2, "the pause truncated no transition"); + let audit = f.harness.coordinator.audit.clone(); + let pause = at(&audit, "pause:2"); + for agent in [fly_a(), fly_b()] { + assert!(at(&audit, &format!("committed:{agent}@1")) < pause); + } + assert!(at(&audit, "publish:2") < pause); + + // A paused worker retains its state and answers Status; the world does not advance. + let env = f.harness.coordinator.environment_ref().clone(); + let status = within("status", f.harness.coordinator.status(&env)).await.unwrap(); + assert_eq!(status.current_scope.unwrap().step.get(), 2); + assert_eq!(f.harness.coordinator.stats().advances, 2); + + f.harness.coordinator.resume().unwrap(); + assert_eq!(f.harness.coordinator.phase(), Phase::Ready(2)); + let report = within("resumed step", f.harness.coordinator.step()).await.unwrap(); + assert_eq!(report.boundary, 3); + assert!(!report.paused, "the pause request was consumed by the pause it caused"); + f.shutdown().await; +} + +/// Bootstrap and warm-up mutate the fake brains but cannot advance the environment or produce +/// a gameplay reward. +async fn bootstrap_cannot_advance_the_world_or_produce_a_reward(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + assert_eq!(f.harness.coordinator.phase(), Phase::Ready(0)); + let observation = f.harness.coordinator.observation().unwrap(); + assert_eq!(observation.boundary.get(), 0); + assert!(observation.world_time.is_zero()); + assert_eq!(f.harness.coordinator.stats().advances, 0); + assert_eq!(f.harness.coordinator.evaluations(), 0); + // The environment advanced nothing, so its status is still at boundary 0 with no batch. + let env = f.harness.coordinator.environment_ref().clone(); + let status = within("status", f.harness.coordinator.status(&env)).await.unwrap(); + assert_eq!(status.current_scope.as_ref().unwrap().step.get(), 0); + assert!(status.last_batch_id.is_none(), "no batch was ever applied"); + // Warm-up did run, with learning disabled, so the models did mutate. + for agent in [fly_a(), fly_b()] { + assert!( + f.harness.agent_mutations(&agent) >= f.harness.config.warmup_ticks, + "warm-up ticks are real mutations" + ); + } + // And the task ledger has no reward yet. + let progress = f.harness.coordinator.task_progress(); + assert_eq!(progress.number("totalReward").unwrap(), 0.0); + assert_eq!(progress.integer("transitions").unwrap(), 0); + f.shutdown().await; +} + +/// The published snapshot represents the committed boundary and labels the transition that +/// just ended. +async fn the_committed_snapshot_names_the_boundary_that_just_ended(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + // The snapshots topic retains its latest value, so a subscriber joining after boundary 0 + // still replays it before the boundaries that follow. + let observer = f.harness.observer().await.unwrap(); + let topic = f.harness.coordinator.topics().snapshots.clone(); + let mut subscription = observer + .subscribe( + &topic, + flybus::SubscriptionConfig::bounded().in_flight(8).replay(true), + ) + .await + .unwrap(); + within("run", f.harness.coordinator.run(2)).await.unwrap(); + + let mut boundaries = Vec::new(); + for _ in 0..3 { + let message = within("snapshot", subscription.next()).await.expect("a snapshot"); + let payload = message.payload().clone(); + let step: u64 = payload["scope"]["step"].as_str().unwrap().parse().unwrap(); + let decisions_present = payload["agents"] + .as_array() + .unwrap() + .iter() + .all(|a| !a["selectedDecision"].is_null()); + boundaries.push((step, decisions_present)); + // The frame the snapshot names travels as an owned attachment. + if step > 0 { + let frame = message.artifact("view.arena").expect("the published frame"); + assert_eq!(frame.reference().byte_length, 4 * 4 * 4); + } + } + assert_eq!(boundaries[0], (0, false), "boundary 0 has no decision or control"); + assert_eq!(boundaries[1], (1, true)); + assert_eq!(boundaries[2], (2, true)); + f.shutdown().await; +} + +/// A terminal task event is evaluated, its rewards committed once, and the session pauses at +/// that boundary before any further gameplay transition. +async fn a_terminal_episode_pauses_at_its_own_boundary(via: Via) { + let config = HarnessConfig { + terminal: fly_session::task::Terminal::AfterTransitions(3), + ..HarnessConfig::default() + }; + let mut f = fixture(via, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + let reports = within("run", f.harness.coordinator.run(5)).await.unwrap(); + let last = reports.last().unwrap(); + assert!(last.terminal, "the counter task asked for a terminal transition"); + assert!(last.paused); + assert_eq!(f.harness.coordinator.phase(), Phase::Paused(last.boundary)); + assert!(f.harness.coordinator.episode_request().is_some()); + // No worker resets itself, and no further gameplay transition is allowed. + let err = f.harness.coordinator.step().await.expect_err("no transition after terminal"); + assert_eq!(err.error.code, fly_session::types::ErrorCode::InvalidPhase); + f.shutdown().await; +} + +/// `Worker.Status` answers with the worker's own committed boundary and progress. +async fn status_answers_with_the_committed_boundary(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(2)).await.unwrap(); + for agent in [fly_a(), fly_b()] { + let worker = f.harness.coordinator.agent_ref(&agent).cloned().unwrap(); + let status = within("status", f.harness.coordinator.status(&worker)).await.unwrap(); + assert_eq!(status.state, fly_session::types::WorkerState::Ready); + assert_eq!(status.current_scope.unwrap().step.get(), 2); + let before = status.progress_counter.get(); + // A status query is not progress. + let again = within("status", f.harness.coordinator.status(&worker)).await.unwrap(); + assert_eq!(again.progress_counter.get(), before); + } + f.shutdown().await; +} + +/// `Agent.Initialize` is allowed only on an uninitialized agent. +async fn a_worker_refuses_a_second_initialize(via: Via) { + let mut f = default_fixture(via).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + let err = within("second bootstrap", f.harness.coordinator.bootstrap()) + .await + .expect_err("the environment is already initialized"); + assert_eq!(err.error.code, fly_session::types::ErrorCode::InvalidPhase); + f.shutdown().await; +} + +// ------------------------------------------------------------------------------------------- +// Dispatch order equivalence: this one builds its own fixtures per order. + +/// `step-v1` section 8: sequential, concurrent and reversed dispatch and completion orders all +/// produce the same behaviour trace, excluding request ids and other operational metadata. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn sequential_concurrent_and_reversed_orders_agree() { + let mut behaviours: BTreeMap> = BTreeMap::new(); + for via in [Via::Memory, Via::Unix] { + for order in [ + DispatchOrder::Sequential, + DispatchOrder::Concurrent, + DispatchOrder::Reversed, + ] { + let mut config = HarnessConfig::default(); + // Deliberately unequal completion times, so a concurrent run really does finish + // out of dispatch order. + config.agents[0].faults = + AgentFaults { prepare_delay_ms: 12, commit_delay_ms: 0, ..AgentFaults::default() }; + config.agents[1].faults = + AgentFaults { prepare_delay_ms: 0, commit_delay_ms: 9, ..AgentFaults::default() }; + let mut f = fixture(via, config).await; + f.harness.coordinator.dispatch = order; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(4)).await.unwrap(); + let behaviour = f.harness.coordinator.trace.behavior(); + assert_eq!(behaviour.len(), 4); + behaviours.insert(format!("{via:?}/{order:?}"), behaviour); + // The operational metadata is recorded but is not part of the comparison. + let operational = f.harness.coordinator.trace.transitions[0].operational(); + assert!(operational.iter().any(|(k, _)| k == "batchId")); + f.shutdown().await; + } + } + let mut iter = behaviours.iter(); + let (first_name, first) = iter.next().expect("at least one run"); + for (name, behaviour) in iter { + assert_eq!( + behaviour, first, + "{name} produced a different behaviour trace from {first_name}" + ); + } +} + +/// A one-agent composition still runs the same transaction, so the barrier is not two-agent +/// specific. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_single_agent_composition_runs_the_same_transaction() { + let config = HarnessConfig { + agents: vec![AgentSpec { + agent_id: Id::lit("fly-a"), + port_id: Id::lit("p1"), + seed: 7, + faults: AgentFaults::default(), + }], + ..HarnessConfig::default() + }; + let mut f: Fixture = fixture(Via::Memory, config).await; + within("bootstrap", f.harness.coordinator.bootstrap()).await.unwrap(); + within("run", f.harness.coordinator.run(3)).await.unwrap(); + assert_eq!(f.harness.coordinator.stats().advances, 3); + let ticks: Vec = f + .harness + .coordinator + .trace + .transitions + .iter() + .map(|t| t.agents[0].ticks_advanced.get()) + .collect(); + assert_eq!(ticks, vec![16, 17, 17]); + f.shutdown().await; +}