diff --git a/services/flysim/crates/flysim/src/frame.rs b/services/flysim/crates/flysim/src/frame.rs new file mode 100644 index 0000000..eb48f61 --- /dev/null +++ b/services/flysim/crates/flysim/src/frame.rs @@ -0,0 +1,618 @@ +//! The legacy frame: one Game Boy frame of the live loop, in its phases, in the one order. +//! +//! This is the order `simloop.rs` runs on the stream, and the order every harness that claims to +//! measure the stream runs: the trap hunt, the palette bench, the room-escape bench, and the +//! stub-readout drivers of the ROM tests and the scene probe. It used to be written out in each of +//! them, and the copies had drifted: the benches ticked the brain through `NeuralAgent::tick`, which +//! installs the previous frame and its rewards *after* the next ticks, so every bench ran the +//! brain one frame behind the stream. There is one copy now, and the parity oracle is this file. +//! +//! The phases are named for the lockstep transaction they become in the session framework +//! (`docs/design/session-framework/legacy-gameboy-v1.md` section 4): +//! +//! | phase | what it does | lockstep | +//! | --- | --- | --- | +//! | (host) | drains commands: sugar and operator pulses are applied here, before the ticks | admission at `Ready(k)` | +//! | [`LegacyFrame::prepare`] | 16 or 17 brain ticks, the remainder carried; decode with the scene's bound channels and the blocked direction | A: `Agent.Prepare` | +//! | [`LegacyFrame::execute`] | the raw mask, then the macro layer decides the mask | B: the executor | +//! | [`LegacyFrame::advance`] | the joypad, one emulator frame, the framebuffer, the audio | B: `Environment.Advance` | +//! | [`LegacyFrame::evaluate`] | reward events, the macro layer's observation, the location, the rank | C: the task | +//! | [`LegacyFrame::commit`] | install the frame, one stimulation per event, one reinforcement | D: `Agent.Commit` | +//! | (host) | the milestone archive | a capture at `Ready(k+1)` | +//! | [`LegacyFrame::boundary`] | the ratchet's capture and decision, and the rollback when it fires | C decides; slot save and rollback at `Ready(k+1)` | +//! +//! Two moves from the order `simloop.rs` used to spell out, both between operations that touch +//! disjoint state, so neither changes a byte: the visual frame is installed in `commit` rather +//! than straight after the emulator frame (nothing reads the network in between), and the +//! stimulation and reinforcement come after the macro layer's observation and the location +//! (which read the emulator and the adapter, never the network). The milestone archive stays +//! where the stream has it -- after the reinforcement, before the ratchet captures -- which is +//! why the ratchet is its own call after `transition`: the host takes its archive between the +//! two. `FLY_TRACE` (`crate::trace`) records every phase, and a trace of the stream from one +//! checkpoint is byte-identical before and after this extraction. + +use anyhow::{Result, anyhow}; +use flybrain_core::agent::NeuralAgent; +use flybrain_core::decoder::gameboy::to_button_mask; +use flybrain_gb::adapter::{GameAdapter, ProgressSnapshot}; +use flybrain_gb::emulator::{Emulator, FRAMEBUFFER_LEN}; +use flybrain_gb::macros::AdapterLedger; +use flybrain_gb::ratchet::{Ratchet, Snapshot}; +use flybrain_gb::recovery::{NeuralRecovery, recover_game}; +use flybrain_gb::RewardEvent; + +use crate::macros::{MacroEvent, MacroLayer, Silence}; +use crate::trace::FrameTrace; + +/// Everything one frame reads and writes besides the frame's own state: the parts the loop owns. +pub struct Parts<'a> { + pub agent: &'a mut NeuralAgent, + pub emulator: &'a mut Emulator, + pub adapter: &'a mut dyn GameAdapter, + pub ratchet: &'a mut Ratchet, + /// `None` in raw mode, where not one line of the macro layer runs. + pub macros: Option<&'a mut MacroLayer>, +} + +/// Where a host may look in, or time a phase. Every method defaults to nothing. +/// +/// The stream's loop uses [`FrameObserver::after`] for its per-phase profile and nothing else. A +/// harness may read the emulator between phases to measure the run, and a stub-readout harness may +/// replace the decision in [`FrameObserver::readout`]; nothing else about the frame is open. +pub trait FrameObserver { + /// A phase has finished. `agent` is lent for the profiler's kernel timings. + fn after(&mut self, _phase: FramePhase, _agent: &mut NeuralAgent) {} + + /// The decoded decision, before the executor sees it. The stream never replaces it; the trap + /// hunt's `FLY_TRAP_STUB` does, and the brain still ticks exactly as it would. + fn readout(&mut self, _ms: f64, _bound: Option<&[String]>, _active: &mut Vec) {} + + /// Just before the executor decides: O[k] is on the emulator, the palette is the one dealt + /// for it. + fn before_execute(&mut self, _frame: &LegacyFrame, _parts: &mut Parts<'_>, _active: &[String]) { + } + + /// The executor has decided and the mask is not yet on the joypad. + fn executed(&mut self, _frame: &LegacyFrame, _parts: &mut Parts<'_>, _executed: &Executed) {} +} + +/// The observer that observes nothing. +impl FrameObserver for () {} + +/// The points [`FrameObserver::after`] is called at, in order. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FramePhase { + /// The brain ticks are done. + Ticked, + /// The decode and the executor's decision are done. + Executed, + /// The emulator frame has run. + Emulated, + /// The framebuffer and the audio are taken. + Advanced, + /// Rewards are sampled, the scene observed and the transition committed to the brain. + Committed, +} + +/// Phase B's result. +#[derive(Debug, Default)] +pub struct Executed { + /// The mask the emulator is given. + pub mask: u32, + /// The macro layer's start and finish events, in order. + pub events: Vec, + /// Why nothing was pressed, when nothing was (`crate::macros::Decision::silence`). + pub silence: Option, +} + +/// Phase C's result. +#[derive(Debug)] +pub struct Evaluated { + /// Reward events from the frame just produced, in adapter order. + pub rewards: Vec, + /// The macro layer's own events from observing that frame (at most one abandonment). + pub abandoned: Vec, + pub progress: ProgressSnapshot, +} + +/// One transition `k -> k+1`, up to and including its commit. +#[derive(Debug)] +pub struct Transition { + /// Brain ticks this frame advanced. + pub ticks: u64, + /// The brain clock after them, which is the clock of every phase that follows. + pub ms: f64, + /// The scene's bound macro channels the decode was masked to; `None` in raw mode. + pub bound: Option>, + /// The decision the executor was given. + pub active: Vec, + pub executed: Executed, + /// The frame's audio, binjgb's unsigned 8-bit interleaved stereo. + pub audio: Vec, + pub evaluated: Evaluated, +} + +/// Why the ratchet rolled the game back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RollbackTrigger { + GameOver, + Stall, +} + +/// A rollback at the boundary. +#[derive(Debug)] +pub struct Rollback { + pub trigger: RollbackTrigger, + /// The running macro's abandonment and the restored scene's observation, in order. + pub events: Vec, +} + +/// What happened at `Ready(k+1)`. +#[derive(Debug, Default)] +pub struct Boundary { + /// The ratchet captured a slot this boundary. + pub captured: bool, + pub rollback: Option, +} + +/// The neural half of a ratchet recovery, wired to `flybrain-core`. +struct AgentRecovery<'a> { + agent: &'a mut NeuralAgent, +} + +impl NeuralRecovery for AgentRecovery<'_> { + fn clear_decoder_holds(&mut self) { + let ms = self.agent.network.ms; + self.agent.decoder.clear_holds(ms); + } + + fn clear_eligibility(&mut self) { + let ms = self.agent.network.ms; + self.agent.network.plasticity.clear_eligibility(ms); + } + + fn set_visual_frame(&mut self, frame: &[u8]) { + let (width, height) = (self.agent.frame.width, self.agent.frame.height); + self.agent.network.set_visual_frame(frame, width, height); + } +} + +/// The frame's own state: the clock remainder, the frame counter, the frame on screen, the mask, +/// and the readout's blocked-direction window. +/// +/// The window (`docs/readout.md`) is the player's area and tile as of the last frame the adapter +/// reported one, the channel the group is holding, and the brain clock at which *either* of those +/// last changed. A direction is only blamed once it has been held for a whole `blocked_ms` with no +/// movement, so a direction that has just won is never blamed for a wall the previous one hit. +/// All three are transient and never checkpointed: one hold of a wall after a restart is cheaper +/// than a stale position surviving a restore (`restore: legacy-transient-reset`). +pub struct LegacyFrame { + /// Fractional millisecond carried into the next frame; checkpointed. + pub remainder: f64, + /// Frames the emulator has run in this fly's life; checkpointed. + pub frame_counter: u64, + /// The frame on screen: the last one produced, or a restored slot's. + pub frame_buffer: Vec, + /// The mask on the joypad; checkpointed. + pub buttons: u32, + pub location: Option<(u32, u32, u32)>, + pub held_channel: Option, + pub blocked_since_ms: f64, + trace: Option, +} + +impl Default for LegacyFrame { + fn default() -> Self { + Self::new() + } +} + +impl LegacyFrame { + /// The state of a fresh process: nothing held, no location, the blocked window starting at + /// brain time 0 (legacy-gameboy-v1 section 14), and a black frame. + pub fn new() -> Self { + Self { + remainder: 0.0, + frame_counter: 0, + frame_buffer: vec![0u8; FRAMEBUFFER_LEN], + buttons: 0, + location: None, + held_channel: None, + blocked_since_ms: 0.0, + trace: None, + } + } + + /// Record every phase into `trace` (`FLY_TRACE`). + pub fn with_trace(mut self, trace: Option) -> Self { + self.trace = trace; + self + } + + /// The trace, when one is on: a host records its admissions and captures through it. + pub fn trace_mut(&mut self) -> Option<&mut FrameTrace> { + self.trace.as_mut() + } + + // -- setup ------------------------------------------------------------------------------- + + /// A fresh start: one frame with no button down, then the brain's warm-up on it + /// (`Environment.Initialize`, legacy-gameboy-v1 section 9). Returns that frame's audio. + pub fn initialize(&mut self, emulator: &mut Emulator, agent: &mut NeuralAgent) -> Result> { + emulator + .run_frame() + .map_err(|error| anyhow!("running the first frame: {error}"))?; + self.frame_buffer.copy_from_slice(emulator.framebuffer()); + self.frame_counter = 1; + let audio = emulator.take_audio_u8(); + agent.warmup(Some(&self.frame_buffer)).map_err(|error| anyhow!("{error}"))?; + Ok(audio) + } + + /// Everything a `FLYSIM01` checkpoint restores into the parts and the frame, in the order the + /// stream restores it; the host checks the cartridge and the compatibility string first. + /// + /// The readout transient is left as a fresh process has it (`legacy-transient-reset`), which + /// is what a restart of the service gives the fly. `import_state` of the agent is + /// self-validating, so a refused checkpoint leaves the agent as it was. + pub fn restore(&mut self, parts: &mut Parts<'_>, checkpoint: &crate::store::Checkpoint) -> Result<()> { + let runtime = &checkpoint.runtime; + if runtime.framebuffer.len() != FRAMEBUFFER_LEN { + anyhow::bail!("checkpoint framebuffer is {} bytes", runtime.framebuffer.len()); + } + parts.agent.import_state(&checkpoint.agent).map_err(|error| anyhow!("{error}"))?; + parts.emulator.import_state(&runtime.emulator).map_err(|error| anyhow!("{error}"))?; + if !runtime.reward.is_null() { + parts.adapter.import_state(&runtime.reward).map_err(|error| anyhow!("{error}"))?; + } + let snapshot = if runtime.ratchet_game.is_empty() { + None + } else { + Some(Snapshot { game: runtime.ratchet_game.clone(), frame: runtime.ratchet_frame.clone() }) + }; + parts + .ratchet + .import(Some(runtime.ratchet), snapshot, parts.adapter.rank_ladder().len()) + .map_err(|error| anyhow!("{error}"))?; + + self.remainder = checkpoint.agent.remainder; + self.frame_counter = runtime.emulator_frame; + self.buttons = runtime.buttons; + self.frame_buffer.copy_from_slice(&runtime.framebuffer); + let (width, height) = (parts.agent.frame.width, parts.agent.frame.height); + parts.agent.network.set_visual_frame(&self.frame_buffer, width, height); + parts.emulator.set_buttons(self.buttons as u8); + Ok(()) + } + + // -- the transition ---------------------------------------------------------------------- + + /// Transition `k -> k+1`, prepare through commit. The host takes its milestone archive after + /// this and then calls [`LegacyFrame::boundary`]. + pub fn transition( + &mut self, + parts: &mut Parts<'_>, + observer: &mut dyn FrameObserver, + ) -> Result { + let ticks = self.tick(parts.agent); + observer.after(FramePhase::Ticked, parts.agent); + + // Not the mode string: the adapter decides what counts as boot, because a platformer + // needs the permissive Start variant in four of its five modes (`GameAdapter::boot`). + let boot = parts.adapter.boot(); + // The scene's own macro buttons, for the macro group's per-decision mask + // (`docs/design/macros.md` section 12: "unbound channels are masked from the decision"). + // They are the bindings the previous frame's `observe` dealt, which is the palette the + // page is showing, so the fly is choosing among exactly the buttons the audience can see. + let bound = parts.macros.as_deref().map(MacroLayer::bound_channels); + let mut active = self.decode(parts.agent, boot, bound.as_deref()); + let ms = parts.agent.network.ms; + observer.readout(ms, bound.as_deref(), &mut active); + + observer.before_execute(self, parts, &active); + let raw = to_button_mask(&active); + let executed = self.execute( + parts.macros.as_deref_mut(), + &active, + raw, + ms, + parts.emulator, + &*parts.adapter, + ); + observer.executed(self, parts, &executed); + observer.after(FramePhase::Executed, parts.agent); + + let audio = self.advance(parts.emulator, parts.agent, observer)?; + observer.after(FramePhase::Advanced, parts.agent); + + let evaluated = self.evaluate(parts.emulator, parts.adapter, parts.macros.as_deref_mut(), ms); + self.commit(parts.agent, &evaluated.rewards, ms); + observer.after(FramePhase::Committed, parts.agent); + + Ok(Transition { ticks, ms, bound, active, executed, audio, evaluated }) + } + + /// Phase A: brain ticks and the decode, masked to `bound`. + pub fn prepare( + &mut self, + agent: &mut NeuralAgent, + boot: bool, + bound: Option<&[String]>, + ) -> (u64, Vec) { + let ticks = self.tick(agent); + (ticks, self.decode(agent, boot, bound)) + } + + /// Phase A, the ticks: 16 or 17 whole milliseconds, the fraction carried to the next frame. + pub fn tick(&mut self, agent: &mut NeuralAgent) -> u64 { + if let Some(trace) = self.trace.as_mut() { + trace.begin(self.frame_counter, agent.network.ms); + } + self.remainder += agent.ms_per_frame; + let steps = self.remainder.floor(); + self.remainder -= steps; + agent.network.step(steps as u64); + if let Some(trace) = self.trace.as_mut() { + trace.ticked(steps as u64, self.remainder, &agent.network); + } + steps as u64 + } + + /// Phase A, the readout: decode the rates with the blocked direction and the bound channels, + /// and restart the blocked window when the held channel changes. + pub fn decode(&mut self, agent: &mut NeuralAgent, boot: bool, bound: Option<&[String]>) -> Vec { + let ms = agent.network.ms; + let rates = agent.network.rates.clone(); + // The readout's blocked-direction cooldown (`docs/readout.md`): the direction the group + // is holding, once the adapter's position has stood still for a whole `blocked_ms`. The + // loop owns the clock and the position; the decoder only learns *which* channel did + // nothing. `blocked_ms == 0` -- the platformer preset, and the Game Boy preset before + // v0.1.1 -- switches the rule off here, before the decoder is asked. + let blocked_ms = agent.decoder.blocked_ms(); + let blocked = (blocked_ms > 0.0 && ms - self.blocked_since_ms >= blocked_ms) + .then(|| agent.decoder.current()) + .flatten() + .map(str::to_string); + let active = agent.decoder.decode_bound(&rates, ms, boot, blocked.as_deref(), bound); + // A new winner starts its own window: it has not had a hold to move in yet. + let held = agent.decoder.current().map(str::to_string); + if held != self.held_channel { + self.held_channel = held; + self.blocked_since_ms = ms; + } + active + } + + /// Phase B: the mask. `raw_mask` is the decision's own buttons (`to_button_mask`); in macros + /// mode the mask that reaches the emulator is the running macro's, or nothing, or -- on the + /// title screen alone -- the raw mask (`docs/design/macros.md` sections 4 and 12). In raw mode + /// it is the raw mask. + pub fn execute( + &mut self, + macros: Option<&mut MacroLayer>, + active: &[String], + raw_mask: u32, + ms: f64, + emulator: &mut Emulator, + adapter: &dyn GameAdapter, + ) -> Executed { + self.buttons = raw_mask; + let executed = match macros { + // The adapter's exploration ledger answers the ways out' "unvisited" -- read-only, by + // `&dyn`, and the only thing the palette is told about the reward side. + Some(layer) => { + let ledger = AdapterLedger(adapter); + let decision = layer.decide(active, self.buttons, ms, emulator, &ledger); + self.buttons = decision.mask; + Executed { mask: decision.mask, events: decision.events, silence: decision.silence } + } + None => Executed { mask: self.buttons, ..Executed::default() }, + }; + if let Some(trace) = self.trace.as_mut() { + trace.decided(active); + trace.executed(self.buttons, &executed.events); + } + executed + } + + /// Phase B, the environment: the joypad, one emulator frame, the frame it drew and its audio. + pub fn advance( + &mut self, + emulator: &mut Emulator, + agent: &mut NeuralAgent, + observer: &mut dyn FrameObserver, + ) -> Result> { + self.run(emulator)?; + observer.after(FramePhase::Emulated, agent); + Ok(self.take_frame(emulator)) + } + + /// [`LegacyFrame::advance`] without a brain, for the stub-readout drivers. + pub fn advance_stub(&mut self, emulator: &mut Emulator) -> Result> { + self.run(emulator)?; + Ok(self.take_frame(emulator)) + } + + fn run(&mut self, emulator: &mut Emulator) -> Result<()> { + emulator.set_buttons(self.buttons as u8); + emulator + .run_frame() + .map_err(|error| anyhow!("frame {}: {error}", self.frame_counter + 1))?; + self.frame_counter += 1; + Ok(()) + } + + fn take_frame(&mut self, emulator: &mut Emulator) -> Vec { + self.frame_buffer.copy_from_slice(emulator.framebuffer()); + if let Some(trace) = self.trace.as_mut() { + trace.advanced(&self.frame_buffer, emulator); + } + emulator.take_audio_u8() + } + + /// Phase C: rewards from the frame just produced, then the scene, then the location. + /// + /// `docs/design/macros.md` section 2: the scene is sampled once per game frame, after the + /// frame, so the palette the fly is offered on the next frame is the one for the frame it can + /// actually see. + pub fn evaluate( + &mut self, + emulator: &mut Emulator, + adapter: &mut dyn GameAdapter, + macros: Option<&mut MacroLayer>, + ms: f64, + ) -> Evaluated { + let rewards = adapter.sample(emulator, ms); + // At most one: a macro that has run into a scene with no palette. + let abandoned = match macros { + Some(layer) => { + let ledger = AdapterLedger(&*adapter); + layer.observe(emulator, &ledger, ms) + } + None => Vec::new(), + }; + // The cooldown's other reset: the player actually moved. `None` -- a battle, a script, a + // map transition -- is no information rather than "still", so the rule cannot fire while + // the fly has no control anyway. + let location = adapter.location(); + if location.is_some() && location != self.location { + self.location = location; + self.blocked_since_ms = ms; + } + let progress = adapter.progress(); + if let Some(trace) = self.trace.as_mut() { + trace.evaluated(&rewards, &abandoned, progress.rank); + } + Evaluated { rewards, abandoned, progress } + } + + /// Phase D: the frame just produced becomes the next ticks' visual drive, each reward event + /// stimulates once, and the summed value reinforces once. + pub fn commit(&mut self, agent: &mut NeuralAgent, rewards: &[RewardEvent], ms: f64) { + let (width, height) = (agent.frame.width, agent.frame.height); + agent.network.set_visual_frame(&self.frame_buffer, width, height); + let mut total = 0.0; + for event in rewards { + agent.network.stimulate(f64::from(event.stimulation_ms)); + total += event.value; + } + if agent.network.plasticity.enabled { + agent.network.plasticity.reinforce(total, ms); + } + } + + // -- the boundary ------------------------------------------------------------------------ + + /// `Ready(k+1)`: the ratchet captures on a safe frame above its best, observes, and rolls the + /// game back when it says so. + pub fn boundary( + &mut self, + parts: &mut Parts<'_>, + progress: &ProgressSnapshot, + ms: f64, + ) -> Result { + let safe = parts.adapter.safe_for_snapshot(); + let capture_due = safe && u64::from(progress.rank) > parts.ratchet.state.best; + let captured = if capture_due { + Some(Snapshot { + game: parts + .emulator + .export_state() + .map_err(|error| anyhow!("capturing a ratchet snapshot: {error}"))?, + frame: self.frame_buffer.clone(), + }) + } else { + None + }; + // The stall window's second progress signal (`docs/design/ladder.md`, the 2026-09-17 + // rule as amended 2026-09-22): the macro layer answers "nearer the objective" with the map + // graph it already walks (`docs/design/macros.md` section 12.15); in raw mode there is no + // layer and no objective, and the answer is false. + let nearer = parts.macros.as_deref().is_some_and(MacroLayer::nearer_the_objective); + let trace = &mut self.trace; + let mut saved = false; + let recover = parts.ratchet.observe_with_progress( + safe, + u64::from(progress.rank), + progress.unique_locations as u64, + ms as u64, + parts.adapter.game_over(), + nearer, + || { + let snapshot = + captured.expect("the ratchet only captures when a snapshot was prepared"); + if let Some(trace) = trace.as_mut() { + trace.slot_saved(&snapshot.game); + } + saved = true; + snapshot + }, + ); + let rollback = if recover { + // Two triggers, two stories on the ticker: a game over ended the run, a stall did not. + let trigger = + if parts.adapter.game_over() { RollbackTrigger::GameOver } else { RollbackTrigger::Stall }; + let events = self.rollback(parts)?; + Some(Rollback { trigger, events }) + } else { + None + }; + Ok(Boundary { captured: saved, rollback }) + } + + /// The ratchet's game-only rollback (`legacy-ratchet-rollback-v1`): the slot is restored, the + /// brain's holds and eligibility are cleared and it is shown the slot's frame, the buttons are + /// released, the blocked window restarts, and a running macro is abandoned and the restored + /// scene observed. The brain clock, its learning and the ratchet's ledger carry on. + pub fn rollback(&mut self, parts: &mut Parts<'_>) -> Result> { + let snapshot = Snapshot { + game: parts + .ratchet + .game() + .ok_or_else(|| anyhow!("the ratchet asked to recover with no snapshot"))? + .to_vec(), + frame: parts + .ratchet + .frame() + .ok_or_else(|| anyhow!("the ratchet snapshot has no framebuffer"))? + .to_vec(), + }; + let frame = { + let mut neural = AgentRecovery { agent: parts.agent }; + recover_game(parts.emulator, parts.adapter, &mut neural, &snapshot) + .map_err(|error| anyhow!("recovering the game: {error}"))? + }; + self.frame_buffer.copy_from_slice(&frame); + self.buttons = 0; + parts.emulator.set_buttons(0); + let ms = parts.agent.network.ms; + self.location = parts.adapter.location(); + self.held_channel = None; + self.blocked_since_ms = ms; + // A rollback restores a game the running macro's plan was never made for, so the macro is + // abandoned rather than carried over a map change it cannot see. The frame's `observe` ran + // before the ratchet decided, so the scene describes the run just thrown away: re-detect + // on the restored game rather than decide the next frame against a map the fly is no + // longer standing on. + let events = match parts.macros.as_deref_mut() { + Some(layer) => { + let mut events = layer.cancel(ms); + let ledger = AdapterLedger(&*parts.adapter); + events.extend(layer.observe(parts.emulator, &ledger, ms)); + events + } + None => Vec::new(), + }; + if let Some(trace) = self.trace.as_mut() { + trace.rolled_back(&events); + } + Ok(events) + } + + /// Write the open transition of the trace, if one is on. + pub fn finish_trace(&mut self) { + if let Some(trace) = self.trace.as_mut() { + trace.finish(); + } + } +} diff --git a/services/flysim/crates/flysim/src/lib.rs b/services/flysim/crates/flysim/src/lib.rs index 56b7523..410742a 100644 --- a/services/flysim/crates/flysim/src/lib.rs +++ b/services/flysim/crates/flysim/src/lib.rs @@ -27,6 +27,7 @@ pub mod config; pub mod eventlog; pub mod feed; pub mod feedbus; +pub mod frame; pub mod macros; pub mod metrics; pub mod pacing; diff --git a/services/flysim/crates/flysim/src/simloop.rs b/services/flysim/crates/flysim/src/simloop.rs index 36c9c55..097f540 100644 --- a/services/flysim/crates/flysim/src/simloop.rs +++ b/services/flysim/crates/flysim/src/simloop.rs @@ -8,7 +8,8 @@ //! //! The per-frame order is the prototype worker's (`fly-plays-pokemon/src/simulation.worker.ts`, //! `tick()`), not `NeuralAgent::tick`'s argument order, because the prototype samples reward -//! inside the same frame it produced: +//! inside the same frame it produced. It lives in `crate::frame::LegacyFrame`, the one copy every +//! harness runs too; in outline: //! //! 1. drain commands //! 2. step the brain 16 or 17 ms (the fractional remainder carries and is checkpointed) @@ -30,14 +31,13 @@ use anyhow::{Context, Result, anyhow, bail}; use flybrain_core::agent::{AgentConfig, NeuralAgent}; use flybrain_core::dataset::load_brain_dataset_from_dir; use flybrain_core::decoder::DecoderConfig; -use flybrain_core::decoder::gameboy::{gameboy_decoder_config_with_macros, to_button_mask}; +use flybrain_core::decoder::gameboy::gameboy_decoder_config_with_macros; use flybrain_core::decoder::platformer::platformer_decoder_config; use flybrain_core::lif::SweepPlan; use flybrain_gb::adapter::{DecoderPresetId, GameAdapter, ProgressSnapshot}; use flybrain_gb::macros::AdapterLedger; -use flybrain_gb::emulator::{DEFAULT_AUDIO_FRAMES, Emulator, FRAMEBUFFER_LEN}; +use flybrain_gb::emulator::{DEFAULT_AUDIO_FRAMES, Emulator}; use flybrain_gb::ratchet::Ratchet; -use flybrain_gb::recovery::{NeuralRecovery, recover_game}; use serde::Serialize; use serde_json::{Map, Value}; use tokio::sync::{mpsc, oneshot, watch}; @@ -45,6 +45,7 @@ use tokio::sync::{mpsc, oneshot, watch}; use crate::chat::{ChatLimiter, ChatRefusal, ChatRing, DenyList, RejectReason}; use crate::config::Config; use crate::eventlog::{EventLog, EventRing, NewEvent, now_wall_ms, utc_day}; +use crate::frame::{FrameObserver, FramePhase, LegacyFrame, Parts, RollbackTrigger}; use crate::macros::{MacroEvent, MacroLayer, macro_layer}; use crate::metrics::Metrics; use crate::pacing::{Pacer, RealtimeWindow}; @@ -274,25 +275,26 @@ pub fn booting_snapshot(seq: u64, wall_ms: u64, mode: MacroMode) -> Snapshot { } } -/// The neural half of a ratchet recovery, wired to `flybrain-core`. -struct AgentRecovery<'a> { - agent: &'a mut NeuralAgent, +/// The stream's only look inside the frame: the per-phase profile (`crate::profile`). +struct Laps<'a> { + profiler: &'a mut Profiler, } -impl NeuralRecovery for AgentRecovery<'_> { - fn clear_decoder_holds(&mut self) { - let ms = self.agent.network.ms; - self.agent.decoder.clear_holds(ms); - } - - fn clear_eligibility(&mut self) { - let ms = self.agent.network.ms; - self.agent.network.plasticity.clear_eligibility(ms); - } - - fn set_visual_frame(&mut self, frame: &[u8]) { - let (width, height) = (self.agent.frame.width, self.agent.frame.height); - self.agent.network.set_visual_frame(frame, width, height); +impl FrameObserver for Laps<'_> { + fn after(&mut self, phase: FramePhase, agent: &mut NeuralAgent) { + match phase { + FramePhase::Ticked => { + self.profiler.lap(Phase::Step); + if self.profiler.enabled() { + self.profiler.absorb_brain(agent.network.timings()); + agent.network.reset_timings(); + } + } + FramePhase::Executed => self.profiler.lap(Phase::Decode), + FramePhase::Emulated => self.profiler.lap(Phase::Emulate), + FramePhase::Advanced => self.profiler.lap(Phase::Retina), + FramePhase::Committed => self.profiler.lap(Phase::Rewards), + } } } @@ -337,24 +339,11 @@ pub struct Sim { next_generation: u64, best_archived_rank: Option, - /// Fractional millisecond carried into the next frame, exactly as `NeuralAgent` keeps it. - remainder: f64, - frame_counter: u64, - frame_buffer: Vec, + /// The frame order and its state: the remainder, the frame counter, the frame on screen, the + /// mask and the readout's blocked-direction window (`crate::frame`). + frame: LegacyFrame, pending_audio: Vec, dc_blocker: DcBlocker, - buttons: u32, - /// The readout's blocked-direction cooldown (`docs/readout.md`), as the loop computes it: the - /// player's area and tile as of the last frame the adapter reported one, the channel the group is - /// holding, and the brain clock at which *either* of those last changed. A direction is only - /// blamed once it has been held for a whole `blocked_ms` with no movement, so a direction that - /// has just won is never blamed for a wall the previous one hit. - /// - /// All three are transient and deliberately not checkpointed: one hold of a wall after a - /// restart is cheaper than a stale position surviving a restore. - location: Option<(u32, u32, u32)>, - held_channel: Option, - blocked_since_ms: f64, /// Palette mode (`docs/design/macros.md`), or `None` in raw mode — which is the default and /// which is byte for byte the behaviour that predates it: every call site below is inside an @@ -398,8 +387,6 @@ pub struct Sim { restored: bool, /// Per-phase timing, off unless `FLY_PROFILE_SECONDS` is set (`crate::profile`). profiler: Profiler, - /// The per-frame trace, off unless `FLY_TRACE` names a file (`crate::trace`). - trace: Option, } /// The checkpoint compatibility string, from the pieces that make it up. @@ -569,16 +556,14 @@ impl Sim { writer_thread: None, next_generation: 1, best_archived_rank: None, - remainder: 0.0, - frame_counter: 0, - frame_buffer: vec![0u8; FRAMEBUFFER_LEN], + // The per-frame trace, off unless `FLY_TRACE` names a file (`crate::trace`). + frame: LegacyFrame::new().with_trace( + FrameTrace::from_env() + .with_context(|| format!("creating the {} file", crate::trace::ENV))?, + ), pending_audio: Vec::new(), dc_blocker: DcBlocker::default(), - buttons: 0, status: FeedStatus::Booting, - location: None, - held_channel: None, - blocked_since_ms: 0.0, pending_recovery: false, seq: 0, started, @@ -606,8 +591,6 @@ impl Sim { semantic_rewards, restored: false, profiler: Profiler::from_env(now), - trace: FrameTrace::from_env() - .with_context(|| format!("creating the {} file", crate::trace::ENV))?, agent, emulator, adapter, @@ -691,7 +674,7 @@ impl Sim { } tracing::info!( origin = %candidate.origin, - frame = self.frame_counter, + frame = self.frame.frame_counter, brain_ms = self.agent.network.ms, rank = self.rank, "restored" @@ -754,46 +737,21 @@ impl Sim { self.compatibility ), } - if runtime.framebuffer.len() != FRAMEBUFFER_LEN { - bail!("checkpoint framebuffer is {} bytes", runtime.framebuffer.len()); - } // `import_state` is self-validating and a no-op on failure, so a refused checkpoint // leaves the agent exactly as it was and the next candidate starts clean. - self.agent - .import_state(&checkpoint.agent) - .map_err(|error| anyhow!("{error}"))?; - self.emulator - .import_state(&runtime.emulator) - .map_err(|error| anyhow!("{error}"))?; - if !runtime.reward.is_null() { - self.adapter - .import_state(&runtime.reward) - .map_err(|error| anyhow!("{error}"))?; + { + let mut parts = Parts { + agent: &mut self.agent, + emulator: &mut self.emulator, + adapter: self.adapter.as_mut(), + ratchet: &mut self.ratchet, + macros: self.macros.as_mut(), + }; + self.frame.restore(&mut parts, &checkpoint)?; } - let snapshot = if runtime.ratchet_game.is_empty() { - None - } else { - Some(flybrain_gb::ratchet::Snapshot { - game: runtime.ratchet_game.clone(), - frame: runtime.ratchet_frame.clone(), - }) - }; - self.ratchet - .import(Some(runtime.ratchet), snapshot, self.adapter.rank_ladder().len()) - .map_err(|error| anyhow!("{error}"))?; - - self.remainder = checkpoint.agent.remainder; - self.frame_counter = runtime.emulator_frame; - self.buttons = runtime.buttons; self.rank_since_ms = runtime.rank_since_ms; - self.frame_buffer.copy_from_slice(&runtime.framebuffer); - let (width, height) = (self.agent.frame.width, self.agent.frame.height); - self.agent - .network - .set_visual_frame(&self.frame_buffer, width, height); self.rank = self.adapter.progress().rank; self.log.resume_from(runtime.last_event_id); - self.emulator.set_buttons(self.buttons as u8); Ok(()) } @@ -801,16 +759,8 @@ impl Sim { /// readout on the settled rates. Never after a restore, which already carries a settled /// network and a calibrated decoder. fn fresh_start(&mut self) -> Result<()> { - self.emulator - .run_frame() - .map_err(|error| anyhow!("running the first frame: {error}"))?; - self.frame_buffer.copy_from_slice(self.emulator.framebuffer()); - self.frame_counter = 1; - let raw = self.emulator.take_audio_u8(); + let raw = self.frame.initialize(&mut self.emulator, &mut self.agent)?; self.dc_blocker.process_into(&raw, &mut self.pending_audio); - self.agent - .warmup(Some(&self.frame_buffer)) - .map_err(|error| anyhow!("{error}"))?; self.emit(NewEvent::new(FeedEventKind::System, "Fresh start: the fly woke up")); Ok(()) } @@ -932,132 +882,27 @@ impl Sim { self.deny_list.maybe_reload(Instant::now(), forced); } - /// One frame, in the prototype's order. + /// One frame, in the prototype's order (`crate::frame`). fn step_frame(&mut self) -> Result<()> { - if let Some(trace) = self.trace.as_mut() { - trace.begin(self.frame_counter, self.agent.network.ms); - } - // 2. Step the brain: 16 or 17 integer ticks, the remainder carried and checkpointed. - self.remainder += self.agent.ms_per_frame; - let steps = self.remainder.floor(); - self.remainder -= steps; - self.agent.network.step(steps as u64); - if let Some(trace) = self.trace.as_mut() { - trace.ticked(steps as u64, self.remainder, &self.agent.network); - } - self.profiler.lap(Phase::Step); - if self.profiler.enabled() { - self.profiler.absorb_brain(self.agent.network.timings()); - self.agent.network.reset_timings(); - } - - // 3. Decode, 4. apply the buttons. - // Not the mode string: the adapter decides what counts as boot, because a platformer needs - // the permissive Start variant in four of its five modes (`GameAdapter::boot`). - let boot = self.adapter.boot(); - let ms = self.agent.network.ms; - let rates = self.agent.network.rates.clone(); - // The readout's blocked-direction cooldown (`docs/readout.md`): the direction the group is - // holding, once the adapter's position has stood still for a whole `blocked_ms`. The loop - // owns the clock and the position; the decoder only learns *which* channel did nothing. - // `blocked_ms == 0` -- the platformer preset, and the Game Boy preset before v0.1.1 -- - // switches the rule off here, before the decoder is asked. - let blocked_ms = self.agent.decoder.blocked_ms(); - let blocked = (blocked_ms > 0.0 && ms - self.blocked_since_ms >= blocked_ms) - .then(|| self.agent.decoder.current()) - .flatten() - .map(str::to_string); - // The scene's own macro buttons, for the macro group's per-decision mask - // (`docs/design/macros.md` section 12: "unbound channels are masked from the decision"). - // They are the bindings the previous frame's `observe` dealt, which is the palette the - // page is showing, so the fly is choosing among exactly the buttons the audience can see. - // `None` in raw mode, where the group has no channels to mask anyway. - let bound = self.macros.as_ref().map(MacroLayer::bound_channels); - let active = self.agent.decoder.decode_bound( - &rates, - ms, - boot, - blocked.as_deref(), - bound.as_deref(), - ); - if let Some(trace) = self.trace.as_mut() { - trace.decided(&active); - } - // A new winner starts its own window: it has not had a hold to move in yet. - let held = self.agent.decoder.current().map(str::to_string); - if held != self.held_channel { - self.held_channel = held; - self.blocked_since_ms = ms; - } - self.buttons = to_button_mask(&active); - // Macros mode (`docs/design/macros.md` sections 4 and 12): the same decode, plus the - // macro group whose winner is in `active` alongside the buttons. The mask that reaches - // the emulator is the running macro's, or nothing, or -- on the title screen alone -- the - // raw mask above. In raw mode `self.macros` is `None` and not one line of this runs. - let started_or_finished = match self.macros.as_mut() { - // Two disjoint fields of the same struct, so the layer can read the emulator while - // the loop still owns both. Taking the layer out and putting it back would leave - // raw mode running silently if anything in between ever panicked. - Some(layer) => { - // Three disjoint fields: the layer decides, the emulator is read, and the - // adapter's exploration ledger answers the ways out' "unvisited" — read-only, by - // `&dyn`, and the only thing the palette is told about the reward side. - let ledger = AdapterLedger(self.adapter.as_ref()); - let decision = layer.decide( - &active, - self.buttons, - ms, - &mut self.emulator, - &ledger, - ); - self.buttons = decision.mask; - decision.events - } - None => Vec::new(), + // Prepare through commit: ticks, decode, the executor's mask, one emulator frame, rewards, + // the scene and the location, then the stimulations and the reinforcement. + let transition = { + let mut parts = Parts { + agent: &mut self.agent, + emulator: &mut self.emulator, + adapter: self.adapter.as_mut(), + ratchet: &mut self.ratchet, + macros: self.macros.as_mut(), + }; + let mut laps = Laps { profiler: &mut self.profiler }; + self.frame.transition(&mut parts, &mut laps)? }; - if let Some(trace) = self.trace.as_mut() { - trace.executed(self.buttons, &started_or_finished); - } - self.emit_macro_events(&started_or_finished); - self.emulator.set_buttons(self.buttons as u8); - self.profiler.lap(Phase::Decode); - - // 5. Run one emulator frame, 6. set the visual frame from it. - self.emulator - .run_frame() - .map_err(|error| anyhow!("frame {}: {error}", self.frame_counter + 1))?; - self.frame_counter += 1; Metrics::incr(&self.shared.metrics.sim_frames); - self.profiler.lap(Phase::Emulate); - self.frame_buffer.copy_from_slice(self.emulator.framebuffer()); - let (width, height) = (self.agent.frame.width, self.agent.frame.height); - self.agent - .network - .set_visual_frame(&self.frame_buffer, width, height); - if let Some(trace) = self.trace.as_mut() { - trace.advanced(&self.frame_buffer, &self.emulator); - } - let raw = self.emulator.take_audio_u8(); - self.dc_blocker.process_into(&raw, &mut self.pending_audio); - self.profiler.lap(Phase::Retina); - - // 7. Sample rewards from the frame just produced. - let ms = self.agent.network.ms; - let events = { - let (adapter, emulator) = (&mut self.adapter, &mut self.emulator); - adapter.sample(emulator, ms) - }; - - // 8. Stimulate once per event, 9. reinforce with the sum. - let mut total = 0.0; - for event in &events { - self.agent.network.stimulate(f64::from(event.stimulation_ms)); - total += event.value; - } - if self.agent.network.plasticity.enabled { - self.agent.network.plasticity.reinforce(total, ms); - } - for event in &events { + self.dc_blocker.process_into(&transition.audio, &mut self.pending_audio); + // The feed events, in the order the phases produced them: the executor's starts and + // finishes, the rewards, then the observation's abandonment. + self.emit_macro_events(&transition.executed.events); + for event in &transition.evaluated.rewards { let kind = RewardKind::from_adapter(event.kind); let mut new = NewEvent::new(FeedEventKind::Reward, event.label.clone()) .value(event.value); @@ -1066,79 +911,27 @@ impl Sim { } self.emit(new); } + self.emit_macro_events(&transition.evaluated.abandoned); - self.profiler.lap(Phase::Rewards); - - // `docs/design/macros.md` section 2: the scene is sampled once per game frame, after the - // frame. So the palette the fly is offered on the next frame is the one for the frame it - // can actually see, and the feed's `game.scene` is never a frame ahead of the screen. - let abandoned = match self.macros.as_mut() { - Some(layer) => { - let ledger = AdapterLedger(self.adapter.as_ref()); - layer.observe(&mut self.emulator, &ledger, ms) - } - None => Vec::new(), - }; - // At most one: a macro that has run into a scene with no palette, abandoned before the - // header this frame publishes can show it beside that scene. - self.emit_macro_events(&abandoned); - - // The cooldown's other reset: the player actually moved. `None` -- a battle, a script, a - // map transition -- is no information rather than "still", so the rule cannot fire while - // the fly has no control anyway. - let location = self.adapter.location(); - if location.is_some() && location != self.location { - self.location = location; - self.blocked_since_ms = ms; - } - - // 10. Ratchet: observe, and recover if it says so. - let progress = self.adapter.progress(); - if let Some(trace) = self.trace.as_mut() { - trace.evaluated(&events, &abandoned, progress.rank); - } + // The milestone archive sits here, after the commit and before the ratchet captures, + // which is the legacy order legacy-gameboy-v1 section 4 declares. + let ms = transition.ms; + let progress = transition.evaluated.progress; self.track_rank(&progress, ms); - let safe = self.adapter.safe_for_snapshot(); - let capture_due = safe && u64::from(progress.rank) > self.ratchet.state.best; - let captured = if capture_due { - Some(flybrain_gb::ratchet::Snapshot { - game: self - .emulator - .export_state() - .map_err(|error| anyhow!("capturing a ratchet snapshot: {error}"))?, - frame: self.frame_buffer.clone(), - }) - } else { - None + + // `Ready(k+1)`: the ratchet captures, observes, and rolls the game back if it says so. + let boundary = { + let mut parts = Parts { + agent: &mut self.agent, + emulator: &mut self.emulator, + adapter: self.adapter.as_mut(), + ratchet: &mut self.ratchet, + macros: self.macros.as_mut(), + }; + self.frame.boundary(&mut parts, &progress, ms)? }; - // The stall window's second progress signal (`docs/design/ladder.md`, the 2026-09-17 - // rule as amended 2026-09-22): coverage is ground never stood on, and a fly crossing a - // town it has already covered to reach the rung's own door earns none of it while it is - // plainly getting somewhere. The macro layer answers with the map graph it already walks - // (`docs/design/macros.md` section 12.15); in raw mode there is no layer and no - // objective, and the answer is false. - let nearer = self.macros.as_ref().is_some_and(MacroLayer::nearer_the_objective); - let trace = &mut self.trace; - let recover = self.ratchet.observe_with_progress( - safe, - u64::from(progress.rank), - progress.unique_locations as u64, - ms as u64, - self.adapter.game_over(), - nearer, - || { - let snapshot = - captured.expect("the ratchet only captures when a snapshot was prepared"); - if let Some(trace) = trace.as_mut() { - trace.slot_saved(&snapshot.game); - } - snapshot - }, - ); - if recover { - // Two triggers, two stories on the ticker: a game over ended the run, a stall did not. - let reason = if self.adapter.game_over() { "Game over" } else { "Stuck" }; - self.recover(reason)?; + if let Some(rollback) = boundary.rollback { + self.recovered(rollback.trigger, &rollback.events); } self.profiler.lap(Phase::Ratchet); Ok(()) @@ -1171,32 +964,14 @@ impl Sim { } } - fn recover(&mut self, reason: &str) -> Result<()> { - let snapshot = flybrain_gb::ratchet::Snapshot { - game: self - .ratchet - .game() - .ok_or_else(|| anyhow!("the ratchet asked to recover with no snapshot"))? - .to_vec(), - frame: self - .ratchet - .frame() - .ok_or_else(|| anyhow!("the ratchet snapshot has no framebuffer"))? - .to_vec(), + /// The host's half of a rollback the frame has already applied: the ticker, the metric, the + /// `recovering` status and a durable checkpoint. + fn recovered(&mut self, trigger: RollbackTrigger, events: &[crate::macros::MacroEvent]) { + // Two triggers, two stories on the ticker: a game over ended the run, a stall did not. + let reason = match trigger { + RollbackTrigger::GameOver => "Game over", + RollbackTrigger::Stall => "Stuck", }; - let frame = { - let mut neural = AgentRecovery { agent: &mut self.agent }; - recover_game( - &mut self.emulator, - self.adapter.as_mut(), - &mut neural, - &snapshot, - ) - .map_err(|error| anyhow!("recovering the game: {error}"))? - }; - self.frame_buffer.copy_from_slice(&frame); - self.buttons = 0; - self.emulator.set_buttons(0); Metrics::incr(&self.shared.metrics.recoveries_total); let attempts = self.ratchet.state.attempts; self.emit( @@ -1209,34 +984,11 @@ impl Sim { ) .value(f64::from(self.rank)), ); - self.location = self.adapter.location(); - self.held_channel = None; - self.blocked_since_ms = self.agent.network.ms; - // A rollback restores a game the running macro's plan was never made for, so the macro is - // abandoned here rather than carried over a map change it cannot see. - let abandoned = match self.macros.as_mut() { - Some(layer) => { - let events = layer.cancel(self.agent.network.ms); - // The frame's `observe` ran before the ratchet decided to roll back, so the scene - // and the palette describe the run that was just thrown away. The restored game - // is in WRAM now, so re-detect here rather than let the next frame's decision be - // made against a map the fly is no longer standing on. - let ledger = AdapterLedger(self.adapter.as_ref()); - let mut events = events; - events.extend(layer.observe(&mut self.emulator, &ledger, self.agent.network.ms)); - events - } - None => Vec::new(), - }; - if let Some(trace) = self.trace.as_mut() { - trace.rolled_back(&abandoned); - } - self.emit_macro_events(&abandoned); + self.emit_macro_events(events); self.pending_recovery = true; if let Err(error) = self.checkpoint(true, None) { tracing::error!(%error, "could not checkpoint after a recovery"); } - Ok(()) } // -- commands ------------------------------------------------------------------------ @@ -1326,7 +1078,7 @@ impl Sim { .clamp(1.0, self.shared.config.control.sugar_max_ms) .min(self.shared.config.control.sugar_max_ms); self.agent.network.stimulate(duration); - if let Some(trace) = self.trace.as_mut() { + if let Some(trace) = self.frame.trace_mut() { trace.sugar(duration); } @@ -1350,7 +1102,7 @@ impl Sim { fn reward(&mut self, value: f64, by: &str, source: &str) -> u64 { let ms = self.agent.network.ms; self.agent.network.plasticity.reinforce(value, ms); - if let Some(trace) = self.trace.as_mut() { + if let Some(trace) = self.frame.trace_mut() { trace.reward_pulse(value); } tracing::info!(by, source, value, "reward pulse applied"); @@ -1430,9 +1182,7 @@ impl Sim { if let Err(error) = self.checkpoint_blocking(true, None) { tracing::error!(%error, "the final checkpoint failed"); } - if let Some(trace) = self.trace.as_mut() { - trace.finish(); - } + self.frame.finish_trace(); if let Err(error) = self.log.flush() { tracing::error!(%error, "the final event log flush failed"); } @@ -1542,8 +1292,9 @@ impl Sim { durable && self.best_archived_rank.is_none_or(|best| *rank > best) }); let (generation, agent, runtime) = self.snapshot_state()?; - if let Some(trace) = self.trace.as_mut() { - trace.capture(generation, self.frame_counter); + let step = self.frame.frame_counter; + if let Some(trace) = self.frame.trace_mut() { + trace.capture(generation, step); } if let Some(rank) = archive_rank { self.best_archived_rank = Some(rank); @@ -1593,15 +1344,15 @@ impl Sim { let mut agent_state = self.agent.export_state(); // The sim loop owns the frame remainder, not `NeuralAgent::tick`, so the exported state // carries the loop's value. - agent_state.remainder = self.remainder; + agent_state.remainder = self.frame.remainder; let runtime = RuntimeState { generation, wall_ms: now_wall_ms(), rom_sha256: self.rom_sha256.clone(), - emulator_frame: self.frame_counter, + emulator_frame: self.frame.frame_counter, compatibility: self.compatibility.clone(), speed: self.shared.config.loop_.speed, - buttons: self.buttons, + buttons: self.frame.buttons, rank_since_ms: self.rank_since_ms, last_event_id: self.log.next_id().saturating_sub(1), reward: self.adapter.export_state(), @@ -1610,7 +1361,7 @@ impl Sim { .emulator .export_state() .map_err(|error| anyhow!("exporting the Game Boy: {error}"))?, - framebuffer: self.frame_buffer.clone(), + framebuffer: self.frame.frame_buffer.clone(), ratchet_game: self.ratchet.game().map(<[u8]>::to_vec).unwrap_or_default(), ratchet_frame: self.ratchet.frame().map(<[u8]>::to_vec).unwrap_or_default(), }; @@ -1683,7 +1434,7 @@ impl Sim { let audio = f32_bytes(&self.pending_audio); self.pending_audio.clear(); ( - Arc::new(self.frame_buffer.clone()), + Arc::new(self.frame.frame_buffer.clone()), Arc::new(audio), Arc::new(bitset), count, @@ -1709,8 +1460,8 @@ impl Sim { uptime_seconds: self.started.elapsed().as_secs_f64(), run_seconds: finite(ms / 1000.0).max(0.0), brain_ms: finite(ms).max(0.0), - frame: self.frame_counter, - buttons: self.buttons & 0xff, + frame: self.frame.frame_counter, + buttons: self.frame.buttons & 0xff, rates, population_rate: finite(self.agent.network.population_rate).max(0.0), spike_count,