diff --git a/services/flysim/Cargo.lock b/services/flysim/Cargo.lock index 0aba9e1..b2a05e3 100644 --- a/services/flysim/Cargo.lock +++ b/services/flysim/Cargo.lock @@ -441,6 +441,18 @@ dependencies = [ "sha2", ] +[[package]] +name = "flybus" +version = "0.1.1" +dependencies = [ + "libc", + "serde", + "serde_json", + "sha2", + "tempfile", + "tokio", +] + [[package]] name = "flysim" version = "0.1.1" diff --git a/services/flysim/Cargo.toml b/services/flysim/Cargo.toml index a403e27..79b775e 100644 --- a/services/flysim/Cargo.toml +++ b/services/flysim/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/flybrain-core", "crates/flybrain-gb", "crates/flysim"] +members = ["crates/flybrain-core", "crates/flybrain-gb", "crates/flybus", "crates/flysim"] [workspace.package] version = "0.1.1" diff --git a/services/flysim/crates/flybus/Cargo.toml b/services/flysim/crates/flybus/Cargo.toml new file mode 100644 index 0000000..fc4bcb1 --- /dev/null +++ b/services/flysim/crates/flybus/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "flybus" +version.workspace = true +edition = "2024" +rust-version.workspace = true +license.workspace = true +publish = false +description = "A small local RPC and pub/sub bus with immutable file-backed artifacts." + +[dependencies] +# `flock`, `posix_fallocate` and `O_NOFOLLOW`; everything else is std or Tokio. +libc = "0.2" +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +tokio = { version = "1", features = ["rt", "net", "io-util", "sync", "time", "macros"] } + +[dev-dependencies] +sha2 = { workspace = true } +tempfile = "3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } diff --git a/services/flysim/crates/flybus/README.md b/services/flysim/crates/flybus/README.md new file mode 100644 index 0000000..d3086f0 --- /dev/null +++ b/services/flysim/crates/flybus/README.md @@ -0,0 +1,370 @@ +# flybus + +A small local RPC and pub/sub bus with immutable file-backed artifacts, for Tokio. + +One library, one router, one wire protocol. Messages are small JSON envelopes carrying +metadata and artifact references. Large immutable bytes live in a file store the router +manages. Ownership follows deliveries and explicit holds. The router moves messages and tracks +ownership; it does not know what the messages mean. + +This crate implements the Flybus v1 draft (session-framework design, `bus-v1`, draft 1 of +2026-09-18), using the scalar encodings of its companion `ipc-v1` (`Id`, `U64`, `Digest`). +Where this crate narrows or extends the draft, the difference is listed under +[Differences from the draft](#differences-from-the-draft). + +Nothing in the crate is specific to a game, a brain or a stream. It is a workspace member and +no other crate depends on it yet. + +## Layout + +| Module | Contents | +| --- | --- | +| `wire` (public) | Scalars, strict JSON, `Envelope`, `ArtifactRef`, `Attachment`, `Location`, framing, `CONTRACT` and `contract_digest()` | +| `error` (public) | `ErrorCode`, `Dispatch`, `BusError` | +| `limits` (public) | `Limits` and its hello encoding | +| `policy` (public) | `Policy`, `Grants`, `Pattern` | +| `router` | `Router`, `RouterConfig`, `RouterStats`, `UnixListenerHandle`; the state machine in `router/state.rs` | +| `client` | `Client` and every handle type | +| `store` | The file store (router side) and location resolution (client side) | +| `transport` | `Transport`, the `Stream` trait | + +Dependencies: `tokio`, `serde`, `serde_json`, `sha2` and `libc` (for `flock`, +`posix_fallocate` and `O_NOFOLLOW`), all already in the workspace lockfile. + +## API + +### Router + +```rust +let mut config = RouterConfig::new("/run/example/bus-store"); // default Limits, closed Policy +config.policy = Policy::closed() + .client("coordinator", Grants { call: vec![Pattern::prefix("agent.")], ..Grants::default() }) + .client("agent-a", Grants { register: vec![Pattern::exact("agent.fly-a")], ..Grants::default() }); +let router = Router::new(config)?; // creates /, cleans orphans +let listener = router.listen_unix_as("/run/example/coordinator.sock", "coordinator").await?; +let transport = router.connect_in_memory_as("coordinator"); +router.serve_as(any_async_read_write_stream, "coordinator"); +router.stats(); // RouterStats (logical registry counters) +router.shutdown(); // notices, then closes every connection +``` + +- `Router` is `Clone`; all clones are one router. `serve_as`, `connect_in_memory_as` and + `listen_unix_as` bind the launcher's expected participant id to the transport before Hello. + A mismatching Hello is refused before registration or routing. +- The unbound `serve`, `connect_in_memory` and `listen_unix` entry points are only for explicit + `Policy::open()` trusted/test deployments. Restricted policies refuse unbound transports. + Identity in open/unbound mode is self-asserted and is not authentication. +- `RouterConfig::hello_timeout` defaults to 5 seconds. `max_clients` counts all accepted + connections, pending and active. +- `router_id()`, `store_id()`, `store_root()` and `store_dir()` report the incarnation and + paths. +- `RouterStats` has `connections`, `services`, `topics`, `subscriptions`, `calls` (correlation + records, detached included), `active_calls`, `artifacts`, `sealed_artifacts`, + `artifact_roots`, `owners`, `reply_capabilities`, `queued`, `store_bytes` and + `retained_bytes`. +- `Limits` (defaults in brackets): `max_clients` [64], `max_services` [256], `max_topics` + [512], `max_subscriptions_per_client` [128], `max_subscriptions` [1024], + `max_active_calls_per_client` [64], `max_service_queued` / `max_service_in_flight` [16 / 16], + `max_latest_in_flight` [2], `max_bounded_queued` / `max_bounded_in_flight` [64 / 16], + `max_owners_per_client` [256], `reserved_owners_per_client` [64], `max_store_bytes` + [512 MiB], `max_artifact_bytes` [128 MiB], `max_retained_bytes` [128 MiB], + `max_queued_bytes_per_client` [1 MiB], `max_control_frames` / `max_control_bytes` + [128 / 1 MiB]. `Router::new` refuses a configuration `Limits::validate` rejects. +- `Policy::open()` explicitly enables trusted unbound transports and admits any client id with + every grant. `Policy::closed()` admits only the + ids added with `.client(id, grants)`. `.with_default(Some(grants))` admits unlisted ids too. + `Grants` lists `register`, `call`, `publish`, `subscribe` and `manage_topics` (declare, clear, + delete) as `Pattern::Any`, `Pattern::exact(name)` or `Pattern::prefix(prefix)`. An empty list + grants nothing. + +### Client + +```rust +let bus = Client::connect(transport, ClientConfig::new("coordinator", store_root)).await?; +let bus = Client::connect_unix(socket_path, ClientConfig::new("coordinator", store_root)).await?; +bus.info(); // SessionInfo: router_id, connection_id, identity, selected_major, + // selected_minor, contract_digest, limits +bus.closed(); // Some(reason) once the connection has closed +bus.control_errors();// fire-and-forget releases/consumes/unregisters the router refused +bus.close().await; // flush queued releases, close, wait for the reader to stop +``` + +`ClientConfig` has `client_id`, `client_incarnation` (generated when `None`; a reconnect needs a +new one), `store_root` (the router's) and `control_lane_capacity` [4096]. `Client` is `Clone`. +The connection closes when the client and every handle made from it are dropped, or on +`close()`. After that, handles are inert and the router has released what they owned. + +#### RPC + +```rust +let mut svc = bus.register("agent.fly-a", ServiceConfig { max_queued: 16, max_in_flight: 16 }).await?; +while let Some(req) = svc.next().await { // Request + req.method(); req.payload(); req.caller(); // authenticated only on a launcher-bound transport + let frame = req.artifact("frame")?; // shares the request's delivery guard + req.reply(outcome_map, &[("result", &artifact)]).await?; // Ok(true) routed, Ok(false) detached +} // dropping a Request consumes its delivery + +let mut pending = bus.call("agent.fly-a", Some(&incarnation), "Agent.Prepare", payload, &[("frame", &frame)]).await?; +let result = pending.result().await?; // RpcResult: outcome(), responder(), artifact() +let state = pending.cancel().await?; // CancelState +let result = bus.call_and_wait(service, pin, method, payload, &attachments).await?; +``` + +- `register` is exclusive; `Service::incarnation()` is the id callers pin. Dropping the + `Service` unregisters it. +- `call` returns once the router has admitted the call. The `PendingCall` then yields exactly + one terminal outcome. `result()` is cancel-safe: wrap it in `tokio::time::timeout`, then call + it again or `cancel()`. There is no deadline parameter; the deadline belongs to the caller. +- `cancel()` returns `CancelledBeforeDispatch`, `ExecutionUnknown`, `Completed` or `CallGone`. + After any state except `Completed`, `result()` fails with `CALL_GONE`. The error's dispatch is + `not-dispatched` only for `CancelledBeforeDispatch`. +- Dropping an unfinished `PendingCall` sends a best-effort `rpc.cancel`. The reactor consumes + any result that still arrives. +- `Request::responder()` returns a `Responder` that can reply after the request itself has + been dropped. Request delivery credit returns normally; a separate router-visible reply + capability preserves correlation until the last `Request`/`Responder` is dropped or a reply + is admitted. If the final capability is dropped while the caller is still attached, the call + terminates with `CALL_GONE` and dispatch `dispatched` while the route remains live, or the + existing `NO_SERVICE`/`dispatched` terminal if the route has ended. Capabilities share the + service connection's owner bound. +- A call fails with `call.failed` data in its `BusError`: + - Queued calls whose service unregisters or disconnects fail with `NO_SERVICE` and dispatch + `not-dispatched`. + - Dispatched calls whose service disconnects without replying fail with `NO_SERVICE` and + dispatch `dispatched`. + - Dispatched calls whose last responder is released without replying fail with `CALL_GONE` + and dispatch `dispatched`. + - A lost connection fails pending calls with `ROUTER_LOST` and dispatch `unknown`. + +#### Pub/sub + +```rust +bus.declare_topic("session.demo.snapshots", Retained::Latest).await?; // TopicInfo +bus.clear_topic(name).await?; // bool: a retained value was released +bus.delete_topic(name).await?; // bool; CONFLICT while it has subscribers +let mut sub = bus.subscribe(name, SubscriptionConfig::latest().in_flight(1).replay(true)).await?; +let receipt = bus.publish(name, payload, &[("frame", &frame)]).await?; // topic_sequence, subscribers, replaced +let msg = sub.next().await; // Option; try_next() does not wait +msg.topic_sequence(); msg.replaced(); msg.topic_incarnation(); msg.payload(); msg.artifact("frame")?; +``` + +- `SubscriptionConfig::latest()` is 1 queued and 2 in flight. `SubscriptionConfig::bounded()` + is 64 queued and 16 in flight. `.queued(n)`, `.in_flight(n)` and `.replay(bool)` adjust them. +- A credit returns only when the delivery is consumed: when the `Message` and every artifact + or `ArtifactFile` taken from it have been dropped. +- A retained replay into a bounded subscription is admitted atomically under the same queued + envelope-byte quota as an ordinary bounded publication. Latest replay remains outside that + byte pool and is bounded by one slot per subscription. +- Dropping a `Subscription` unsubscribes and discards its queue. Messages already handed out + stay valid. + +#### Artifacts + +```rust +let mut writer = bus.artifacts().allocate(len, "image/x-rgba").await?; // ArtifactWriter: io::Write +writer.write_all(&pixels)?; +let frame = writer.seal().await?; // or seal_with_digest(Some(sha256_hex)) +frame.reference(); // ArtifactRef +let bytes = frame.read_all().await?; // Vec +let file = frame.open().await?; // ArtifactFile: io::Read + io::Seek, keeps the handle +let kept = frame.retain().await?; // an independent explicit hold +``` + +- `Artifact` is a read-only, cloneable handle on one owner: a delivery or an explicit hold. + Its last clone (including any `ArtifactFile`) releases that owner. +- `ArtifactWriter` is unique. Dropping it unsealed releases the staging storage. Writes past + the allocated length fail. Unwritten bytes read as zeros. +- Sealing copies staging into a fresh read-only (0444) file and unlinks staging. A descriptor + the producer kept or duplicated afterwards writes only to the unlinked staging inode. Sealing + therefore needs quota for both copies while it runs. + +### Errors + +`BusError { code: ErrorCode, message, dispatch: Dispatch }`. The codes are the draft's +`INVALID_ENVELOPE`, `VERSION_MISMATCH`, `NOT_AUTHORIZED`, `NO_SERVICE`, `TARGET_CHANGED`, +`BACKPRESSURE`, `CALL_GONE`, `ARTIFACT_UNSEALED`, `ARTIFACT_GONE`, `OWNER_INVALID`, +`QUOTA_EXCEEDED`, `STORE_FAILURE` and `ROUTER_LOST`, plus `CONFLICT`, `NO_TOPIC` and +`ARTIFACT_MISMATCH`. `Dispatch` is `not-dispatched`, `dispatched` or `unknown`. Refusals +before admission are `not-dispatched`. A command in flight when the connection is lost reports +`unknown`. + +## Wire summary + +- Frames are a `u32` little-endian length followed by 1..=65,536 bytes of UTF-8 JSON. The length + is checked before any allocation. +- Every frame is parsed strictly: + - Duplicate keys at any depth, invalid UTF-8, non-finite numbers and trailing bytes are + refused. Nesting is limited to 128 levels. + - Unknown fields are refused in the envelope, attachments, references and every management + body. `payload` and `outcome` are opaque objects. +- Frame or envelope errors close the connection after a `connection.closing` notice. They + include a bad length, bad JSON, an unknown envelope field, a wrong kind, a non-null `replyTo`, + a command id that is not canonical or does not increase, a command before `bus.hello`, a + second hello, and a major/minor other than 1.0 after hello. A body-level error (unknown op, + bad field, out-of-range value, attachments on a management op) gets an `INVALID_ENVELOPE` + reply and the connection stays up. +- The SDK applies symmetric checks to router frames: exact 1.0 version, strictly increasing + canonical `bus-` ids, kind/replyTo/attachment direction rules, expected reply operation, + delivery correlations and complete strict reply, delivery and notice body parsing. +- Ids: + - Commands: `msg-`, strictly increasing per connection. + - Router envelopes: `bus-`. Connections: `conn-`. Services: `svc-`. Topics: + `top-`. Subscriptions: `sub-`. Calls: `call-`, increasing per client. + - Artifacts: `a-`. Deliveries: `dlv-`. Holds and writers: `own-`. + - `routerId` is `router-<16 hex>` and `storeId` is `store-<16 hex>` (same tag). + - Delivery and hold serials have separate per-connection watermarks. +- Notices: + - `call.failed {callId, code, message, dispatch}` + - `route.removed {name, serviceIncarnation, reason}` + - `subscription.closed {subscriptionId, topic, topicIncarnation, reason}` + - `connection.closing {code, message}` +- The hello reply's `limits` object gives counts as JSON integers and byte sizes as U64 strings. + Its fields: `maxEnvelopeBytes`, `maxAttachments`, `maxBatch`, `maxClients`, `maxServices`, + `maxTopics`, `maxSubscriptionsPerClient`, `maxSubscriptions`, `maxActiveCallsPerClient`, + `maxServiceQueued`, `maxServiceInFlight`, `maxLatestInFlight`, `maxBoundedQueued`, + `maxBoundedInFlight`, `maxOwnersPerClient`, `reservedOwnersPerClient`, `maxControlFrames`, + `maxStoreBytes`, `maxArtifactBytes`, `maxRetainedBytes`, `maxQueuedBytesPerClient` and + `maxControlBytes`. +- `contractDigest` is the SHA-256 of `wire::CONTRACT`, a text listing of every operation, + delivery and notice shape. The client refuses a router whose digest differs from its own. + +## Semantics worth knowing + +- **Admission is all or nothing.** `rpc.call`, `rpc.reply` and `publish` validate every + attachment and every bound before changing any state. A refused publication creates no + delivery, does not move the retained value and spends no topic sequence number. A refused + syntactically valid call id advances the issued-id watermark even when semantic admission is + refused, while creating no call, delivery or root. Reuse or decrease is rejected. +- **Ownership.** Each sealed artifact has a root count. Roots are held by queued deliveries, + delivered-but-unconsumed deliveries, explicit holds and a retained topic value. A delivery + that names one artifact twice holds one root. Fan-out adds roots and never copies bytes. When + the last root goes, the artifact leaves the quota and its file is unlinked, outside the router + lock. A process that still has the file open keeps its pages until it closes it. +- **Owner budget.** `max_owners_per_client` counts deliveries handed to the client and not + consumed, explicit holds and writers. Topic deliveries, holds and writers may use all but + `reserved_owners_per_client`; the reserve is for RPC requests and results. A topic delivery + over budget waits in its queue. A hold or allocation over budget fails with `QUOTA_EXCEEDED`. + Request reply capabilities consume the same finite bound independently of request delivery + credit, preventing retained responders from growing detached correlation state without limit. +- **Lanes.** The router sends each client its replies and notices first, then RPC requests and + results, then topic messages. Topic messages still get one frame after 16 consecutive + higher-priority frames. Router envelope ids are assigned only after this scheduler selects + the next item, so they increase in actual write order. Replies and notices waiting for one client are bounded by + `max_control_frames` / `max_control_bytes`; exceeding either closes that client with a + `connection.closing` notice. The client sends consumes, releases and cancels ahead of + ordinary commands and batches up to 64 ids per command. A full client control lane closes the + connection rather than drop a release. +- **No waiting on readers.** A client that stops reading stalls only its own writer task. The + router keeps admitting until that client's queues refuse, then returns `BACKPRESSURE` to + publishers of bounded topics. Connection teardown is synchronized with each synchronous + `poll_write`/`poll_flush` call without holding a mutex across an await. Teardown either observes + a complete frame before reclaiming its delivery owner, or marks a partial frame canceled before + cleanup and appends no final notice to the truncated stream. At a frame boundary, normal final + notices are still attempted. +- **RPC.** First dispatch is FIFO per service, which includes per caller. A call is marked + dispatched when its bytes are about to be written. Replies correlate by call id and may + arrive in any order. A second reply to one call fails with `CALL_GONE`. A reply to a detached + call returns `routed:false` and creates no roots. Unregistering a service fails its queued + calls; dispatched calls can still be answered while a `Request` or `Responder` retains reply + capability. Both cancel/consume orderings retire cleanly. +- **Seals** copy (and hash, when a digest was given) on Tokio's blocking pool, alongside the + connection's reader, so a large copy does not hold up that client's releases. If the writer is released or disconnects + mid-seal, the copy is discarded and the artifact is never published. + +## Differences from the draft + +1. **Extra error codes.** `CONFLICT` covers a duplicate registration, a conflicting topic + redeclaration and deleting a topic that has subscribers. `NO_TOPIC` covers publishing or + subscribing to an undeclared topic. `ARTIFACT_MISMATCH` covers a seal whose length or digest + is wrong, and a reference that disagrees with the artifact it names. +2. **Topics must be declared.** Publish and subscribe fail with `NO_TOPIC` otherwise. + `topic.delete` of an unknown topic returns `deleted:false`. `topic.clear` of an unknown + topic returns `NO_TOPIC`. +3. **When notices are sent.** + - `subscription.closed` is sent only when the router shuts down, because `topic.delete` + requires no subscribers and nothing else closes a subscription. + - `route.removed` goes to callers with queued or dispatched calls on the removed + registration, not to every client. + - `connection.closing` is an extra notice that precedes every router-initiated close. +4. **Byte budgets.** + - `max_queued_bytes_per_client` counts only `bounded` subscriptions. A `latest` slot is bounded + by subscription count times envelope size. + - `max_retained_bytes` (not in the draft's table) counts the artifact bytes pinned by + retained values, once per topic. +5. **Delivery size.** Admission computes the delivery's size with the router-added ids at their + longest. If that would exceed 65,536 bytes it refuses with `INVALID_ENVELOPE`, so an inbound + envelope near the limit can be refused even though it fits. +6. **Identity.** One live connection per client id. The router remembers each client id's last + incarnation and refuses its reuse. Launcher-bound `*_as` endpoints authenticate the Hello id; + trusted open/unbound endpoints do not. With an open policy this is one small record per + distinct client id ever seen. +7. **Seal reply.** The reply's `ownerId` is the writer's own id, now an explicit hold. +8. **No `budget` argument on calls, and no router executable.** Timeouts are the caller's + (`tokio::time::timeout` plus `cancel`). The draft's executable is optional; embed `Router`. +9. **Wire strictness.** Management bodies reject unknown fields. After hello, envelopes must + carry `minor: 0`. +10. **Reply capability release.** The SDK sends `rpc.responder.release {callId, + requestDeliveryId} -> {released}` when the last local reply capability is dropped. This + keeps request consumption independent from late-reply correlation while bounding that + correlation under the service connection's owner limit. For an attached dispatched call, + final release atomically retires the correlation and caller slot and emits `call.failed` + with dispatch `dispatched`: `CALL_GONE` while the route remains live, or `NO_SERVICE` after + route loss. + +## Limitations + +- **One host, one user.** Clients must run as the router's user and see the same store root. + The store directory is 0700, staging files 0600, sealed files 0444 and the socket 0600. Bus + authority (grants, connection-scoped owner ids, location grants) is enforced on bus + operations, not on the filesystem: a process running as the same user can read, mutate, + replace, chmod or unlink store files directly and can deny service. Mode 0600 authenticates + only the shared OS user, not a Flybus participant. +- **No memory maps.** Artifacts are read through `std::fs::File` (`ArtifactFile`) or + `read_all()`; there is no mmap API. `ArtifactFile` reads are blocking I/O. +- **Stats are logical.** `RouterStats` counts registry entries. Unlinked files that are still + open, client memory and OS pages are not in it. +- **Fairness is modest.** Fairness between clients is Tokio's scheduling plus a yield after + every 32 commands a connection sends. Within a connection, subscriptions and services are + served round robin. There is no weighting. +- **Orphan cleanup needs a restart.** Store directories left by a stopped router are removed + only when a new router starts on the same root. A directory counts as orphaned when it carries + the store marker and its `flock` is free. +- **Rust only.** There are no other language bindings. +- **Two perf runs.** `tests/perf.rs` (ignored by default) measured 640x480 RGBA frames at + 60 Hz over a Unix socket to three latest-mode consumers, one delayed 40 ms per frame. It ran + twice, on a laptop under WSL, release build, router and clients in one process. Allocate, + write and seal took p50 1.2 to 1.3 ms and p99 1.5 to 2.0 ms per 1.2 MB frame. Publish + admission took p50 0.2 ms. The RPC round trip took p50 0.24 / 0.29 to 0.31 / 0.50 ms with + 1 / 2 / 4 agents. The whole process used 0.18 to 0.25 cores and about 15 MB RSS. The store + peaked at 3.7 MB. These are two runs, not capacity data. + +## Tests + +```text +cargo test -p flybus # unit + integration +cargo test --release -p flybus --test perf -- --ignored --nocapture # the measurement above +cargo run -p flybus --example demo # counter RPC, observer, held frame +``` + +Every integration test runs twice, once over the in-memory transport and once over a Unix +socket, through the same router code: + +- `tests/wire.rs`: hello negotiation and refusals, malformed frames of every kind, body errors + that keep the connection, the exact 65,536-byte limit and control-lane exhaustion. +- `tests/rpc.rs`: exclusive and pinned registration, authority, FIFO dispatch with + out-of-order completion, backpressure, all four cancel states, single replies, service + disconnect and unregister, forged replies and call ids, an endpoint cache replaying an + artifact result, and abandoned calls. +- `tests/sol_review_races.rs`: launcher-bound identity and Hello resource bounds, detached RPC + orderings, responder/service teardown, reply/cancel races, replay admission, call-id + watermarking, strict client router validation, unsent rollback and shared task shutdown. +- `tests/pubsub.rs`: bounded FIFO with atomic backpressure, latest coalescing, credits, + retention and incarnations, zero-subscriber publication, unsubscribe, validation, quotas, a + saturated subscriber that does not block RPC, and shutdown notices. +- `tests/artifacts.rs`: allocate, seal and read; unsealed use; immutability against live + writable descriptors; length and digest checks; quotas; one physical object shared by + fan-out and collected after the last consumer; extracted artifacts and holds; atomic + admission; release watermarks; connection-scoped owners; abandoned futures; disconnects; and + router restarts. +- `tests/integration.rs`: two agents called in parallel with a forwarded frame, an environment + service, committed snapshot publication, a slow latest consumer and a bounded recorder. diff --git a/services/flysim/crates/flybus/examples/demo.rs b/services/flysim/crates/flybus/examples/demo.rs new file mode 100644 index 0000000..1e124eb --- /dev/null +++ b/services/flysim/crates/flybus/examples/demo.rs @@ -0,0 +1,99 @@ +//! A counter RPC, a pub/sub observer and a frame artifact held past its message, in one +//! process over the in-memory transport (bus-v1 section 11). +//! +//! ```text +//! cargo run -p flybus --example demo +//! ``` + +use std::io::Write; + +use flybus::{ + Client, ClientConfig, Policy, Retained, Router, RouterConfig, ServiceConfig, SubscriptionConfig, +}; +use serde_json::{Map, Value, json}; + +fn obj(v: Value) -> Map { + v.as_object().cloned().unwrap_or_default() +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let root = std::env::temp_dir().join(format!("flybus-demo-{}", std::process::id())); + let mut config = RouterConfig::new(&root); + config.policy = Policy::open(); + let router = Router::new(config)?; + let connect = |id: &str| { + Client::connect( + router.connect_in_memory_as(id), + ClientConfig::new(id, &root), + ) + }; + + // A counter service. + let counter = connect("counter").await?; + let mut svc = counter + .register("example.counter", ServiceConfig::default()) + .await?; + tokio::spawn(async move { + let mut total = 0; + while let Some(req) = svc.next().await { + total += req.payload()["amount"].as_i64().unwrap_or(0); + let _ = req.reply(obj(json!({ "total": total })), &[]).await; + } + }); + let app = connect("app").await?; + for _ in 0..3 { + let res = app + .call_and_wait( + "example.counter", + None, + "Counter.Increment", + obj(json!({"amount": 2})), + &[], + ) + .await?; + println!("counter total = {}", res.outcome()["total"]); + } + + // An observer of a frame topic. + app.declare_topic("world.demo.frame", Retained::None) + .await?; + let observer = connect("observer").await?; + let mut frames = observer + .subscribe("world.demo.frame", SubscriptionConfig::latest()) + .await?; + + let mut writer = app + .artifacts() + .allocate(160 * 144 * 4, "image/x-rgba") + .await?; + writer.write_all(&vec![0x7f; 160 * 144 * 4])?; + let frame = writer.seal().await?; + let receipt = app + .publish( + "world.demo.frame", + obj(json!({"width": 160, "height": 144})), + &[("frame", &frame)], + ) + .await?; + println!( + "published sequence {} to {} subscriber(s)", + receipt.topic_sequence, receipt.subscribers + ); + drop(frame); + + let message = frames.next().await.ok_or("subscription closed")?; + let image = message.artifact("frame")?; + drop(message); // the extracted handle still owns the delivery + let bytes = image.read_all().await?; + println!( + "read {} bytes after the message was dropped; router: {:?}", + bytes.len(), + router.stats() + ); + drop(image); // the last handle: the delivery is consumed and the frame collected + + router.shutdown(); + std::fs::remove_dir_all(&root)?; + Ok(()) +} diff --git a/services/flysim/crates/flybus/src/client/handles.rs b/services/flysim/crates/flybus/src/client/handles.rs new file mode 100644 index 0000000..b10674b --- /dev/null +++ b/services/flysim/crates/flybus/src/client/handles.rs @@ -0,0 +1,677 @@ +//! RAII handles: artifacts, writers, deliveries, services, subscriptions and pending calls. +//! +//! Every owner the router tracks for this client is behind one `OwnerGuard`. Its last clone +//! dropping queues `delivery.consumed` (for a delivery) or `artifact.release` (for a hold or +//! writer). A message, the artifacts extracted from it and any open files share the message's +//! guard, so a delivery is consumed only when all of them are gone. + +use std::fs::File; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::sync::Arc; + +use serde_json::{Map, Value}; +use tokio::sync::{mpsc, oneshot}; + +use super::reactor::{CallSlot, ClientConn, Control, Extra, Hook, OutCommand}; +use crate::error::{BusError, Dispatch, ErrorCode}; +use crate::store::{open_read, resolve}; +use crate::wire::{ArtifactRef, Attachment, Fields, GENERATION, Identity, Location}; + +pub(crate) struct OwnerGuard { + pub id: String, + pub delivery: bool, + pub conn: Arc, +} + +impl Drop for OwnerGuard { + fn drop(&mut self) { + let id = std::mem::take(&mut self.id); + let item = if self.delivery { + Control::Consumed(id) + } else { + Control::Release(id) + }; + self.conn.shared.push_control(item); + } +} + +pub(crate) fn attachment_list( + list: &[(&str, &Artifact)], +) -> (Vec, Vec>) { + let atts = list + .iter() + .map(|(name, a)| Attachment { + name: (*name).to_owned(), + reference: a.reference.as_ref().clone(), + owner_id: a.owner.id.clone(), + }) + .collect(); + let keep = list.iter().map(|(_, a)| a.owner.clone()).collect(); + (atts, keep) +} + +fn find( + list: &[(String, ArtifactRef)], + guard: &Arc, + name: &str, +) -> Result { + list.iter() + .find(|(n, _)| n == name) + .map(|(_, r)| Artifact { + reference: Arc::new(r.clone()), + owner: guard.clone(), + }) + .ok_or_else(|| BusError::invalid(format!("no attachment named {name:?}"))) +} + +// --------------------------------------------------------------------------------------------- +// Artifacts + +/// A read-only, cloneable handle on an immutable artifact. While any clone (or a file opened +/// from it) lives, the owner it came from keeps the bytes alive. +#[derive(Clone)] +pub struct Artifact { + pub(crate) reference: Arc, + pub(crate) owner: Arc, +} + +impl std::fmt::Debug for Artifact { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Artifact") + .field("reference", &self.reference) + .field("owner", &self.owner.id) + .finish() + } +} + +impl Artifact { + pub fn reference(&self) -> &ArtifactRef { + &self.reference + } + + /// The router-side owner this handle rides on: a delivery id or an explicit hold id. + pub fn owner_id(&self) -> &str { + &self.owner.id + } + + /// Opens the sealed bytes read-only. The file keeps this handle (and so its owner) alive. + pub async fn open(&self) -> Result { + let mut body = Map::new(); + body.insert("ref".into(), self.reference.to_json()); + body.insert("ownerId".into(), self.owner.id.clone().into()); + let shared = &self.owner.conn.shared; + let reply = shared + .command( + "artifact.open", + body, + Vec::new(), + vec![self.owner.clone()], + Hook::None, + ) + .await?; + let loc = Location::from_json(Fields::of(&reply.value, "reply").value("readLocation")?)?; + if loc.store_id != self.reference.store_id { + return Err(BusError::new( + ErrorCode::ArtifactGone, + "read location names another store", + )); + } + let path = resolve(&shared.store_root, &loc.store_id, &loc)?; + let file = open_read(&path) + .map_err(|e| BusError::new(ErrorCode::StoreFailure, format!("open: {e}")))?; + let len = file + .metadata() + .map_err(|e| BusError::new(ErrorCode::StoreFailure, e.to_string()))? + .len(); + if len != self.reference.byte_length { + return Err(BusError::new( + ErrorCode::ArtifactMismatch, + "sealed file length disagrees with the reference", + )); + } + Ok(ArtifactFile { + file, + _artifact: self.clone(), + }) + } + + /// Reads the whole artifact (on the blocking pool). + pub async fn read_all(&self) -> Result, BusError> { + let mut file = self.open().await?; + tokio::task::spawn_blocking(move || { + let mut out = Vec::with_capacity(file.len() as usize); + file.read_to_end(&mut out).map(|_| out) + }) + .await + .map_err(|e| BusError::new(ErrorCode::StoreFailure, e.to_string()))? + .map_err(|e| BusError::new(ErrorCode::StoreFailure, e.to_string())) + } + + /// Creates an independent explicit hold, so the bytes outlive this handle's delivery. + pub async fn retain(&self) -> Result { + let mut body = Map::new(); + body.insert("ref".into(), self.reference.to_json()); + body.insert("ownerId".into(), self.owner.id.clone().into()); + let shared = &self.owner.conn.shared; + let reply = shared + .command( + "artifact.retain", + body, + Vec::new(), + vec![self.owner.clone()], + Hook::Owner, + ) + .await?; + match reply.extra { + Extra::Owner(owner) => Ok(Artifact { + reference: self.reference.clone(), + owner, + }), + _ => Err(BusError::lost("retain reply without an owner")), + } + } +} + +/// An open, read-only sealed file. Holds its [`Artifact`]. +pub struct ArtifactFile { + file: File, + _artifact: Artifact, +} + +impl ArtifactFile { + pub fn len(&self) -> u64 { + self._artifact.reference.byte_length + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn artifact(&self) -> &Artifact { + &self._artifact + } +} + +impl Read for ArtifactFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.file.read(buf) + } +} + +impl Seek for ArtifactFile { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.file.seek(pos) + } +} + +/// The unique writer of a freshly allocated artifact. Not cloneable. Dropping it unsealed +/// releases the staging storage; [`ArtifactWriter::seal`] consumes it. +pub struct ArtifactWriter { + pub(crate) file: Option, + pub(crate) owner: Arc, + pub(crate) artifact_id: String, + pub(crate) byte_length: u64, + pub(crate) written: u64, +} + +impl ArtifactWriter { + pub fn artifact_id(&self) -> &str { + &self.artifact_id + } + + pub fn byte_length(&self) -> u64 { + self.byte_length + } + + /// Closes the writable handle and seals, returning the immutable artifact on an explicit + /// hold. Bytes not written read as zeros: the staging file is preallocated. + pub async fn seal(self) -> Result { + self.seal_with_digest(None).await + } + + /// As [`seal`](Self::seal), and the router refuses the seal unless the content's SHA-256 + /// (lowercase hex) equals `digest`. + pub async fn seal_with_digest(mut self, digest: Option) -> Result { + drop(self.file.take()); + let mut body = Map::new(); + body.insert("artifactId".into(), self.artifact_id.clone().into()); + body.insert("generation".into(), GENERATION.to_string().into()); + body.insert("ownerId".into(), self.owner.id.clone().into()); + body.insert("digest".into(), digest.map_or(Value::Null, Value::String)); + let shared = &self.owner.conn.shared; + let reply = shared + .command( + "artifact.seal", + body, + Vec::new(), + vec![self.owner.clone()], + Hook::None, + ) + .await?; + let reference = ArtifactRef::from_json(Fields::of(&reply.value, "reply").value("ref")?)?; + Ok(Artifact { + reference: Arc::new(reference), + owner: self.owner.clone(), + }) + } +} + +impl Write for ArtifactWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let room = self.byte_length - self.written; + if buf.len() as u64 > room { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "write past the allocated length", + )); + } + let file = self + .file + .as_mut() + .ok_or_else(|| io::Error::other("writer is closed"))?; + let n = file.write(buf)?; + self.written += n as u64; + Ok(n) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.as_mut().map_or(Ok(()), |f| f.flush()) + } +} + +// --------------------------------------------------------------------------------------------- +// Deliveries + +/// One `topic.message` delivery. Dropping it, and every artifact taken from it, consumes the +/// delivery and returns its credit. +pub struct Message { + pub(crate) guard: Arc, + pub(crate) subscription_id: String, + pub(crate) topic: String, + pub(crate) topic_incarnation: String, + pub(crate) topic_sequence: u64, + pub(crate) replaced: u64, + pub(crate) payload: Map, + pub(crate) attachments: Vec<(String, ArtifactRef)>, +} + +impl Message { + pub fn delivery_id(&self) -> &str { + &self.guard.id + } + pub fn subscription_id(&self) -> &str { + &self.subscription_id + } + pub fn topic(&self) -> &str { + &self.topic + } + pub fn topic_incarnation(&self) -> &str { + &self.topic_incarnation + } + pub fn topic_sequence(&self) -> u64 { + self.topic_sequence + } + /// Undelivered messages coalesced into this one since the previous delivery. + pub fn replaced(&self) -> u64 { + self.replaced + } + pub fn payload(&self) -> &Map { + &self.payload + } + pub fn attachment_names(&self) -> impl Iterator { + self.attachments.iter().map(|(n, _)| n.as_str()) + } + /// An artifact handle sharing this delivery's guard. + pub fn artifact(&self, name: &str) -> Result { + find(&self.attachments, &self.guard, name) + } +} + +/// An incoming `rpc.request`. Dropping it (and its artifacts) consumes the request delivery; +/// that is independent of replying. +pub struct Request { + pub(crate) guard: Arc, + pub(crate) reply_guard: Arc, + pub(crate) call_id: String, + pub(crate) caller: Identity, + pub(crate) target: String, + pub(crate) service_incarnation: String, + pub(crate) method: String, + pub(crate) payload: Map, + pub(crate) attachments: Vec<(String, ArtifactRef)>, +} + +impl Request { + pub fn delivery_id(&self) -> &str { + &self.guard.id + } + pub fn call_id(&self) -> &str { + &self.call_id + } + /// The caller identity supplied by the router. It is authenticated only when the router + /// accepted this connection through a launcher-bound `*_as` transport entry point. + pub fn caller(&self) -> &Identity { + &self.caller + } + pub fn target(&self) -> &str { + &self.target + } + pub fn service_incarnation(&self) -> &str { + &self.service_incarnation + } + pub fn method(&self) -> &str { + &self.method + } + pub fn payload(&self) -> &Map { + &self.payload + } + pub fn artifact(&self, name: &str) -> Result { + find(&self.attachments, &self.guard, name) + } + /// A reply capability that keeps bounded router correlation alive independently of the + /// request delivery credit. + pub fn responder(&self) -> Responder { + Responder { + guard: self.reply_guard.clone(), + call_id: self.call_id.clone(), + request_delivery_id: self.guard.id.clone(), + } + } + /// Replies. `Ok(true)` if routed to the caller, `Ok(false)` if the caller had detached. + pub async fn reply( + &self, + outcome: Map, + attachments: &[(&str, &Artifact)], + ) -> Result { + self.responder().reply(outcome, attachments).await + } +} + +/// Replies to one request, whether or not the request delivery is still held. +#[derive(Clone)] +pub struct Responder { + guard: Arc, + call_id: String, + request_delivery_id: String, +} + +impl Responder { + pub async fn reply( + &self, + outcome: Map, + attachments: &[(&str, &Artifact)], + ) -> Result { + let (atts, keep) = attachment_list(attachments); + let mut body = Map::new(); + body.insert("callId".into(), self.call_id.clone().into()); + body.insert( + "requestDeliveryId".into(), + self.request_delivery_id.clone().into(), + ); + body.insert("outcome".into(), Value::Object(outcome)); + let reply = self + .guard + .conn + .shared + .command("rpc.reply", body, atts, keep, Hook::None) + .await?; + Ok(Fields::of(&reply.value, "reply").boolean("routed")?) + } +} + +pub(crate) struct ReplyGuard { + pub conn: Arc, + pub call_id: String, + pub request_delivery_id: String, +} + +impl Drop for ReplyGuard { + fn drop(&mut self) { + self.conn.shared.push_control(Control::ResponderReleased { + call_id: self.call_id.clone(), + request_delivery_id: self.request_delivery_id.clone(), + }); + } +} + +/// A terminal `rpc.result`. Dropping it (and its artifacts) consumes the result delivery. +pub struct RpcResult { + pub(crate) guard: Arc, + pub(crate) call_id: String, + pub(crate) responder: Identity, + pub(crate) service_incarnation: String, + pub(crate) outcome: Map, + pub(crate) attachments: Vec<(String, ArtifactRef)>, +} + +impl RpcResult { + pub fn delivery_id(&self) -> &str { + &self.guard.id + } + pub fn call_id(&self) -> &str { + &self.call_id + } + pub fn responder(&self) -> &Identity { + &self.responder + } + pub fn service_incarnation(&self) -> &str { + &self.service_incarnation + } + pub fn outcome(&self) -> &Map { + &self.outcome + } + pub fn artifact(&self, name: &str) -> Result { + find(&self.attachments, &self.guard, name) + } +} + +// --------------------------------------------------------------------------------------------- +// Services, subscriptions, calls + +pub(crate) struct ServiceGuard { + pub conn: Arc, + pub incarnation: String, +} + +/// A registered service endpoint. Dropping it unregisters. +pub struct Service { + pub(crate) name: String, + pub(crate) rx: mpsc::UnboundedReceiver, + pub(crate) guard: ServiceGuard, +} + +impl Service { + pub fn name(&self) -> &str { + &self.name + } + pub fn incarnation(&self) -> &str { + &self.guard.incarnation + } + /// The next request, or `None` once the connection is closed. + pub async fn next(&mut self) -> Option { + self.rx.recv().await + } +} + +impl Drop for Service { + fn drop(&mut self) { + let shared = &self.guard.conn.shared; + let tx = shared.lock().services.remove(&self.guard.incarnation); + drop(tx); + // Route removal must precede abandoning buffered requests, whose last responder drops + // queue responder-release controls. This preserves NO_SERVICE as their terminal cause. + shared.push_control(Control::Unregister { + name: self.name.clone(), + service_incarnation: self.guard.incarnation.clone(), + }); + self.rx.close(); + while let Ok(request) = self.rx.try_recv() { + drop(request); + } + } +} + +pub(crate) struct SubscriptionGuard { + pub conn: Arc, + pub id: String, +} + +/// An exact-topic subscription. Dropping it unsubscribes; messages already handed out stay +/// valid. +pub struct Subscription { + pub(crate) rx: mpsc::UnboundedReceiver, + pub(crate) guard: SubscriptionGuard, + pub(crate) topic_incarnation: String, +} + +impl Subscription { + pub fn id(&self) -> &str { + &self.guard.id + } + pub fn topic_incarnation(&self) -> &str { + &self.topic_incarnation + } + /// The next message, or `None` once the subscription or connection is closed. + pub async fn next(&mut self) -> Option { + self.rx.recv().await + } + /// A message already delivered to this client, without waiting. + pub fn try_next(&mut self) -> Option { + self.rx.try_recv().ok() + } +} + +impl Drop for Subscription { + fn drop(&mut self) { + let shared = &self.guard.conn.shared; + let tx = shared.lock().subscriptions.remove(&self.guard.id); + drop(tx); + let mut body = Map::new(); + body.insert("subscriptionId".into(), self.guard.id.clone().into()); + let cmd = OutCommand { + op: "unsubscribe", + body, + attachments: Vec::new(), + respond: None, + hook: Hook::None, + keep: Vec::new(), + }; + let _ = shared.enqueue(cmd); + } +} + +pub(crate) struct CallGuard { + pub conn: Arc, + pub call_id: String, + pub done: bool, +} + +impl Drop for CallGuard { + fn drop(&mut self) { + if self.done { + return; + } + let shared = &self.conn.shared; + let waiting = { + let mut st = shared.lock(); + match st.calls.remove(&self.call_id) { + Some(slot @ CallSlot::Waiting(_)) => { + st.calls.insert(self.call_id.clone(), CallSlot::Abandoned); + Some(slot) + } + other => { + if let Some(o) = other { + st.calls.insert(self.call_id.clone(), o); + } + None + } + } + }; + if waiting.is_some() { + // Best effort; a result that still arrives is consumed by the reactor. + shared.push_control(Control::Cancel(self.call_id.clone(), None)); + } + } +} + +macro_rules! debug_fields { + ($ty:ident, |$s:ident| { $($name:literal => $val:expr),* $(,)? }) => { + impl std::fmt::Debug for $ty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let $s = self; + f.debug_struct(stringify!($ty))$(.field($name, &$val))*.finish() + } + } + }; +} + +debug_fields!(ArtifactFile, |s| { "artifact" => s._artifact }); +debug_fields!(ArtifactWriter, |s| { "artifact_id" => s.artifact_id, "byte_length" => s.byte_length, "owner" => s.owner.id }); +debug_fields!(Message, |s| { + "delivery_id" => s.guard.id, "topic" => s.topic, "topic_sequence" => s.topic_sequence, "replaced" => s.replaced, +}); +debug_fields!(Request, |s| { "delivery_id" => s.guard.id, "call_id" => s.call_id, "method" => s.method }); +debug_fields!(RpcResult, |s| { "delivery_id" => s.guard.id, "call_id" => s.call_id }); +debug_fields!(Responder, |s| { "call_id" => s.call_id, "request_delivery_id" => s.request_delivery_id }); +debug_fields!(Service, |s| { "name" => s.name, "incarnation" => s.guard.incarnation }); +debug_fields!(Subscription, |s| { "id" => s.guard.id, "topic_incarnation" => s.topic_incarnation }); +debug_fields!(PendingCall, |s| { "call_id" => s.guard.call_id, "service_incarnation" => s.service_incarnation }); + +/// The router's answer to `rpc.cancel`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CancelState { + CancelledBeforeDispatch, + ExecutionUnknown, + Completed, + CallGone, +} + +/// An admitted call. [`result`](Self::result) waits for the terminal result. Dropping it +/// unfinished sends a best-effort `rpc.cancel` and consumes any result that still arrives. +pub struct PendingCall { + pub(crate) guard: CallGuard, + pub(crate) rx: oneshot::Receiver>, + pub(crate) service_incarnation: String, +} + +impl PendingCall { + pub fn call_id(&self) -> &str { + &self.guard.call_id + } + + pub fn service_incarnation(&self) -> &str { + &self.service_incarnation + } + + /// Waits for the result. Cancel-safe: wrap it in a timeout and call it again, or cancel. + pub async fn result(&mut self) -> Result { + if self.guard.done { + return Err(BusError::new(ErrorCode::CallGone, "result already taken")); + } + let r = (&mut self.rx) + .await + .unwrap_or_else(|_| Err(BusError::lost("connection closed"))); + self.guard.done = true; + r + } + + /// Asks the router to cancel. Only `CancelledBeforeDispatch` establishes that the handler + /// never ran. After any state but `Completed`, [`result`](Self::result) fails with + /// `CALL_GONE`. + pub async fn cancel(&self) -> Result { + let (tx, rx) = oneshot::channel(); + let shared = &self.guard.conn.shared; + shared.push_control(Control::Cancel(self.guard.call_id.clone(), Some(tx))); + let reply = rx + .await + .unwrap_or_else(|_| Err(BusError::lost("connection closed")))?; + match Fields::of(&reply.value, "reply").string("state")? { + "cancelled-before-dispatch" => Ok(CancelState::CancelledBeforeDispatch), + "execution-unknown" => Ok(CancelState::ExecutionUnknown), + "completed" => Ok(CancelState::Completed), + "call-gone" => Ok(CancelState::CallGone), + other => Err(BusError::lost(format!("unknown cancel state {other:?}")) + .with_dispatch(Dispatch::Unknown)), + } + } +} diff --git a/services/flysim/crates/flybus/src/client/mod.rs b/services/flysim/crates/flybus/src/client/mod.rs new file mode 100644 index 0000000..fb043f3 --- /dev/null +++ b/services/flysim/crates/flybus/src/client/mod.rs @@ -0,0 +1,598 @@ +//! The client SDK: one connection for RPC, pub/sub and artifacts. + +mod handles; +mod reactor; + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde_json::{Map, Value}; +use sha2::{Digest as _, Sha256}; +use tokio::sync::oneshot; + +use handles::attachment_list; +pub use handles::{ + Artifact, ArtifactFile, ArtifactWriter, CancelState, Message, PendingCall, Request, Responder, + RpcResult, Service, Subscription, +}; +use reactor::{CallSlot, ClientConn, Extra, Hook, OutCommand, Shared}; + +use crate::error::{BusError, Dispatch, ErrorCode}; +use crate::limits::Limits; +use crate::store::{open_write, resolve}; +use crate::transport::Transport; +use crate::wire::{ + Envelope, Fields, Identity, Kind, Location, MAJOR, contract_digest, hex, is_content_type, + is_id, parse_serial_id, read_frame, serial_id, write_frame, +}; + +/// How a participant connects. +#[derive(Clone, Debug)] +pub struct ClientConfig { + /// The configured participant identity the launcher's policy knows. + pub client_id: String, + /// This SDK client's lifetime; generated when `None`. Reconnecting needs a new one. + pub client_incarnation: Option, + /// The router's store root, as configured for the router. + pub store_root: PathBuf, + /// Queued consumes, releases and cancels before the client gives up on the connection. + pub control_lane_capacity: usize, +} + +impl ClientConfig { + pub fn new(client_id: &str, store_root: impl Into) -> ClientConfig { + ClientConfig { + client_id: client_id.to_owned(), + client_incarnation: None, + store_root: store_root.into(), + control_lane_capacity: 4096, + } + } +} + +/// What `bus.hello` negotiated. +#[derive(Clone, Debug)] +pub struct SessionInfo { + pub router_id: String, + pub connection_id: String, + pub identity: Identity, + pub selected_major: u64, + pub selected_minor: u64, + pub contract_digest: String, + pub limits: Limits, +} + +/// `publish` admission: counts, not consumption. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PublishReceipt { + pub topic_sequence: u64, + pub subscribers: u64, + pub replaced: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TopicInfo { + /// False when an identical declaration already existed. + pub declared: bool, + pub topic_incarnation: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Retained { + None, + Latest, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + /// One replaceable queued value; `max_queued` is always 1. + Latest, + /// FIFO; a full queue refuses the whole publication. + Bounded, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SubscriptionConfig { + pub mode: Mode, + pub max_queued: u32, + pub max_in_flight: u32, + pub replay_latest: bool, +} + +impl SubscriptionConfig { + /// `latest`, 1 queued, 2 in flight. + pub fn latest() -> SubscriptionConfig { + SubscriptionConfig { + mode: Mode::Latest, + max_queued: 1, + max_in_flight: 2, + replay_latest: false, + } + } + + /// `bounded`, 64 queued, 16 in flight. + pub fn bounded() -> SubscriptionConfig { + SubscriptionConfig { + mode: Mode::Bounded, + max_queued: 64, + max_in_flight: 16, + replay_latest: false, + } + } + + pub fn queued(mut self, n: u32) -> SubscriptionConfig { + self.max_queued = n; + self + } + + pub fn in_flight(mut self, n: u32) -> SubscriptionConfig { + self.max_in_flight = n; + self + } + + pub fn replay(mut self, replay: bool) -> SubscriptionConfig { + self.replay_latest = replay; + self + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ServiceConfig { + pub max_queued: u32, + pub max_in_flight: u32, +} + +impl Default for ServiceConfig { + fn default() -> ServiceConfig { + ServiceConfig { + max_queued: 16, + max_in_flight: 16, + } + } +} + +/// A connected participant. Cheap to clone; the connection closes when the client, and every +/// handle made from it, is dropped, or on [`close`](Client::close). +#[derive(Clone)] +pub struct Client { + conn: Arc, +} + +impl std::fmt::Debug for Client { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Client").field("info", self.info()).finish() + } +} + +fn fresh_incarnation() -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let mut h = Sha256::new(); + h.update(std::process::id().to_le_bytes()); + h.update(nanos.to_le_bytes()); + h.update(COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes()); + format!("inc-{}", hex(&h.finalize()[..8])) +} + +fn field<'a>(m: &'a Map) -> Fields<'a> { + Fields::of(m, "reply") +} + +impl Client { + /// Connects over a transport and negotiates `bus.hello`. Must run inside a Tokio runtime. + pub async fn connect(transport: Transport, config: ClientConfig) -> Result { + if !is_id(&config.client_id) { + return Err(BusError::invalid("client id is not a valid id")); + } + let incarnation = config + .client_incarnation + .clone() + .unwrap_or_else(fresh_incarnation); + if !is_id(&incarnation) { + return Err(BusError::invalid("client incarnation is not a valid id")); + } + let (mut rd, mut wr) = tokio::io::split(transport); + let mut body = Map::new(); + body.insert("clientId".into(), config.client_id.clone().into()); + body.insert("clientIncarnation".into(), incarnation.clone().into()); + body.insert("supportedMajors".into(), Value::Array(vec![MAJOR.into()])); + let hello = Envelope::new(serial_id("msg", 1), Kind::Command, "bus.hello", body); + let lost = |e: String| BusError::lost(format!("hello: {e}")); + write_frame(&mut wr, &hello.encode()?) + .await + .map_err(|e| lost(e.to_string()))?; + let bytes = read_frame(&mut rd) + .await + .map_err(|e| lost(e.to_string()))? + .ok_or_else(|| lost("connection closed".into()))?; + let env = Envelope::decode(&bytes).map_err(|e| lost(e.0))?; + let hello_router_serial = parse_serial_id("bus", &env.id) + .filter(|n| *n > 0) + .ok_or_else(|| lost("hello reply id is not canonical bus-".into()))?; + if env.major != MAJOR + || env.minor != crate::wire::MINOR + || env.kind != Kind::Reply + || env.reply_to.as_deref() != Some("msg-1") + || env.op != "bus.hello" + || !env.attachments.is_empty() + { + return Err(lost("expected the hello reply".into())); + } + let mut f = Fields::of(&env.body, "reply"); + if !f.boolean("ok")? { + let e = f.object("error")?; + let mut g = Fields::of(e, "error"); + let code = ErrorCode::parse(g.string("code")?) + .ok_or_else(|| BusError::lost("hello error has unknown code"))?; + let message = g.string("message")?.to_owned(); + let dispatch = Dispatch::parse(g.string("dispatch")?) + .ok_or_else(|| BusError::lost("hello error has unknown dispatch"))?; + g.finish()?; + f.finish()?; + return Err(BusError { + code, + message, + dispatch, + }); + } + let value = f.object("value")?; + let mut v = Fields::of(value, "hello"); + let info = SessionInfo { + router_id: v.id("routerId")?, + connection_id: v.id("connectionId")?, + identity: Identity { + client_id: config.client_id.clone(), + client_incarnation: incarnation, + }, + selected_major: v.int("selectedMajor", 0, u64::MAX)?, + selected_minor: v.int("selectedMinor", 0, u64::MAX)?, + contract_digest: v.string("contractDigest")?.to_owned(), + limits: Limits::from_json(v.value("limits")?)?, + }; + v.finish()?; + f.finish()?; + if info.selected_major != MAJOR + || info.selected_minor != crate::wire::MINOR + || info.contract_digest != contract_digest() + { + return Err(BusError::new( + ErrorCode::VersionMismatch, + "router speaks a different contract", + )); + } + let shared = Arc::new(Shared::new( + info, + config.store_root, + config.control_lane_capacity.max(1), + 1, + hello_router_serial, + )); + let conn = Arc::new(ClientConn { + shared: shared.clone(), + }); + tokio::spawn(reactor::read_loop( + shared.clone(), + Arc::downgrade(&conn), + rd, + )); + tokio::spawn(reactor::write_loop(shared, wr)); + Ok(Client { conn }) + } + + /// Connects to a router's Unix-domain socket. + pub async fn connect_unix( + path: impl AsRef, + config: ClientConfig, + ) -> Result { + let t = Transport::unix(path) + .await + .map_err(|e| BusError::lost(format!("connect: {e}")))?; + Client::connect(t, config).await + } + + fn shared(&self) -> &Shared { + &self.conn.shared + } + + pub fn info(&self) -> &SessionInfo { + &self.shared().info + } + + /// Why the connection closed, once it has. + pub fn closed(&self) -> Option { + self.shared().lock().closed.clone() + } + + /// Replies to fire-and-forget releases, consumes and unregisters that came back as + /// errors. Nonzero means the client and router disagreed about ownership. + pub fn control_errors(&self) -> u64 { + self.shared().lock().control_errors + } + + /// Flushes queued releases, closes the connection and waits until the reader has stopped. + /// Handles that outlive this become inert; the router releases what they owned. + pub async fn close(self) { + let shared = self.conn.shared.clone(); + shared.begin_shutdown(); + let mut done = shared.done.subscribe(); + drop(self); + let _ = done.wait_for(|v| *v).await; + } + + // ---- services and RPC + + /// Registers an exclusive service endpoint. + pub async fn register(&self, name: &str, config: ServiceConfig) -> Result { + let mut body = Map::new(); + body.insert("name".into(), name.into()); + body.insert("maxQueued".into(), config.max_queued.into()); + body.insert("maxInFlight".into(), config.max_in_flight.into()); + let reply = self + .shared() + .command( + "service.register", + body, + Vec::new(), + Vec::new(), + Hook::Register, + ) + .await?; + match reply.extra { + Extra::Service(guard, rx) => Ok(Service { + name: name.to_owned(), + rx, + guard, + }), + _ => Err(BusError::lost("register reply without a service")), + } + } + + /// Calls `method` on `service`, pinned to `expected_incarnation` when given. Returns once + /// the router has admitted the call; the result arrives through the [`PendingCall`]. + pub async fn call( + &self, + service: &str, + expected_incarnation: Option<&str>, + method: &str, + payload: Map, + attachments: &[(&str, &Artifact)], + ) -> Result { + let (atts, keep) = attachment_list(attachments); + let (tx, rx) = oneshot::channel(); + let (reply_tx, reply_rx) = oneshot::channel(); + let shared = self.shared(); + // The call id, its result slot and its queue position are fixed together, so call ids + // reach the router in increasing order. + let rejected = { + let mut st = shared.lock(); + if let Some(e) = &st.closed { + Some((e.clone(), keep)) + } else { + st.next_call += 1; + let call_id = serial_id("call", st.next_call); + let mut body = Map::new(); + body.insert("callId".into(), call_id.clone().into()); + body.insert("target".into(), service.into()); + body.insert( + "expectedIncarnation".into(), + expected_incarnation.map_or(Value::Null, Value::from), + ); + body.insert("method".into(), method.into()); + body.insert("payload".into(), Value::Object(payload)); + st.calls + .insert(call_id.clone(), CallSlot::Waiting(Some(tx))); + st.ordinary.push_back(OutCommand { + op: "rpc.call", + body, + attachments: atts, + respond: Some(reply_tx), + hook: Hook::Call(call_id), + keep, + }); + None + } + }; + if let Some((e, keep)) = rejected { + drop(keep); + return Err(e); + } + shared.wake(); + let reply = reply_rx + .await + .unwrap_or_else(|_| Err(BusError::lost("connection closed")))?; + let service_incarnation = field(&reply.value).id("serviceIncarnation")?; + match reply.extra { + Extra::Call(guard) => Ok(PendingCall { + guard, + rx, + service_incarnation, + }), + _ => Err(BusError::lost("call reply without a call handle")), + } + } + + /// Admits a call and waits for its result. + pub async fn call_and_wait( + &self, + service: &str, + expected_incarnation: Option<&str>, + method: &str, + payload: Map, + attachments: &[(&str, &Artifact)], + ) -> Result { + let mut pending = self + .call(service, expected_incarnation, method, payload, attachments) + .await?; + pending.result().await + } + + // ---- topics + + pub async fn declare_topic( + &self, + name: &str, + retained: Retained, + ) -> Result { + let mut body = Map::new(); + body.insert("name".into(), name.into()); + body.insert( + "retained".into(), + if retained == Retained::Latest { + "latest" + } else { + "none" + } + .into(), + ); + let reply = self + .shared() + .command("topic.declare", body, Vec::new(), Vec::new(), Hook::None) + .await?; + let mut f = field(&reply.value); + Ok(TopicInfo { + declared: f.boolean("declared")?, + topic_incarnation: f.id("topicIncarnation")?, + }) + } + + /// Releases the retained value. `false` if there was none. + pub async fn clear_topic(&self, name: &str) -> Result { + let mut body = Map::new(); + body.insert("name".into(), name.into()); + let reply = self + .shared() + .command("topic.clear", body, Vec::new(), Vec::new(), Hook::None) + .await?; + Ok(field(&reply.value).boolean("cleared")?) + } + + /// Deletes a topic with no subscribers. `false` if it did not exist. + pub async fn delete_topic(&self, name: &str) -> Result { + let mut body = Map::new(); + body.insert("name".into(), name.into()); + let reply = self + .shared() + .command("topic.delete", body, Vec::new(), Vec::new(), Hook::None) + .await?; + Ok(field(&reply.value).boolean("deleted")?) + } + + pub async fn subscribe( + &self, + topic: &str, + config: SubscriptionConfig, + ) -> Result { + let mut body = Map::new(); + body.insert("topic".into(), topic.into()); + body.insert( + "mode".into(), + if config.mode == Mode::Latest { + "latest" + } else { + "bounded" + } + .into(), + ); + body.insert("maxQueued".into(), config.max_queued.into()); + body.insert("maxInFlight".into(), config.max_in_flight.into()); + body.insert("replayLatest".into(), config.replay_latest.into()); + let reply = self + .shared() + .command("subscribe", body, Vec::new(), Vec::new(), Hook::Subscribe) + .await?; + let topic_incarnation = field(&reply.value).id("topicIncarnation")?; + match reply.extra { + Extra::Subscription(guard, rx) => Ok(Subscription { + rx, + guard, + topic_incarnation, + }), + _ => Err(BusError::lost("subscribe reply without a subscription")), + } + } + + /// Publishes. The attachments' owners are held until the router has answered. + pub async fn publish( + &self, + topic: &str, + payload: Map, + attachments: &[(&str, &Artifact)], + ) -> Result { + let (atts, keep) = attachment_list(attachments); + let mut body = Map::new(); + body.insert("topic".into(), topic.into()); + body.insert("payload".into(), Value::Object(payload)); + let reply = self + .shared() + .command("publish", body, atts, keep, Hook::None) + .await?; + let mut f = field(&reply.value); + Ok(PublishReceipt { + topic_sequence: f.u64_string("topicSequence")?, + subscribers: f.u64_string("subscribers")?, + replaced: f.u64_string("replaced")?, + }) + } + + // ---- artifacts + + pub fn artifacts(&self) -> Artifacts<'_> { + Artifacts { client: self } + } +} + +/// The artifact half of the client API. +pub struct Artifacts<'a> { + client: &'a Client, +} + +impl Artifacts<'_> { + /// Reserves `byte_length` bytes of private staging storage and opens it for writing. + pub async fn allocate( + &self, + byte_length: u64, + content_type: &str, + ) -> Result { + if !is_content_type(content_type) { + return Err(BusError::invalid( + "contentType must be 1..=127 printable ASCII characters", + )); + } + let shared = self.client.shared(); + let mut body = Map::new(); + body.insert("byteLength".into(), byte_length.to_string().into()); + body.insert("contentType".into(), content_type.into()); + let reply = shared + .command( + "artifact.allocate", + body, + Vec::new(), + Vec::new(), + Hook::Owner, + ) + .await?; + let Extra::Owner(owner) = reply.extra else { + return Err(BusError::lost("allocate reply without an owner")); + }; + let mut f = field(&reply.value); + let artifact_id = f.id("artifactId")?; + if parse_serial_id("a", &artifact_id).is_none() { + return Err(BusError::lost("router issued a malformed artifact id")); + } + let loc = Location::from_json(f.value("writeLocation")?)?; + let path = resolve(&shared.store_root, &loc.store_id, &loc)?; + let file = open_write(&path) + .map_err(|e| BusError::new(ErrorCode::StoreFailure, format!("staging: {e}")))?; + Ok(ArtifactWriter { + file: Some(file), + owner, + artifact_id, + byte_length, + written: 0, + }) + } +} diff --git a/services/flysim/crates/flybus/src/client/reactor.rs b/services/flysim/crates/flybus/src/client/reactor.rs new file mode 100644 index 0000000..abdf9fa --- /dev/null +++ b/services/flysim/crates/flybus/src/client/reactor.rs @@ -0,0 +1,992 @@ +//! The client's connection reactor: one reader task, one writer task, and the shared state +//! they and every handle use. +//! +//! Lanes: the writer sends queued control items (consumes, releases, cancels) before ordinary +//! commands, so dropping handles is never stuck behind a burst of publishes. Command ids are +//! assigned when a command is written, which keeps them strictly increasing on the wire +//! whatever order the lanes interleave in. +//! +//! Locking rule: nothing that can own a guard is dropped while the state mutex is held, because +//! guard drops lock it. Values are taken out under the lock and dropped after it is released. + +use std::collections::{HashMap, VecDeque}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, MutexGuard, Weak}; +use std::time::Duration; + +use serde_json::{Map, Value}; +use tokio::io::{AsyncWriteExt, ReadHalf, WriteHalf}; +use tokio::sync::{Notify, mpsc, oneshot, watch}; + +use super::SessionInfo; +use super::handles::{ + CallGuard, Message, OwnerGuard, ReplyGuard, Request, RpcResult, ServiceGuard, SubscriptionGuard, +}; +use crate::error::{BusError, Dispatch, ErrorCode}; +use crate::transport::Transport; +use crate::wire::{ + ArtifactRef, Attachment, Envelope, Fields, GENERATION, Identity, Kind, MAJOR, MAX_BATCH, MINOR, + WireError, parse_serial_id, parse_u64, read_frame, serial_id, write_frame, +}; + +pub(crate) type Responder = oneshot::Sender>; + +/// A successful reply, plus whatever handle the reactor built from it. +pub(crate) struct Reply { + pub value: Map, + pub extra: Extra, +} + +pub(crate) enum Extra { + None, + Owner(Arc), + Service(ServiceGuard, mpsc::UnboundedReceiver), + Subscription(SubscriptionGuard, mpsc::UnboundedReceiver), + Call(CallGuard), +} + +/// What the reactor does with a reply before handing it over. Handles are built here, inside +/// the reactor, so a caller that abandoned its future cannot leak an owner it never saw: the +/// handle drops with the undelivered reply and releases itself. +pub(crate) enum Hook { + None, + /// `value.ownerId` is a new explicit hold. + Owner, + Register, + Subscribe, + Call(String), + Cancel(String), +} + +pub(crate) struct OutCommand { + pub op: &'static str, + pub body: Map, + pub attachments: Vec, + pub respond: Option, + pub hook: Hook, + /// Source owners held until the router has answered (bus-v1 section 8.2). + pub keep: Vec>, +} + +pub(crate) enum Control { + Consumed(String), + Release(String), + Cancel(String, Option), + Unregister { + name: String, + service_incarnation: String, + }, + ResponderReleased { + call_id: String, + request_delivery_id: String, + }, +} + +pub(crate) enum CallSlot { + Waiting(Option>>), + Abandoned, +} + +struct Pending { + op: &'static str, + respond: Option, + hook: Hook, + _keep: Vec>, +} + +pub(crate) struct ClientState { + pub closed: Option, + pub shutting_down: bool, + pub control: VecDeque, + pub ordinary: VecDeque, + next_msg: u64, + last_router: u64, + last_delivery: u64, + pending: HashMap, + pub services: HashMap>, + pub subscriptions: HashMap>, + pub calls: HashMap, + pub next_call: u64, + pub control_errors: u64, +} + +pub(crate) struct Shared { + pub info: SessionInfo, + pub store_root: PathBuf, + state: Mutex, + wake: Notify, + control_capacity: usize, + pub done: watch::Sender, + shutdown: watch::Sender, +} + +/// The connection's lifetime token. Every public handle holds one; the connection closes when +/// the last is dropped (or on [`crate::Client::close`]). +pub(crate) struct ClientConn { + pub shared: Arc, +} + +impl Drop for ClientConn { + fn drop(&mut self) { + self.shared.begin_shutdown(); + } +} + +impl Shared { + pub(crate) fn new( + info: SessionInfo, + store_root: PathBuf, + control_capacity: usize, + next_msg: u64, + last_router: u64, + ) -> Shared { + Shared { + info, + store_root, + state: Mutex::new(ClientState { + closed: None, + shutting_down: false, + control: VecDeque::new(), + ordinary: VecDeque::new(), + next_msg, + last_router, + last_delivery: 0, + pending: HashMap::new(), + services: HashMap::new(), + subscriptions: HashMap::new(), + calls: HashMap::new(), + next_call: 0, + control_errors: 0, + }), + wake: Notify::new(), + control_capacity, + done: watch::Sender::new(false), + shutdown: watch::Sender::new(false), + } + } + + pub(crate) fn lock(&self) -> MutexGuard<'_, ClientState> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + pub(crate) fn wake(&self) { + self.wake.notify_one(); + } + + pub(crate) fn begin_shutdown(&self) { + self.lock().shutting_down = true; + self.wake.notify_one(); + } + + /// Queues a control item. A full control lane closes the connection rather than lose a + /// release (bus-v1 section 8.3). + pub(crate) fn push_control(&self, item: Control) { + let overflow = { + let mut st = self.lock(); + if st.closed.is_some() { + Some(item) + } else if st.control.len() >= self.control_capacity { + st.closed = Some(BusError::lost("client control lane exhausted")); + Some(item) + } else { + st.control.push_back(item); + None + } + }; + self.wake.notify_one(); + drop(overflow); + } + + /// Queues an ordinary command, or hands it back with the reason it cannot be sent. + pub(crate) fn enqueue(&self, cmd: OutCommand) -> Result<(), BusError> { + let rejected = { + let mut st = self.lock(); + match &st.closed { + Some(e) => Some((e.clone().with_dispatch(Dispatch::NotDispatched), cmd)), + None => { + st.ordinary.push_back(cmd); + None + } + } + }; + self.wake.notify_one(); + match rejected { + Some((e, cmd)) => { + drop(cmd); + Err(e) + } + None => Ok(()), + } + } + + /// Queues a command and waits for its reply. + pub(crate) async fn command( + &self, + op: &'static str, + body: Map, + attachments: Vec, + keep: Vec>, + hook: Hook, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.enqueue(OutCommand { + op, + body, + attachments, + respond: Some(tx), + hook, + keep, + })?; + rx.await + .unwrap_or_else(|_| Err(BusError::lost("connection closed"))) + } + + /// Fails everything outstanding. Values are dropped after the lock is released. + fn fail_all(&self, reason: BusError) { + let (pending, calls, services, subscriptions, control, ordinary, reason) = { + let mut st = self.lock(); + let reason = st.closed.get_or_insert(reason).clone(); + ( + std::mem::take(&mut st.pending), + std::mem::take(&mut st.calls), + std::mem::take(&mut st.services), + std::mem::take(&mut st.subscriptions), + std::mem::take(&mut st.control), + std::mem::take(&mut st.ordinary), + reason, + ) + }; + for (_, p) in pending { + if let Some(tx) = p.respond { + let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::Unknown))); + } + } + for (_, slot) in calls { + if let CallSlot::Waiting(Some(tx)) = slot { + let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::Unknown))); + } + } + for cmd in ordinary { + if let Some(tx) = cmd.respond { + let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::NotDispatched))); + } + } + for item in control { + if let Control::Cancel(_, Some(tx)) = item { + let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::NotDispatched))); + } + } + drop((services, subscriptions)); + self.done.send_replace(true); + } + + fn terminate(&self, reason: BusError) { + self.fail_all(reason); + self.shutdown.send_replace(true); + self.wake.notify_waiters(); + } +} + +// --------------------------------------------------------------------------------------------- +// Writer + +enum Next { + Send(Vec), + Failed(OutCommand, BusError), + Idle, + Exit, +} + +fn take_batch(control: &mut VecDeque) -> Option { + let first = control.pop_front()?; + let (op, key, mut ids) = match first { + Control::Cancel(call_id, respond) => { + let mut body = Map::new(); + body.insert("callId".into(), call_id.clone().into()); + let hook = Hook::Cancel(call_id); + return Some(OutCommand { + op: "rpc.cancel", + body, + attachments: Vec::new(), + respond, + hook, + keep: Vec::new(), + }); + } + Control::ResponderReleased { + call_id, + request_delivery_id, + } => { + let mut body = Map::new(); + body.insert("callId".into(), call_id.into()); + body.insert("requestDeliveryId".into(), request_delivery_id.into()); + return Some(OutCommand { + op: "rpc.responder.release", + body, + attachments: Vec::new(), + respond: None, + hook: Hook::None, + keep: Vec::new(), + }); + } + Control::Unregister { + name, + service_incarnation, + } => { + let mut body = Map::new(); + body.insert("name".into(), name.into()); + body.insert("serviceIncarnation".into(), service_incarnation.into()); + return Some(OutCommand { + op: "service.unregister", + body, + attachments: Vec::new(), + respond: None, + hook: Hook::None, + keep: Vec::new(), + }); + } + Control::Consumed(id) => ("delivery.consumed", "deliveryIds", vec![id]), + Control::Release(id) => ("artifact.release", "ownerIds", vec![id]), + }; + while ids.len() < MAX_BATCH { + match (control.front(), op) { + (Some(Control::Consumed(_)), "delivery.consumed") + | (Some(Control::Release(_)), "artifact.release") => match control.pop_front() { + Some(Control::Consumed(id) | Control::Release(id)) => ids.push(id), + _ => unreachable!("front was checked"), + }, + _ => break, + } + } + let mut body = Map::new(); + body.insert( + key.into(), + Value::Array(ids.into_iter().map(Value::String).collect()), + ); + Some(OutCommand { + op, + body, + attachments: Vec::new(), + respond: None, + hook: Hook::None, + keep: Vec::new(), + }) +} + +fn next_outgoing(shared: &Shared) -> Next { + let mut st = shared.lock(); + if st.closed.is_some() { + return Next::Exit; + } + let cmd = match take_batch(&mut st.control).or_else(|| st.ordinary.pop_front()) { + Some(cmd) => cmd, + None if st.shutting_down => { + st.closed = Some(BusError::lost("client closed")); + return Next::Exit; + } + None => return Next::Idle, + }; + st.next_msg += 1; + let n = st.next_msg; + let mut env = Envelope::new(serial_id("msg", n), Kind::Command, cmd.op, cmd.body.clone()); + env.attachments = cmd.attachments.clone(); + match env.encode() { + Ok(bytes) => { + st.pending.insert( + n, + Pending { + op: cmd.op, + respond: cmd.respond, + hook: cmd.hook, + _keep: cmd.keep, + }, + ); + Next::Send(bytes) + } + Err(e) => Next::Failed(cmd, e.into()), + } +} + +pub(crate) async fn write_loop(shared: Arc, mut wr: WriteHalf) { + let mut shutdown = shared.shutdown.subscribe(); + loop { + match next_outgoing(&shared) { + Next::Send(bytes) => { + let result = tokio::select! { + result = write_frame(&mut wr, &bytes) => result, + _ = shutdown.wait_for(|v| *v) => break, + }; + if result.is_err() { + shared.terminate(BusError::lost("write to router failed")); + break; + } + } + Next::Failed(cmd, e) => { + let slot = match &cmd.hook { + Hook::Call(call_id) => shared.lock().calls.remove(call_id), + _ => None, + }; + drop(slot); + if let Some(tx) = cmd.respond { + let _ = tx.send(Err(e)); + } + } + Next::Idle => { + tokio::select! { + _ = shared.wake.notified() => {} + _ = shutdown.wait_for(|v| *v) => break, + } + } + Next::Exit => break, + } + } + tokio::select! { + _ = wr.shutdown() => {} + _ = shutdown.wait_for(|v| *v) => {} + _ = tokio::time::sleep(Duration::from_secs(1)) => {} + } + let reason = shared + .lock() + .closed + .clone() + .unwrap_or_else(|| BusError::lost("client writer stopped")); + shared.terminate(reason); +} + +// --------------------------------------------------------------------------------------------- +// Reader + +pub(crate) async fn read_loop( + shared: Arc, + conn: Weak, + mut rd: ReadHalf, +) { + let mut shutdown = shared.shutdown.subscribe(); + let reason = loop { + let frame = tokio::select! { + frame = read_frame(&mut rd) => frame, + _ = shutdown.wait_for(|v| *v) => return, + }; + let bytes = match frame { + Ok(Some(b)) => b, + Ok(None) => break BusError::lost("router closed the connection"), + Err(e) => break BusError::lost(format!("router connection failed: {e}")), + }; + let env = match Envelope::decode(&bytes) { + Ok(env) => env, + Err(e) => break BusError::lost(format!("router sent an invalid envelope: {e}")), + }; + if let Err(e) = on_envelope(&shared, &conn, env) { + break BusError::lost(format!("router protocol violation: {e}")); + } + }; + shared.terminate(reason); +} + +fn on_envelope( + shared: &Arc, + conn: &Weak, + env: Envelope, +) -> Result<(), WireError> { + if env.major != MAJOR || env.minor != MINOR { + return Err(WireError(format!( + "router envelope version is not {MAJOR}.{MINOR}" + ))); + } + let router_serial = parse_serial_id("bus", &env.id) + .filter(|n| *n > 0) + .ok_or_else(|| WireError("router envelope id is not canonical bus-".into()))?; + { + let mut st = shared.lock(); + if router_serial <= st.last_router { + return Err(WireError( + "router envelope ids must strictly increase".into(), + )); + } + st.last_router = router_serial; + } + match env.kind { + Kind::Reply => { + if !env.attachments.is_empty() { + return Err(WireError("router replies cannot carry attachments".into())); + } + let n = env + .reply_to + .as_deref() + .and_then(|r| parse_serial_id("msg", r)); + let n = n.ok_or_else(|| WireError("reply without a command id".into()))?; + let pending = { + let mut st = shared.lock(); + let Some(pending) = st.pending.get(&n) else { + return Err(WireError(format!("reply to unknown command msg-{n}"))); + }; + if env.op != pending.op { + return Err(WireError(format!( + "reply operation {:?} does not match {:?}", + env.op, pending.op + ))); + } + st.pending.remove(&n) + }; + let pending = + pending.ok_or_else(|| WireError(format!("reply to unknown command msg-{n}")))?; + let result = parse_reply(&env.body)?; + if let Ok(value) = &result { + validate_reply_value(pending.op, value)?; + } + complete(shared, conn, pending, result) + } + Kind::Delivery => { + if env.reply_to.is_some() { + return Err(WireError("deliveries must have null replyTo".into())); + } + on_delivery(shared, conn, env) + } + Kind::Notice => { + if env.reply_to.is_some() || !env.attachments.is_empty() { + return Err(WireError( + "notices must have null replyTo and no attachments".into(), + )); + } + on_notice(shared, env) + } + Kind::Command => Err(WireError("routers do not send commands".into())), + } +} + +fn parse_reply( + body: &Map, +) -> Result, BusError>, WireError> { + let mut f = Fields::of(body, "reply"); + if f.boolean("ok")? { + let value = f.object("value")?.clone(); + f.finish()?; + return Ok(Ok(value)); + } + let e = parse_error(f.object("error")?)?; + f.finish()?; + Ok(Err(e)) +} + +fn parse_error(m: &Map) -> Result { + let mut g = Fields::of(m, "error"); + let code = ErrorCode::parse(g.string("code")?) + .ok_or_else(|| WireError("unknown error code".into()))?; + let message = g.string("message")?.to_owned(); + let dispatch = Dispatch::parse(g.string("dispatch")?) + .ok_or_else(|| WireError("unknown dispatch".into()))?; + g.finish()?; + Ok(BusError { + code, + message, + dispatch, + }) +} + +fn serial_field(f: &mut Fields<'_>, key: &'static str, prefix: &str) -> Result { + let id = f.id(key)?; + parse_serial_id(prefix, &id) + .ok_or_else(|| WireError(format!("{key} is not canonical {prefix}-")))?; + Ok(id) +} + +fn validate_reply_value(op: &str, value: &Map) -> Result<(), WireError> { + let mut f = Fields::of(value, "reply value"); + match op { + "service.register" => { + serial_field(&mut f, "serviceIncarnation", "svc")?; + } + "service.unregister" => { + f.boolean("removed")?; + } + "rpc.call" => { + if !f.boolean("accepted")? { + return Err(WireError( + "an admitted rpc.call must say accepted:true".into(), + )); + } + serial_field(&mut f, "serviceIncarnation", "svc")?; + } + "rpc.reply" => { + f.boolean("routed")?; + } + "rpc.responder.release" => { + f.boolean("released")?; + } + "rpc.cancel" => match f.string("state")? { + "cancelled-before-dispatch" | "execution-unknown" | "completed" | "call-gone" => {} + _ => return Err(WireError("unknown rpc.cancel state".into())), + }, + "topic.declare" => { + f.boolean("declared")?; + serial_field(&mut f, "topicIncarnation", "top")?; + } + "topic.clear" => { + f.boolean("cleared")?; + } + "topic.delete" => { + f.boolean("deleted")?; + } + "subscribe" => { + serial_field(&mut f, "subscriptionId", "sub")?; + serial_field(&mut f, "topicIncarnation", "top")?; + } + "unsubscribe" => { + f.boolean("removed")?; + } + "publish" => { + f.u64_string("topicSequence")?; + f.u64_string("subscribers")?; + f.u64_string("replaced")?; + } + "delivery.consumed" | "artifact.release" => { + f.u64_string("released")?; + } + "artifact.allocate" => { + serial_field(&mut f, "artifactId", "a")?; + if f.u64_string("generation")? != GENERATION { + return Err(WireError("unsupported artifact generation".into())); + } + serial_field(&mut f, "ownerId", "own")?; + crate::wire::Location::from_json(f.value("writeLocation")?)?; + } + "artifact.seal" => { + ArtifactRef::from_json(f.value("ref")?)?; + serial_field(&mut f, "ownerId", "own")?; + } + "artifact.open" => { + crate::wire::Location::from_json(f.value("readLocation")?)?; + } + "artifact.retain" => { + serial_field(&mut f, "ownerId", "own")?; + } + _ => return Err(WireError(format!("reply for unknown operation {op:?}"))), + } + f.finish() +} + +fn owner_guard(conn: &Weak, id: String, delivery: bool) -> Option> { + Some(Arc::new(OwnerGuard { + id, + delivery, + conn: conn.upgrade()?, + })) +} + +fn complete( + shared: &Arc, + conn: &Weak, + pending: Pending, + result: Result, BusError>, +) -> Result<(), WireError> { + let Pending { + respond, + hook, + _keep, + .. + } = pending; + let reply = match (hook, result) { + (Hook::Call(call_id), Err(e)) => { + let slot = shared.lock().calls.remove(&call_id); + drop(slot); + Err(e) + } + (_, Err(e)) => Err(e), + (Hook::None, Ok(value)) => Ok(Reply { + value, + extra: Extra::None, + }), + (Hook::Owner, Ok(value)) => { + let id = Fields::of(&value, "reply").id("ownerId")?; + match owner_guard(conn, id, false) { + Some(g) => Ok(Reply { + value, + extra: Extra::Owner(g), + }), + None => Err(BusError::lost("client closed")), + } + } + (Hook::Register, Ok(value)) => { + let inc = Fields::of(&value, "reply").id("serviceIncarnation")?; + match conn.upgrade() { + Some(conn) => { + let (tx, rx) = mpsc::unbounded_channel(); + shared.lock().services.insert(inc.clone(), tx); + let guard = ServiceGuard { + conn, + incarnation: inc, + }; + Ok(Reply { + value, + extra: Extra::Service(guard, rx), + }) + } + None => Err(BusError::lost("client closed")), + } + } + (Hook::Subscribe, Ok(value)) => { + let id = Fields::of(&value, "reply").id("subscriptionId")?; + match conn.upgrade() { + Some(conn) => { + let (tx, rx) = mpsc::unbounded_channel(); + shared.lock().subscriptions.insert(id.clone(), tx); + let guard = SubscriptionGuard { conn, id }; + Ok(Reply { + value, + extra: Extra::Subscription(guard, rx), + }) + } + None => Err(BusError::lost("client closed")), + } + } + (Hook::Call(call_id), Ok(value)) => match conn.upgrade() { + Some(conn) => Ok(Reply { + value, + extra: Extra::Call(CallGuard { + conn, + call_id, + done: false, + }), + }), + None => Err(BusError::lost("client closed")), + }, + (Hook::Cancel(call_id), Ok(value)) => { + let state = Fields::of(&value, "reply").string("state")?.to_owned(); + if state != "completed" { + // No result will follow: resolve whoever still waits for one. + let slot = shared.lock().calls.remove(&call_id); + if let Some(CallSlot::Waiting(Some(tx))) = slot { + let dispatch = if state == "cancelled-before-dispatch" { + Dispatch::NotDispatched + } else { + Dispatch::Unknown + }; + let e = BusError::new(ErrorCode::CallGone, format!("cancelled: {state}")) + .with_dispatch(dispatch); + let _ = tx.send(Err(e)); + } + } + Ok(Reply { + value, + extra: Extra::None, + }) + } + }; + match respond { + Some(tx) => { + // If the caller is gone the reply, and any handle in it, drops here and releases. + let _ = tx.send(reply); + } + None => { + if reply.is_err() { + shared.lock().control_errors += 1; + } + } + } + Ok(()) +} + +fn attachments(env: &Envelope, delivery_id: &str) -> Result, WireError> { + env.attachments + .iter() + .map(|a| { + if a.owner_id != delivery_id { + return Err(WireError( + "delivery attachment owner is not the delivery".into(), + )); + } + Ok((a.name.clone(), a.reference.clone())) + }) + .collect() +} + +fn u64_field(f: &mut Fields<'_>, key: &'static str) -> Result { + let s = f.string(key)?; + parse_u64(s).ok_or_else(|| WireError(format!("{key} is not a U64"))) +} + +fn on_delivery( + shared: &Arc, + conn: &Weak, + env: Envelope, +) -> Result<(), WireError> { + let mut f = Fields::of(&env.body, "delivery"); + let delivery_id = f.id("deliveryId")?; + let delivery_serial = parse_serial_id("dlv", &delivery_id) + .filter(|n| *n > 0) + .ok_or_else(|| WireError("delivery id is not canonical dlv-".into()))?; + let atts = attachments(&env, &delivery_id)?; + // Without a live connection handle there is nobody to hand this to. + let Some(guard) = owner_guard(conn, delivery_id.clone(), true) else { + return Ok(()); + }; + match env.op.as_str() { + "rpc.request" => { + let call_id = f.id("callId")?; + if parse_serial_id("call", &call_id).is_none() { + return Err(WireError("rpc.request callId is not call-".into())); + } + let caller = Identity::from_json(f.value("caller")?)?; + let target = f.name("target")?; + let service_incarnation = f.id("serviceIncarnation")?; + if parse_serial_id("svc", &service_incarnation).is_none() { + return Err(WireError("service incarnation is not svc-".into())); + } + let method = f.method("method")?; + let payload = f.object("payload")?.clone(); + f.finish()?; + accept_delivery_serial(shared, delivery_serial)?; + let req = Request { + reply_guard: Arc::new(ReplyGuard { + conn: guard.conn.clone(), + call_id: call_id.clone(), + request_delivery_id: delivery_id.clone(), + }), + guard, + call_id, + caller, + target, + service_incarnation, + method, + payload, + attachments: atts, + }; + let tx = shared + .lock() + .services + .get(&req.service_incarnation) + .cloned(); + if let Some(tx) = tx { + let _ = tx.send(req); + } + } + "rpc.result" => { + let call_id = f.id("callId")?; + if parse_serial_id("call", &call_id).is_none() { + return Err(WireError("rpc.result callId is not call-".into())); + } + let responder = Identity::from_json(f.value("responder")?)?; + let service_incarnation = f.id("serviceIncarnation")?; + if parse_serial_id("svc", &service_incarnation).is_none() { + return Err(WireError("service incarnation is not svc-".into())); + } + let outcome = f.object("outcome")?.clone(); + f.finish()?; + accept_delivery_serial(shared, delivery_serial)?; + let res = RpcResult { + guard, + call_id, + responder, + service_incarnation, + outcome, + attachments: atts, + }; + let slot = shared.lock().calls.remove(&res.call_id); + match slot { + Some(CallSlot::Waiting(Some(tx))) => { + let _ = tx.send(Ok(res)); + } + // Abandoned or unknown: dropping it consumes the delivery. + other => drop((other, res)), + } + } + "topic.message" => { + let subscription_id = f.id("subscriptionId")?; + if parse_serial_id("sub", &subscription_id).is_none() { + return Err(WireError("subscription id is not sub-".into())); + } + let topic = f.name("topic")?; + let topic_incarnation = f.id("topicIncarnation")?; + if parse_serial_id("top", &topic_incarnation).is_none() { + return Err(WireError("topic incarnation is not top-".into())); + } + let topic_sequence = u64_field(&mut f, "topicSequence")?; + let replaced = u64_field(&mut f, "replaced")?; + let payload = f.object("payload")?.clone(); + f.finish()?; + accept_delivery_serial(shared, delivery_serial)?; + let msg = Message { + guard, + subscription_id, + topic, + topic_incarnation, + topic_sequence, + replaced, + payload, + attachments: atts, + }; + let tx = shared + .lock() + .subscriptions + .get(&msg.subscription_id) + .cloned(); + if let Some(tx) = tx { + let _ = tx.send(msg); + } + } + other => return Err(WireError(format!("unknown delivery {other:?}"))), + } + Ok(()) +} + +fn accept_delivery_serial(shared: &Shared, serial: u64) -> Result<(), WireError> { + let mut st = shared.lock(); + if serial <= st.last_delivery { + return Err(WireError("delivery ids must strictly increase".into())); + } + st.last_delivery = serial; + Ok(()) +} + +fn on_notice(shared: &Arc, env: Envelope) -> Result<(), WireError> { + let mut f = Fields::of(&env.body, "notice"); + match env.op.as_str() { + "call.failed" => { + let call_id = f.id("callId")?; + if parse_serial_id("call", &call_id).is_none() { + return Err(WireError("call.failed callId is not call-".into())); + } + let code = ErrorCode::parse(f.string("code")?) + .ok_or_else(|| WireError("unknown error code".into()))?; + let message = f.string("message")?.to_owned(); + let dispatch = Dispatch::parse(f.string("dispatch")?) + .ok_or_else(|| WireError("unknown dispatch".into()))?; + f.finish()?; + let slot = shared.lock().calls.remove(&call_id); + if let Some(CallSlot::Waiting(Some(tx))) = slot { + let _ = tx.send(Err(BusError { + code, + message, + dispatch, + })); + } + } + "route.removed" => { + f.name("name")?; + serial_field(&mut f, "serviceIncarnation", "svc")?; + f.string("reason")?; + f.finish()?; + } + "subscription.closed" => { + let id = serial_field(&mut f, "subscriptionId", "sub")?; + f.name("topic")?; + serial_field(&mut f, "topicIncarnation", "top")?; + f.string("reason")?; + f.finish()?; + let tx = shared.lock().subscriptions.remove(&id); + drop(tx); + } + "connection.closing" => { + let code = ErrorCode::parse(f.string("code")?) + .ok_or_else(|| WireError("unknown error code".into()))?; + let message = f.string("message")?.to_owned(); + f.finish()?; + let mut st = shared.lock(); + if st.closed.is_none() { + st.closed = Some(BusError::new(code, message).with_dispatch(Dispatch::Unknown)); + } + } + other => return Err(WireError(format!("unknown notice {other:?}"))), + } + Ok(()) +} diff --git a/services/flysim/crates/flybus/src/error.rs b/services/flysim/crates/flybus/src/error.rs new file mode 100644 index 0000000..5e1d7aa --- /dev/null +++ b/services/flysim/crates/flybus/src/error.rs @@ -0,0 +1,154 @@ +//! Transport error codes and the error value every fallible call returns. + +use std::fmt; + +/// Transport error codes (bus-v1 section 9, plus the three this crate adds; see the README). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ErrorCode { + InvalidEnvelope, + VersionMismatch, + NotAuthorized, + NoService, + TargetChanged, + Backpressure, + CallGone, + ArtifactUnsealed, + ArtifactGone, + OwnerInvalid, + QuotaExceeded, + StoreFailure, + RouterLost, + /// Addition: a name is already registered, a redeclaration disagrees, or a topic still has + /// subscribers. + Conflict, + /// Addition: publish or subscribe named a topic nobody declared. + NoTopic, + /// Addition: a seal found the wrong length or digest, or a reference disagrees with the + /// artifact it names. + ArtifactMismatch, +} + +impl ErrorCode { + pub const ALL: [ErrorCode; 16] = [ + ErrorCode::InvalidEnvelope, + ErrorCode::VersionMismatch, + ErrorCode::NotAuthorized, + ErrorCode::NoService, + ErrorCode::TargetChanged, + ErrorCode::Backpressure, + ErrorCode::CallGone, + ErrorCode::ArtifactUnsealed, + ErrorCode::ArtifactGone, + ErrorCode::OwnerInvalid, + ErrorCode::QuotaExceeded, + ErrorCode::StoreFailure, + ErrorCode::RouterLost, + ErrorCode::Conflict, + ErrorCode::NoTopic, + ErrorCode::ArtifactMismatch, + ]; + + pub fn as_str(self) -> &'static str { + match self { + ErrorCode::InvalidEnvelope => "INVALID_ENVELOPE", + ErrorCode::VersionMismatch => "VERSION_MISMATCH", + ErrorCode::NotAuthorized => "NOT_AUTHORIZED", + ErrorCode::NoService => "NO_SERVICE", + ErrorCode::TargetChanged => "TARGET_CHANGED", + ErrorCode::Backpressure => "BACKPRESSURE", + ErrorCode::CallGone => "CALL_GONE", + ErrorCode::ArtifactUnsealed => "ARTIFACT_UNSEALED", + ErrorCode::ArtifactGone => "ARTIFACT_GONE", + ErrorCode::OwnerInvalid => "OWNER_INVALID", + ErrorCode::QuotaExceeded => "QUOTA_EXCEEDED", + ErrorCode::StoreFailure => "STORE_FAILURE", + ErrorCode::RouterLost => "ROUTER_LOST", + ErrorCode::Conflict => "CONFLICT", + ErrorCode::NoTopic => "NO_TOPIC", + ErrorCode::ArtifactMismatch => "ARTIFACT_MISMATCH", + } + } + + pub fn parse(s: &str) -> Option { + ErrorCode::ALL.into_iter().find(|c| c.as_str() == s) + } +} + +impl fmt::Display for ErrorCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Whether the operation may have reached its target (bus-v1 section 5). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Dispatch { + NotDispatched, + Dispatched, + Unknown, +} + +impl Dispatch { + pub fn as_str(self) -> &'static str { + match self { + Dispatch::NotDispatched => "not-dispatched", + Dispatch::Dispatched => "dispatched", + Dispatch::Unknown => "unknown", + } + } + + pub fn parse(s: &str) -> Option { + match s { + "not-dispatched" => Some(Dispatch::NotDispatched), + "dispatched" => Some(Dispatch::Dispatched), + "unknown" => Some(Dispatch::Unknown), + _ => None, + } + } +} + +/// A transport error: code, a short message and the dispatch certainty. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BusError { + pub code: ErrorCode, + pub message: String, + pub dispatch: Dispatch, +} + +impl BusError { + /// An error raised before anything could have been dispatched. + pub fn new(code: ErrorCode, message: impl Into) -> BusError { + BusError { + code, + message: message.into(), + dispatch: Dispatch::NotDispatched, + } + } + + pub fn with_dispatch(mut self, dispatch: Dispatch) -> BusError { + self.dispatch = dispatch; + self + } + + pub(crate) fn invalid(message: impl Into) -> BusError { + BusError::new(ErrorCode::InvalidEnvelope, message) + } + + pub(crate) fn lost(message: impl Into) -> BusError { + BusError::new(ErrorCode::RouterLost, message).with_dispatch(Dispatch::Unknown) + } +} + +impl fmt::Display for BusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} ({}): {}", + self.code, + self.dispatch.as_str(), + self.message + ) + } +} + +impl std::error::Error for BusError {} diff --git a/services/flysim/crates/flybus/src/lib.rs b/services/flysim/crates/flybus/src/lib.rs new file mode 100644 index 0000000..fd8b4f9 --- /dev/null +++ b/services/flysim/crates/flybus/src/lib.rs @@ -0,0 +1,37 @@ +//! `flybus`: a small local RPC and pub/sub bus with immutable file-backed artifacts. +//! +//! One library, one router, one wire protocol. Small JSON messages carry metadata and +//! artifact references; large immutable bytes live in a file store the router manages, and +//! ownership follows deliveries and explicit holds. The router moves messages and tracks +//! ownership; it knows nothing about what the messages mean. +//! +//! ```text +//! Client ── Transport (in-memory pipe or Unix socket) ── Router ── State (one mutex) +//! │ │ +//! └── Artifact / ArtifactWriter ── files ──────────── Store (//...) +//! ``` +//! +//! The draft contract this implements is bus-v1 (the session-framework design); the README +//! lists the API, the limits and every place the implementation narrows or extends the draft. + +pub mod error; +pub mod limits; +pub mod policy; +pub mod wire; + +mod client; +mod router; +mod store; +mod transport; + +pub use client::{ + Artifact, ArtifactFile, ArtifactWriter, Artifacts, CancelState, Client, ClientConfig, Message, + Mode, PendingCall, PublishReceipt, Request, Responder, Retained, RpcResult, Service, + ServiceConfig, SessionInfo, Subscription, SubscriptionConfig, TopicInfo, +}; +pub use error::{BusError, Dispatch, ErrorCode}; +pub use limits::Limits; +pub use policy::{Grants, Pattern, Policy}; +pub use router::{Router, RouterConfig, RouterStats, UnixListenerHandle}; +pub use transport::{Stream, Transport}; +pub use wire::{ArtifactRef, Identity}; diff --git a/services/flysim/crates/flybus/src/limits.rs b/services/flysim/crates/flybus/src/limits.rs new file mode 100644 index 0000000..2ab041c --- /dev/null +++ b/services/flysim/crates/flybus/src/limits.rs @@ -0,0 +1,183 @@ +//! Router limits (bus-v1 section 9). The defaults are the draft's prototype starting point, +//! not capacity data. + +use serde_json::{Map, Value}; + +use crate::wire::{Fields, MAX_ATTACHMENTS, MAX_BATCH, MAX_CREDIT, MAX_ENVELOPE_BYTES, WireError}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Limits { + pub max_clients: usize, + pub max_services: usize, + pub max_topics: usize, + pub max_subscriptions_per_client: usize, + pub max_subscriptions: usize, + /// Calls a client may have admitted and not yet finished (result consumed, failed or + /// cancelled before dispatch). A call detached by a post-dispatch cancel no longer counts. + pub max_active_calls_per_client: usize, + /// Upper bounds on what `service.register` may ask for. + pub max_service_queued: u64, + pub max_service_in_flight: u64, + /// Upper bound on a `latest` subscription's credits; its queue is always exactly 1. + pub max_latest_in_flight: u64, + pub max_bounded_queued: u64, + pub max_bounded_in_flight: u64, + /// Owners (deliveries handed to the client and not consumed, explicit holds and staging + /// writers) a connection may have at once. + pub max_owners_per_client: usize, + /// Of those, how many only RPC request/result deliveries may use. Topic deliveries, holds + /// and writers are refused (holds, writers) or wait (deliveries) beyond + /// `max_owners_per_client - reserved_owners_per_client`. + pub reserved_owners_per_client: usize, + /// Bytes of staging, sealing copies and sealed objects the store may hold. + pub max_store_bytes: u64, + pub max_artifact_bytes: u64, + /// Artifact bytes pinned by retained topic values, counted once per topic. + pub max_retained_bytes: u64, + /// Envelope bytes queued for a client's `bounded` subscriptions and not yet delivered. + pub max_queued_bytes_per_client: usize, + /// Replies and notices waiting to be written to one client. Exceeding either closes it. + pub max_control_frames: usize, + pub max_control_bytes: usize, +} + +impl Default for Limits { + fn default() -> Limits { + Limits { + max_clients: 64, + max_services: 256, + max_topics: 512, + max_subscriptions_per_client: 128, + max_subscriptions: 1024, + max_active_calls_per_client: 64, + max_service_queued: 16, + max_service_in_flight: 16, + max_latest_in_flight: 2, + max_bounded_queued: 64, + max_bounded_in_flight: 16, + max_owners_per_client: 256, + reserved_owners_per_client: 64, + max_store_bytes: 512 << 20, + max_artifact_bytes: 128 << 20, + max_retained_bytes: 128 << 20, + max_queued_bytes_per_client: 1 << 20, + max_control_frames: 128, + max_control_bytes: 1 << 20, + } + } +} + +impl Limits { + /// Refuses a configuration the router could not honour. + pub fn validate(&self) -> Result<(), String> { + let credits = [ + ("max_service_queued", self.max_service_queued), + ("max_service_in_flight", self.max_service_in_flight), + ("max_latest_in_flight", self.max_latest_in_flight), + ("max_bounded_queued", self.max_bounded_queued), + ("max_bounded_in_flight", self.max_bounded_in_flight), + ]; + for (name, v) in credits { + if !(1..=MAX_CREDIT).contains(&v) { + return Err(format!("{name} must be in 1..={MAX_CREDIT}")); + } + } + if self.reserved_owners_per_client >= self.max_owners_per_client { + return Err("reserved_owners_per_client must be below max_owners_per_client".into()); + } + if self.max_artifact_bytes > self.max_store_bytes { + return Err("max_artifact_bytes must not exceed max_store_bytes".into()); + } + if self.max_control_frames == 0 || self.max_control_bytes < MAX_ENVELOPE_BYTES { + return Err("the control lane must hold at least one full envelope".into()); + } + Ok(()) + } + + /// The `limits` object in the hello reply. Counts are JSON integers; byte sizes are U64 + /// strings. + pub fn to_json(&self) -> Value { + let mut m = Map::new(); + let mut n = |k: &str, v: u64| { + m.insert(k.into(), Value::from(v)); + }; + n("maxEnvelopeBytes", MAX_ENVELOPE_BYTES as u64); + n("maxAttachments", MAX_ATTACHMENTS as u64); + n("maxBatch", MAX_BATCH as u64); + n("maxClients", self.max_clients as u64); + n("maxServices", self.max_services as u64); + n("maxTopics", self.max_topics as u64); + n( + "maxSubscriptionsPerClient", + self.max_subscriptions_per_client as u64, + ); + n("maxSubscriptions", self.max_subscriptions as u64); + n( + "maxActiveCallsPerClient", + self.max_active_calls_per_client as u64, + ); + n("maxServiceQueued", self.max_service_queued); + n("maxServiceInFlight", self.max_service_in_flight); + n("maxLatestInFlight", self.max_latest_in_flight); + n("maxBoundedQueued", self.max_bounded_queued); + n("maxBoundedInFlight", self.max_bounded_in_flight); + n("maxOwnersPerClient", self.max_owners_per_client as u64); + n( + "reservedOwnersPerClient", + self.reserved_owners_per_client as u64, + ); + n("maxControlFrames", self.max_control_frames as u64); + let mut s = |k: &str, v: u64| { + m.insert(k.into(), Value::from(v.to_string())); + }; + s("maxStoreBytes", self.max_store_bytes); + s("maxArtifactBytes", self.max_artifact_bytes); + s("maxRetainedBytes", self.max_retained_bytes); + s( + "maxQueuedBytesPerClient", + self.max_queued_bytes_per_client as u64, + ); + s("maxControlBytes", self.max_control_bytes as u64); + Value::Object(m) + } + + /// Reads the hello reply's `limits` object back. + pub fn from_json(v: &Value) -> Result { + let mut f = Fields::new(v, "limits")?; + let big = u64::MAX; + for (k, want) in [ + ("maxEnvelopeBytes", MAX_ENVELOPE_BYTES as u64), + ("maxAttachments", MAX_ATTACHMENTS as u64), + ("maxBatch", MAX_BATCH as u64), + ] { + if f.int(k, 0, big)? != want { + return Err(WireError(format!( + "limits: {k} disagrees with this implementation" + ))); + } + } + let limits = Limits { + max_clients: f.int("maxClients", 0, big)? as usize, + max_services: f.int("maxServices", 0, big)? as usize, + max_topics: f.int("maxTopics", 0, big)? as usize, + max_subscriptions_per_client: f.int("maxSubscriptionsPerClient", 0, big)? as usize, + max_subscriptions: f.int("maxSubscriptions", 0, big)? as usize, + max_active_calls_per_client: f.int("maxActiveCallsPerClient", 0, big)? as usize, + max_service_queued: f.int("maxServiceQueued", 1, MAX_CREDIT)?, + max_service_in_flight: f.int("maxServiceInFlight", 1, MAX_CREDIT)?, + max_latest_in_flight: f.int("maxLatestInFlight", 1, MAX_CREDIT)?, + max_bounded_queued: f.int("maxBoundedQueued", 1, MAX_CREDIT)?, + max_bounded_in_flight: f.int("maxBoundedInFlight", 1, MAX_CREDIT)?, + max_owners_per_client: f.int("maxOwnersPerClient", 0, big)? as usize, + reserved_owners_per_client: f.int("reservedOwnersPerClient", 0, big)? as usize, + max_control_frames: f.int("maxControlFrames", 0, big)? as usize, + max_store_bytes: f.u64_string("maxStoreBytes")?, + max_artifact_bytes: f.u64_string("maxArtifactBytes")?, + max_retained_bytes: f.u64_string("maxRetainedBytes")?, + max_queued_bytes_per_client: f.u64_string("maxQueuedBytesPerClient")? as usize, + max_control_bytes: f.u64_string("maxControlBytes")? as usize, + }; + f.finish()?; + Ok(limits) + } +} diff --git a/services/flysim/crates/flybus/src/policy.rs b/services/flysim/crates/flybus/src/policy.rs new file mode 100644 index 0000000..58fedf6 --- /dev/null +++ b/services/flysim/crates/flybus/src/policy.rs @@ -0,0 +1,108 @@ +//! Launcher-provided access policy (bus-v1 section 3): which configured participants may +//! connect, and what each may register, call, publish, subscribe to and manage. Naming a +//! target is not authority to control it. + +use std::collections::HashMap; + +/// A set of service or topic names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Pattern { + Any, + Exact(String), + /// Every name that starts with this string, e.g. `"session.demo."`. + Prefix(String), +} + +impl Pattern { + pub fn exact(name: &str) -> Pattern { + Pattern::Exact(name.to_owned()) + } + + pub fn prefix(prefix: &str) -> Pattern { + Pattern::Prefix(prefix.to_owned()) + } + + pub fn matches(&self, name: &str) -> bool { + match self { + Pattern::Any => true, + Pattern::Exact(n) => n == name, + Pattern::Prefix(p) => name.starts_with(p.as_str()), + } + } +} + +/// What one participant may do. Empty lists grant nothing. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Grants { + /// Service names it may register. + pub register: Vec, + /// Service names it may call. + pub call: Vec, + pub publish: Vec, + pub subscribe: Vec, + /// Topic names it may declare, clear and delete. + pub manage_topics: Vec, +} + +impl Grants { + pub fn all() -> Grants { + let any = vec![Pattern::Any]; + Grants { + register: any.clone(), + call: any.clone(), + publish: any.clone(), + subscribe: any.clone(), + manage_topics: any, + } + } + + pub(crate) fn allows(list: &[Pattern], name: &str) -> bool { + list.iter().any(|p| p.matches(name)) + } +} + +/// Who may connect and with what grants. +#[derive(Clone, Debug, Default)] +pub struct Policy { + clients: HashMap, + default: Option, + trusted_unbound: bool, +} + +impl Policy { + /// Admits any client id with every grant. For tests and single-purpose local deployments. + pub fn open() -> Policy { + Policy { + clients: HashMap::new(), + default: Some(Grants::all()), + trusted_unbound: true, + } + } + + /// Admits only the clients added with [`Policy::client`]. + pub fn closed() -> Policy { + Policy::default() + } + + pub fn client(mut self, client_id: &str, grants: Grants) -> Policy { + self.clients.insert(client_id.to_owned(), grants); + self + } + + /// Grants for clients not listed by id; `None` refuses them at hello. + pub fn with_default(mut self, grants: Option) -> Policy { + self.default = grants; + self + } + + pub(crate) fn grants_for(&self, client_id: &str) -> Option { + self.clients + .get(client_id) + .cloned() + .or_else(|| self.default.clone()) + } + + pub(crate) fn permits_unbound_transport(&self) -> bool { + self.trusted_unbound + } +} diff --git a/services/flysim/crates/flybus/src/router/mod.rs b/services/flysim/crates/flybus/src/router/mod.rs new file mode 100644 index 0000000..2429a4b --- /dev/null +++ b/services/flysim/crates/flybus/src/router/mod.rs @@ -0,0 +1,502 @@ +//! The embeddable router: connection tasks around the [`state`] machine. +//! +//! Each connection has a reader task, which decodes one command at a time and applies it under +//! the state lock, and a writer task, which pulls its next frame from the state and writes it +//! with the lock released. Staging creation and seal copies run on the blocking pool; seals +//! run beside the reader, so a large copy never holds up that client's releases. The router +//! never waits on a subscriber: a slow reader only stalls its own writer task. + +mod state; + +use std::io; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +use tokio::net::UnixListener; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +pub use state::RouterStats; +use state::{ConnKey, ConnSignals, NextFrame, Outcome, State}; + +use crate::limits::Limits; +use crate::policy::Policy; +use crate::store::{SealFailure, Store}; +use crate::transport::{Stream, Transport}; +use crate::wire::{Envelope, FrameError, contract_digest, hex, read_frame, write_frame}; + +/// How the launcher configures a router. +#[derive(Clone, Debug)] +pub struct RouterConfig { + /// Directory under which the store creates its per-incarnation directory. Put it on tmpfs + /// for transient media. Clients must be given the same root. + pub store_root: PathBuf, + pub limits: Limits, + pub policy: Policy, + /// Maximum time an accepted transport may remain pending before completing `bus.hello`. + pub hello_timeout: Duration, +} + +impl RouterConfig { + /// Default limits and a closed policy: add clients with [`Policy::client`]. + pub fn new(store_root: impl Into) -> RouterConfig { + RouterConfig { + store_root: store_root.into(), + limits: Limits::default(), + policy: Policy::closed(), + hello_timeout: Duration::from_secs(5), + } + } +} + +struct Inner { + state: Mutex, + store: Store, + store_root: PathBuf, + router_id: String, + store_id: String, + stop: watch::Sender, + hello_timeout: Duration, +} + +impl Inner { + /// Runs `f` under the state lock, then unlinks whatever it released, outside the lock. + fn with_state(&self, f: impl FnOnce(&mut State) -> R) -> R { + let mut st = self.state.lock().unwrap_or_else(|e| e.into_inner()); + let r = f(&mut st); + let unlinks = st.take_unlinks(); + drop(st); + for rel in unlinks { + self.store.remove(&rel); + } + r + } +} + +/// A router and its artifact store. Cheap to clone; every clone is the same router. +#[derive(Clone)] +pub struct Router { + inner: Arc, +} + +fn fresh_tag() -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let mut h = Sha256::new(); + h.update(std::process::id().to_le_bytes()); + h.update(nanos.to_le_bytes()); + h.update(COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes()); + hex(&h.finalize()[..8]) +} + +impl Router { + /// Creates the store directory (deleting orphans of stopped routers under the same root) + /// and a router with a fresh `routerId`/`storeId`. + pub fn new(config: RouterConfig) -> io::Result { + config.limits.validate().map_err(io::Error::other)?; + let tag = fresh_tag(); + let router_id = format!("router-{tag}"); + let store_id = format!("store-{tag}"); + let store = Store::create(&config.store_root, &store_id)?; + let state = State::new( + router_id.clone(), + store_id.clone(), + contract_digest(), + config.limits, + config.policy, + ); + Ok(Router { + inner: Arc::new(Inner { + state: Mutex::new(state), + store, + store_root: config.store_root, + router_id, + store_id, + stop: watch::Sender::new(false), + hello_timeout: config.hello_timeout, + }), + }) + } + + pub fn router_id(&self) -> &str { + &self.inner.router_id + } + + pub fn store_id(&self) -> &str { + &self.inner.store_id + } + + pub fn store_root(&self) -> &Path { + &self.inner.store_root + } + + /// `/`. + pub fn store_dir(&self) -> &Path { + self.inner.store.dir() + } + + pub fn stats(&self) -> RouterStats { + self.inner.with_state(|s| s.stats()) + } + + /// Serves an explicitly trusted, unbound connection. This is refused unless the router + /// uses [`Policy::open`]; Hello identity is self-asserted in this mode. + pub fn serve(&self, stream: S) { + tokio::spawn(run_connection(self.inner.clone(), stream, None)); + } + + /// Serves one launcher-bound participant. Hello must name `expected_client_id`. + pub fn serve_as(&self, stream: S, expected_client_id: &str) { + tokio::spawn(run_connection( + self.inner.clone(), + stream, + Some(expected_client_id.to_owned()), + )); + } + + /// A connected in-memory transport for a client in this process. + pub fn connect_in_memory(&self) -> Transport { + let (client, router) = Transport::pair(); + self.serve(router); + client + } + + /// A connected in-memory transport bound to `expected_client_id` out of band. + pub fn connect_in_memory_as(&self, expected_client_id: &str) -> Transport { + let (client, router) = Transport::pair(); + self.serve_as(router, expected_client_id); + client + } + + /// Listens on a new Unix-domain socket (mode 0600) until the returned handle is dropped or + /// the router shuts down. + pub async fn listen_unix(&self, path: impl AsRef) -> io::Result { + self.listen_unix_inner(path.as_ref(), None).await + } + + /// Listens on a launcher-created endpoint dedicated to `expected_client_id`. + pub async fn listen_unix_as( + &self, + path: impl AsRef, + expected_client_id: &str, + ) -> io::Result { + self.listen_unix_inner(path.as_ref(), Some(expected_client_id.to_owned())) + .await + } + + async fn listen_unix_inner( + &self, + path: &Path, + expected_client_id: Option, + ) -> io::Result { + let path = path.to_path_buf(); + let listener = UnixListener::bind(&path)?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))?; + let inner = self.inner.clone(); + let mut stop = self.inner.stop.subscribe(); + let task = tokio::spawn(async move { + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((stream, _)) => { + tokio::spawn(run_connection(inner.clone(), stream, expected_client_id.clone())); + } + // Out of descriptors and the like: back off rather than spin. + Err(_) => tokio::time::sleep(Duration::from_millis(10)).await, + }, + _ = stopped(&mut stop) => break, + } + } + }); + Ok(UnixListenerHandle { path, task }) + } + + /// Closes every connection, releases retained values and refuses new connections. A + /// connection at a frame boundary gets final notices; one inside a partial frame is closed + /// without appending another frame. The store directory is removed once the last connection + /// task and seal have finished and every `Router` clone is dropped. + pub fn shutdown(&self) { + self.inner.stop.send_replace(true); + self.inner.with_state(|s| s.shutdown()); + } +} + +/// Stops the listener and removes its socket file when dropped. +pub struct UnixListenerHandle { + path: PathBuf, + task: JoinHandle<()>, +} + +impl UnixListenerHandle { + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for UnixListenerHandle { + fn drop(&mut self) { + self.task.abort(); + let _ = std::fs::remove_file(&self.path); + } +} + +/// Resolves once the flag is true (or its sender is gone). +async fn stopped(rx: &mut watch::Receiver) { + let _ = rx.wait_for(|v| *v).await; +} + +async fn run_connection( + inner: Arc, + stream: S, + expected_client_id: Option, +) { + let signals = Arc::new(ConnSignals::new()); + let Some(c) = inner.with_state(|s| s.add_conn(signals.clone(), expected_client_id)) else { + return; + }; + let (rd, wr) = tokio::io::split(stream); + let writer = tokio::spawn(write_loop(inner.clone(), c, signals.clone(), wr)); + read_loop(&inner, c, &signals, rd).await; + // A no-op if the state already closed it. + inner.with_state(|s| s.close_conn(c, [])); + let _ = writer.await; +} + +async fn read_loop( + inner: &Arc, + c: ConnKey, + signals: &ConnSignals, + mut rd: R, +) { + let mut shutdown = signals.shutdown.subscribe(); + let hello_deadline = tokio::time::sleep(inner.hello_timeout); + tokio::pin!(hello_deadline); + let mut negotiated = false; + let mut handled: u32 = 0; + loop { + let frame = tokio::select! { + f = read_frame(&mut rd) => f, + _ = stopped(&mut shutdown) => return, + _ = &mut hello_deadline, if !negotiated => { + inner.with_state(|s| s.reject_frame(c, "bus.hello deadline expired".into())); + return; + } + }; + let bytes = match frame { + Ok(Some(b)) => b, + Ok(None) | Err(FrameError::Io(_)) | Err(FrameError::Truncated) => return, + Err(e) => { + inner.with_state(|s| s.reject_frame(c, e.to_string())); + return; + } + }; + let env = match Envelope::decode(&bytes) { + Ok(env) => env, + Err(e) => { + inner.with_state(|s| s.reject_frame(c, e.0)); + return; + } + }; + let was_hello = !negotiated && env.op == "bus.hello"; + match inner.with_state(|s| s.handle(c, env)) { + Outcome::Done => {} + Outcome::Close => return, + Outcome::Allocate(job) => { + let (i, serial, len) = (inner.clone(), job.serial, job.len); + let created = + tokio::task::spawn_blocking(move || i.store.create_staging(serial, len)) + .await + .unwrap_or_else(|e| Err(io::Error::other(e))); + inner.with_state(|s| s.finish_allocate(job, created)); + } + Outcome::Seal(job) => { + let inner = inner.clone(); + tokio::spawn(async move { + let (i, serial, len, digest) = + (inner.clone(), job.serial, job.len, job.digest.clone()); + let result = tokio::task::spawn_blocking(move || { + let r = i.store.seal(serial, len, digest.as_deref()); + // Staging is finished with either way; unlink it before anyone can + // see the outcome. + i.store.remove(&crate::store::staging_rel(serial)); + r + }) + .await + .unwrap_or_else(|e| Err(SealFailure::Io(io::Error::other(e)))); + inner.with_state(|s| s.finish_seal(job, result)); + }); + } + } + if was_hello { + negotiated = true; + } + handled = handled.wrapping_add(1); + if handled.is_multiple_of(32) { + // Fairness: a client flooding commands yields to the others. + tokio::task::yield_now().await; + } + } +} + +/// How long a closing connection gets to write final notices and shut down the transport. +const CLOSING_WRITE: Duration = Duration::from_secs(2); + +enum SelectedWrite { + Complete, + Closing { partial: bool }, + Failed, +} + +/// Writes one selected frame while serializing each synchronous transport poll with teardown. +/// The standard mutex is released after every poll and is never held across an await. +async fn write_selected( + wr: &mut W, + bytes: &[u8], + signals: &ConnSignals, +) -> SelectedWrite { + let mut frame = Vec::with_capacity(bytes.len() + 4); + frame.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + frame.extend_from_slice(bytes); + { + let mut gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner()); + if !gate.begin_frame(frame.len()) { + return SelectedWrite::Closing { + partial: gate.cut_partial(), + }; + } + } + + let mut written = 0; + while written < frame.len() { + let polled = std::future::poll_fn(|cx| { + let mut gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner()); + if gate.closing() { + return std::task::Poll::Ready(Err(io::Error::new( + io::ErrorKind::Interrupted, + "connection closing", + ))); + } + match std::pin::Pin::new(&mut *wr).poll_write(cx, &frame[written..]) { + std::task::Poll::Ready(Ok(0)) => std::task::Poll::Ready(Err(io::Error::new( + io::ErrorKind::WriteZero, + "failed to write router frame", + ))), + std::task::Poll::Ready(Ok(n)) => { + gate.wrote(n); + std::task::Poll::Ready(Ok(n)) + } + other => other, + } + }) + .await; + match polled { + Ok(n) => written += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => { + let gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner()); + return SelectedWrite::Closing { + partial: gate.cut_partial(), + }; + } + Err(_) => return SelectedWrite::Failed, + } + } + + let flushed = std::future::poll_fn(|cx| { + let gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner()); + if gate.closing() { + return std::task::Poll::Ready(Err(io::Error::new( + io::ErrorKind::Interrupted, + "connection closing", + ))); + } + std::pin::Pin::new(&mut *wr).poll_flush(cx) + }) + .await; + if let Err(e) = flushed { + if e.kind() == io::ErrorKind::Interrupted { + let gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner()); + return SelectedWrite::Closing { + partial: gate.cut_partial(), + }; + } + return SelectedWrite::Failed; + } + signals + .write_gate + .lock() + .unwrap_or_else(|e| e.into_inner()) + .finish_frame(); + SelectedWrite::Complete +} + +async fn write_loop( + inner: Arc, + c: ConnKey, + signals: Arc, + mut wr: W, +) { + let mut shutdown = signals.shutdown.subscribe(); + // False once a frame was cut short: nothing more may be written on this stream. + let mut aligned = true; + loop { + match inner.with_state(|s| s.next_frame(c)) { + NextFrame::Frame(bytes) => { + let write = write_selected(&mut wr, &bytes, &signals); + tokio::pin!(write); + let selected = tokio::select! { + result = &mut write => result, + _ = stopped(&mut shutdown) => { + let gate = signals + .write_gate + .lock() + .unwrap_or_else(|e| e.into_inner()); + SelectedWrite::Closing { partial: gate.cut_partial() } + } + }; + match selected { + SelectedWrite::Complete => {} + SelectedWrite::Closing { partial } => { + aligned = !partial; + break; + } + SelectedWrite::Failed => { + aligned = false; + inner.with_state(|s| s.close_conn(c, [])); + break; + } + } + } + NextFrame::Idle => { + tokio::select! { + _ = signals.wake.notified() => {} + _ = stopped(&mut shutdown) => {} + } + } + NextFrame::Gone => break, + } + } + let finals = std::mem::take( + &mut *signals + .final_frames + .lock() + .unwrap_or_else(|e| e.into_inner()), + ); + if aligned { + let _ = tokio::time::timeout(CLOSING_WRITE, async { + for f in &finals { + if write_frame(&mut wr, f).await.is_err() { + break; + } + } + }) + .await; + } + let _ = tokio::time::timeout(CLOSING_WRITE, wr.shutdown()).await; +} diff --git a/services/flysim/crates/flybus/src/router/state.rs b/services/flysim/crates/flybus/src/router/state.rs new file mode 100644 index 0000000..4dc52e6 --- /dev/null +++ b/services/flysim/crates/flybus/src/router/state.rs @@ -0,0 +1,2606 @@ +//! The router's state machine: every routing, queueing and ownership decision, with no I/O. +//! +//! One `State` sits behind one mutex. Connection tasks call in with a decoded command, get +//! their reply queued on the connection's control lane, and do any file I/O (staging +//! creation, seal copies, unlinks) only after the lock is released. Writers pull their next +//! frame from here, so a delivery is marked dispatched and assigned its delivery id at the +//! moment its bytes are about to be written, never earlier. +//! +//! Ownership: every sealed artifact has a root count. A root is held by each queued delivery, +//! each delivery handed to a client and not yet consumed, each explicit hold and each retained +//! topic value that references it; a delivery naming the same artifact twice holds one root. +//! The last root to go collects the artifact: its bytes leave the quota and its file is +//! unlinked. Staging writers are not roots; an unsealed artifact is owned by its writer alone. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; + +use serde_json::{Map, Value}; +use tokio::sync::{Notify, watch}; + +use crate::error::{BusError, Dispatch, ErrorCode}; +use crate::limits::Limits; +use crate::policy::{Grants, Policy}; +use crate::store::{sealed_rel, staging_rel}; +use crate::wire::{ + ArtifactRef, Attachment, Envelope, Fields, GENERATION, Identity, Kind, MAJOR, MAX_BATCH, + MAX_CREDIT, MAX_ENVELOPE_BYTES, MINOR, WireError, id_batch, parse_serial_id, serial_id, +}; + +pub(crate) type ConnKey = u64; +type CallKey = u64; + +/// After this many consecutive control or RPC frames while topic data waits, one topic frame +/// goes out, so telemetry is deprioritised but never starved outright. +const TOPIC_STARVATION_LIMIT: u32 = 16; +/// Placeholder that is as long as any router-issued serial id can be. +const MAX_SERIAL: u64 = u64::MAX; + +/// Signals between a connection's tasks and the state. +pub(crate) struct ConnSignals { + /// Wakes the writer: something may be ready to send. + pub wake: Notify, + /// Flips to true when the connection is closed; both tasks watch it. + pub shutdown: watch::Sender, + /// Serializes synchronous transport polls with connection teardown. It is never held + /// across an await. + pub write_gate: std::sync::Mutex, + /// The last frames to write before closing: a refusal or `connection.closing` notice, + /// preceded on router shutdown by `subscription.closed` notices. + pub final_frames: std::sync::Mutex>>, +} + +#[derive(Default)] +pub(crate) struct WriteGate { + closing: bool, + frame_len: usize, + written: usize, + cut_partial: bool, +} + +impl WriteGate { + pub(crate) fn begin_frame(&mut self, len: usize) -> bool { + if self.closing { + return false; + } + self.frame_len = len; + self.written = 0; + true + } + + pub(crate) fn wrote(&mut self, len: usize) { + self.written += len; + debug_assert!(self.written <= self.frame_len); + } + + pub(crate) fn finish_frame(&mut self) { + if !self.closing { + debug_assert_eq!(self.written, self.frame_len); + self.frame_len = 0; + self.written = 0; + } + } + + fn begin_close(&mut self) { + self.closing = true; + self.cut_partial = self.written > 0 && self.written < self.frame_len; + } + + pub(crate) fn closing(&self) -> bool { + self.closing + } + + pub(crate) fn cut_partial(&self) -> bool { + self.cut_partial + } +} + +impl ConnSignals { + pub(crate) fn new() -> ConnSignals { + ConnSignals { + wake: Notify::new(), + shutdown: watch::Sender::new(false), + write_gate: std::sync::Mutex::new(WriteGate::default()), + final_frames: std::sync::Mutex::new(Vec::new()), + } + } +} + +/// What the connection task must do after `handle` returns. +pub(crate) enum Outcome { + Done, + /// Create the staging file, then call `finish_allocate`. + Allocate(AllocJob), + /// Copy staging into the sealed file (off the routing path), then call `finish_seal`. + Seal(SealJob), + /// The connection was closed; stop reading. + Close, +} + +pub(crate) struct AllocJob { + pub conn: ConnKey, + pub reply_to: String, + pub serial: u64, + pub len: u64, + pub owner: u64, +} + +pub(crate) struct SealJob { + pub conn: ConnKey, + pub reply_to: String, + pub serial: u64, + pub len: u64, + pub digest: Option, + pub owner: u64, +} + +pub(crate) enum NextFrame { + Frame(Vec), + Idle, + Gone, +} + +pub(crate) struct Outbound { + kind: Kind, + op: String, + reply_to: Option, + body: Map, + attachments: Vec, + /// Conservative encoded size using the longest router envelope id. + bytes: usize, +} + +/// Registry counters, for tests and resource measurement. They are logical: an unlinked file +/// a process still has open keeps its pages until that process closes it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RouterStats { + pub connections: usize, + pub services: usize, + pub topics: usize, + pub subscriptions: usize, + /// Call correlation records the router still keeps, detached ones included. + pub calls: usize, + /// Calls counted against their callers' active-call limits (admitted, not finished, + /// not detached). + pub active_calls: usize, + /// Artifacts in any state: writing, sealing or sealed. + pub artifacts: usize, + pub sealed_artifacts: usize, + /// Sum of root counts over sealed artifacts. + pub artifact_roots: u64, + /// Live owners (deliveries, holds, writers) over all connections. + pub owners: usize, + /// Live request reply capabilities. They are independent of delivery credit but share the + /// per-client bound. + pub reply_capabilities: usize, + /// Admitted and not yet dispatched: topic messages in subscription queues, calls in + /// service queues and results waiting for their caller. + pub queued: usize, + /// Staging, sealing-copy and sealed bytes currently charged to the store. + pub store_bytes: u64, + pub retained_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum OwnerKey { + Delivery(u64), + Hold(u64), +} + +enum Owner { + Writer { + artifact: u64, + sealing: bool, + }, + Hold { + artifact: u64, + }, + Delivery { + artifacts: Vec, + kind: DeliveryKind, + }, +} + +impl Owner { + /// Counted against the ordinary owner budget (everything but RPC deliveries). + fn ordinary(&self) -> bool { + !matches!( + self, + Owner::Delivery { + kind: DeliveryKind::Request { .. } | DeliveryKind::Result { .. }, + .. + } + ) + } +} + +enum DeliveryKind { + Topic { + sub: u64, + }, + Request { + call: CallKey, + service: String, + incarnation: String, + }, + Result { + call: CallKey, + }, +} + +struct Hello { + identity: Identity, + grants: Grants, +} + +struct Conn { + signals: Arc, + connection_id: String, + hello: Option, + expected_client_id: Option, + last_command: Option, + out_serial: u64, + delivery_issued: u64, + hold_issued: u64, + sub_issued: u64, + owners: HashMap, + ordinary_owners: usize, + subs: HashMap, + sub_order: Vec, + sub_cursor: usize, + services: Vec, + svc_cursor: usize, + /// Request delivery serial -> call, for `rpc.reply` correlation. + requests: HashMap, + reply_capabilities: usize, + call_watermark: Option, + /// Admitted calls not yet finished or detached, by call serial. + active_calls: HashMap, + result_queue: VecDeque, + control: VecDeque, + control_bytes: usize, + streak: u32, + queued_bytes: usize, +} + +struct Art { + state: ArtState, + byte_length: u64, + content_type: String, + digest: Option, + roots: u64, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ArtState { + Writing, + Sealing, + Sealed, +} + +struct TopicMsg { + topic: String, + topic_incarnation: String, + sequence: u64, + payload: Map, + attachments: Vec<(String, ArtifactRef)>, + artifacts: Vec, + bytes: usize, + artifact_bytes: u64, +} + +struct Sub { + topic: String, + topic_incarnation: String, + latest: bool, + max_queued: u64, + max_in_flight: u64, + in_flight: u64, + queue: VecDeque>, + replaced: u64, +} + +struct Topic { + incarnation: String, + retain: bool, + sequence: u64, + retained: Option>, + subscribers: Vec<(ConnKey, u64)>, +} + +struct Svc { + incarnation: String, + conn: ConnKey, + max_queued: u64, + max_in_flight: u64, + in_flight: u64, + queue: VecDeque, +} + +struct ResultMsg { + responder: Identity, + outcome: Map, + attachments: Vec<(String, ArtifactRef)>, + artifacts: Vec, +} + +enum Phase { + Queued, + Dispatched { delivery: u64 }, + Replied { delivery: u64, result: CallResult }, +} + +enum CallResult { + Queued(ResultMsg), + Delivered, +} + +struct Call { + caller: ConnKey, + caller_identity: Identity, + call_serial: u64, + call_id: String, + target: String, + service_incarnation: String, + service_conn: ConnKey, + method: String, + payload: Map, + attachments: Vec<(String, ArtifactRef)>, + artifacts: Vec, + phase: Phase, + detached: bool, + reply_capability: bool, +} + +#[derive(Default)] +struct ClientRecord { + live: Option, + last_incarnation: Option, +} + +pub(crate) struct State { + pub router_id: String, + pub store_id: String, + contract_digest: String, + limits: Limits, + policy: Policy, + closed: bool, + conns: HashMap, + identities: HashMap, + services: HashMap, + topics: HashMap, + artifacts: HashMap, + calls: HashMap, + next_conn: u64, + next_service: u64, + next_topic: u64, + next_artifact: u64, + next_call: u64, + store_bytes: u64, + retained_bytes: u64, + subscriptions: usize, + unlinks: Vec, + notices: Vec<(ConnKey, &'static str, Map)>, +} + +fn obj(pairs: Vec<(&str, Value)>) -> Map { + pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect() +} + +fn err(code: ErrorCode, message: impl Into) -> Result { + Err(BusError::new(code, message)) +} + +fn wire(e: WireError) -> BusError { + e.into() +} + +fn dedup(serials: Vec) -> Vec { + let mut seen = HashSet::new(); + serials.into_iter().filter(|s| seen.insert(*s)).collect() +} + +fn attachments_with_owner(list: &[(String, ArtifactRef)], owner: &str) -> Vec { + list.iter() + .map(|(name, r)| Attachment { + name: name.clone(), + reference: r.clone(), + owner_id: owner.to_owned(), + }) + .collect() +} + +fn frame_len( + kind: Kind, + op: &str, + body: Map, + attachments: Vec, +) -> usize { + let mut env = Envelope::new(serial_id("bus", MAX_SERIAL), kind, op, body); + env.attachments = attachments; + serde_json::to_vec(&env.to_value()) + .map(|b| b.len()) + .unwrap_or(usize::MAX) +} + +impl Call { + fn request_body(&self, delivery_id: &str) -> Map { + obj(vec![ + ("deliveryId", delivery_id.into()), + ("callId", self.call_id.clone().into()), + ("caller", self.caller_identity.to_json()), + ("target", self.target.clone().into()), + ( + "serviceIncarnation", + self.service_incarnation.clone().into(), + ), + ("method", self.method.clone().into()), + ("payload", Value::Object(self.payload.clone())), + ]) + } + + fn result_body(&self, delivery_id: &str, result: &ResultMsg) -> Map { + obj(vec![ + ("deliveryId", delivery_id.into()), + ("callId", self.call_id.clone().into()), + ("responder", result.responder.to_json()), + ( + "serviceIncarnation", + self.service_incarnation.clone().into(), + ), + ("outcome", Value::Object(result.outcome.clone())), + ]) + } +} + +impl TopicMsg { + fn body(&self, delivery_id: &str, subscription_id: &str, replaced: u64) -> Map { + obj(vec![ + ("deliveryId", delivery_id.into()), + ("subscriptionId", subscription_id.into()), + ("topic", self.topic.clone().into()), + ("topicIncarnation", self.topic_incarnation.clone().into()), + ("topicSequence", self.sequence.to_string().into()), + ("replaced", replaced.to_string().into()), + ("payload", Value::Object(self.payload.clone())), + ]) + } +} + +impl State { + pub(crate) fn new( + router_id: String, + store_id: String, + contract_digest: String, + limits: Limits, + policy: Policy, + ) -> State { + State { + router_id, + store_id, + contract_digest, + limits, + policy, + closed: false, + conns: HashMap::new(), + identities: HashMap::new(), + services: HashMap::new(), + topics: HashMap::new(), + artifacts: HashMap::new(), + calls: HashMap::new(), + next_conn: 0, + next_service: 0, + next_topic: 0, + next_artifact: 0, + next_call: 0, + store_bytes: 0, + retained_bytes: 0, + subscriptions: 0, + unlinks: Vec::new(), + notices: Vec::new(), + } + } + + /// Store files (relative paths) to unlink once the lock is released. + pub(crate) fn take_unlinks(&mut self) -> Vec { + std::mem::take(&mut self.unlinks) + } + + pub(crate) fn stats(&self) -> RouterStats { + RouterStats { + connections: self.conns.len(), + services: self.services.len(), + topics: self.topics.len(), + subscriptions: self.subscriptions, + calls: self.calls.len(), + active_calls: self.conns.values().map(|c| c.active_calls.len()).sum(), + artifacts: self.artifacts.len(), + sealed_artifacts: self + .artifacts + .values() + .filter(|a| a.state == ArtState::Sealed) + .count(), + artifact_roots: self.artifacts.values().map(|a| a.roots).sum(), + owners: self.conns.values().map(|c| c.owners.len()).sum(), + reply_capabilities: self.conns.values().map(|c| c.reply_capabilities).sum(), + queued: self + .conns + .values() + .map(|c| { + c.subs.values().map(|s| s.queue.len()).sum::() + c.result_queue.len() + }) + .sum::() + + self.services.values().map(|s| s.queue.len()).sum::(), + store_bytes: self.store_bytes, + retained_bytes: self.retained_bytes, + } + } + + // ----------------------------------------------------------------------------------------- + // Connections + + pub(crate) fn add_conn( + &mut self, + signals: Arc, + expected_client_id: Option, + ) -> Option { + if self.closed + || self.conns.len() >= self.limits.max_clients + || (expected_client_id.is_none() && !self.policy.permits_unbound_transport()) + { + return None; + } + self.next_conn += 1; + let key = self.next_conn; + self.conns.insert( + key, + Conn { + signals, + connection_id: serial_id("conn", key), + hello: None, + expected_client_id, + last_command: None, + out_serial: 0, + delivery_issued: 0, + hold_issued: 0, + sub_issued: 0, + owners: HashMap::new(), + ordinary_owners: 0, + subs: HashMap::new(), + sub_order: Vec::new(), + sub_cursor: 0, + services: Vec::new(), + svc_cursor: 0, + requests: HashMap::new(), + reply_capabilities: 0, + call_watermark: None, + active_calls: HashMap::new(), + result_queue: VecDeque::new(), + control: VecDeque::new(), + control_bytes: 0, + streak: 0, + queued_bytes: 0, + }, + ); + Some(key) + } + + fn wake(&self, c: ConnKey) { + if let Some(conn) = self.conns.get(&c) { + conn.signals.wake.notify_one(); + } + } + + /// Builds a router-originated item without assigning its envelope id. IDs are assigned only + /// after the scheduler selects an item, so fairness cannot reorder numbered envelopes. + fn outbound( + &self, + kind: Kind, + op: &str, + reply_to: Option, + body: Map, + attachments: Vec, + ) -> Option { + let mut env = Envelope::new(serial_id("bus", MAX_SERIAL), kind, op, body.clone()); + env.reply_to = reply_to; + env.attachments = attachments.clone(); + let bytes = env.encode().ok()?.len(); + Some(Outbound { + kind, + op: op.to_owned(), + reply_to: env.reply_to, + body, + attachments, + bytes, + }) + } + + fn encode_outbound(&mut self, c: ConnKey, item: Outbound) -> Option> { + let conn = self.conns.get_mut(&c)?; + conn.out_serial += 1; + let mut env = Envelope::new( + serial_id("bus", conn.out_serial), + item.kind, + &item.op, + item.body, + ); + env.reply_to = item.reply_to; + env.attachments = item.attachments; + env.encode().ok() + } + + /// Queues a reply or notice on the control lane; closes the connection if the lane is full. + fn push_control(&mut self, c: ConnKey, frame: Outbound) { + let (frames, bytes) = ( + self.limits.max_control_frames, + self.limits.max_control_bytes, + ); + let Some(conn) = self.conns.get_mut(&c) else { + return; + }; + if conn.control.len() >= frames || conn.control_bytes + frame.bytes > bytes { + let body = obj(vec![ + ("code", ErrorCode::QuotaExceeded.as_str().into()), + ( + "message", + "control lane exhausted: replies and notices are not being read".into(), + ), + ]); + let last = self.outbound(Kind::Notice, "connection.closing", None, body, Vec::new()); + self.close_conn(c, last); + return; + } + conn.control_bytes += frame.bytes; + conn.control.push_back(frame); + conn.signals.wake.notify_one(); + } + + fn reply( + &mut self, + c: ConnKey, + reply_to: &str, + op: &str, + result: Result, BusError>, + ) { + let body = reply_body(result); + if let Some(frame) = + self.outbound(Kind::Reply, op, Some(reply_to.to_owned()), body, Vec::new()) + { + self.push_control(c, frame); + } + } + + fn notice(&mut self, c: ConnKey, op: &'static str, body: Map) { + self.notices.push((c, op, body)); + } + + /// Sends deferred notices. Pushing one may close its connection, whose cleanup may defer + /// more, so this loops until none are left. + fn flush_notices(&mut self) { + while !self.notices.is_empty() { + for (c, op, body) in std::mem::take(&mut self.notices) { + if let Some(frame) = self.outbound(Kind::Notice, op, None, body, Vec::new()) { + self.push_control(c, frame); + } + } + } + } + + /// Closes `c`: records the final frames, signals both tasks and releases everything the + /// connection owned. Replies still queued on its control lane are dropped. + pub(crate) fn close_conn( + &mut self, + c: ConnKey, + final_frames: impl IntoIterator, + ) { + let Some(conn) = self.conns.get(&c) else { + return; + }; + let signals = conn.signals.clone(); + let final_frames = final_frames + .into_iter() + .filter_map(|frame| self.encode_outbound(c, frame)); + signals + .final_frames + .lock() + .unwrap_or_else(|e| e.into_inner()) + .extend(final_frames); + // A transport poll that is already in progress finishes before this lock is acquired. + // Once acquired, teardown marks the stream closing before reclaiming any owner, and no + // later normal-frame poll is allowed through. + let mut write_gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner()); + write_gate.begin_close(); + self.disconnect(c); + signals.shutdown.send_replace(true); + signals.wake.notify_one(); + drop(write_gate); + self.flush_notices(); + } + + fn violation(&mut self, c: ConnKey, message: String) -> Outcome { + let body = obj(vec![ + ("code", ErrorCode::InvalidEnvelope.as_str().into()), + ("message", message.into()), + ]); + let last = self.outbound(Kind::Notice, "connection.closing", None, body, Vec::new()); + self.close_conn(c, last); + Outcome::Close + } + + /// A frame that failed to decode: close with a notice. + pub(crate) fn reject_frame(&mut self, c: ConnKey, message: String) { + self.violation(c, message); + } + + /// Closes every connection and refuses new ones. + pub(crate) fn shutdown(&mut self) { + self.closed = true; + let keys: Vec = self.conns.keys().copied().collect(); + for c in keys { + let mut finals = Vec::new(); + let subs: Vec<(u64, String, String)> = self.conns[&c] + .subs + .iter() + .map(|(id, s)| (*id, s.topic.clone(), s.topic_incarnation.clone())) + .collect(); + for (id, topic, inc) in subs { + let body = obj(vec![ + ("subscriptionId", serial_id("sub", id).into()), + ("topic", topic.into()), + ("topicIncarnation", inc.into()), + ("reason", "router-stopping".into()), + ]); + finals.extend(self.outbound( + Kind::Notice, + "subscription.closed", + None, + body, + Vec::new(), + )); + } + let body = obj(vec![ + ("code", ErrorCode::RouterLost.as_str().into()), + ("message", "router stopping".into()), + ]); + finals.extend(self.outbound( + Kind::Notice, + "connection.closing", + None, + body, + Vec::new(), + )); + self.close_conn(c, finals); + } + let topics: Vec = self.topics.keys().cloned().collect(); + for t in topics { + if let Some(msg) = self.topics.get_mut(&t).and_then(|t| t.retained.take()) { + self.retained_bytes -= msg.artifact_bytes; + self.drop_roots(&msg.artifacts); + } + } + } + + /// Releases everything `c` owned. Retained topic values stay; they are router-owned. + pub(crate) fn disconnect(&mut self, c: ConnKey) { + let Some(conn) = self.conns.get(&c) else { + return; + }; + if let Some(h) = &conn.hello + && let Some(rec) = self.identities.get_mut(&h.identity.client_id) + && rec.live == Some(c) + { + rec.live = None; + } + // Services: queued calls fail; dispatched ones are settled with the owners below. + for name in conn.services.clone() { + self.remove_service(&name, "disconnected"); + } + // Subscriptions. + let conn = self.conns.get_mut(&c).expect("present"); + let subs: Vec = conn.sub_order.clone(); + for id in subs { + self.drop_subscription(c, id); + } + // Owners. + let conn = self.conns.get_mut(&c).expect("present"); + let owners: Vec<(OwnerKey, Owner)> = conn.owners.drain().collect(); + conn.ordinary_owners = 0; + for (_, owner) in owners { + self.release_owner(c, owner); + } + // Calls dispatched to this connection that it never answered, whether or not it still + // held the request: the caller learns the call may have executed. + let served: Vec = self + .calls + .iter() + .filter(|(_, call)| { + call.service_conn == c && matches!(call.phase, Phase::Dispatched { .. }) + }) + .map(|(k, _)| *k) + .collect(); + for key in served { + let call = &self.calls[&key]; + if !call.detached { + let body = obj(vec![ + ("callId", call.call_id.clone().into()), + ("code", ErrorCode::NoService.as_str().into()), + ("message", "the service disconnected before replying".into()), + ("dispatch", Dispatch::Dispatched.as_str().into()), + ]); + let caller = call.caller; + self.notice(caller, "call.failed", body); + } + self.remove_call(key); + } + // Calls this connection made. + let mine: Vec = self + .calls + .iter() + .filter(|(_, call)| call.caller == c) + .map(|(k, _)| *k) + .collect(); + for key in mine { + let Some(call) = self.calls.get_mut(&key) else { + continue; + }; + match call.phase { + Phase::Queued => { + let (target, inc) = (call.target.clone(), call.service_incarnation.clone()); + if let Some(svc) = self + .services + .get_mut(&target) + .filter(|s| s.incarnation == inc) + { + svc.queue.retain(|k| *k != key); + } + self.remove_call(key); + } + // The service may still reply (it will hear `routed:false`) or consume the + // request; the record goes when either happens. + Phase::Dispatched { .. } => { + call.detached = true; + if !call.reply_capability { + self.remove_call(key); + } + } + Phase::Replied { .. } => self.remove_call(key), + } + } + if let Some(conn) = self.conns.remove(&c) { + debug_assert!(conn.owners.is_empty()); + } + } + + // ----------------------------------------------------------------------------------------- + // Roots and artifacts + + fn add_roots(&mut self, serials: &[u64]) { + for s in serials { + if let Some(a) = self.artifacts.get_mut(s) { + a.roots += 1; + } + } + } + + fn drop_roots(&mut self, serials: &[u64]) { + for s in serials { + let Some(a) = self.artifacts.get_mut(s) else { + continue; + }; + a.roots -= 1; + if a.roots == 0 && a.state == ArtState::Sealed { + let a = self.artifacts.remove(s).expect("present"); + self.store_bytes -= a.byte_length; + self.unlinks.push(sealed_rel(*s)); + } + } + } + + fn reference(&self, serial: u64) -> Option { + let a = self.artifacts.get(&serial)?; + Some(ArtifactRef { + store_id: self.store_id.clone(), + artifact_id: serial_id("a", serial), + generation: GENERATION, + byte_length: a.byte_length, + content_type: a.content_type.clone(), + digest: a.digest.clone(), + }) + } + + /// Drops an unsealed writer's artifact and its staging reservation. + fn abandon_writer(&mut self, serial: u64) { + if let Some(a) = self.artifacts.remove(&serial) { + self.store_bytes -= a.byte_length; + self.unlinks.push(staging_rel(serial)); + } + } + + fn take_owner(&mut self, c: ConnKey, key: OwnerKey) -> Option { + let conn = self.conns.get_mut(&c)?; + let owner = conn.owners.remove(&key)?; + if owner.ordinary() { + conn.ordinary_owners -= 1; + } + Some(owner) + } + + fn insert_owner(&mut self, c: ConnKey, key: OwnerKey, owner: Owner) { + let conn = self.conns.get_mut(&c).expect("present"); + if owner.ordinary() { + conn.ordinary_owners += 1; + } + conn.owners.insert(key, owner); + } + + fn ordinary_budget_left(&self, c: ConnKey) -> bool { + let conn = &self.conns[&c]; + conn.ordinary_owners + < self.limits.max_owners_per_client - self.limits.reserved_owners_per_client + && conn.owners.len() < self.limits.max_owners_per_client + } + + /// Releases a removed owner's roots and returns its credit. + fn release_owner(&mut self, c: ConnKey, owner: Owner) { + match owner { + Owner::Hold { artifact } => self.drop_roots(&[artifact]), + // A sealing writer's artifact is reclaimed by `finish_seal`, which finds its owner gone. + Owner::Writer { + artifact, + sealing: false, + } => self.abandon_writer(artifact), + Owner::Writer { sealing: true, .. } => {} + Owner::Delivery { artifacts, kind } => { + self.drop_roots(&artifacts); + match kind { + DeliveryKind::Topic { sub } => { + if let Some(s) = self + .conns + .get_mut(&c) + .and_then(|conn| conn.subs.get_mut(&sub)) + { + s.in_flight -= 1; + } + } + DeliveryKind::Request { + call, + service, + incarnation, + } => { + if let Some(svc) = self + .services + .get_mut(&service) + .filter(|s| s.incarnation == incarnation) + { + svc.in_flight -= 1; + } + // A detached call without a remaining reply capability is terminal. + if self + .calls + .get(&call) + .is_some_and(|e| e.detached && !e.reply_capability) + { + self.remove_call(call); + } + } + DeliveryKind::Result { call } => self.remove_call(call), + } + } + } + self.wake(c); + } + + /// Checks that `owner_id` is a live owner on `c` holding the sealed artifact `r`. + fn check_owned(&self, c: ConnKey, r: &ArtifactRef, owner_id: &str) -> Result { + if r.store_id != self.store_id { + return err( + ErrorCode::ArtifactGone, + "reference from another store incarnation", + ); + } + let Some(serial) = parse_serial_id("a", &r.artifact_id) else { + return err(ErrorCode::ArtifactGone, "unknown artifact id"); + }; + let key = if let Some(n) = parse_serial_id("dlv", owner_id) { + OwnerKey::Delivery(n) + } else if let Some(n) = parse_serial_id("own", owner_id) { + OwnerKey::Hold(n) + } else { + return err(ErrorCode::OwnerInvalid, "not an owner id"); + }; + let conn = &self.conns[&c]; + let holds = match conn.owners.get(&key) { + None => { + return err( + ErrorCode::OwnerInvalid, + format!("{owner_id} is not a live owner on this connection"), + ); + } + Some(Owner::Writer { artifact, .. }) if *artifact == serial => { + return err(ErrorCode::ArtifactUnsealed, "artifact is not sealed"); + } + Some(Owner::Writer { .. }) => false, + Some(Owner::Hold { artifact }) => *artifact == serial, + Some(Owner::Delivery { artifacts, .. }) => artifacts.contains(&serial), + }; + if !holds { + return err( + ErrorCode::OwnerInvalid, + format!("{owner_id} does not own {}", r.artifact_id), + ); + } + let Some(art) = self.artifacts.get(&serial) else { + return err(ErrorCode::ArtifactGone, "artifact was collected"); + }; + if art.state != ArtState::Sealed { + return err(ErrorCode::ArtifactUnsealed, "artifact is not sealed"); + } + if self.reference(serial).as_ref() != Some(r) { + return err( + ErrorCode::ArtifactMismatch, + "reference does not match the artifact", + ); + } + Ok(serial) + } + + /// Validates every attachment and returns the distinct artifacts, in first-seen order. + fn check_attachments(&self, c: ConnKey, list: &[Attachment]) -> Result, BusError> { + let mut serials = Vec::with_capacity(list.len()); + for a in list { + serials.push(self.check_owned(c, &a.reference, &a.owner_id)?); + } + Ok(dedup(serials)) + } + + // ----------------------------------------------------------------------------------------- + // Commands + + pub(crate) fn handle(&mut self, c: ConnKey, env: Envelope) -> Outcome { + let Some(conn) = self.conns.get_mut(&c) else { + return Outcome::Close; + }; + if env.kind != Kind::Command || env.reply_to.is_some() { + return self.violation(c, "clients send commands only, with a null replyTo".into()); + } + let Some(serial) = parse_serial_id("msg", &env.id) else { + return self.violation(c, "command ids are msg-".into()); + }; + if conn.last_command.is_some_and(|last| serial <= last) { + return self.violation(c, "command ids must increase".into()); + } + conn.last_command = Some(serial); + if conn.hello.is_none() { + if env.op != "bus.hello" { + return self.violation(c, "the first command must be bus.hello".into()); + } + return self.hello(c, env); + } + if env.op == "bus.hello" { + return self.violation(c, "bus.hello was already negotiated".into()); + } + if env.major != MAJOR || env.minor != MINOR { + return self.violation( + c, + format!("envelopes on this connection are {MAJOR}.{MINOR}"), + ); + } + let op = env.op.clone(); + let result = if !env.attachments.is_empty() + && !matches!(op.as_str(), "rpc.call" | "rpc.reply" | "publish") + { + err( + ErrorCode::InvalidEnvelope, + format!("{op} takes no attachments"), + ) + } else { + match op.as_str() { + "service.register" => self.op_register(c, &env), + "service.unregister" => self.op_unregister(c, &env), + "rpc.call" => self.op_call(c, &env), + "rpc.reply" => self.op_reply(c, &env), + "rpc.responder.release" => self.op_responder_release(c, &env), + "rpc.cancel" => self.op_cancel(c, &env), + "topic.declare" => self.op_declare(c, &env), + "topic.clear" => self.op_clear(c, &env), + "topic.delete" => self.op_delete(c, &env), + "subscribe" => self.op_subscribe(c, &env), + "unsubscribe" => self.op_unsubscribe(c, &env), + "publish" => self.op_publish(c, &env), + "delivery.consumed" => self.op_consumed(c, &env), + "artifact.release" => self.op_release(c, &env), + "artifact.open" => self.op_open(c, &env), + "artifact.retain" => self.op_retain(c, &env), + "artifact.allocate" => match self.op_allocate(c, &env) { + Ok(job) => return Outcome::Allocate(job), + Err(e) => Err(e), + }, + "artifact.seal" => match self.op_seal(c, &env) { + Ok(job) => return Outcome::Seal(job), + Err(e) => Err(e), + }, + _ => err( + ErrorCode::InvalidEnvelope, + format!("unknown operation {op:?}"), + ), + } + }; + self.reply(c, &env.id, &op, result); + self.flush_notices(); + if self.conns.contains_key(&c) { + Outcome::Done + } else { + Outcome::Close + } + } + + fn grants(&self, c: ConnKey) -> &Grants { + &self.conns[&c].hello.as_ref().expect("after hello").grants + } + + fn identity(&self, c: ConnKey) -> Identity { + self.conns[&c] + .hello + .as_ref() + .expect("after hello") + .identity + .clone() + } + + fn hello(&mut self, c: ConnKey, env: Envelope) -> Outcome { + let parsed = (|| -> Result<(String, String, Vec), WireError> { + let mut f = Fields::of(&env.body, "bus.hello"); + let client_id = f.id("clientId")?; + let incarnation = f.id("clientIncarnation")?; + let majors = f.array("supportedMajors", 1, 16)?; + let mut out = Vec::new(); + for m in majors { + match m.as_u64() { + Some(n) if (1..=MAX_CREDIT).contains(&n) => out.push(n), + _ => { + return Err(WireError( + "bus.hello: supportedMajors holds integers 1..=65535".into(), + )); + } + } + } + f.finish()?; + Ok((client_id, incarnation, out)) + })(); + let refuse = |s: &mut State, e: BusError| { + let body = reply_body(Err(e)); + let last = s.outbound( + Kind::Reply, + "bus.hello", + Some(env.id.clone()), + body, + Vec::new(), + ); + s.close_conn(c, last); + Outcome::Close + }; + let (client_id, incarnation, majors) = match parsed { + Ok(p) => p, + Err(e) => return refuse(self, e.into()), + }; + if !env.attachments.is_empty() { + return refuse(self, BusError::invalid("bus.hello takes no attachments")); + } + if !majors.contains(&MAJOR) { + return refuse( + self, + BusError::new( + ErrorCode::VersionMismatch, + format!("this router speaks major {MAJOR} only"), + ), + ); + } + let Some(grants) = self.policy.grants_for(&client_id) else { + return refuse( + self, + BusError::new(ErrorCode::NotAuthorized, "client id is not configured"), + ); + }; + let rec = self.identities.entry(client_id.clone()).or_default(); + if rec.live.is_some() { + return refuse( + self, + BusError::new(ErrorCode::NotAuthorized, "client id is already connected"), + ); + } + if rec.last_incarnation.as_deref() == Some(incarnation.as_str()) { + return refuse( + self, + BusError::new( + ErrorCode::NotAuthorized, + "client incarnation was already used", + ), + ); + } + if let Some(expected) = self.conns[&c].expected_client_id.as_deref() + && expected != client_id + { + return refuse( + self, + BusError::new( + ErrorCode::NotAuthorized, + "client id does not match the launcher's transport binding", + ), + ); + } + let rec = self.identities.get_mut(&client_id).expect("inserted"); + rec.live = Some(c); + rec.last_incarnation = Some(incarnation.clone()); + let identity = Identity { + client_id, + client_incarnation: incarnation, + }; + let conn = self.conns.get_mut(&c).expect("present"); + conn.hello = Some(Hello { identity, grants }); + let value = obj(vec![ + ("routerId", self.router_id.clone().into()), + ("connectionId", conn.connection_id.clone().into()), + ("selectedMajor", MAJOR.into()), + ("selectedMinor", MINOR.into()), + ("contractDigest", self.contract_digest.clone().into()), + ("limits", self.limits.to_json()), + ]); + self.reply(c, &env.id, "bus.hello", Ok(value)); + if self.conns.contains_key(&c) { + Outcome::Done + } else { + Outcome::Close + } + } + + // ---- services and RPC + + fn op_register(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "service.register"); + let name = f.name("name").map_err(wire)?; + let max_queued = f.int("maxQueued", 1, MAX_CREDIT).map_err(wire)?; + let max_in_flight = f.int("maxInFlight", 1, MAX_CREDIT).map_err(wire)?; + f.finish().map_err(wire)?; + if !Grants::allows(&self.grants(c).register, &name) { + return err(ErrorCode::NotAuthorized, format!("may not register {name}")); + } + if max_queued > self.limits.max_service_queued + || max_in_flight > self.limits.max_service_in_flight + { + return err( + ErrorCode::QuotaExceeded, + "maxQueued or maxInFlight exceeds the configured limit", + ); + } + if self.services.contains_key(&name) { + return err(ErrorCode::Conflict, format!("{name} is already registered")); + } + if self.services.len() >= self.limits.max_services { + return err(ErrorCode::QuotaExceeded, "too many services"); + } + self.next_service += 1; + let incarnation = serial_id("svc", self.next_service); + self.services.insert( + name.clone(), + Svc { + incarnation: incarnation.clone(), + conn: c, + max_queued, + max_in_flight, + in_flight: 0, + queue: VecDeque::new(), + }, + ); + self.conns.get_mut(&c).expect("present").services.push(name); + Ok(obj(vec![("serviceIncarnation", incarnation.into())])) + } + + fn op_unregister( + &mut self, + c: ConnKey, + env: &Envelope, + ) -> Result, BusError> { + let mut f = Fields::of(&env.body, "service.unregister"); + let name = f.name("name").map_err(wire)?; + let incarnation = f.id("serviceIncarnation").map_err(wire)?; + f.finish().map_err(wire)?; + let mine = self + .services + .get(&name) + .is_some_and(|s| s.conn == c && s.incarnation == incarnation); + if mine { + self.remove_service(&name, "unregistered"); + } + Ok(obj(vec![("removed", mine.into())])) + } + + /// Removes a registration. Queued calls fail with `call.failed`; each affected caller also + /// hears `route.removed`. Dispatched calls stay correlated: the service may still reply. + fn remove_service(&mut self, name: &str, reason: &str) { + let Some(svc) = self.services.remove(name) else { + return; + }; + if let Some(conn) = self.conns.get_mut(&svc.conn) { + conn.services.retain(|n| n != name); + } + let mut callers = Vec::new(); + for (_, call) in self.calls.iter() { + let open = matches!(call.phase, Phase::Queued | Phase::Dispatched { .. }); + if call.service_incarnation == svc.incarnation + && open + && !call.detached + && !callers.contains(&call.caller) + { + callers.push(call.caller); + } + } + for caller in callers { + let body = obj(vec![ + ("name", name.into()), + ("serviceIncarnation", svc.incarnation.clone().into()), + ("reason", reason.into()), + ]); + self.notice(caller, "route.removed", body); + } + for key in svc.queue { + let Some(call) = self.calls.get(&key) else { + continue; + }; + let body = obj(vec![ + ("callId", call.call_id.clone().into()), + ("code", ErrorCode::NoService.as_str().into()), + ( + "message", + format!("{name} was {reason} before the call was dispatched").into(), + ), + ("dispatch", Dispatch::NotDispatched.as_str().into()), + ]); + let caller = call.caller; + self.notice(caller, "call.failed", body); + self.remove_call(key); + } + let abandoned: Vec = self + .calls + .iter() + .filter(|(_, call)| { + call.service_incarnation == svc.incarnation + && matches!(call.phase, Phase::Dispatched { .. }) + && !call.reply_capability + }) + .map(|(key, _)| *key) + .collect(); + for key in abandoned { + let call = &self.calls[&key]; + if !call.detached { + self.notice( + call.caller, + "call.failed", + obj(vec![ + ("callId", call.call_id.clone().into()), + ("code", ErrorCode::NoService.as_str().into()), + ( + "message", + format!("{name} was {reason} after dispatch").into(), + ), + ("dispatch", Dispatch::Dispatched.as_str().into()), + ]), + ); + } + self.remove_call(key); + } + } + + /// Forgets a call: frees the caller's slot, the reply correlation and any roots the call + /// itself still holds (queued request, queued result). + fn remove_call(&mut self, key: CallKey) { + let Some(call) = self.calls.remove(&key) else { + return; + }; + if let Some(conn) = self.conns.get_mut(&call.caller) { + if conn.active_calls.get(&call.call_serial) == Some(&key) { + conn.active_calls.remove(&call.call_serial); + } + conn.result_queue.retain(|k| *k != key); + } + match call.phase { + Phase::Queued => self.drop_roots(&call.artifacts), + Phase::Dispatched { delivery } => { + if let Some(conn) = self.conns.get_mut(&call.service_conn) { + conn.requests.remove(&delivery); + if call.reply_capability { + conn.reply_capabilities -= 1; + } + } + } + Phase::Replied { delivery, result } => { + if let Some(conn) = self.conns.get_mut(&call.service_conn) { + conn.requests.remove(&delivery); + if call.reply_capability { + conn.reply_capabilities -= 1; + } + } + if let CallResult::Queued(msg) = result { + self.drop_roots(&msg.artifacts); + } + } + } + } + + fn op_call(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "rpc.call"); + let call_id = f.id("callId").map_err(wire)?; + let target = f.name("target").map_err(wire)?; + let expected = f.nullable_id("expectedIncarnation").map_err(wire)?; + let method = f.method("method").map_err(wire)?; + let payload = f.object("payload").map_err(wire)?.clone(); + f.finish().map_err(wire)?; + let Some(call_serial) = parse_serial_id("call", &call_id) else { + return err(ErrorCode::InvalidEnvelope, "callId must be call-"); + }; + if self.conns[&c] + .call_watermark + .is_some_and(|w| call_serial <= w) + { + return err( + ErrorCode::InvalidEnvelope, + "callId was already used or is not increasing", + ); + } + // The issued-id history advances after syntactic validation, before semantic admission. + self.conns.get_mut(&c).expect("present").call_watermark = Some(call_serial); + if !Grants::allows(&self.grants(c).call, &target) { + return err(ErrorCode::NotAuthorized, format!("may not call {target}")); + } + let conn = &self.conns[&c]; + if conn.active_calls.len() >= self.limits.max_active_calls_per_client { + return err(ErrorCode::Backpressure, "too many active calls"); + } + let Some(svc) = self.services.get(&target) else { + return err(ErrorCode::NoService, format!("{target} is not registered")); + }; + if expected.as_ref().is_some_and(|e| *e != svc.incarnation) { + return err( + ErrorCode::TargetChanged, + format!("{target} is now {}", svc.incarnation), + ); + } + if svc.queue.len() as u64 >= svc.max_queued { + return err(ErrorCode::Backpressure, format!("{target} queue is full")); + } + let artifacts = self.check_attachments(c, &env.attachments)?; + let call = Call { + caller: c, + caller_identity: self.identity(c), + call_serial, + call_id, + target: target.clone(), + service_incarnation: svc.incarnation.clone(), + service_conn: svc.conn, + method, + payload, + attachments: env + .attachments + .iter() + .map(|a| (a.name.clone(), a.reference.clone())) + .collect(), + artifacts, + phase: Phase::Queued, + detached: false, + reply_capability: false, + }; + let placeholder = serial_id("dlv", MAX_SERIAL); + let size = frame_len( + Kind::Delivery, + "rpc.request", + call.request_body(&placeholder), + attachments_with_owner(&call.attachments, &placeholder), + ); + if size > MAX_ENVELOPE_BYTES { + return err( + ErrorCode::InvalidEnvelope, + "the request delivery would exceed the envelope limit", + ); + } + // Admission: roots first, then the queue entry, then the reply. + self.add_roots(&call.artifacts); + self.next_call += 1; + let key = self.next_call; + let incarnation = call.service_incarnation.clone(); + let service_conn = call.service_conn; + self.calls.insert(key, call); + self.services + .get_mut(&target) + .expect("checked") + .queue + .push_back(key); + let conn = self.conns.get_mut(&c).expect("present"); + conn.active_calls.insert(call_serial, key); + self.wake(service_conn); + Ok(obj(vec![ + ("accepted", true.into()), + ("serviceIncarnation", incarnation.into()), + ])) + } + + fn op_reply(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "rpc.reply"); + let call_id = f.id("callId").map_err(wire)?; + let delivery_id = f.id("requestDeliveryId").map_err(wire)?; + let outcome = f.object("outcome").map_err(wire)?.clone(); + f.finish().map_err(wire)?; + let Some(n) = parse_serial_id("dlv", &delivery_id) else { + return err( + ErrorCode::InvalidEnvelope, + "requestDeliveryId must be dlv-", + ); + }; + let conn = &self.conns[&c]; + if n > conn.delivery_issued { + return err( + ErrorCode::OwnerInvalid, + "requestDeliveryId was never issued to this connection", + ); + } + let Some(&key) = conn.requests.get(&n) else { + return err( + ErrorCode::CallGone, + "no call awaits a reply for that request", + ); + }; + let call = &self.calls[&key]; + if call.call_id != call_id { + return err( + ErrorCode::InvalidEnvelope, + "callId does not match the request", + ); + } + if matches!(call.phase, Phase::Replied { .. }) { + return err(ErrorCode::CallGone, "the call was already replied to"); + } + let artifacts = self.check_attachments(c, &env.attachments)?; + let result = ResultMsg { + responder: self.identity(c), + outcome, + attachments: env + .attachments + .iter() + .map(|a| (a.name.clone(), a.reference.clone())) + .collect(), + artifacts, + }; + let placeholder = serial_id("dlv", MAX_SERIAL); + let size = frame_len( + Kind::Delivery, + "rpc.result", + call.result_body(&placeholder, &result), + attachments_with_owner(&result.attachments, &placeholder), + ); + if size > MAX_ENVELOPE_BYTES { + return err( + ErrorCode::InvalidEnvelope, + "the result delivery would exceed the envelope limit", + ); + } + if call.detached { + self.remove_call(key); + return Ok(obj(vec![("routed", false.into())])); + } + let caller = call.caller; + self.add_roots(&result.artifacts); + let released_capability = self.calls[&key].reply_capability; + if released_capability { + self.conns + .get_mut(&c) + .expect("service is connected") + .reply_capabilities -= 1; + } + let call = self.calls.get_mut(&key).expect("present"); + call.reply_capability = false; + call.phase = Phase::Replied { + delivery: n, + result: CallResult::Queued(result), + }; + self.conns + .get_mut(&caller) + .expect("attached caller is connected") + .result_queue + .push_back(key); + self.wake(caller); + Ok(obj(vec![("routed", true.into())])) + } + + fn op_responder_release( + &mut self, + c: ConnKey, + env: &Envelope, + ) -> Result, BusError> { + let mut f = Fields::of(&env.body, "rpc.responder.release"); + let call_id = f.id("callId").map_err(wire)?; + let delivery_id = f.id("requestDeliveryId").map_err(wire)?; + f.finish().map_err(wire)?; + let Some(delivery) = parse_serial_id("dlv", &delivery_id) else { + return err( + ErrorCode::InvalidEnvelope, + "requestDeliveryId must be dlv-", + ); + }; + if delivery > self.conns[&c].delivery_issued { + return err( + ErrorCode::OwnerInvalid, + "requestDeliveryId was never issued to this connection", + ); + } + let Some(&key) = self.conns[&c].requests.get(&delivery) else { + return Ok(obj(vec![("released", false.into())])); + }; + if self.calls[&key].call_id != call_id { + return err( + ErrorCode::InvalidEnvelope, + "callId does not match the request", + ); + } + if !self.calls[&key].reply_capability { + return Ok(obj(vec![("released", false.into())])); + } + let detached = self.calls[&key].detached; + let service_live = { + let call = &self.calls[&key]; + self.services.get(&call.target).is_some_and(|service| { + service.conn == call.service_conn && service.incarnation == call.service_incarnation + }) + }; + self.calls + .get_mut(&key) + .expect("request maps to a call") + .reply_capability = false; + self.conns + .get_mut(&c) + .expect("service is connected") + .reply_capabilities -= 1; + if detached { + self.remove_call(key); + } else { + let call = &self.calls[&key]; + let (code, message) = if service_live { + ( + ErrorCode::CallGone, + "the last responder was released without replying", + ) + } else { + ( + ErrorCode::NoService, + "the service route ended after dispatch without a reply", + ) + }; + self.notice( + call.caller, + "call.failed", + obj(vec![ + ("callId", call.call_id.clone().into()), + ("code", code.as_str().into()), + ("message", message.into()), + ("dispatch", Dispatch::Dispatched.as_str().into()), + ]), + ); + self.remove_call(key); + } + Ok(obj(vec![("released", true.into())])) + } + + fn op_cancel(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "rpc.cancel"); + let call_id = f.id("callId").map_err(wire)?; + f.finish().map_err(wire)?; + let Some(serial) = parse_serial_id("call", &call_id) else { + return err(ErrorCode::InvalidEnvelope, "callId must be call-"); + }; + let state = match self.conns[&c].active_calls.get(&serial).copied() { + None => "call-gone", + Some(key) => match self.calls[&key].phase { + Phase::Queued => { + let call = &self.calls[&key]; + let (target, inc) = (call.target.clone(), call.service_incarnation.clone()); + if let Some(svc) = self + .services + .get_mut(&target) + .filter(|s| s.incarnation == inc) + { + svc.queue.retain(|k| *k != key); + } + self.remove_call(key); + "cancelled-before-dispatch" + } + Phase::Dispatched { .. } => { + self.calls.get_mut(&key).expect("present").detached = true; + self.conns + .get_mut(&c) + .expect("present") + .active_calls + .remove(&serial); + if !self.calls[&key].reply_capability { + self.remove_call(key); + } + "execution-unknown" + } + Phase::Replied { .. } => "completed", + }, + }; + Ok(obj(vec![("state", state.into())])) + } + + // ---- topics + + fn op_declare(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "topic.declare"); + let name = f.name("name").map_err(wire)?; + let retain = match f.string("retained").map_err(wire)? { + "none" => false, + "latest" => true, + _ => { + return err( + ErrorCode::InvalidEnvelope, + "retained is \"none\" or \"latest\"", + ); + } + }; + f.finish().map_err(wire)?; + if !Grants::allows(&self.grants(c).manage_topics, &name) { + return err(ErrorCode::NotAuthorized, format!("may not declare {name}")); + } + if let Some(t) = self.topics.get(&name) { + if t.retain != retain { + return err( + ErrorCode::Conflict, + format!("{name} is declared with other settings"), + ); + } + return Ok(obj(vec![ + ("declared", false.into()), + ("topicIncarnation", t.incarnation.clone().into()), + ])); + } + if self.topics.len() >= self.limits.max_topics { + return err(ErrorCode::QuotaExceeded, "too many topics"); + } + self.next_topic += 1; + let incarnation = serial_id("top", self.next_topic); + self.topics.insert( + name, + Topic { + incarnation: incarnation.clone(), + retain, + sequence: 0, + retained: None, + subscribers: Vec::new(), + }, + ); + Ok(obj(vec![ + ("declared", true.into()), + ("topicIncarnation", incarnation.into()), + ])) + } + + fn op_clear(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "topic.clear"); + let name = f.name("name").map_err(wire)?; + f.finish().map_err(wire)?; + if !Grants::allows(&self.grants(c).manage_topics, &name) { + return err(ErrorCode::NotAuthorized, format!("may not clear {name}")); + } + let Some(t) = self.topics.get_mut(&name) else { + return err(ErrorCode::NoTopic, format!("{name} is not declared")); + }; + let old = t.retained.take(); + let cleared = old.is_some(); + if let Some(msg) = old { + self.retained_bytes -= msg.artifact_bytes; + self.drop_roots(&msg.artifacts); + } + Ok(obj(vec![("cleared", cleared.into())])) + } + + fn op_delete(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "topic.delete"); + let name = f.name("name").map_err(wire)?; + f.finish().map_err(wire)?; + if !Grants::allows(&self.grants(c).manage_topics, &name) { + return err(ErrorCode::NotAuthorized, format!("may not delete {name}")); + } + let Some(t) = self.topics.get(&name) else { + return Ok(obj(vec![("deleted", false.into())])); + }; + if !t.subscribers.is_empty() { + return err(ErrorCode::Conflict, format!("{name} still has subscribers")); + } + let t = self.topics.remove(&name).expect("present"); + if let Some(msg) = t.retained { + self.retained_bytes -= msg.artifact_bytes; + self.drop_roots(&msg.artifacts); + } + Ok(obj(vec![("deleted", true.into())])) + } + + fn op_subscribe(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "subscribe"); + let topic = f.name("topic").map_err(wire)?; + let latest = match f.string("mode").map_err(wire)? { + "latest" => true, + "bounded" => false, + _ => { + return err( + ErrorCode::InvalidEnvelope, + "mode is \"latest\" or \"bounded\"", + ); + } + }; + let max_queued = f.int("maxQueued", 1, MAX_CREDIT).map_err(wire)?; + let max_in_flight = f.int("maxInFlight", 1, MAX_CREDIT).map_err(wire)?; + let replay = f.boolean("replayLatest").map_err(wire)?; + f.finish().map_err(wire)?; + if !Grants::allows(&self.grants(c).subscribe, &topic) { + return err( + ErrorCode::NotAuthorized, + format!("may not subscribe to {topic}"), + ); + } + if latest && max_queued != 1 { + return err( + ErrorCode::InvalidEnvelope, + "latest subscriptions have maxQueued 1", + ); + } + let (queue_cap, credit_cap) = if latest { + (1, self.limits.max_latest_in_flight) + } else { + ( + self.limits.max_bounded_queued, + self.limits.max_bounded_in_flight, + ) + }; + if max_queued > queue_cap || max_in_flight > credit_cap { + return err( + ErrorCode::QuotaExceeded, + "maxQueued or maxInFlight exceeds the configured limit", + ); + } + let Some(t) = self.topics.get(&topic) else { + return err(ErrorCode::NoTopic, format!("{topic} is not declared")); + }; + let conn = &self.conns[&c]; + if conn.subs.len() >= self.limits.max_subscriptions_per_client + || self.subscriptions >= self.limits.max_subscriptions + { + return err(ErrorCode::QuotaExceeded, "too many subscriptions"); + } + let incarnation = t.incarnation.clone(); + let replayed = if replay { t.retained.clone() } else { None }; + if !latest + && replayed.as_ref().is_some_and(|msg| { + conn.queued_bytes.saturating_add(msg.bytes) + > self.limits.max_queued_bytes_per_client + }) + { + return err( + ErrorCode::Backpressure, + "retained replay exceeds the bounded subscriber's queued-byte limit", + ); + } + let mut sub = Sub { + topic: topic.clone(), + topic_incarnation: incarnation.clone(), + latest, + max_queued, + max_in_flight, + in_flight: 0, + queue: VecDeque::new(), + replaced: 0, + }; + let mut bytes = 0; + if let Some(msg) = replayed { + self.add_roots(&msg.artifacts); + if !latest { + bytes = msg.bytes; + } + sub.queue.push_back(msg); + } + let conn = self.conns.get_mut(&c).expect("present"); + conn.sub_issued += 1; + let id = conn.sub_issued; + conn.queued_bytes += bytes; + conn.subs.insert(id, sub); + conn.sub_order.push(id); + self.subscriptions += 1; + self.topics + .get_mut(&topic) + .expect("present") + .subscribers + .push((c, id)); + self.wake(c); + Ok(obj(vec![ + ("subscriptionId", serial_id("sub", id).into()), + ("topicIncarnation", incarnation.into()), + ])) + } + + /// Removes a subscription and releases its queue. Deliveries already handed out keep their + /// roots until consumed. + fn drop_subscription(&mut self, c: ConnKey, id: u64) -> bool { + let Some(conn) = self.conns.get_mut(&c) else { + return false; + }; + let Some(sub) = conn.subs.remove(&id) else { + return false; + }; + conn.sub_order.retain(|s| *s != id); + self.subscriptions -= 1; + if !sub.latest { + conn.queued_bytes -= sub.queue.iter().map(|m| m.bytes).sum::(); + } + if let Some(t) = self + .topics + .get_mut(&sub.topic) + .filter(|t| t.incarnation == sub.topic_incarnation) + { + t.subscribers.retain(|s| *s != (c, id)); + } + for msg in sub.queue { + self.drop_roots(&msg.artifacts); + } + true + } + + fn op_unsubscribe( + &mut self, + c: ConnKey, + env: &Envelope, + ) -> Result, BusError> { + let mut f = Fields::of(&env.body, "unsubscribe"); + let sid = f.id("subscriptionId").map_err(wire)?; + f.finish().map_err(wire)?; + let removed = parse_serial_id("sub", &sid).is_some_and(|id| self.drop_subscription(c, id)); + Ok(obj(vec![("removed", removed.into())])) + } + + fn op_publish(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "publish"); + let topic = f.name("topic").map_err(wire)?; + let payload = f.object("payload").map_err(wire)?.clone(); + f.finish().map_err(wire)?; + if !Grants::allows(&self.grants(c).publish, &topic) { + return err( + ErrorCode::NotAuthorized, + format!("may not publish to {topic}"), + ); + } + let Some(t) = self.topics.get(&topic) else { + return err(ErrorCode::NoTopic, format!("{topic} is not declared")); + }; + let artifacts = self.check_attachments(c, &env.attachments)?; + let artifact_bytes: u64 = artifacts + .iter() + .map(|s| self.artifacts[s].byte_length) + .sum(); + let mut msg = TopicMsg { + topic: topic.clone(), + topic_incarnation: t.incarnation.clone(), + sequence: t.sequence + 1, + payload, + attachments: env + .attachments + .iter() + .map(|a| (a.name.clone(), a.reference.clone())) + .collect(), + artifacts, + bytes: 0, + artifact_bytes, + }; + let placeholder = serial_id("dlv", MAX_SERIAL); + msg.bytes = frame_len( + Kind::Delivery, + "topic.message", + msg.body(&placeholder, &serial_id("sub", MAX_SERIAL), MAX_SERIAL), + attachments_with_owner(&msg.attachments, &placeholder), + ); + if msg.bytes > MAX_ENVELOPE_BYTES { + return err( + ErrorCode::InvalidEnvelope, + "the delivery would exceed the envelope limit", + ); + } + // Admission is all or nothing: check every subscriber and the retention quota before + // touching anything. + let mut extra: HashMap = HashMap::new(); + for (sc, sid) in &t.subscribers { + let conn = &self.conns[sc]; + let sub = &conn.subs[sid]; + if sub.latest { + continue; + } + if sub.queue.len() as u64 >= sub.max_queued { + return err( + ErrorCode::Backpressure, + "a bounded subscriber's queue is full", + ); + } + let add = extra.entry(*sc).or_default(); + *add += msg.bytes; + if conn.queued_bytes + *add > self.limits.max_queued_bytes_per_client { + return err( + ErrorCode::Backpressure, + "a bounded subscriber's queued bytes are at the limit", + ); + } + } + if t.retain { + let old = t.retained.as_ref().map_or(0, |m| m.artifact_bytes); + if self.retained_bytes - old + msg.artifact_bytes > self.limits.max_retained_bytes { + return err(ErrorCode::QuotaExceeded, "retained bytes are at the limit"); + } + } + // Accept. + let msg = Arc::new(msg); + let t = self.topics.get_mut(&topic).expect("present"); + t.sequence += 1; + let subscribers = t.subscribers.clone(); + let retain = t.retain; + let mut replaced = 0u64; + for (sc, sid) in &subscribers { + let conn = self.conns.get_mut(sc).expect("subscriber is connected"); + let sub = conn.subs.get_mut(sid).expect("subscription is live"); + let dropped = if sub.latest { + let old = sub.queue.pop_front(); + if old.is_some() { + sub.replaced += 1; + replaced += 1; + } + old + } else { + conn.queued_bytes += msg.bytes; + None + }; + sub.queue.push_back(msg.clone()); + self.add_roots(&msg.artifacts); + if let Some(old) = dropped { + self.drop_roots(&old.artifacts); + } + self.wake(*sc); + } + if retain { + self.add_roots(&msg.artifacts); + self.retained_bytes += msg.artifact_bytes; + let old = self + .topics + .get_mut(&topic) + .expect("present") + .retained + .replace(msg.clone()); + if let Some(old) = old { + self.retained_bytes -= old.artifact_bytes; + self.drop_roots(&old.artifacts); + } + } + Ok(obj(vec![ + ("topicSequence", msg.sequence.to_string().into()), + ("subscribers", subscribers.len().to_string().into()), + ("replaced", replaced.to_string().into()), + ])) + } + + // ---- ownership + + fn op_consumed(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "delivery.consumed"); + let ids = id_batch( + f.array("deliveryIds", 1, MAX_BATCH).map_err(wire)?, + "deliveryIds", + ) + .map_err(wire)?; + f.finish().map_err(wire)?; + let issued = self.conns[&c].delivery_issued; + let mut serials = Vec::with_capacity(ids.len()); + for id in &ids { + match parse_serial_id("dlv", id) { + Some(n) if n <= issued => serials.push(n), + Some(_) => return err(ErrorCode::OwnerInvalid, format!("{id} was never issued")), + None => { + return err( + ErrorCode::OwnerInvalid, + format!("{id} is not a delivery id"), + ); + } + } + } + let mut released = 0u64; + for n in serials { + if let Some(owner) = self.take_owner(c, OwnerKey::Delivery(n)) { + released += 1; + self.release_owner(c, owner); + } + } + Ok(obj(vec![("released", released.to_string().into())])) + } + + fn op_release(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "artifact.release"); + let ids = + id_batch(f.array("ownerIds", 1, MAX_BATCH).map_err(wire)?, "ownerIds").map_err(wire)?; + f.finish().map_err(wire)?; + let issued = self.conns[&c].hold_issued; + let mut serials = Vec::with_capacity(ids.len()); + for id in &ids { + match parse_serial_id("own", id) { + Some(n) if n <= issued => serials.push(n), + Some(_) => return err(ErrorCode::OwnerInvalid, format!("{id} was never issued")), + None => { + return err( + ErrorCode::OwnerInvalid, + format!("{id} is not a hold; deliveries end with delivery.consumed"), + ); + } + } + } + let mut released = 0u64; + for n in serials { + if let Some(owner) = self.take_owner(c, OwnerKey::Hold(n)) { + released += 1; + self.release_owner(c, owner); + } + } + Ok(obj(vec![("released", released.to_string().into())])) + } + + fn op_open(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "artifact.open"); + let r = ArtifactRef::from_json(f.value("ref").map_err(wire)?).map_err(wire)?; + let owner = f.id("ownerId").map_err(wire)?; + f.finish().map_err(wire)?; + let serial = self.check_owned(c, &r, &owner)?; + let loc = crate::wire::Location { + store_id: self.store_id.clone(), + relative_path: sealed_rel(serial), + }; + Ok(obj(vec![("readLocation", loc.to_json())])) + } + + fn op_retain(&mut self, c: ConnKey, env: &Envelope) -> Result, BusError> { + let mut f = Fields::of(&env.body, "artifact.retain"); + let r = ArtifactRef::from_json(f.value("ref").map_err(wire)?).map_err(wire)?; + let owner = f.id("ownerId").map_err(wire)?; + f.finish().map_err(wire)?; + let serial = self.check_owned(c, &r, &owner)?; + if !self.ordinary_budget_left(c) { + return err(ErrorCode::QuotaExceeded, "owner budget exhausted"); + } + let conn = self.conns.get_mut(&c).expect("present"); + conn.hold_issued += 1; + let n = conn.hold_issued; + self.insert_owner(c, OwnerKey::Hold(n), Owner::Hold { artifact: serial }); + self.add_roots(&[serial]); + Ok(obj(vec![("ownerId", serial_id("own", n).into())])) + } + + fn op_allocate(&mut self, c: ConnKey, env: &Envelope) -> Result { + let mut f = Fields::of(&env.body, "artifact.allocate"); + let len = f.u64_string("byteLength").map_err(wire)?; + let content_type = f.string("contentType").map_err(wire)?.to_owned(); + f.finish().map_err(wire)?; + if !crate::wire::is_content_type(&content_type) { + return err( + ErrorCode::InvalidEnvelope, + "contentType must be 1..=127 printable ASCII characters", + ); + } + if len > self.limits.max_artifact_bytes { + return err( + ErrorCode::QuotaExceeded, + "byteLength exceeds the per-object limit", + ); + } + if self.store_bytes.saturating_add(len) > self.limits.max_store_bytes { + return err(ErrorCode::QuotaExceeded, "the store is full"); + } + if !self.ordinary_budget_left(c) { + return err(ErrorCode::QuotaExceeded, "owner budget exhausted"); + } + self.next_artifact += 1; + let serial = self.next_artifact; + self.artifacts.insert( + serial, + Art { + state: ArtState::Writing, + byte_length: len, + content_type, + digest: None, + roots: 0, + }, + ); + self.store_bytes += len; + let conn = self.conns.get_mut(&c).expect("present"); + conn.hold_issued += 1; + let owner = conn.hold_issued; + self.insert_owner( + c, + OwnerKey::Hold(owner), + Owner::Writer { + artifact: serial, + sealing: false, + }, + ); + Ok(AllocJob { + conn: c, + reply_to: env.id.clone(), + serial, + len, + owner, + }) + } + + pub(crate) fn finish_allocate(&mut self, job: AllocJob, created: std::io::Result<()>) { + let live = self.conns.get(&job.conn).is_some_and(|conn| { + matches!( + conn.owners.get(&OwnerKey::Hold(job.owner)), + Some(Owner::Writer { .. }) + ) + }); + match created { + Err(e) => { + if live { + self.take_owner(job.conn, OwnerKey::Hold(job.owner)); + } + if let Some(a) = self.artifacts.remove(&job.serial) { + self.store_bytes -= a.byte_length; + } + self.reply( + job.conn, + &job.reply_to, + "artifact.allocate", + err(ErrorCode::StoreFailure, format!("staging: {e}")), + ); + } + Ok(()) if !live || !self.artifacts.contains_key(&job.serial) => { + // The connection went away while the file was being made. + self.unlinks.push(staging_rel(job.serial)); + } + Ok(()) => { + let loc = crate::wire::Location { + store_id: self.store_id.clone(), + relative_path: staging_rel(job.serial), + }; + let value = obj(vec![ + ("artifactId", serial_id("a", job.serial).into()), + ("generation", GENERATION.to_string().into()), + ("ownerId", serial_id("own", job.owner).into()), + ("writeLocation", loc.to_json()), + ]); + self.reply(job.conn, &job.reply_to, "artifact.allocate", Ok(value)); + } + } + self.flush_notices(); + } + + fn op_seal(&mut self, c: ConnKey, env: &Envelope) -> Result { + let mut f = Fields::of(&env.body, "artifact.seal"); + let artifact_id = f.id("artifactId").map_err(wire)?; + let generation = f.u64_string("generation").map_err(wire)?; + let owner_id = f.id("ownerId").map_err(wire)?; + let digest = match f.value("digest").map_err(wire)? { + Value::Null => None, + Value::String(s) if crate::wire::is_digest(s) => Some(s.clone()), + _ => { + return err( + ErrorCode::InvalidEnvelope, + "digest must be null or 64 lowercase hex digits", + ); + } + }; + f.finish().map_err(wire)?; + let Some(serial) = parse_serial_id("a", &artifact_id) else { + return err(ErrorCode::ArtifactGone, "unknown artifact id"); + }; + let Some(owner) = parse_serial_id("own", &owner_id) else { + return err(ErrorCode::OwnerInvalid, "ownerId is not a writer"); + }; + match self.conns[&c].owners.get(&OwnerKey::Hold(owner)) { + Some(Owner::Writer { + artifact, + sealing: false, + }) if *artifact == serial => {} + Some(Owner::Writer { + artifact, + sealing: true, + }) if *artifact == serial => { + return err(ErrorCode::OwnerInvalid, "a seal is already in progress"); + } + Some(Owner::Hold { artifact }) if *artifact == serial => { + return err(ErrorCode::OwnerInvalid, "the artifact is already sealed"); + } + _ => { + return err( + ErrorCode::OwnerInvalid, + format!("{owner_id} is not the writer of {artifact_id}"), + ); + } + } + if generation != GENERATION { + return err(ErrorCode::ArtifactGone, "unknown generation"); + } + let len = self.artifacts[&serial].byte_length; + if self.store_bytes.saturating_add(len) > self.limits.max_store_bytes { + self.take_owner(c, OwnerKey::Hold(owner)); + self.abandon_writer(serial); + return err( + ErrorCode::QuotaExceeded, + "no room for the sealing copy; the staging object was released", + ); + } + self.store_bytes += len; + self.artifacts.get_mut(&serial).expect("present").state = ArtState::Sealing; + let conn = self.conns.get_mut(&c).expect("present"); + conn.owners.insert( + OwnerKey::Hold(owner), + Owner::Writer { + artifact: serial, + sealing: true, + }, + ); + Ok(SealJob { + conn: c, + reply_to: env.id.clone(), + serial, + len, + digest, + owner, + }) + } + + pub(crate) fn finish_seal( + &mut self, + job: SealJob, + result: Result<(), crate::store::SealFailure>, + ) { + let owner_live = self.conns.get(&job.conn).is_some_and(|conn| { + matches!(conn.owners.get(&OwnerKey::Hold(job.owner)), Some(Owner::Writer { artifact, sealing: true }) if *artifact == job.serial) + }); + let reply = match (result, owner_live) { + (Ok(()), true) => { + let art = self + .artifacts + .get_mut(&job.serial) + .expect("sealing artifact is registered"); + art.state = ArtState::Sealed; + art.digest = job.digest.clone(); + art.roots = 1; + // The seal task already unlinked staging; its reservation ends here. + self.store_bytes -= job.len; + let conn = self.conns.get_mut(&job.conn).expect("present"); + conn.owners.insert( + OwnerKey::Hold(job.owner), + Owner::Hold { + artifact: job.serial, + }, + ); + let r = self.reference(job.serial).expect("present"); + Ok(obj(vec![ + ("ref", r.to_json()), + ("ownerId", serial_id("own", job.owner).into()), + ])) + } + (result, _) => { + if self.artifacts.remove(&job.serial).is_some() { + self.store_bytes -= 2 * job.len; + } + self.unlinks.push(sealed_rel(job.serial)); + if owner_live { + self.take_owner(job.conn, OwnerKey::Hold(job.owner)); + } + match result { + Ok(()) => err( + ErrorCode::ArtifactGone, + "the writer was released during the seal", + ), + Err(crate::store::SealFailure::Mismatch(m)) => { + err(ErrorCode::ArtifactMismatch, m) + } + Err(crate::store::SealFailure::Io(e)) => { + err(ErrorCode::StoreFailure, format!("seal: {e}")) + } + } + } + }; + self.reply(job.conn, &job.reply_to, "artifact.seal", reply); + self.flush_notices(); + } + + // ----------------------------------------------------------------------------------------- + // Writers pull frames + + fn topic_ready(&self, c: ConnKey) -> bool { + if !self.ordinary_budget_left(c) { + return false; + } + let conn = &self.conns[&c]; + conn.subs + .values() + .any(|s| s.in_flight < s.max_in_flight && !s.queue.is_empty()) + } + + /// The next frame for `c`: control first, then RPC, then topic data, with topic data + /// guaranteed a turn after [`TOPIC_STARVATION_LIMIT`] higher-priority frames. + pub(crate) fn next_frame(&mut self, c: ConnKey) -> NextFrame { + if !self.conns.contains_key(&c) { + return NextFrame::Gone; + } + let topic_ready = self.topic_ready(c); + let starving = topic_ready && self.conns[&c].streak >= TOPIC_STARVATION_LIMIT; + if !starving && let Some(f) = self.pop_control(c).or_else(|| self.dispatch_rpc(c)) { + if topic_ready { + self.conns.get_mut(&c).expect("present").streak += 1; + } + return self.selected_frame(c, f); + } + if let Some(f) = self.dispatch_topic(c) { + self.conns.get_mut(&c).expect("present").streak = 0; + return self.selected_frame(c, f); + } + match self.pop_control(c).or_else(|| self.dispatch_rpc(c)) { + Some(f) => self.selected_frame(c, f), + None => NextFrame::Idle, + } + } + + fn selected_frame(&mut self, c: ConnKey, frame: Outbound) -> NextFrame { + match self.encode_outbound(c, frame) { + Some(bytes) => NextFrame::Frame(bytes), + None => NextFrame::Gone, + } + } + + fn pop_control(&mut self, c: ConnKey) -> Option { + let conn = self.conns.get_mut(&c)?; + let f = conn.control.pop_front()?; + conn.control_bytes -= f.bytes; + Some(f) + } + + fn dispatch_rpc(&mut self, c: ConnKey) -> Option { + if self.conns[&c].owners.len() + self.conns[&c].reply_capabilities + >= self.limits.max_owners_per_client + { + return None; + } + // Results for calls this connection made. + if let Some(key) = self.conns.get_mut(&c)?.result_queue.pop_front() { + let call = self.calls.get_mut(&key)?; + let Phase::Replied { delivery, result } = + std::mem::replace(&mut call.phase, Phase::Queued) + else { + unreachable!("queued results belong to replied calls"); + }; + let CallResult::Queued(msg) = result else { + unreachable!("result queued once") + }; + call.phase = Phase::Replied { + delivery, + result: CallResult::Delivered, + }; + let conn = self.conns.get_mut(&c).expect("present"); + conn.delivery_issued += 1; + let n = conn.delivery_issued; + let did = serial_id("dlv", n); + let call = &self.calls[&key]; + let body = call.result_body(&did, &msg); + let atts = attachments_with_owner(&msg.attachments, &did); + self.insert_owner( + c, + OwnerKey::Delivery(n), + Owner::Delivery { + artifacts: msg.artifacts, + kind: DeliveryKind::Result { call: key }, + }, + ); + return self.outbound(Kind::Delivery, "rpc.result", None, body, atts); + } + // Requests for services this connection registered, round robin. + let conn = &self.conns[&c]; + let n_svcs = conn.services.len(); + for i in 0..n_svcs { + let idx = (conn.svc_cursor + i) % n_svcs; + let name = conn.services[idx].clone(); + let svc = self.services.get_mut(&name)?; + if svc.in_flight >= svc.max_in_flight || svc.queue.is_empty() { + continue; + } + if self.conns[&c].owners.len() + self.conns[&c].reply_capabilities + 2 + > self.limits.max_owners_per_client + { + continue; + } + let key = svc.queue.pop_front().expect("nonempty"); + svc.in_flight += 1; + let incarnation = svc.incarnation.clone(); + let conn = self.conns.get_mut(&c).expect("present"); + conn.svc_cursor = idx + 1; + conn.delivery_issued += 1; + let n = conn.delivery_issued; + conn.requests.insert(n, key); + conn.reply_capabilities += 1; + let call = self.calls.get_mut(&key).expect("queued call is registered"); + // Dispatched before a byte of it is written. + call.phase = Phase::Dispatched { delivery: n }; + call.reply_capability = true; + let did = serial_id("dlv", n); + let body = call.request_body(&did); + let atts = attachments_with_owner(&call.attachments, &did); + let artifacts = call.artifacts.clone(); + self.insert_owner( + c, + OwnerKey::Delivery(n), + Owner::Delivery { + artifacts, + kind: DeliveryKind::Request { + call: key, + service: name, + incarnation, + }, + }, + ); + return self.outbound(Kind::Delivery, "rpc.request", None, body, atts); + } + None + } + + fn dispatch_topic(&mut self, c: ConnKey) -> Option { + if !self.ordinary_budget_left(c) { + return None; + } + let conn = self.conns.get_mut(&c)?; + let n_subs = conn.sub_order.len(); + for i in 0..n_subs { + let idx = (conn.sub_cursor + i) % n_subs; + let id = conn.sub_order[idx]; + let sub = conn.subs.get_mut(&id).expect("ordered subscription exists"); + if sub.in_flight >= sub.max_in_flight || sub.queue.is_empty() { + continue; + } + let msg = sub.queue.pop_front().expect("nonempty"); + sub.in_flight += 1; + let replaced = std::mem::take(&mut sub.replaced); + if !sub.latest { + conn.queued_bytes -= msg.bytes; + } + conn.sub_cursor = idx + 1; + conn.delivery_issued += 1; + let n = conn.delivery_issued; + let did = serial_id("dlv", n); + let body = msg.body(&did, &serial_id("sub", id), replaced); + let atts = attachments_with_owner(&msg.attachments, &did); + self.insert_owner( + c, + OwnerKey::Delivery(n), + Owner::Delivery { + artifacts: msg.artifacts.clone(), + kind: DeliveryKind::Topic { sub: id }, + }, + ); + return self.outbound(Kind::Delivery, "topic.message", None, body, atts); + } + None + } +} + +fn reply_body(result: Result, BusError>) -> Map { + match result { + Ok(value) => obj(vec![("ok", true.into()), ("value", Value::Object(value))]), + Err(e) => obj(vec![ + ("ok", false.into()), + ( + "error", + Value::Object(obj(vec![ + ("code", e.code.as_str().into()), + ("message", e.message.into()), + ("dispatch", e.dispatch.as_str().into()), + ])), + ), + ]), + } +} diff --git a/services/flysim/crates/flybus/src/store.rs b/services/flysim/crates/flybus/src/store.rs new file mode 100644 index 0000000..4964ac6 --- /dev/null +++ b/services/flysim/crates/flybus/src/store.rs @@ -0,0 +1,366 @@ +//! The file-backed artifact store (bus-v1 section 8). +//! +//! Layout under the configured root, one directory per store incarnation: +//! +//! ```text +//! //.flybus-store marker: this directory belongs to a flybus store +//! //.lock flock()ed by the live router for its whole life +//! //staging/a- producer-writable staging file, preallocated +//! //sealed/a- immutable copy, mode 0444, never rewritten +//! ``` +//! +//! Sealing copies staging into a fresh inode rather than renaming it, so a writable handle a +//! producer kept (or duplicated) after sealing reaches only the unlinked staging inode, never +//! the sealed bytes. A store directory whose lock nobody holds is an orphan of a stopped +//! router and is deleted when the next router starts on the same root. + +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; + +use sha2::{Digest as _, Sha256}; + +use crate::error::{BusError, ErrorCode}; +use crate::wire::{Location, hex}; + +const MARKER: &str = ".flybus-store"; +const LOCK: &str = ".lock"; + +pub(crate) fn staging_rel(serial: u64) -> String { + format!("staging/a-{serial}") +} + +pub(crate) fn sealed_rel(serial: u64) -> String { + format!("sealed/a-{serial}") +} + +pub(crate) enum SealFailure { + Mismatch(String), + Io(io::Error), +} + +impl From for SealFailure { + fn from(e: io::Error) -> SealFailure { + SealFailure::Io(e) + } +} + +pub(crate) struct Store { + dir: PathBuf, + // Held open for the flock; closing it releases the lock. + _lock: File, +} + +fn try_lock(file: &File) -> bool { + // SAFETY: flock on a valid, owned descriptor. + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) == 0 } +} + +impl Store { + /// Creates `/`, first deleting orphaned store directories under `root`. + pub(crate) fn create(root: &Path, store_id: &str) -> io::Result { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(root)?; + clean_orphans(root)?; + let dir = root.join(store_id); + let mut builder = fs::DirBuilder::new(); + builder.mode(0o700); + builder.create(&dir)?; + builder.create(dir.join("staging"))?; + builder.create(dir.join("sealed"))?; + File::create(dir.join(MARKER))?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .open(dir.join(LOCK))?; + if !try_lock(&lock) { + return Err(io::Error::other("could not lock a fresh store directory")); + } + Ok(Store { dir, _lock: lock }) + } + + pub(crate) fn dir(&self) -> &Path { + &self.dir + } + + pub(crate) fn path(&self, rel: &str) -> PathBuf { + self.dir.join(rel) + } + + /// Creates the staging file and reserves its blocks, so a full disk fails here and not in + /// the producer's write. + pub(crate) fn create_staging(&self, serial: u64, len: u64) -> io::Result<()> { + let path = self.path(&staging_rel(serial)); + let file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&path)?; + let reserve = || -> io::Result<()> { + if len == 0 { + return Ok(()); + } + let off_len = libc::off_t::try_from(len) + .map_err(|_| io::Error::other("length overflows off_t"))?; + // SAFETY: posix_fallocate on a valid descriptor opened for writing. + let rc = unsafe { libc::posix_fallocate(file.as_raw_fd(), 0, off_len) }; + match rc { + 0 => Ok(()), + libc::EOPNOTSUPP | libc::EINVAL => file.set_len(len), + e => Err(io::Error::from_raw_os_error(e)), + } + }; + let result = reserve(); + if result.is_err() { + let _ = fs::remove_file(&path); + } + result + } + + /// Copies exactly `len` staging bytes into a fresh sealed file, checking the length and, + /// when given, the SHA-256. On failure the partial sealed file is removed; the staging file + /// is left for the caller either way. + pub(crate) fn seal( + &self, + serial: u64, + len: u64, + digest: Option<&str>, + ) -> Result<(), SealFailure> { + let mut src = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(self.path(&staging_rel(serial)))?; + let dst_path = self.path(&sealed_rel(serial)); + let mut dst = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&dst_path)?; + let result = copy_exact(&mut src, &mut dst, len, digest).and_then(|()| { + dst.set_permissions(fs::Permissions::from_mode(0o444))?; + Ok(()) + }); + if result.is_err() { + let _ = fs::remove_file(&dst_path); + } + result + } + + /// Unlinks a store file; a missing file is not an error. + pub(crate) fn remove(&self, rel: &str) { + let _ = fs::remove_file(self.path(rel)); + } +} + +impl Drop for Store { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +fn read_retry(r: &mut File, buf: &mut [u8]) -> io::Result { + loop { + match r.read(buf) { + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + other => return other, + } + } +} + +fn copy_exact( + src: &mut File, + dst: &mut File, + len: u64, + digest: Option<&str>, +) -> Result<(), SealFailure> { + let mut hasher = digest.map(|_| Sha256::new()); + let mut buf = vec![0u8; 256 * 1024]; + let mut remaining = len; + while remaining > 0 { + let want = remaining.min(buf.len() as u64) as usize; + let n = read_retry(src, &mut buf[..want])?; + if n == 0 { + return Err(SealFailure::Mismatch(format!( + "staging holds {} of {len} declared bytes", + len - remaining + ))); + } + if let Some(h) = hasher.as_mut() { + h.update(&buf[..n]); + } + dst.write_all(&buf[..n])?; + remaining -= n as u64; + } + if read_retry(src, &mut buf[..1])? != 0 { + return Err(SealFailure::Mismatch(format!( + "staging holds more than the declared {len} bytes" + ))); + } + if let (Some(h), Some(want)) = (hasher, digest) { + let got = hex(&h.finalize()); + if got != want { + return Err(SealFailure::Mismatch(format!( + "digest mismatch: content hashes to {got}" + ))); + } + } + Ok(()) +} + +/// Deletes store directories under `root` that carry the marker and whose lock is free. +/// Directories without the marker are never touched. +fn clean_orphans(root: &Path) -> io::Result<()> { + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if !entry.file_type()?.is_dir() || !path.join(MARKER).is_file() { + continue; + } + let stale = match OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path.join(LOCK)) + { + Ok(lock) => try_lock(&lock), + Err(e) if e.kind() == io::ErrorKind::NotFound => true, + Err(e) => return Err(e), + }; + if stale { + fs::remove_dir_all(&path)?; + } + } + Ok(()) +} + +// --------------------------------------------------------------------------------------------- +// Client side + +/// Resolves a location grant beneath `/`. Absolute paths, `..`, anything but +/// plain `[a-z0-9._-]` components, and symlinks that lead outside the store are refused. +pub(crate) fn resolve(root: &Path, store_id: &str, loc: &Location) -> Result { + if loc.store_id != store_id { + return Err(BusError::new( + ErrorCode::ArtifactGone, + "location names another store incarnation", + )); + } + let refuse = + |why: &str| BusError::new(ErrorCode::StoreFailure, format!("location refused: {why}")); + let rel = Path::new(&loc.relative_path); + if loc.relative_path.is_empty() || rel.is_absolute() { + return Err(refuse("not a relative path")); + } + for c in rel.components() { + match c { + Component::Normal(s) => { + let s = s.to_str().unwrap_or(""); + if s.is_empty() + || !s.bytes().all(|b| { + b.is_ascii_lowercase() + || b.is_ascii_digit() + || matches!(b, b'.' | b'_' | b'-') + }) + { + return Err(refuse("unexpected path component")); + } + } + _ => return Err(refuse("parent, root or current-directory component")), + } + } + let base = root.join(&loc.store_id); + let base = base + .canonicalize() + .map_err(|e| refuse(&format!("store directory: {e}")))?; + let full = base.join(rel).canonicalize().map_err(|e| match e.kind() { + io::ErrorKind::NotFound => BusError::new(ErrorCode::ArtifactGone, "artifact file is gone"), + _ => refuse(&e.to_string()), + })?; + if !full.starts_with(&base) { + return Err(refuse("escapes the store")); + } + Ok(full) +} + +pub(crate) fn open_read(path: &Path) -> io::Result { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) +} + +pub(crate) fn open_write(path: &Path) -> io::Result { + OpenOptions::new() + .write(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn loc(p: &str) -> Location { + Location { + store_id: "store-x".into(), + relative_path: p.into(), + } + } + + #[test] + fn resolve_refuses_escapes() { + let root = tempfile::tempdir().unwrap(); + let store = root.path().join("store-x"); + fs::create_dir_all(store.join("sealed")).unwrap(); + fs::write(store.join("sealed/a-1"), b"ok").unwrap(); + fs::write(root.path().join("secret"), b"no").unwrap(); + std::os::unix::fs::symlink(root.path().join("secret"), store.join("sealed/a-2")).unwrap(); + assert!(resolve(root.path(), "store-x", &loc("sealed/a-1")).is_ok()); + for bad in [ + "", + "/etc/passwd", + "../secret", + "sealed/../../secret", + "./sealed/a-1", + "sealed/A-1", + "sealed/a-2", + ] { + assert!( + resolve(root.path(), "store-x", &loc(bad)).is_err(), + "{bad:?}" + ); + } + let other = Location { + store_id: "store-y".into(), + relative_path: "sealed/a-1".into(), + }; + assert_eq!( + resolve(root.path(), "store-x", &other).unwrap_err().code, + ErrorCode::ArtifactGone + ); + } + + #[test] + fn orphans_are_removed_and_live_stores_kept() { + let root = tempfile::tempdir().unwrap(); + let live = Store::create(root.path(), "store-live").unwrap(); + let orphan = root.path().join("store-dead"); + fs::create_dir_all(orphan.join("sealed")).unwrap(); + fs::write(orphan.join(MARKER), b"").unwrap(); + fs::write(orphan.join(LOCK), b"").unwrap(); + let unrelated = root.path().join("keep-me"); + fs::create_dir_all(&unrelated).unwrap(); + let second = Store::create(root.path(), "store-next").unwrap(); + assert!(!orphan.exists()); + assert!(live.dir().exists() && second.dir().exists() && unrelated.exists()); + drop(live); + assert!(!root.path().join("store-live").exists()); + } +} diff --git a/services/flysim/crates/flybus/src/transport.rs b/services/flysim/crates/flybus/src/transport.rs new file mode 100644 index 0000000..efb2d03 --- /dev/null +++ b/services/flysim/crates/flybus/src/transport.rs @@ -0,0 +1,67 @@ +//! Byte-stream transports. Both carry the same framed protocol to the same router code: the +//! in-memory transport is a Tokio duplex pipe, the other a Unix-domain stream socket. + +use std::io; +use std::path::Path; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::UnixStream; + +/// Anything a connection can run over. +pub trait Stream: AsyncRead + AsyncWrite + Send + Unpin + 'static {} + +impl Stream for T {} + +/// One end of a connection to a router. +pub struct Transport { + inner: Box, +} + +impl Transport { + pub fn from_stream(stream: S) -> Transport { + Transport { + inner: Box::new(stream), + } + } + + /// Connects to a router's Unix-domain socket. + pub async fn unix(path: impl AsRef) -> io::Result { + Ok(Transport::from_stream(UnixStream::connect(path).await?)) + } + + /// An in-memory pipe pair: one end for a router, the other for a client. + pub(crate) fn pair() -> (Transport, Transport) { + let (a, b) = tokio::io::duplex(64 * 1024); + (Transport::from_stream(a), Transport::from_stream(b)) + } +} + +impl AsyncRead for Transport { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut *self.inner).poll_read(cx, buf) + } +} + +impl AsyncWrite for Transport { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut *self.inner).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.inner).poll_shutdown(cx) + } +} diff --git a/services/flysim/crates/flybus/src/wire.rs b/services/flysim/crates/flybus/src/wire.rs new file mode 100644 index 0000000..3e2e1bb --- /dev/null +++ b/services/flysim/crates/flybus/src/wire.rs @@ -0,0 +1,816 @@ +//! The wire: scalar encodings, strict JSON, the envelope and its framing (bus-v1 section 4). +//! +//! A frame is a `u32` little-endian byte count followed by that many bytes of UTF-8 JSON. The +//! JSON is parsed strictly: duplicate keys at any depth, invalid UTF-8, non-finite numbers, +//! trailing bytes and unknown envelope fields are all refused. Both ends use this module, so +//! the router and the client agree byte for byte on what is valid. + +use std::collections::HashSet; +use std::fmt; + +use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use serde_json::{Map, Value}; +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +use crate::error::BusError; + +pub const PROTOCOL: &str = "flybus"; +pub const MAJOR: u64 = 1; +pub const MINOR: u64 = 0; +/// Largest JSON envelope, in bytes, not counting the four-byte length prefix. +pub const MAX_ENVELOPE_BYTES: usize = 65_536; +pub const MAX_ATTACHMENTS: usize = 32; +/// Release and consume batches carry 1..=64 ids. +pub const MAX_BATCH: usize = 64; +/// Queue and credit requests are integers in 1..=65535. +pub const MAX_CREDIT: u64 = 65_535; +pub const MAX_NAME_LEN: usize = 192; +pub const MAX_METHOD_LEN: usize = 128; +pub const MAX_CONTENT_TYPE_LEN: usize = 127; +/// The one artifact generation v1 issues; ids and inodes are never reused. +pub const GENERATION: u64 = 1; + +/// The operations, delivery and notice shapes this implementation speaks. `contractDigest` is +/// the SHA-256 of this text, so any change to it changes the digest a client sees in hello. +pub const CONTRACT: &str = "flybus 1.0 +frame: u32le length, 1..=65536 bytes of strict UTF-8 JSON +envelope: protocol major minor id replyTo kind op body attachments +attachment: name ref ownerId +ref: storeId artifactId generation byteLength contentType digest +reply: ok value | ok error{code message dispatch} +bus.hello: clientId clientIncarnation supportedMajors -> routerId connectionId selectedMajor selectedMinor contractDigest limits +service.register: name maxQueued maxInFlight -> serviceIncarnation +service.unregister: name serviceIncarnation -> removed +rpc.call: callId target expectedIncarnation method payload -> accepted serviceIncarnation +rpc.reply: callId requestDeliveryId outcome -> routed +rpc.responder.release: callId requestDeliveryId -> released; final attached release emits call.failed dispatched (CALL_GONE, or NO_SERVICE after route loss) +rpc.cancel: callId -> state(cancelled-before-dispatch|execution-unknown|completed|call-gone) +topic.declare: name retained(none|latest) -> declared topicIncarnation +topic.clear: name -> cleared +topic.delete: name -> deleted +subscribe: topic mode(latest|bounded) maxQueued maxInFlight replayLatest -> subscriptionId topicIncarnation +unsubscribe: subscriptionId -> removed +publish: topic payload -> topicSequence subscribers replaced +delivery.consumed: deliveryIds -> released +artifact.allocate: byteLength contentType -> artifactId generation ownerId writeLocation +artifact.seal: artifactId generation ownerId digest -> ref ownerId +artifact.open: ref ownerId -> readLocation +artifact.retain: ref ownerId -> ownerId +artifact.release: ownerIds -> released +delivery rpc.request: deliveryId callId caller target serviceIncarnation method payload +delivery rpc.result: deliveryId callId responder serviceIncarnation outcome +delivery topic.message: deliveryId subscriptionId topic topicIncarnation topicSequence replaced payload +notice call.failed: callId code message dispatch +notice route.removed: name serviceIncarnation reason +notice subscription.closed: subscriptionId topic topicIncarnation reason +notice connection.closing: code message +"; + +/// SHA-256 of [`CONTRACT`], lowercase hex. +pub fn contract_digest() -> String { + hex(&Sha256::digest(CONTRACT.as_bytes())) +} + +pub(crate) fn hex(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push_str(&format!("{b:02x}")); + } + s +} + +/// A wire-level validation failure. Always maps to `INVALID_ENVELOPE`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WireError(pub String); + +impl fmt::Display for WireError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for WireError {} + +impl From for BusError { + fn from(e: WireError) -> BusError { + BusError::invalid(e.0) + } +} + +fn err(message: impl Into) -> Result { + Err(WireError(message.into())) +} + +// --------------------------------------------------------------------------------------------- +// Scalars + +/// `Id`: `^[a-z0-9][a-z0-9._-]{0,63}$`. +pub fn is_id(s: &str) -> bool { + let b = s.as_bytes(); + !b.is_empty() + && b.len() <= 64 + && (b[0].is_ascii_lowercase() || b[0].is_ascii_digit()) + && b.iter().all(|&c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, b'.' | b'_' | b'-') + }) +} + +/// `U64`: `"0"` or `[1-9][0-9]*`, at most `u64::MAX`. +pub fn parse_u64(s: &str) -> Option { + let b = s.as_bytes(); + if b.is_empty() || b.len() > 20 || (b.len() > 1 && b[0] == b'0') { + return None; + } + let mut n: u64 = 0; + for &c in b { + if !c.is_ascii_digit() { + return None; + } + n = n.checked_mul(10)?.checked_add(u64::from(c - b'0'))?; + } + Some(n) +} + +/// `Digest`: 64 lowercase hexadecimal digits. +pub fn is_digest(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) +} + +/// Service and topic names: 1..=192 of `[a-z0-9._-]`, no empty dot-separated segment. +pub fn is_name(s: &str) -> bool { + !s.is_empty() + && s.len() <= MAX_NAME_LEN + && s.bytes().all(|c| { + c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, b'.' | b'_' | b'-') + }) + && s.split('.').all(|seg| !seg.is_empty()) +} + +/// RPC method: 1..=128 printable ASCII characters. +pub fn is_method(s: &str) -> bool { + !s.is_empty() && s.len() <= MAX_METHOD_LEN && s.bytes().all(|c| (0x20..=0x7e).contains(&c)) +} + +/// Content type: 1..=127 printable ASCII characters. +pub fn is_content_type(s: &str) -> bool { + !s.is_empty() + && s.len() <= MAX_CONTENT_TYPE_LEN + && s.bytes().all(|c| (0x20..=0x7e).contains(&c)) +} + +/// Operation names: 1..=64 of `[a-z.]`. +pub fn is_op(s: &str) -> bool { + !s.is_empty() && s.len() <= 64 && s.bytes().all(|c| c.is_ascii_lowercase() || c == b'.') +} + +/// `-`, the canonical form of every serial-numbered id on the bus. +pub fn serial_id(prefix: &str, n: u64) -> String { + format!("{prefix}-{n}") +} + +/// Parses `-`; `None` unless canonical. +pub fn parse_serial_id(prefix: &str, s: &str) -> Option { + s.strip_prefix(prefix)? + .strip_prefix('-') + .and_then(parse_u64) +} + +// --------------------------------------------------------------------------------------------- +// Strict JSON + +/// Parses one JSON value, refusing duplicate object keys at any depth, invalid UTF-8, +/// non-finite numbers and trailing data. Nesting depth is bounded by serde_json's recursion +/// limit (128). +pub fn parse_json_strict(bytes: &[u8]) -> Result { + if std::str::from_utf8(bytes).is_err() { + return err("invalid UTF-8"); + } + let mut de = serde_json::Deserializer::from_slice(bytes); + let value = StrictSeed + .deserialize(&mut de) + .map_err(|e| WireError(format!("invalid JSON: {e}")))?; + de.end() + .map_err(|e| WireError(format!("invalid JSON: {e}")))?; + Ok(value) +} + +struct StrictSeed; + +impl<'de> DeserializeSeed<'de> for StrictSeed { + type Value = Value; + + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(StrictVisitor) + } +} + +struct StrictVisitor; + +impl<'de> Visitor<'de> for StrictVisitor { + type Value = Value; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a JSON value") + } + + fn visit_bool(self, v: bool) -> Result { + Ok(Value::Bool(v)) + } + + fn visit_i64(self, v: i64) -> Result { + Ok(Value::from(v)) + } + + fn visit_u64(self, v: u64) -> Result { + Ok(Value::from(v)) + } + + fn visit_f64(self, v: f64) -> Result { + serde_json::Number::from_f64(v) + .map(Value::Number) + .ok_or_else(|| E::custom("non-finite number")) + } + + fn visit_str(self, v: &str) -> Result { + Ok(Value::String(v.to_owned())) + } + + fn visit_string(self, v: String) -> Result { + Ok(Value::String(v)) + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = Vec::new(); + while let Some(v) = seq.next_element_seed(StrictSeed)? { + out.push(v); + } + Ok(Value::Array(out)) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut out = Map::new(); + while let Some(key) = map.next_key::()? { + if out.contains_key(&key) { + return Err(de::Error::custom(format!("duplicate key {key:?}"))); + } + let v = map.next_value_seed(StrictSeed)?; + out.insert(key, v); + } + Ok(Value::Object(out)) + } +} + +// --------------------------------------------------------------------------------------------- +// Exact-field object reader + +/// Reads the fields of one JSON object, then refuses any it did not read. +pub struct Fields<'a> { + map: &'a Map, + seen: HashSet<&'static str>, + what: &'static str, +} + +impl<'a> Fields<'a> { + pub fn new(v: &'a Value, what: &'static str) -> Result, WireError> { + match v { + Value::Object(map) => Ok(Fields::of(map, what)), + _ => err(format!("{what} must be an object")), + } + } + + pub fn of(map: &'a Map, what: &'static str) -> Fields<'a> { + Fields { + map, + seen: HashSet::new(), + what, + } + } + + pub fn value(&mut self, key: &'static str) -> Result<&'a Value, WireError> { + self.seen.insert(key); + match self.map.get(key) { + Some(v) => Ok(v), + None => err(format!("{}: missing field {key:?}", self.what)), + } + } + + pub fn string(&mut self, key: &'static str) -> Result<&'a str, WireError> { + let what = self.what; + self.value(key)? + .as_str() + .ok_or_else(|| WireError(format!("{what}: {key} must be a string"))) + } + + fn checked( + &mut self, + key: &'static str, + ok: fn(&str) -> bool, + kind: &str, + ) -> Result { + let s = self.string(key)?; + if ok(s) { + Ok(s.to_owned()) + } else { + err(format!("{}: {key} is not a valid {kind}", self.what)) + } + } + + pub fn id(&mut self, key: &'static str) -> Result { + self.checked(key, is_id, "id") + } + + pub fn nullable_id(&mut self, key: &'static str) -> Result, WireError> { + match self.value(key)? { + Value::Null => Ok(None), + _ => self.id(key).map(Some), + } + } + + pub fn name(&mut self, key: &'static str) -> Result { + self.checked(key, is_name, "name") + } + + pub fn method(&mut self, key: &'static str) -> Result { + self.checked(key, is_method, "method") + } + + pub fn u64_string(&mut self, key: &'static str) -> Result { + let s = self.string(key)?; + parse_u64(s).ok_or_else(|| { + WireError(format!( + "{}: {key} is not a canonical U64 string", + self.what + )) + }) + } + + /// A JSON integer in `lo..=hi`. + pub fn int(&mut self, key: &'static str, lo: u64, hi: u64) -> Result { + let what = self.what; + match self.value(key)?.as_u64() { + Some(n) if (lo..=hi).contains(&n) => Ok(n), + _ => err(format!("{what}: {key} must be an integer in {lo}..={hi}")), + } + } + + pub fn boolean(&mut self, key: &'static str) -> Result { + let what = self.what; + self.value(key)? + .as_bool() + .ok_or_else(|| WireError(format!("{what}: {key} must be a boolean"))) + } + + pub fn object(&mut self, key: &'static str) -> Result<&'a Map, WireError> { + let what = self.what; + self.value(key)? + .as_object() + .ok_or_else(|| WireError(format!("{what}: {key} must be an object"))) + } + + pub fn array( + &mut self, + key: &'static str, + lo: usize, + hi: usize, + ) -> Result<&'a Vec, WireError> { + let what = self.what; + match self.value(key)?.as_array() { + Some(a) if (lo..=hi).contains(&a.len()) => Ok(a), + _ => err(format!( + "{what}: {key} must be an array of {lo}..={hi} items" + )), + } + } + + /// Refuses fields that were not read. + pub fn finish(self) -> Result<(), WireError> { + if let Some(extra) = self.map.keys().find(|k| !self.seen.contains(k.as_str())) { + return err(format!("{}: unknown field {extra:?}", self.what)); + } + Ok(()) + } +} + +/// An array of ids, each `-`, 1..=64 of them. +pub fn id_batch(v: &[Value], what: &str) -> Result, WireError> { + let mut out = Vec::with_capacity(v.len()); + for item in v { + match item.as_str() { + Some(s) if is_id(s) => out.push(s.to_owned()), + _ => return err(format!("{what}: every entry must be an id")), + } + } + Ok(out) +} + +// --------------------------------------------------------------------------------------------- +// Shared structures + +/// An immutable artifact's identity. Not an address and not authority to read. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ArtifactRef { + pub store_id: String, + pub artifact_id: String, + pub generation: u64, + pub byte_length: u64, + pub content_type: String, + pub digest: Option, +} + +impl ArtifactRef { + pub fn to_json(&self) -> Value { + let mut m = Map::new(); + m.insert("storeId".into(), self.store_id.clone().into()); + m.insert("artifactId".into(), self.artifact_id.clone().into()); + m.insert("generation".into(), self.generation.to_string().into()); + m.insert("byteLength".into(), self.byte_length.to_string().into()); + m.insert("contentType".into(), self.content_type.clone().into()); + m.insert( + "digest".into(), + self.digest.clone().map_or(Value::Null, Value::String), + ); + Value::Object(m) + } + + pub fn from_json(v: &Value) -> Result { + let mut f = Fields::new(v, "ref")?; + let store_id = f.id("storeId")?; + let artifact_id = f.id("artifactId")?; + let generation = f.u64_string("generation")?; + let byte_length = f.u64_string("byteLength")?; + let content_type = f.string("contentType")?; + if !is_content_type(content_type) { + return err("ref: contentType must be 1..=127 printable ASCII characters"); + } + let digest = match f.value("digest")? { + Value::Null => None, + Value::String(s) if is_digest(s) => Some(s.clone()), + _ => return err("ref: digest must be null or 64 lowercase hex digits"), + }; + f.finish()?; + Ok(ArtifactRef { + store_id, + artifact_id, + generation, + byte_length, + content_type: content_type.to_owned(), + digest, + }) + } +} + +/// One attachment entry: an application name, the reference and the sender's owner token. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Attachment { + pub name: String, + pub reference: ArtifactRef, + pub owner_id: String, +} + +impl Attachment { + pub fn to_json(&self) -> Value { + let mut m = Map::new(); + m.insert("name".into(), self.name.clone().into()); + m.insert("ref".into(), self.reference.to_json()); + m.insert("ownerId".into(), self.owner_id.clone().into()); + Value::Object(m) + } + + pub fn from_json(v: &Value) -> Result { + let mut f = Fields::new(v, "attachment")?; + let name = f.id("name")?; + let reference = ArtifactRef::from_json(f.value("ref")?)?; + let owner_id = f.id("ownerId")?; + f.finish()?; + Ok(Attachment { + name, + reference, + owner_id, + }) + } +} + +/// A store location grant: a path relative to `/`. SDK-private. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Location { + pub store_id: String, + pub relative_path: String, +} + +impl Location { + pub fn to_json(&self) -> Value { + let mut m = Map::new(); + m.insert("storeId".into(), self.store_id.clone().into()); + m.insert("relativePath".into(), self.relative_path.clone().into()); + Value::Object(m) + } + + pub fn from_json(v: &Value) -> Result { + let mut f = Fields::new(v, "location")?; + let store_id = f.id("storeId")?; + let relative_path = f.string("relativePath")?.to_owned(); + f.finish()?; + Ok(Location { + store_id, + relative_path, + }) + } +} + +/// A participant identity as the router reports it on deliveries. It is authenticated only +/// when the connection was accepted through a launcher-bound transport entry point. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Identity { + pub client_id: String, + pub client_incarnation: String, +} + +impl Identity { + pub fn to_json(&self) -> Value { + let mut m = Map::new(); + m.insert("clientId".into(), self.client_id.clone().into()); + m.insert( + "clientIncarnation".into(), + self.client_incarnation.clone().into(), + ); + Value::Object(m) + } + + pub fn from_json(v: &Value) -> Result { + let mut f = Fields::new(v, "identity")?; + let client_id = f.id("clientId")?; + let client_incarnation = f.id("clientIncarnation")?; + f.finish()?; + Ok(Identity { + client_id, + client_incarnation, + }) + } +} + +// --------------------------------------------------------------------------------------------- +// Envelope + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + Command, + Reply, + Delivery, + Notice, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Kind::Command => "command", + Kind::Reply => "reply", + Kind::Delivery => "delivery", + Kind::Notice => "notice", + } + } + + fn parse(s: &str) -> Option { + match s { + "command" => Some(Kind::Command), + "reply" => Some(Kind::Reply), + "delivery" => Some(Kind::Delivery), + "notice" => Some(Kind::Notice), + _ => None, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Envelope { + pub major: u64, + pub minor: u64, + pub id: String, + pub reply_to: Option, + pub kind: Kind, + pub op: String, + pub body: Map, + pub attachments: Vec, +} + +impl Envelope { + pub fn new(id: String, kind: Kind, op: &str, body: Map) -> Envelope { + Envelope { + major: MAJOR, + minor: MINOR, + id, + reply_to: None, + kind, + op: op.to_owned(), + body, + attachments: Vec::new(), + } + } + + /// Decodes and validates one frame's bytes. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return err("empty envelope"); + } + if bytes.len() > MAX_ENVELOPE_BYTES { + return err(format!( + "envelope of {} bytes exceeds {MAX_ENVELOPE_BYTES}", + bytes.len() + )); + } + let v = parse_json_strict(bytes)?; + let mut f = Fields::new(&v, "envelope")?; + if f.string("protocol")? != PROTOCOL { + return err("envelope: protocol must be \"flybus\""); + } + let major = f.int("major", 0, MAX_CREDIT)?; + let minor = f.int("minor", 0, MAX_CREDIT)?; + let id = f.id("id")?; + let reply_to = f.nullable_id("replyTo")?; + let kind = Kind::parse(f.string("kind")?) + .ok_or_else(|| WireError("envelope: unknown kind".into()))?; + let op = f.string("op")?; + if !is_op(op) { + return err("envelope: op must be 1..=64 of [a-z.]"); + } + let op = op.to_owned(); + let body = f.object("body")?.clone(); + let raw = f.array("attachments", 0, MAX_ATTACHMENTS)?; + let mut attachments = Vec::with_capacity(raw.len()); + let mut names = HashSet::new(); + for a in raw { + let a = Attachment::from_json(a)?; + if !names.insert(a.name.clone()) { + return err(format!("envelope: duplicate attachment name {:?}", a.name)); + } + attachments.push(a); + } + f.finish()?; + Ok(Envelope { + major, + minor, + id, + reply_to, + kind, + op, + body, + attachments, + }) + } + + pub fn to_value(&self) -> Value { + let mut m = Map::new(); + m.insert("protocol".into(), PROTOCOL.into()); + m.insert("major".into(), self.major.into()); + m.insert("minor".into(), self.minor.into()); + m.insert("id".into(), self.id.clone().into()); + m.insert( + "replyTo".into(), + self.reply_to.clone().map_or(Value::Null, Value::String), + ); + m.insert("kind".into(), self.kind.as_str().into()); + m.insert("op".into(), self.op.clone().into()); + m.insert("body".into(), Value::Object(self.body.clone())); + m.insert( + "attachments".into(), + Value::Array(self.attachments.iter().map(Attachment::to_json).collect()), + ); + Value::Object(m) + } + + /// Serializes, refusing anything over [`MAX_ENVELOPE_BYTES`]. + pub fn encode(&self) -> Result, WireError> { + let bytes = serde_json::to_vec(&self.to_value()).map_err(|e| WireError(e.to_string()))?; + if bytes.len() > MAX_ENVELOPE_BYTES { + return err(format!( + "envelope of {} bytes exceeds {MAX_ENVELOPE_BYTES}", + bytes.len() + )); + } + Ok(bytes) + } +} + +// --------------------------------------------------------------------------------------------- +// Framing + +#[derive(Debug)] +pub enum FrameError { + Io(std::io::Error), + /// A zero length prefix. + Empty, + /// The length prefix exceeds the limit; nothing was allocated for it. + TooLarge(u32), + /// End of stream inside a frame. + Truncated, +} + +impl fmt::Display for FrameError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FrameError::Io(e) => write!(f, "I/O error: {e}"), + FrameError::Empty => f.write_str("zero-length frame"), + FrameError::TooLarge(n) => write!(f, "frame of {n} bytes exceeds {MAX_ENVELOPE_BYTES}"), + FrameError::Truncated => f.write_str("stream ended inside a frame"), + } + } +} + +impl std::error::Error for FrameError {} + +/// Reads one frame. `Ok(None)` is a clean end of stream at a frame boundary. The length is +/// checked against [`MAX_ENVELOPE_BYTES`] before any buffer is allocated. +pub async fn read_frame(r: &mut R) -> Result>, FrameError> { + let mut len = [0u8; 4]; + let mut got = 0; + while got < 4 { + let n = r.read(&mut len[got..]).await.map_err(FrameError::Io)?; + if n == 0 { + return if got == 0 { + Ok(None) + } else { + Err(FrameError::Truncated) + }; + } + got += n; + } + let len = u32::from_le_bytes(len); + if len == 0 { + return Err(FrameError::Empty); + } + if len as usize > MAX_ENVELOPE_BYTES { + return Err(FrameError::TooLarge(len)); + } + let mut buf = vec![0u8; len as usize]; + r.read_exact(&mut buf).await.map_err(|e| match e.kind() { + std::io::ErrorKind::UnexpectedEof => FrameError::Truncated, + _ => FrameError::Io(e), + })?; + Ok(Some(buf)) +} + +/// Writes one frame: the length prefix and the bytes, as one buffer. +pub async fn write_frame(w: &mut W, bytes: &[u8]) -> std::io::Result<()> { + debug_assert!(!bytes.is_empty() && bytes.len() <= MAX_ENVELOPE_BYTES); + let mut buf = Vec::with_capacity(bytes.len() + 4); + buf.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(bytes); + w.write_all(&buf).await?; + w.flush().await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scalars() { + assert!(is_id("a") && is_id("msg-7") && is_id("0.a_b-c")); + assert!(!is_id("") && !is_id("-a") && !is_id("A") && !is_id(&"a".repeat(65))); + assert!(is_id(&"a".repeat(64))); + assert_eq!(parse_u64("0"), Some(0)); + assert_eq!(parse_u64("18446744073709551615"), Some(u64::MAX)); + for bad in ["", "01", "18446744073709551616", "-1", "1.0", "+1", " 1"] { + assert_eq!(parse_u64(bad), None, "{bad:?}"); + } + assert!(is_name("session.demo.snapshots") && is_name(&"a".repeat(192))); + for bad in ["", ".a", "a.", "a..b", "A", "a/b", &"a".repeat(193)] { + assert!(!is_name(bad), "{bad:?}"); + } + assert!(is_digest(&"0f".repeat(32)) && !is_digest(&"0F".repeat(32)) && !is_digest("00")); + assert_eq!(parse_serial_id("msg", "msg-12"), Some(12)); + assert_eq!(parse_serial_id("msg", "msg-012"), None); + assert_eq!(parse_serial_id("msg", "msgx-1"), None); + assert!(is_method("Counter.Increment") && !is_method("a\u{7f}") && !is_method("")); + } + + #[test] + fn strict_json() { + assert!(parse_json_strict(br#"{"a":{"b":[{"c":1,"d":2}]}}"#).is_ok()); + for bad in [ + &br#"{"a":1,"a":2}"#[..], + br#"{"a":{"b":[{"c":1,"c":2}]}}"#, + br#"[{"x":1},{"y":{"z":1,"z":1}}]"#, + br#"{"a":NaN}"#, + br#"{"a":Infinity}"#, + br#"{"a":1e400}"#, + br#"{"a":"\ud800"}"#, + br#"{"a":1} x"#, + b"{\"a\":\"\xff\"}", + ] { + assert!( + parse_json_strict(bad).is_err(), + "{:?}", + String::from_utf8_lossy(bad) + ); + } + let deep = format!("{}{}", "[".repeat(200), "]".repeat(200)); + assert!(parse_json_strict(deep.as_bytes()).is_err()); + } +} diff --git a/services/flysim/crates/flybus/tests/artifacts.rs b/services/flysim/crates/flybus/tests/artifacts.rs new file mode 100644 index 0000000..c3937ed --- /dev/null +++ b/services/flysim/crates/flybus/tests/artifacts.rs @@ -0,0 +1,704 @@ +//! Artifacts: allocate/seal/read, immutability against live writable handles, ownership +//! through fan-out and retention, quotas, atomic admission, watermarked releases, abandoned +//! futures, disconnects and router restarts. + +mod common; + +use std::io::{Seek, SeekFrom, Write}; +use std::os::unix::fs::PermissionsExt; +use std::time::Duration; + +use common::{Via, code, env, env_with, obj, quiet, sealed, within}; +use flybus::{ + ErrorCode, Limits, Policy, Retained, Router, RouterConfig, ServiceConfig, SubscriptionConfig, +}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +fn sha(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +async fn allocate_write_seal_read(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + let data: Vec = (0..100_000u32).map(|i| (i % 251) as u8).collect(); + let mut w = c + .artifacts() + .allocate(data.len() as u64, "application/octet-stream") + .await + .unwrap(); + assert!( + w.write_all(&[0; 100_001]).is_err(), + "writes past the allocation are refused" + ); + w.write_all(&data).unwrap(); + assert_eq!(e.files("staging"), 1); + let art = w.seal_with_digest(Some(sha(&data))).await.unwrap(); + let r = art.reference(); + assert_eq!( + (r.generation, r.byte_length, r.digest.clone()), + (1, 100_000, Some(sha(&data))) + ); + assert_eq!(r.store_id, e.router.store_id()); + assert_eq!(art.read_all().await.unwrap(), data); + assert_eq!((e.files("staging"), e.files("sealed")), (0, 1)); + let meta = std::fs::metadata(e.router.store_dir().join("sealed").join(&r.artifact_id)).unwrap(); + assert_eq!( + meta.permissions().mode() & 0o222, + 0, + "sealed files are read-only" + ); + + let empty = c + .artifacts() + .allocate(0, "text/plain") + .await + .unwrap() + .seal() + .await + .unwrap(); + assert!(empty.read_all().await.unwrap().is_empty()); + assert_eq!( + c.artifacts().allocate(1, "").await.unwrap_err().code, + ErrorCode::InvalidEnvelope + ); + drop((art, empty)); + e.settle("collected", |s| s.artifacts == 0 && s.store_bytes == 0) + .await; + e.settle_files("sealed", 0).await; +} + +async fn unsealed_artifacts_cannot_be_used(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("raw").await; + assert_eq!( + code( + &raw.call("topic.declare", json!({"name": "t.x", "retained": "none"})) + .await + ), + "OK" + ); + let (a, _) = raw.allocate(8).await; + let store = a["writeLocation"]["storeId"].clone(); + let reference = json!({"storeId": store, "artifactId": a["artifactId"], "generation": "1", "byteLength": "8", + "contentType": "application/octet-stream", "digest": null}); + let att = json!([{"name": "x", "ref": reference, "ownerId": a["ownerId"]}]); + let r = raw + .call_with("publish", json!({"topic": "t.x", "payload": {}}), att) + .await; + assert_eq!(code(&r), "ARTIFACT_UNSEALED"); + let r = raw + .call( + "artifact.open", + json!({"ref": reference, "ownerId": a["ownerId"]}), + ) + .await; + assert_eq!(code(&r), "ARTIFACT_UNSEALED"); + let r = raw + .call( + "artifact.retain", + json!({"ref": reference, "ownerId": a["ownerId"]}), + ) + .await; + assert_eq!(code(&r), "ARTIFACT_UNSEALED"); + assert_eq!(code(&raw.seal(&a, Value::Null).await), "OK"); + assert_eq!( + code(&raw.seal(&a, Value::Null).await), + "OWNER_INVALID", + "a writer seals once" + ); + assert_eq!( + code( + &raw.call( + "artifact.allocate", + json!({"byteLength": "01", "contentType": "x"}) + ) + .await + ), + "INVALID_ENVELOPE" + ); +} + +/// A producer that kept (or duplicated) its writable descriptor cannot change sealed bytes. +async fn seal_is_immune_to_live_writable_handles(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("raw").await; + let (a, path) = raw.allocate(8).await; + let mut kept = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + kept.write_all(b"AAAAAAAA").unwrap(); + let mut dup = kept.try_clone().unwrap(); + let sealed = raw.seal(&a, json!(sha(b"AAAAAAAA"))).await.unwrap(); + // Both descriptors still write, into an inode the store no longer uses. + dup.seek(SeekFrom::Start(0)).unwrap(); + let _ = dup.write_all(b"BBBBBBBB"); + let _ = kept.write_all(b"CCCC"); + assert!(!path.exists(), "staging is unlinked after sealing"); + let open = raw + .call( + "artifact.open", + json!({"ref": sealed["ref"], "ownerId": sealed["ownerId"]}), + ) + .await + .unwrap(); + let read_path = raw.path(&open["readLocation"]); + assert_eq!(std::fs::read(&read_path).unwrap(), b"AAAAAAAA"); + assert_eq!( + std::fs::metadata(&read_path).unwrap().permissions().mode() & 0o222, + 0 + ); +} + +async fn seal_checks_length_and_digest(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("raw").await; + let (short, path) = raw.allocate(4).await; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(2) + .unwrap(); + assert_eq!( + code(&raw.seal(&short, Value::Null).await), + "ARTIFACT_MISMATCH" + ); + let (long, path) = raw.allocate(4).await; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(6) + .unwrap(); + assert_eq!( + code(&raw.seal(&long, Value::Null).await), + "ARTIFACT_MISMATCH" + ); + let (bad, _) = raw.allocate(4).await; + assert_eq!( + code(&raw.seal(&bad, json!("XYZ")).await), + "INVALID_ENVELOPE" + ); + assert_eq!( + code(&raw.seal(&bad, json!("0".repeat(64))).await), + "ARTIFACT_MISMATCH" + ); + // Failed seals clean up both the staging file and the copy. + let s = e.settle("failed seals cleaned", |s| s.artifacts == 0).await; + assert_eq!((s.store_bytes, s.owners), (0, 0)); + e.settle_files("staging", 0).await; + e.settle_files("sealed", 0).await; + + let c = e.client("c").await; + let mut w = c.artifacts().allocate(3, "text/plain").await.unwrap(); + w.write_all(b"abc").unwrap(); + let err = w.seal_with_digest(Some(sha(b"abd"))).await.unwrap_err(); + assert_eq!(err.code, ErrorCode::ArtifactMismatch); + e.settle("sdk seal cleaned", |s| { + s.artifacts == 0 && s.store_bytes == 0 + }) + .await; +} + +async fn quotas_are_enforced(via: Via) { + let limits = Limits { + max_store_bytes: 1000, + max_artifact_bytes: 600, + ..Limits::default() + }; + let e = env_with(via, limits, Policy::open()).await; + let c = e.client("c").await; + assert_eq!( + c.artifacts().allocate(601, "x/y").await.unwrap_err().code, + ErrorCode::QuotaExceeded + ); + let w = c.artifacts().allocate(600, "x/y").await.unwrap(); + assert_eq!( + c.artifacts().allocate(401, "x/y").await.unwrap_err().code, + ErrorCode::QuotaExceeded + ); + // Sealing copies, so it needs room for both; failing releases the staging object. + assert_eq!(w.seal().await.unwrap_err().code, ErrorCode::QuotaExceeded); + e.settle("staging released", |s| s.store_bytes == 0 && s.owners == 0) + .await; + + let limits = Limits { + max_store_bytes: 1200, + max_artifact_bytes: 600, + max_owners_per_client: 4, + reserved_owners_per_client: 1, + ..Limits::default() + }; + let e = env_with(via, limits, Policy::open()).await; + let c = e.client("c").await; + let art = c + .artifacts() + .allocate(600, "x/y") + .await + .unwrap() + .seal() + .await + .unwrap(); + assert_eq!(e.stats().store_bytes, 600); + let _w1 = c.artifacts().allocate(1, "x/y").await.unwrap(); + let _w2 = c.artifacts().allocate(1, "x/y").await.unwrap(); + assert_eq!( + c.artifacts().allocate(1, "x/y").await.unwrap_err().code, + ErrorCode::QuotaExceeded + ); + assert_eq!( + art.retain().await.unwrap_err().code, + ErrorCode::QuotaExceeded + ); +} + +async fn fan_out_shares_one_object_and_the_last_consumer_collects(via: Via) { + let e = env(via).await; + let camera = e.client("camera").await; + camera + .declare_topic("world.demo.frame", Retained::None) + .await + .unwrap(); + let mut viewers = Vec::new(); + for i in 0..3 { + let c = e.client(&format!("viewer-{i}")).await; + let s = c + .subscribe("world.demo.frame", SubscriptionConfig::bounded()) + .await + .unwrap(); + viewers.push((c, s)); + } + let frame: Vec = (0..640 * 480 * 4).map(|i| (i % 256) as u8).collect(); + let art = sealed(&camera, &frame, "image/x-rgba").await; + let r = camera + .publish( + "world.demo.frame", + obj(json!({"width": 640, "height": 480})), + &[("frame", &art)], + ) + .await + .unwrap(); + assert_eq!(r.subscribers, 3); + drop(art); + let mut messages = Vec::new(); + for (_, s) in viewers.iter_mut() { + let m = within("frame", s.next()).await.unwrap(); + assert_eq!( + m.artifact("frame").unwrap().read_all().await.unwrap(), + frame + ); + messages.push(m); + } + let s = e + .settle("three deliveries of one object", |s| s.artifact_roots == 3) + .await; + assert_eq!((s.sealed_artifacts, s.store_bytes), (1, frame.len() as u64)); + assert_eq!(e.files("sealed"), 1, "fan-out never copies bytes"); + let path = e.router.store_dir().join("sealed").join( + &messages[0] + .artifact("frame") + .unwrap() + .reference() + .artifact_id, + ); + for (i, m) in messages.drain(..).enumerate() { + drop(m); + let left = 2 - i as u64; + e.settle("one root per consumer", |s| s.artifact_roots == left) + .await; + if left > 0 { + assert!( + path.exists(), + "collected while {left} consumers still hold it" + ); + } + } + e.settle_files("sealed", 0).await; + e.settle("collected", |s| s.artifacts == 0 && s.store_bytes == 0) + .await; +} + +async fn extracted_artifacts_outlive_their_message(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + let r = e.client("r").await; + p.declare_topic("t.hold", Retained::None).await.unwrap(); + let mut sub = r + .subscribe("t.hold", SubscriptionConfig::bounded().in_flight(1)) + .await + .unwrap(); + for i in 0..3 { + let a = sealed(&p, format!("f{i}").as_bytes(), "text/plain").await; + p.publish("t.hold", obj(json!({})), &[("f", &a)]) + .await + .unwrap(); + } + let m = within("first", sub.next()).await.unwrap(); + let image = m.artifact("f").unwrap(); + drop(m); + // The extracted handle still owns the delivery, so the credit is not back yet. + quiet("credit held by the extracted artifact", sub.next()).await; + assert_eq!(image.read_all().await.unwrap(), b"f0"); + let file = image.open().await.unwrap(); + drop(image); + quiet("credit held by the open file", sub.next()).await; + drop(file); + let m = within("second", sub.next()).await.unwrap(); + // An explicit hold outlives the delivery and does not keep its credit. + let kept = m.artifact("f").unwrap().retain().await.unwrap(); + drop(m); + let third = within("third", sub.next()).await.unwrap(); + assert_eq!(kept.read_all().await.unwrap(), b"f1"); + drop((third, kept)); + e.settle("collected", |s| s.artifacts == 0 && s.owners == 0) + .await; +} + +async fn failed_admission_is_atomic(via: Via) { + let e = env(via).await; + let reader = e.client("reader").await; + let server = e.client("server").await; + let mut raw = e.raw_hello("raw").await; + assert_eq!( + code( + &raw.call( + "topic.declare", + json!({"name": "t.atomic", "retained": "latest"}) + ) + .await + ), + "OK" + ); + let mut sub = reader + .subscribe("t.atomic", SubscriptionConfig::bounded()) + .await + .unwrap(); + let mut svc = server + .register("example.atomic", ServiceConfig::default()) + .await + .unwrap(); + let (a, path) = raw.allocate(3).await; + std::fs::write(&path, b"abc").unwrap(); + let s = raw.seal(&a, Value::Null).await.unwrap(); + let good = |name: &str| json!({"name": name, "ref": s["ref"], "ownerId": s["ownerId"]}); + let before = e.stats(); + let mut wrong_len = s["ref"].clone(); + wrong_len["byteLength"] = json!("4"); + let mut other_store = s["ref"].clone(); + other_store["storeId"] = json!("store-0000000000000000"); + let cases = [ + ( + json!([good("a"), {"name": "b", "ref": s["ref"], "ownerId": "own-7"}]), + "OWNER_INVALID", + ), + ( + json!([good("a"), {"name": "b", "ref": wrong_len, "ownerId": s["ownerId"]}]), + "ARTIFACT_MISMATCH", + ), + ( + json!([good("a"), {"name": "b", "ref": other_store, "ownerId": s["ownerId"]}]), + "ARTIFACT_GONE", + ), + ]; + for (i, (atts, want)) in cases.iter().enumerate() { + let r = raw + .call_with( + "publish", + json!({"topic": "t.atomic", "payload": {}}), + atts.clone(), + ) + .await; + assert_eq!(code(&r), *want); + let call = json!({"callId": format!("call-{}", i + 1), "target": "example.atomic", "expectedIncarnation": null, "method": "M", "payload": {}}); + let r = raw.call_with("rpc.call", call, atts.clone()).await; + assert_eq!( + code(&r), + *want, + "semantic refusal advances correlation history but preserves the refusal reason" + ); + } + assert_eq!( + e.stats(), + before, + "refused admissions leave no roots, calls or retained values" + ); + quiet("no partial delivery", sub.next()).await; + quiet("no partial request", svc.next()).await; + // Two names for one object: both are delivered, one root is held. + let r = raw + .call_with( + "publish", + json!({"topic": "t.atomic", "payload": {}}), + json!([good("a"), good("b")]), + ) + .await + .unwrap(); + assert_eq!( + r["topicSequence"], "1", + "refused publications spent no sequence number" + ); + let m = within("delivery", sub.next()).await.unwrap(); + assert_eq!(m.attachment_names().collect::>(), ["a", "b"]); + assert_eq!(m.artifact("b").unwrap().read_all().await.unwrap(), b"abc"); + // Hold, retained value, delivery: three roots, not four. + e.settle("one root per holder", |s| s.artifact_roots == 3) + .await; +} + +async fn release_ids_are_watermarked(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("raw").await; + let (a1, _) = raw.allocate(4).await; + let (a2, _) = raw.allocate(4).await; + let rel = |ids: Value| json!({ "ownerIds": ids }); + assert_eq!( + code( + &raw.call("artifact.release", rel(json!([a2["ownerId"], "own-99"]))) + .await + ), + "OWNER_INVALID" + ); + assert_eq!( + e.stats().artifacts, + 2, + "a batch with a future id releases nothing" + ); + assert_eq!( + raw.call("artifact.release", rel(json!([a1["ownerId"]]))) + .await + .unwrap()["released"], + "1" + ); + assert_eq!( + raw.call("artifact.release", rel(json!([a1["ownerId"]]))) + .await + .unwrap()["released"], + "0" + ); + assert_eq!( + code(&raw.call("artifact.release", rel(json!(["dlv-1"]))).await), + "OWNER_INVALID" + ); + assert_eq!( + code(&raw.call("artifact.release", rel(json!([]))).await), + "INVALID_ENVELOPE" + ); + let many: Vec = (1..=65).map(|i| format!("own-{i}")).collect(); + assert_eq!( + code(&raw.call("artifact.release", rel(json!(many))).await), + "INVALID_ENVELOPE" + ); + let consumed = |ids: Value| json!({ "deliveryIds": ids }); + assert_eq!( + code( + &raw.call("delivery.consumed", consumed(json!(["dlv-3"]))) + .await + ), + "OWNER_INVALID" + ); + assert_eq!( + code( + &raw.call("delivery.consumed", consumed(json!([a2["ownerId"]]))) + .await + ), + "OWNER_INVALID" + ); + assert_eq!( + raw.call( + "artifact.release", + rel(json!([a2["ownerId"], a2["ownerId"]])) + ) + .await + .unwrap()["released"], + "1" + ); + e.settle("all released", |s| s.artifacts == 0 && s.store_bytes == 0) + .await; +} + +async fn owners_are_scoped_to_their_connection(via: Via) { + let e = env(via).await; + let owner = e.client("owner").await; + let art = sealed(&owner, b"private", "text/plain").await; + let reference = art.reference().to_json(); + let mut thief = e.raw_hello("thief").await; + let open = |owner_id: &str| json!({"ref": reference, "ownerId": owner_id}); + // Naming the victim's owner id means nothing on another connection. + assert_eq!( + code(&thief.call("artifact.open", open(art.owner_id())).await), + "OWNER_INVALID" + ); + let (mine, _) = thief.allocate(1).await; + let my_owner = mine["ownerId"].as_str().unwrap().to_owned(); + assert_eq!( + code(&thief.call("artifact.open", open(&my_owner)).await), + "OWNER_INVALID" + ); + assert_eq!( + code(&thief.call("artifact.retain", open(&my_owner)).await), + "OWNER_INVALID" + ); + assert_eq!( + code( + &thief + .call("artifact.release", json!({"ownerIds": [art.owner_id()]})) + .await + ), + "OK" + ); + assert_eq!( + art.read_all().await.unwrap(), + b"private", + "another connection cannot release it" + ); +} + +async fn abandoned_futures_do_not_leak_owners(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + // Each future is polled once (its command is sent) and then dropped. + let _ = tokio::time::timeout(Duration::ZERO, c.artifacts().allocate(1000, "x/y")).await; + e.settle("abandoned allocation released", |s| { + s.artifacts == 0 && s.owners == 0 && s.store_bytes == 0 + }) + .await; + e.settle_files("staging", 0).await; + + let mut w = c.artifacts().allocate(4, "x/y").await.unwrap(); + w.write_all(b"seal").unwrap(); + let _ = tokio::time::timeout(Duration::ZERO, w.seal()).await; + e.settle("abandoned seal released", |s| { + s.artifacts == 0 && s.owners == 0 + }) + .await; + + let art = sealed(&c, b"keep", "x/y").await; + let _ = tokio::time::timeout(Duration::ZERO, art.retain()).await; + e.settle("abandoned retain released", |s| s.owners == 1) + .await; + drop(art); + e.settle("all released", |s| s.artifacts == 0 && s.owners == 0) + .await; + assert_eq!(c.control_errors(), 0); +} + +async fn writer_drop_releases_staging(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + let mut w = c.artifacts().allocate(1000, "x/y").await.unwrap(); + w.write_all(&[1; 10]).unwrap(); + assert_eq!(e.files("staging"), 1); + drop(w); + e.settle("staging released", |s| { + s.artifacts == 0 && s.store_bytes == 0 + }) + .await; + e.settle_files("staging", 0).await; +} + +async fn disconnect_releases_all_but_retained(via: Via) { + let e = env(via).await; + let producer = e.client("producer").await; + let reader = e.client("reader").await; + producer + .declare_topic("t.keep", Retained::Latest) + .await + .unwrap(); + let mut sub = reader + .subscribe("t.keep", SubscriptionConfig::bounded()) + .await + .unwrap(); + let a1 = sealed(&producer, b"retained", "x/y").await; + let a2 = sealed(&producer, b"held", "x/y").await; + let w3 = producer.artifacts().allocate(10, "x/y").await.unwrap(); + producer + .publish("t.keep", obj(json!({})), &[("a", &a1)]) + .await + .unwrap(); + let m = within("delivery", sub.next()).await.unwrap(); + // Close both while handles are still alive; the router releases what they owned. + producer.close().await; + reader.close().await; + let s = e + .settle("connections gone", |s| s.connections == 0 && s.owners == 0) + .await; + assert_eq!( + (s.artifacts, s.artifact_roots, s.store_bytes), + (1, 1, 8), + "only the retained value survives" + ); + // Handles that outlived their connection are inert. + assert_eq!(a2.read_all().await.unwrap_err().code, ErrorCode::RouterLost); + drop((a1, a2, w3, m, sub)); + let admin = e.client("admin").await; + assert!(admin.clear_topic("t.keep").await.unwrap()); + e.settle("retained released", |s| { + s.artifacts == 0 && s.store_bytes == 0 + }) + .await; +} + +async fn router_restart_invalidates_old_handles(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + c.declare_topic("t.r", Retained::None).await.unwrap(); + let mut sub = c + .subscribe("t.r", SubscriptionConfig::latest()) + .await + .unwrap(); + let art = sealed(&c, b"old", "x/y").await; + e.router.shutdown(); + assert!(within("subscription closed", sub.next()).await.is_none()); + let err = art.read_all().await.unwrap_err(); + assert_eq!(err.code, ErrorCode::RouterLost); + assert!(c.closed().is_some()); + assert_eq!( + e.try_client("late").await.unwrap_err().code, + ErrorCode::RouterLost + ); + + // A restarted router on the same root is a new store incarnation. + let mut cfg = RouterConfig::new(e.router.store_root()); + cfg.policy = Policy::open(); + let second = Router::new(cfg).unwrap(); + assert_ne!(second.store_id(), e.router.store_id()); + let mut raw = common::Raw::over(second.connect_in_memory(), second.store_root()); + raw.hello("new").await.unwrap(); + let (mine, _) = raw.allocate(1).await; + // An old reference is refused even alongside an owner that is live on the new router. + let r = raw + .call( + "artifact.open", + json!({"ref": art.reference().to_json(), "ownerId": mine["ownerId"]}), + ) + .await; + assert_eq!(code(&r), "ARTIFACT_GONE"); + let client = flybus::Client::connect( + second.connect_in_memory(), + flybus::ClientConfig::new("sdk", second.store_root()), + ) + .await + .unwrap(); + let fresh = sealed(&client, b"new", "x/y").await; + assert_eq!(fresh.read_all().await.unwrap(), b"new"); +} + +both_transports!( + allocate_write_seal_read, + unsealed_artifacts_cannot_be_used, + seal_is_immune_to_live_writable_handles, + seal_checks_length_and_digest, + quotas_are_enforced, + fan_out_shares_one_object_and_the_last_consumer_collects, + extracted_artifacts_outlive_their_message, + failed_admission_is_atomic, + release_ids_are_watermarked, + owners_are_scoped_to_their_connection, + abandoned_futures_do_not_leak_owners, + writer_drop_releases_staging, + disconnect_releases_all_but_retained, + router_restart_invalidates_old_handles, +); diff --git a/services/flysim/crates/flybus/tests/common/mod.rs b/services/flysim/crates/flybus/tests/common/mod.rs new file mode 100644 index 0000000..2b2e804 --- /dev/null +++ b/services/flysim/crates/flybus/tests/common/mod.rs @@ -0,0 +1,416 @@ +//! Shared harness: a router on a temporary store, reached over either transport, plus a raw +//! protocol client for adversarial frames the SDK would never send. + +#![allow(dead_code)] + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use flybus::wire::{Envelope, Kind, Location, read_frame}; +use flybus::{ + Artifact, Client, ClientConfig, Limits, Policy, Router, RouterConfig, RouterStats, Transport, + UnixListenerHandle, +}; +use serde_json::{Map, Value, json}; +use tokio::io::{AsyncWriteExt, ReadHalf, WriteHalf}; + +pub const WAIT: Duration = Duration::from_secs(10); + +/// Generates one test per transport from an `async fn name(via: Via)`. +#[macro_export] +macro_rules! both_transports { + ($($name:ident),* $(,)?) => { + mod in_memory { + $( + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn $name() { + super::$name($crate::common::Via::Memory).await + } + )* + } + mod unix_socket { + $( + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn $name() { + super::$name($crate::common::Via::Unix).await + } + )* + } + }; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Via { + Memory, + Unix, +} + +pub struct Env { + pub router: Router, + pub via: Via, + pub dir: tempfile::TempDir, + listeners: Mutex>, + next_socket: AtomicU64, +} + +pub async fn env(via: Via) -> Env { + env_with(via, Limits::default(), Policy::open()).await +} + +pub async fn env_with(via: Via, limits: Limits, policy: Policy) -> Env { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path().join("store")); + config.limits = limits; + config.policy = policy; + let router = Router::new(config).unwrap(); + Env { + router, + via, + dir, + listeners: Mutex::new(Vec::new()), + next_socket: AtomicU64::new(0), + } +} + +impl Env { + pub async fn transport(&self) -> Transport { + match self.via { + Via::Memory => self.router.connect_in_memory(), + Via::Unix => { + let n = self.next_socket.fetch_add(1, Ordering::Relaxed); + let path = self.dir.path().join(format!("unbound-{n}.sock")); + let listener = self.router.listen_unix(&path).await.unwrap(); + let transport = Transport::unix(&path).await.unwrap(); + self.listeners.lock().unwrap().push(listener); + transport + } + } + } + + pub async fn transport_as(&self, id: &str) -> Transport { + self.try_transport_as(id).await.unwrap() + } + + async fn try_transport_as(&self, id: &str) -> std::io::Result { + match self.via { + Via::Memory => Ok(self.router.connect_in_memory_as(id)), + Via::Unix => { + let n = self.next_socket.fetch_add(1, Ordering::Relaxed); + let path = self.dir.path().join(format!("bound-{n}.sock")); + let listener = self.router.listen_unix_as(&path, id).await?; + let transport = Transport::unix(&path).await?; + self.listeners.lock().unwrap().push(listener); + Ok(transport) + } + } + } + + pub fn config(&self, id: &str) -> ClientConfig { + ClientConfig::new(id, self.router.store_root()) + } + + pub async fn try_client(&self, id: &str) -> Result { + let transport = self.try_transport_as(id).await.map_err(|e| { + flybus::BusError::new(flybus::ErrorCode::RouterLost, format!("connect: {e}")) + })?; + Client::connect(transport, self.config(id)).await + } + + pub async fn client(&self, id: &str) -> Client { + self.try_client(id).await.unwrap() + } + + pub async fn raw(&self) -> Raw { + Raw::over(self.transport().await, self.router.store_root()) + } + + pub async fn raw_as(&self, id: &str) -> Raw { + Raw::over(self.transport_as(id).await, self.router.store_root()) + } + + pub async fn raw_hello(&self, id: &str) -> Raw { + let mut raw = self.raw_as(id).await; + raw.hello(id).await.unwrap(); + raw + } + + pub fn stats(&self) -> RouterStats { + self.router.stats() + } + + /// Polls the router until `ok` holds; panics with the last stats after [`WAIT`]. + pub async fn settle(&self, what: &str, ok: impl Fn(&RouterStats) -> bool) -> RouterStats { + let deadline = tokio::time::Instant::now() + WAIT; + loop { + let s = self.stats(); + if ok(&s) { + return s; + } + if tokio::time::Instant::now() > deadline { + panic!("{what}: router never settled: {s:?}"); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + /// Files currently in the store's `sealed/` or `staging/` directory. + pub fn files(&self, sub: &str) -> usize { + std::fs::read_dir(self.router.store_dir().join(sub)) + .map(|d| d.count()) + .unwrap_or(0) + } + + /// Waits for the file count to reach `n`. Unlinks follow the registry update, outside the + /// router lock, so a file can briefly outlive its entry. + pub async fn settle_files(&self, sub: &str, n: usize) { + let deadline = tokio::time::Instant::now() + WAIT; + while self.files(sub) != n { + assert!( + tokio::time::Instant::now() < deadline, + "{sub}: {} files, wanted {n}", + self.files(sub) + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + } +} + +pub fn obj(v: Value) -> Map { + match v { + Value::Object(m) => m, + _ => panic!("not an object"), + } +} + +pub async fn within(what: &str, f: impl Future) -> T { + match tokio::time::timeout(WAIT, f).await { + Ok(v) => v, + Err(_) => panic!("{what}: timed out"), + } +} + +/// Asserts nothing arrives for a short while. +pub async fn quiet(what: &str, f: impl Future>) { + if let Ok(Some(v)) = tokio::time::timeout(Duration::from_millis(150), f).await { + panic!("{what}: unexpected {v:?}"); + } +} + +pub async fn sealed(client: &Client, bytes: &[u8], content_type: &str) -> Artifact { + use std::io::Write; + let mut w = client + .artifacts() + .allocate(bytes.len() as u64, content_type) + .await + .unwrap(); + w.write_all(bytes).unwrap(); + w.seal().await.unwrap() +} + +/// The write half of a [`Raw`], for a task that floods the router. +pub struct RawWriter { + wr: WriteHalf, +} + +impl RawWriter { + /// Sends one frame; false once the router has gone. + pub async fn send(&mut self, bytes: &[u8]) -> bool { + let mut buf = (bytes.len() as u32).to_le_bytes().to_vec(); + buf.extend_from_slice(bytes); + self.wr.write_all(&buf).await.is_ok() + } +} + +/// A client speaking the wire protocol by hand. +pub struct Raw { + rd: ReadHalf, + wr: Option>, + pub next: u64, + pub stash: Vec, + store_root: PathBuf, +} + +pub type Reply = Result, (String, String)>; + +impl Raw { + pub fn over(transport: Transport, store_root: &Path) -> Raw { + let (rd, wr) = tokio::io::split(transport); + Raw { + rd, + wr: Some(wr), + next: 0, + stash: Vec::new(), + store_root: store_root.to_path_buf(), + } + } + + pub fn take_writer(&mut self) -> RawWriter { + RawWriter { + wr: self.wr.take().expect("writer already taken"), + } + } + + fn wr(&mut self) -> &mut WriteHalf { + self.wr.as_mut().expect("writer was taken") + } + + pub async fn send_bytes(&mut self, bytes: &[u8]) { + let mut buf = (bytes.len() as u32).to_le_bytes().to_vec(); + buf.extend_from_slice(bytes); + let _ = self.wr().write_all(&buf).await; + let _ = self.wr().flush().await; + } + + pub async fn send_prefix(&mut self, len: u32) { + let _ = self.wr().write_all(&len.to_le_bytes()).await; + let _ = self.wr().flush().await; + } + + /// Sends a command with the next id; returns the id. + pub async fn command(&mut self, op: &str, body: Value, attachments: Value) -> String { + self.next += 1; + let id = format!("msg-{}", self.next); + let env = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": id, "replyTo": null, "kind": "command", "op": op, + "body": body, "attachments": attachments, + }); + self.send_bytes(&serde_json::to_vec(&env).unwrap()).await; + id + } + + /// The next envelope, `None` at end of stream. + pub async fn recv(&mut self) -> Option { + if !self.stash.is_empty() { + return Some(self.stash.remove(0)); + } + self.read().await + } + + async fn read(&mut self) -> Option { + match within("raw read", read_frame(&mut self.rd)).await { + Ok(Some(bytes)) => Some(Envelope::decode(&bytes).expect("router frames are valid")), + _ => None, + } + } + + /// Waits for the reply to `id`, stashing anything else. + pub async fn reply(&mut self, id: &str) -> Reply { + if let Some(i) = self + .stash + .iter() + .position(|e| e.reply_to.as_deref() == Some(id)) + { + return parse_reply(self.stash.remove(i)); + } + loop { + let env = self + .read() + .await + .unwrap_or_else(|| panic!("closed while waiting for {id}")); + if env.reply_to.as_deref() == Some(id) { + return parse_reply(env); + } + self.stash.push(env); + } + } + + pub async fn call(&mut self, op: &str, body: Value) -> Reply { + let id = self.command(op, body, json!([])).await; + self.reply(&id).await + } + + pub async fn call_with(&mut self, op: &str, body: Value, attachments: Value) -> Reply { + let id = self.command(op, body, attachments).await; + self.reply(&id).await + } + + pub async fn hello(&mut self, id: &str) -> Reply { + self.call( + "bus.hello", + json!({"clientId": id, "clientIncarnation": "inc-raw", "supportedMajors": [1]}), + ) + .await + } + + /// Reads to end of stream and returns the last `connection.closing` notice, if any. + pub async fn closing(&mut self) -> Option> { + let mut last = None; + while let Some(env) = self.recv().await { + if env.kind == Kind::Notice && env.op == "connection.closing" { + last = Some(env.body); + } + } + last + } + + /// The next delivery or notice, from the stash first. + pub async fn event(&mut self) -> Envelope { + if let Some(i) = self.stash.iter().position(|e| e.kind != Kind::Reply) { + return self.stash.remove(i); + } + loop { + let env = self + .read() + .await + .expect("closed while waiting for an event"); + if env.kind != Kind::Reply { + return env; + } + self.stash.push(env); + } + } + + pub fn path(&self, loc: &Value) -> PathBuf { + let loc = Location::from_json(loc).unwrap(); + self.store_root.join(loc.store_id).join(loc.relative_path) + } + + /// Allocates and writes an artifact by hand; returns (ref-less allocate value, staging path). + pub async fn allocate(&mut self, len: u64) -> (Map, PathBuf) { + let v = self + .call( + "artifact.allocate", + json!({"byteLength": len.to_string(), "contentType": "application/octet-stream"}), + ) + .await + .unwrap(); + let path = self.path(&v["writeLocation"]); + (v, path) + } + + pub async fn seal(&mut self, alloc: &Map, digest: Value) -> Reply { + self.call( + "artifact.seal", + json!({"artifactId": alloc["artifactId"], "generation": "1", "ownerId": alloc["ownerId"], "digest": digest}), + ) + .await + } +} + +pub fn parse_reply(env: Envelope) -> Reply { + assert_eq!(env.kind, Kind::Reply); + if env.body["ok"] == json!(true) { + Ok(env.body["value"].as_object().unwrap().clone()) + } else { + let e = &env.body["error"]; + Err(( + e["code"].as_str().unwrap().to_owned(), + e["dispatch"].as_str().unwrap().to_owned(), + )) + } +} + +pub fn code(r: &Reply) -> &str { + match r { + Ok(_) => "OK", + Err((c, _)) => c, + } +} + +pub fn store_path(root: &Path, store_id: &str, rel: &str) -> PathBuf { + root.join(store_id).join(rel) +} diff --git a/services/flysim/crates/flybus/tests/conformance_artifacts.rs b/services/flysim/crates/flybus/tests/conformance_artifacts.rs new file mode 100644 index 0000000..cf8a8d2 --- /dev/null +++ b/services/flysim/crates/flybus/tests/conformance_artifacts.rs @@ -0,0 +1,788 @@ +//! Black-box conformance tests for artifact ownership (bus-v1 section 8): allocate/seal/open, +//! immutability past a stale writable handle, ownership surviving message drop, explicit +//! retain/release, final-owner collection, root accounting across queued/latest/retained +//! delivery, forward-before-release ordering, admission rollback, disconnect cleanup, and +//! rejection of stale or forged references and owners. Every test runs over both transports. +//! +//! These tests use only the public `flybus` API (plus the `common::Raw` protocol harness for +//! adversarial frames no well-behaved SDK client would send) and generic bytes over synthetic +//! messages; nothing here depends on any application on top of the bus. + +mod common; + +use std::io::Write; + +use common::{Via, code, env, env_with, obj, sealed, within}; +use flybus::{ErrorCode, Limits, Policy, Retained, ServiceConfig, SubscriptionConfig}; +use serde_json::{Value, json}; + +// ------------------------------------------------------------------------------------------- +// Allocate / seal / open + +async fn allocate_write_seal_open_roundtrip_and_mismatches(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + + let mut w = c.artifacts().allocate(5, "text/plain").await.unwrap(); + assert_eq!(w.byte_length(), 5); + w.write_all(b"hello").unwrap(); + assert!( + w.write(b"!").is_err(), + "a write past the declared length must fail locally" + ); + let art = w.seal().await.unwrap(); + assert_eq!(art.reference().byte_length, 5); + assert_eq!(art.reference().content_type, "text/plain"); + assert_eq!(art.read_all().await.unwrap(), b"hello"); + drop(art); + e.settle("sealed object collected", |s| s.sealed_artifacts == 0) + .await; + + // Sealing declares more bytes than were actually staged: rejected, not silently truncated. + let mut raw = e.raw_hello("raw").await; + let (alloc, path) = raw.allocate(10).await; + std::fs::write(&path, b"short").unwrap(); + assert_eq!( + code(&raw.seal(&alloc, Value::Null).await), + "ARTIFACT_MISMATCH" + ); + + // A syntactically valid but wrong digest is refused too. + let (alloc2, path2) = raw.allocate(4).await; + std::fs::write(&path2, b"data").unwrap(); + assert_eq!( + code(&raw.seal(&alloc2, json!("0".repeat(64))).await), + "ARTIFACT_MISMATCH" + ); +} + +/// Sealing copies staging into a fresh inode; a producer's writable handle that survives the +/// seal (or a duplicate of it) can only reach the abandoned staging file, never the sealed +/// bytes (bus-v1 section 8.1 and the store module's own contract). +async fn seal_is_immutable_despite_a_stale_writable_handle(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("producer").await; + let (alloc, path) = raw.allocate(7).await; + let mut stale = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); + stale.write_all(b"before1").unwrap(); + stale.flush().unwrap(); + + let sealed_reply = raw.seal(&alloc, Value::Null).await.unwrap(); + // Tamper through the handle only after the router has answered the seal: the sealed copy + // already exists and the staging file is already unlinked. + stale.write_all(b"AFTER!!").unwrap(); + + let open_reply = raw + .call( + "artifact.open", + json!({"ref": sealed_reply["ref"], "ownerId": sealed_reply["ownerId"]}), + ) + .await + .unwrap(); + let sealed_path = raw.path(&open_reply["readLocation"]); + assert_eq!( + std::fs::read(sealed_path).unwrap(), + b"before1", + "a post-seal write must never reach the sealed copy" + ); +} + +// ------------------------------------------------------------------------------------------- +// Ownership surviving drop, explicit retain/release, final-owner collection + +/// The doc's own illustrative snippet (bus-v1 section 2): `drop(message)` alone must not +/// consume the delivery while an artifact extracted from it is still held. +async fn extracted_artifact_outlives_the_message_it_came_from(via: Via) { + let e = env(via).await; + let camera = e.client("camera").await; + let viewer = e.client("viewer").await; + camera + .declare_topic("world.demo.frame", Retained::None) + .await + .unwrap(); + let mut sub = viewer + .subscribe("world.demo.frame", SubscriptionConfig::bounded()) + .await + .unwrap(); + let frame = sealed(&camera, b"pixels", "image/x-rgba").await; + camera + .publish("world.demo.frame", obj(json!({})), &[("frame", &frame)]) + .await + .unwrap(); + drop(frame); + + let message = within("frame delivered", sub.next()).await.unwrap(); + let image = message.artifact("frame").unwrap(); + drop(message); + e.settle("the delivery still owns its artifact", |s| { + s.owners == 1 && s.sealed_artifacts == 1 + }) + .await; + assert_eq!( + image.read_all().await.unwrap(), + b"pixels", + "the surviving handle still reads fine" + ); + drop(image); + e.settle("dropping the last handle finally consumes it", |s| { + s.owners == 0 && s.sealed_artifacts == 0 + }) + .await; +} + +async fn explicit_retain_outlives_the_original_hold(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + let original = sealed(&c, b"payload", "application/octet-stream").await; + let retained = original.retain().await.unwrap(); + assert_eq!( + retained.reference().artifact_id, + original.reference().artifact_id + ); + assert_ne!( + retained.owner_id(), + original.owner_id(), + "retain creates an independent owner" + ); + + drop(original); + e.settle("the retained hold keeps it sealed", |s| { + s.sealed_artifacts == 1 && s.artifact_roots == 1 + }) + .await; + assert_eq!(retained.read_all().await.unwrap(), b"payload"); + drop(retained); + e.settle("collected once the last owner drops", |s| { + s.sealed_artifacts == 0 && s.owners == 0 + }) + .await; +} + +async fn collection_waits_for_every_retained_owner(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + let a = sealed(&c, b"payload", "application/octet-stream").await; + let h1 = a.retain().await.unwrap(); + let h2 = a.retain().await.unwrap(); + drop(a); + e.settle("two independent holds remain", |s| { + s.owners == 2 && s.sealed_artifacts == 1 + }) + .await; + drop(h1); + e.settle("one hold remains", |s| { + s.owners == 1 && s.sealed_artifacts == 1 + }) + .await; + drop(h2); + e.settle("the final owner's drop collects it", |s| { + s.owners == 0 && s.sealed_artifacts == 0 && s.store_bytes == 0 + }) + .await; + assert_eq!(e.files("sealed"), 0); +} + +// ------------------------------------------------------------------------------------------- +// Root accounting across queued, latest and retained delivery (bus-v1 section 8.2) + +/// A message queued behind a saturated credit already holds a root at admission time, before +/// it is ever dispatched to the client; dispatch transfers that root into an owner, it does +/// not add a second one. +async fn queued_deliveries_hold_roots_before_dispatch(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + let r = e.client("r").await; + p.declare_topic("t.queued", Retained::None).await.unwrap(); + let mut sub = r + .subscribe( + "t.queued", + SubscriptionConfig::bounded().in_flight(1).queued(4), + ) + .await + .unwrap(); + + let a1 = sealed(&p, b"m1", "application/octet-stream").await; + let a2 = sealed(&p, b"m2", "application/octet-stream").await; + p.publish("t.queued", obj(json!({})), &[("x", &a1)]) + .await + .unwrap(); + p.publish("t.queued", obj(json!({})), &[("x", &a2)]) + .await + .unwrap(); + drop((a1, a2)); + e.settle("one dispatched owner, two roots", |s| { + s.artifact_roots == 2 && s.owners == 1 && s.sealed_artifacts == 2 + }) + .await; + + let held = within("dispatched message", sub.next()).await.unwrap(); + assert_eq!(held.artifact("x").unwrap().read_all().await.unwrap(), b"m1"); + drop(held); + let second = within("previously queued message", sub.next()) + .await + .unwrap(); + assert_eq!( + second.artifact("x").unwrap().read_all().await.unwrap(), + b"m2" + ); + e.settle("dispatch moved the root, it did not add one", |s| { + s.artifact_roots == 1 && s.owners == 1 + }) + .await; + drop(second); + e.settle("fully collected", |s| { + s.artifact_roots == 0 && s.owners == 0 && s.sealed_artifacts == 0 + }) + .await; +} + +/// `latest` never queues more than one undelivered value; coalescing releases the replaced +/// root immediately, and the already-dispatched value stays independent of it. +async fn latest_mode_holds_at_most_two_roots_delivered_plus_queued(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + let r = e.client("r").await; + p.declare_topic("t.latest", Retained::None).await.unwrap(); + let mut sub = r + .subscribe("t.latest", SubscriptionConfig::latest().in_flight(1)) + .await + .unwrap(); + for i in 0..3u8 { + let a = sealed(&p, &[i], "application/octet-stream").await; + p.publish("t.latest", obj(json!({})), &[("x", &a)]) + .await + .unwrap(); + } + // i=0 was already dispatched (an owner); i=1 was coalesced away by i=2 while still queued. + e.settle( + "a delivered root plus one coalesced-survivor root, no more", + |s| s.artifact_roots == 2 && s.owners == 1 && s.sealed_artifacts == 2, + ) + .await; + + let first = within("delivered", sub.next()).await.unwrap(); + assert_eq!(first.artifact("x").unwrap().read_all().await.unwrap(), &[0]); + drop(first); + let last = within("coalesced survivor", sub.next()).await.unwrap(); + assert_eq!(last.artifact("x").unwrap().read_all().await.unwrap(), &[2]); + drop(last); + e.settle("fully collected", |s| { + s.artifact_roots == 0 && s.owners == 0 && s.sealed_artifacts == 0 + }) + .await; +} + +/// A retained topic value is a root the router itself keeps; it has no per-connection owner, +/// so nothing about a subscriber connecting, reading or disconnecting can move it. +async fn retained_topic_value_holds_a_root_independent_of_subscribers(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + p.declare_topic("t.retained", Retained::Latest) + .await + .unwrap(); + let a = sealed(&p, b"snapshot", "application/octet-stream").await; + let r = p + .publish("t.retained", obj(json!({})), &[("x", &a)]) + .await + .unwrap(); + assert_eq!(r.subscribers, 0); + drop(a); + e.settle("a retained root with no owner", |s| { + s.artifact_roots == 1 && s.owners == 0 && s.sealed_artifacts == 1 + }) + .await; + assert!(p.clear_topic("t.retained").await.unwrap()); + e.settle("clearing releases the retained root", |s| { + s.artifact_roots == 0 && s.sealed_artifacts == 0 + }) + .await; +} + +// ------------------------------------------------------------------------------------------- +// Forward before release (bus-v1 section 8.2: destination roots before source release) + +/// A forward or reply must use a still-live source owner; once a delivery is released its +/// artifacts cannot source a new admission, but forwarding *before* releasing is exactly what +/// keeps bytes alive once the original delivery finally goes. +async fn forward_requires_the_source_owner_still_live(via: Via) { + let e = env(via).await; + let caller = e.client("caller").await; + let consumer = e.client("consumer").await; + consumer + .declare_topic("t.relay", Retained::None) + .await + .unwrap(); + let mut sub = consumer + .subscribe("t.relay", SubscriptionConfig::bounded()) + .await + .unwrap(); + let mut server = e.raw_hello("server").await; + assert_eq!( + code( + &server + .call( + "service.register", + json!({"name": "agent.relay", "maxQueued": 4, "maxInFlight": 4}) + ) + .await + ), + "OK" + ); + + // Round 1: release the request delivery, then try to forward using its stale owner. + let art1 = sealed(&caller, b"payload-1", "application/octet-stream").await; + let _c1 = caller + .call( + "agent.relay", + None, + "Relay", + obj(json!({})), + &[("in", &art1)], + ) + .await + .unwrap(); + let req1 = within("request 1", server.event()).await; + let delivery1 = req1.body["deliveryId"].as_str().unwrap().to_owned(); + let att1 = req1.attachments[0].clone(); + assert_eq!( + code( + &server + .call("delivery.consumed", json!({"deliveryIds": [delivery1]})) + .await + ), + "OK" + ); + e.settle("the request delivery is released", |s| s.owners == 1) + .await; // only art1's own hold remains + let refused = server + .call_with( + "publish", + json!({"topic": "t.relay", "payload": {}}), + json!([{"name": "out", "ref": att1.reference.to_json(), "ownerId": att1.owner_id}]), + ) + .await; + assert_eq!( + code(&refused), + "OWNER_INVALID", + "a released delivery cannot source a forward" + ); + drop(art1); + + // Round 2: forward while the request delivery is still live, then release it. + let art2 = sealed(&caller, b"payload-2", "application/octet-stream").await; + let _c2 = caller + .call( + "agent.relay", + None, + "Relay", + obj(json!({})), + &[("in", &art2)], + ) + .await + .unwrap(); + let req2 = within("request 2", server.event()).await; + let delivery2 = req2.body["deliveryId"].as_str().unwrap().to_owned(); + let att2 = req2.attachments[0].clone(); + let forwarded = server + .call_with( + "publish", + json!({"topic": "t.relay", "payload": {}}), + json!([{"name": "out", "ref": att2.reference.to_json(), "ownerId": att2.owner_id}]), + ) + .await + .unwrap(); + assert_eq!(forwarded["subscribers"], "1"); + assert_eq!( + code( + &server + .call("delivery.consumed", json!({"deliveryIds": [delivery2]})) + .await + ), + "OK" + ); + drop(art2); + + let msg = within("relayed", sub.next()).await.unwrap(); + assert_eq!( + msg.artifact("out").unwrap().read_all().await.unwrap(), + b"payload-2" + ); +} + +// ------------------------------------------------------------------------------------------- +// Failed admission rollback (bus-v1 section 6/7: rejection establishes no root) + +async fn rejected_publish_creates_no_roots(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + let r = e.client("r").await; + p.declare_topic("t.rollback", Retained::None).await.unwrap(); + let mut sub = r + .subscribe( + "t.rollback", + SubscriptionConfig::bounded().queued(1).in_flight(1), + ) + .await + .unwrap(); + p.publish("t.rollback", obj(json!({})), &[]).await.unwrap(); + let held = within("first delivered", sub.next()).await.unwrap(); // consumes the one in-flight credit + p.publish("t.rollback", obj(json!({})), &[]).await.unwrap(); // fills the one queue slot + + let art = sealed(&p, b"payload", "application/octet-stream").await; + let before = e.stats(); + let refused = p + .publish("t.rollback", obj(json!({})), &[("x", &art)]) + .await + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::Backpressure); + assert_eq!( + e.stats(), + before, + "a refused publish must not touch any existing root, owner or byte count" + ); + + drop(held); + p.declare_topic("t.rollback.ok", Retained::None) + .await + .unwrap(); + let receipt = p + .publish("t.rollback.ok", obj(json!({})), &[("x", &art)]) + .await + .unwrap(); + assert_eq!( + receipt.subscribers, 0, + "the untouched artifact is still usable after the rejection" + ); +} + +async fn rejected_call_creates_no_roots(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register( + "agent.rollback", + ServiceConfig { + max_queued: 1, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + let _c1 = caller + .call("agent.rollback", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let dispatched = within("dispatched", svc.next()).await.unwrap(); + let _c2 = caller + .call("agent.rollback", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); // fills the queue + + let art = sealed(&caller, b"payload", "application/octet-stream").await; + let before = e.stats(); + let refused = caller + .call( + "agent.rollback", + None, + "Work", + obj(json!({})), + &[("x", &art)], + ) + .await + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::Backpressure); + assert_eq!( + e.stats(), + before, + "a refused call must not touch any existing root, owner or byte count" + ); + drop(dispatched); +} + +// ------------------------------------------------------------------------------------------- +// Disconnect cleanup (bus-v1 section 8.4) + +/// An abrupt disconnect (no `artifact.release` ever sent) must still abandon an unsealed +/// writer's staging reservation. +async fn disconnect_abandons_an_unsealed_writer(via: Via) { + let e = env(via).await; + { + let mut raw = e.raw_hello("producer").await; + let (_alloc, path) = raw.allocate(64).await; + std::fs::write(&path, [7u8; 64]).unwrap(); + e.settle("writer visible", |s| { + s.artifacts == 1 && s.store_bytes == 64 + }) + .await; + // `raw` drops here without ever releasing anything. + } + e.settle("disconnect abandoned the writer", |s| { + s.artifacts == 0 && s.store_bytes == 0 && s.owners == 0 + }) + .await; + assert_eq!(e.files("staging"), 0); +} + +/// An abrupt disconnect must release an explicit hold too, when it was the object's only root. +async fn disconnect_releases_an_explicit_hold(via: Via) { + let e = env(via).await; + { + let mut raw = e.raw_hello("producer").await; + let (alloc, path) = raw.allocate(5).await; + std::fs::write(&path, b"hello").unwrap(); + let reply = raw.seal(&alloc, Value::Null).await.unwrap(); + assert!(reply.contains_key("ref")); + } + e.settle("disconnect released the only hold", |s| { + s.sealed_artifacts == 0 && s.owners == 0 && s.store_bytes == 0 + }) + .await; + assert_eq!(e.files("sealed"), 0); +} + +/// A vanished subscriber must give up both a delivery it already holds and one still queued +/// behind it, never issued to the wire. +async fn disconnect_releases_queued_and_dispatched_deliveries(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + p.declare_topic("t.gone", Retained::None).await.unwrap(); + { + let mut raw = e.raw_hello("subscriber").await; + assert_eq!( + code(&raw.call("subscribe", json!({"topic": "t.gone", "mode": "bounded", "maxQueued": 4, "maxInFlight": 1, "replayLatest": false})).await), + "OK" + ); + let a1 = sealed(&p, b"m1", "application/octet-stream").await; + let a2 = sealed(&p, b"m2", "application/octet-stream").await; + p.publish("t.gone", obj(json!({})), &[("x", &a1)]) + .await + .unwrap(); + p.publish("t.gone", obj(json!({})), &[("x", &a2)]) + .await + .unwrap(); + drop((a1, a2)); + let delivered = within("dispatched delivery", raw.event()).await; + assert_eq!(delivered.op, "topic.message"); + e.settle("one dispatched, one still queued", |s| { + s.artifact_roots == 2 && s.sealed_artifacts == 2 + }) + .await; + // `raw` drops here without ever consuming the dispatched delivery. + } + e.settle("disconnect released both", |s| { + s.artifact_roots == 0 && s.owners == 0 && s.sealed_artifacts == 0 && s.store_bytes == 0 + }) + .await; +} + +// ------------------------------------------------------------------------------------------- +// Stale references, stale generations and forged/borrowed owners + +async fn stale_store_incarnation_and_generation_are_rejected(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("c").await; + let (alloc, path) = raw.allocate(4).await; + std::fs::write(&path, b"data").unwrap(); + let sealed_reply = raw.seal(&alloc, Value::Null).await.unwrap(); + + let mut bad_store_ref = sealed_reply["ref"].clone(); + bad_store_ref["storeId"] = json!("store-does-not-exist"); + let r = raw + .call( + "artifact.open", + json!({"ref": bad_store_ref, "ownerId": sealed_reply["ownerId"]}), + ) + .await; + assert_eq!( + code(&r), + "ARTIFACT_GONE", + "a reference naming another store incarnation is stale" + ); + + let (alloc2, path2) = raw.allocate(4).await; + std::fs::write(&path2, b"more!").unwrap(); + let bad_generation = raw + .call("artifact.seal", json!({"artifactId": alloc2["artifactId"], "generation": "2", "ownerId": alloc2["ownerId"], "digest": Value::Null})) + .await; + assert_eq!( + code(&bad_generation), + "ARTIFACT_GONE", + "v1 has exactly one generation" + ); +} + +async fn owner_ids_are_scoped_to_their_connection(via: Via) { + let e = env(via).await; + let mut a = e.raw_hello("a").await; + let mut b = e.raw_hello("b").await; + let (alloc, path) = a.allocate(4).await; + std::fs::write(&path, b"data").unwrap(); + let sealed_reply = a.seal(&alloc, Value::Null).await.unwrap(); + let (reference, owner_id) = (sealed_reply["ref"].clone(), sealed_reply["ownerId"].clone()); + + // `b` was never issued this owner id; a real owner id from another connection is just as + // invalid as one that was never issued at all. + assert_eq!( + code( + &b.call( + "artifact.open", + json!({"ref": reference, "ownerId": owner_id}) + ) + .await + ), + "OWNER_INVALID" + ); + assert_eq!( + code( + &b.call( + "artifact.retain", + json!({"ref": reference, "ownerId": owner_id}) + ) + .await + ), + "OWNER_INVALID" + ); + + // The writer's own owner id cannot open before sealing. + let (alloc2, path2) = a.allocate(4).await; + std::fs::write(&path2, b"data").unwrap(); + let unsealed_ref = json!({ + "storeId": alloc2["writeLocation"]["storeId"], "artifactId": alloc2["artifactId"], "generation": "1", + "byteLength": "4", "contentType": "application/octet-stream", "digest": Value::Null, + }); + assert_eq!( + code( + &a.call( + "artifact.open", + json!({"ref": unsealed_ref, "ownerId": alloc2["ownerId"]}) + ) + .await + ), + "ARTIFACT_UNSEALED" + ); + + // A malformed owner id is refused outright, not treated as an unknown serial. + assert_eq!( + code( + &a.call( + "artifact.open", + json!({"ref": reference, "ownerId": "not-an-owner"}) + ) + .await + ), + "OWNER_INVALID" + ); +} + +// ------------------------------------------------------------------------------------------- +// Bounds, quotas and path containment (bus-v1 section 4/9) + +async fn artifact_bounds_and_owner_budget_are_enforced(via: Via) { + let limits = Limits { + max_artifact_bytes: 100, + max_store_bytes: 160, + max_owners_per_client: 4, + reserved_owners_per_client: 1, + ..Limits::default() + }; + let e = env_with(via, limits, Policy::open()).await; + let c = e.client("c").await; + + let too_big = c + .artifacts() + .allocate(101, "application/octet-stream") + .await + .unwrap_err(); + assert_eq!( + too_big.code, + ErrorCode::QuotaExceeded, + "byteLength exceeds the per-object limit" + ); + + let a = sealed(&c, &[0u8; 80], "application/octet-stream").await; + let before = e.stats(); + let no_room = c + .artifacts() + .allocate(90, "application/octet-stream") + .await + .unwrap_err(); + assert_eq!(no_room.code, ErrorCode::QuotaExceeded, "the store is full"); + assert_eq!( + e.stats(), + before, + "a rejected allocation must not charge the store" + ); + + // Owner budget: 4 total minus 1 reserved leaves room for 3 ordinary owners; `a` itself is + // already one of them, so exactly two more retains fit. + let h1 = a.retain().await.unwrap(); + let h2 = a.retain().await.unwrap(); + let over = a.retain().await.unwrap_err(); + assert_eq!( + over.code, + ErrorCode::QuotaExceeded, + "owner budget exhausted" + ); + drop((h1, h2, a)); +} + +/// Every location the router issues stays a plain relative path under the store; the router +/// never hands the client anything to escape with (store.rs enforces this on the resolving +/// side; this checks the issuing side never even offers an unsafe shape). +async fn issued_locations_are_relative_and_contained(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("c").await; + let (alloc, path) = raw.allocate(4).await; + let assert_safe = |loc: &Value| { + assert_eq!(loc["storeId"].as_str().unwrap(), e.router.store_id()); + let rel = loc["relativePath"].as_str().unwrap(); + assert!( + !rel.starts_with('/') && !rel.contains(".."), + "{rel:?} escapes the store" + ); + assert!( + rel.bytes().all(|b| b.is_ascii_lowercase() + || b.is_ascii_digit() + || matches!(b, b'.' | b'_' | b'-' | b'/')), + "{rel:?} has an unexpected character" + ); + }; + assert_safe(&alloc["writeLocation"]); + + std::fs::write(&path, b"data").unwrap(); + let sealed_reply = raw.seal(&alloc, Value::Null).await.unwrap(); + let open_reply = raw + .call( + "artifact.open", + json!({"ref": sealed_reply["ref"], "ownerId": sealed_reply["ownerId"]}), + ) + .await + .unwrap(); + assert_safe(&open_reply["readLocation"]); + assert_eq!( + std::fs::read(raw.path(&open_reply["readLocation"])).unwrap(), + b"data" + ); +} + +both_transports!( + allocate_write_seal_open_roundtrip_and_mismatches, + seal_is_immutable_despite_a_stale_writable_handle, + extracted_artifact_outlives_the_message_it_came_from, + explicit_retain_outlives_the_original_hold, + collection_waits_for_every_retained_owner, + queued_deliveries_hold_roots_before_dispatch, + latest_mode_holds_at_most_two_roots_delivered_plus_queued, + retained_topic_value_holds_a_root_independent_of_subscribers, + forward_requires_the_source_owner_still_live, + rejected_publish_creates_no_roots, + rejected_call_creates_no_roots, + disconnect_abandons_an_unsealed_writer, + disconnect_releases_an_explicit_hold, + disconnect_releases_queued_and_dispatched_deliveries, + stale_store_incarnation_and_generation_are_rejected, + owner_ids_are_scoped_to_their_connection, + artifact_bounds_and_owner_budget_are_enforced, + issued_locations_are_relative_and_contained, +); diff --git a/services/flysim/crates/flybus/tests/conformance_routing.rs b/services/flysim/crates/flybus/tests/conformance_routing.rs new file mode 100644 index 0000000..3c0ef30 --- /dev/null +++ b/services/flysim/crates/flybus/tests/conformance_routing.rs @@ -0,0 +1,596 @@ +//! Routing conformance: adversarial contract cases for bus-v1 sections 5-8, deliberately not +//! duplicating the happy paths already covered by `rpc.rs` and `pubsub.rs`. Generic synthetic +//! services and topics only; no application/session semantics. Every test runs over both the +//! in-memory transport and a Unix socket. + +mod common; + +use common::{Via, env, obj, quiet, sealed, within}; +use flybus::{CancelState, Dispatch, ErrorCode, Retained, ServiceConfig, SubscriptionConfig}; +use serde_json::json; + +/// bus-v1 section 5: "One live registration owns a service name." A client re-registering a +/// name it already owns is a duplicate too, not an idempotent no-op, and the failed attempt +/// must not disturb the live registration. +async fn duplicate_registration_by_owner_itself_is_rejected(via: Via) { + let e = env(via).await; + let a = e.client("a").await; + let caller = e.client("caller").await; + let mut svc = a + .register("agent.self", ServiceConfig::default()) + .await + .unwrap(); + let incarnation = svc.incarnation().to_owned(); + let dup = a + .register("agent.self", ServiceConfig::default()) + .await + .unwrap_err(); + assert_eq!(dup.code, ErrorCode::Conflict); + assert_eq!( + svc.incarnation(), + incarnation, + "the failed self-duplicate did not replace the live registration" + ); + let mut pending = caller + .call("agent.self", Some(&incarnation), "M", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + assert!(req.reply(obj(json!({"ok": true})), &[]).await.unwrap()); + assert_eq!( + within("result", pending.result()).await.unwrap().outcome()["ok"], + true + ); +} + +/// bus-v1 section 3: "Callers pin it after discovery." An unpinned call made *after* the name +/// changes hands must resolve to the new incarnation, never linger on the old one. +async fn unpinned_call_after_incarnation_replacement_reaches_the_new_holder(via: Via) { + let e = env(via).await; + let a = e.client("a").await; + let b = e.client("b").await; + let caller = e.client("caller").await; + let first = a + .register("agent.fly", ServiceConfig::default()) + .await + .unwrap(); + let old = first.incarnation().to_owned(); + drop(first); + // Unregistration is asynchronous; wait for the name to come free. + let mut second = loop { + match b.register("agent.fly", ServiceConfig::default()).await { + Ok(s) => break s, + Err(err) => assert_eq!(err.code, ErrorCode::Conflict), + } + }; + assert_ne!(second.incarnation(), old); + let mut pending = caller + .call("agent.fly", None, "M", obj(json!({})), &[]) + .await + .unwrap(); + assert_eq!( + pending.service_incarnation(), + second.incarnation(), + "unpinned discovery resolves to the current holder" + ); + let req = within("request", second.next()).await.unwrap(); + assert_eq!(req.service_incarnation(), second.incarnation()); + assert!( + req.reply(obj(json!({"from": "second"})), &[]) + .await + .unwrap() + ); + assert_eq!( + within("result", pending.result()).await.unwrap().outcome()["from"], + "second" + ); +} + +/// bus-v1 section 6: "responses may complete out of order and correlate by callId." Two callers +/// interleave calls to the same service and the service answers in reverse admission order; +/// every result must reach the caller it belongs to, never a sibling's. +async fn out_of_order_replies_correlate_across_concurrent_callers(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let alice = e.client("alice").await; + let bob = e.client("bob").await; + let mut svc = server + .register( + "example.multi", + ServiceConfig { + max_queued: 8, + max_in_flight: 4, + }, + ) + .await + .unwrap(); + let mut a1 = alice + .call( + "example.multi", + None, + "M", + obj(json!({"who": "alice", "n": 1})), + &[], + ) + .await + .unwrap(); + let mut b1 = bob + .call( + "example.multi", + None, + "M", + obj(json!({"who": "bob", "n": 1})), + &[], + ) + .await + .unwrap(); + let mut a2 = alice + .call( + "example.multi", + None, + "M", + obj(json!({"who": "alice", "n": 2})), + &[], + ) + .await + .unwrap(); + let mut b2 = bob + .call( + "example.multi", + None, + "M", + obj(json!({"who": "bob", "n": 2})), + &[], + ) + .await + .unwrap(); + let mut reqs = Vec::new(); + for _ in 0..4 { + reqs.push(within("request", svc.next()).await.unwrap()); + } + // Reply in the reverse of admission order. + for r in reqs.iter().rev() { + let who = r.payload()["who"].clone(); + let n = r.payload()["n"].clone(); + assert!( + r.reply(obj(json!({"who": who, "n": n})), &[]) + .await + .unwrap() + ); + } + drop(reqs); + let ra1 = within("a1", a1.result()).await.unwrap(); + let rb1 = within("b1", b1.result()).await.unwrap(); + let ra2 = within("a2", a2.result()).await.unwrap(); + let rb2 = within("b2", b2.result()).await.unwrap(); + assert_eq!( + (ra1.outcome()["who"].as_str(), ra1.outcome()["n"].clone()), + (Some("alice"), json!(1)) + ); + assert_eq!( + (rb1.outcome()["who"].as_str(), rb1.outcome()["n"].clone()), + (Some("bob"), json!(1)) + ); + assert_eq!( + (ra2.outcome()["who"].as_str(), ra2.outcome()["n"].clone()), + (Some("alice"), json!(2)) + ); + assert_eq!( + (rb2.outcome()["who"].as_str(), rb2.outcome()["n"].clone()), + (Some("bob"), json!(2)) + ); +} + +/// bus-v1 section 5: "Queued cancellation releases its queued artifact roots." Not exercised by +/// `cancellation_states`, which never attaches artifacts. +async fn cancel_before_dispatch_releases_queued_artifact_roots(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register( + "example.holdup", + ServiceConfig { + max_queued: 4, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + let _held = caller + .call("example.holdup", None, "Work", obj(json!({"n": 1})), &[]) + .await + .unwrap(); + let req1 = within("request 1", svc.next()).await.unwrap(); + let art = sealed(&caller, b"queued-payload", "text/plain").await; + let queued = caller + .call( + "example.holdup", + None, + "Work", + obj(json!({"n": 2})), + &[("x", &art)], + ) + .await + .unwrap(); + drop(art); // only the queued call's own root should keep the bytes alive now + e.settle("the queued call's attachment is rooted", |s| { + s.artifacts == 1 && s.artifact_roots >= 1 + }) + .await; + assert_eq!( + queued.cancel().await.unwrap(), + CancelState::CancelledBeforeDispatch + ); + e.settle("cancelling before dispatch released the queued root", |s| { + s.artifacts == 0 && s.store_bytes == 0 + }) + .await; + drop(req1); +} + +/// bus-v1 section 6: after a post-dispatch cancel, "a later reply to a detached call returns +/// `routed:false`, with no caller-result roots" — including when that late reply carries an +/// artifact the caller must never see rooted. +async fn cancel_after_dispatch_then_late_reply_with_artifact_is_not_routed(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("example.late", ServiceConfig::default()) + .await + .unwrap(); + let mut dispatched = caller + .call("example.late", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + assert_eq!( + dispatched.cancel().await.unwrap(), + CancelState::ExecutionUnknown + ); + let art = sealed(&server, b"late-with-artifact", "text/plain").await; + let routed = req.reply(obj(json!({})), &[("a", &art)]).await.unwrap(); + assert!( + !routed, + "a detached call must not be routed, artifact attached or not" + ); + drop((req, art)); + e.settle("no caller-side roots leaked from a detached reply", |s| { + s.artifacts == 0 && s.owners == 0 + }) + .await; + let after = within("result of a cancelled call", dispatched.result()) + .await + .unwrap_err(); + assert_eq!( + (after.code, after.dispatch), + (ErrorCode::CallGone, Dispatch::Unknown) + ); +} + +/// bus-v1 section 8.4: a caller's full disconnection (not merely dropping its pending future) +/// detaches its dispatched call, and the service itself keeps serving other callers afterward. +async fn caller_disconnect_detaches_dispatched_call_but_service_keeps_serving(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("example.vanish", ServiceConfig::default()) + .await + .unwrap(); + let pending = caller + .call("example.vanish", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + caller.close().await; + e.settle("caller gone", |s| s.connections == 1).await; + drop(pending); // inert now: must not panic or double-release + let routed = req.reply(obj(json!({"late": true})), &[]).await.unwrap(); + assert!(!routed, "the vanished caller cannot receive the reply"); + drop(req); + e.settle("everything the vanished caller held is cleaned up", |s| { + s.calls == 0 && s.owners == 0 + }) + .await; + let other = e.client("other").await; + let mut fresh = other + .call("example.vanish", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req2 = within("second request", svc.next()).await.unwrap(); + assert!(req2.reply(obj(json!({"ok": true})), &[]).await.unwrap()); + assert_eq!( + within("result", fresh.result()).await.unwrap().outcome()["ok"], + true + ); +} + +/// bus-v1 section 8.4: disconnect "release[s] that connection's active writers/explicit/delivery +/// roots", for a subscriber holding both delivered-but-unconsumed and still-queued artifacts, +/// while a topic's retained value (owned by the topic, not the connection) survives untouched. +async fn subscriber_disconnect_releases_queued_and_delivered_artifacts_but_not_retention(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + let reader = e.client("reader").await; + publisher + .declare_topic("t.gone", Retained::Latest) + .await + .unwrap(); + let mut sub = reader + .subscribe( + "t.gone", + SubscriptionConfig::bounded().queued(4).in_flight(4), + ) + .await + .unwrap(); + for i in 0..3 { + let a = sealed(&publisher, format!("m{i}").as_bytes(), "text/plain").await; + publisher + .publish("t.gone", obj(json!({})), &[("m", &a)]) + .await + .unwrap(); + } + let first = within("first delivered", sub.next()).await.unwrap(); + e.settle("three sealed objects, all rooted", |s| { + s.sealed_artifacts == 3 + }) + .await; + drop(first); + reader.close().await; + e.settle( + "everything the reader held is gone; only retention remains", + |s| s.sealed_artifacts == 1 && s.subscriptions == 0 && s.connections == 1, + ) + .await; + let art = sealed(&publisher, b"after-disconnect", "text/plain").await; + publisher + .publish("t.gone", obj(json!({})), &[("m", &art)]) + .await + .unwrap(); + drop(art); + e.settle("retention alone carries the new value", |s| { + s.sealed_artifacts == 1 && s.artifact_roots == 1 + }) + .await; +} + +/// bus-v1 section 7: "reject the whole publish; no partial fan-out or retained-latest update." +/// Not exercised with attachments by `bounded_fifo_and_atomic_backpressure`: a refused publish +/// must leave no artifact root anywhere, on any subscriber. `maxQueued` and `maxInFlight` are +/// separate counters, so filling `tight`'s single in-flight credit does not yet overflow it; +/// its one queue slot has to fill too before a third publish overflows it. +async fn bounded_overflow_rolls_back_all_artifact_roots(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + let roomy = e.client("roomy").await; + let tight = e.client("tight").await; + publisher + .declare_topic("t.atomic", Retained::None) + .await + .unwrap(); + let mut a = roomy + .subscribe( + "t.atomic", + SubscriptionConfig::bounded().queued(4).in_flight(4), + ) + .await + .unwrap(); + let _tight_sub = tight + .subscribe( + "t.atomic", + SubscriptionConfig::bounded().queued(1).in_flight(1), + ) + .await + .unwrap(); + let art1 = sealed(&publisher, b"one", "text/plain").await; + publisher + .publish("t.atomic", obj(json!({})), &[("x", &art1)]) + .await + .unwrap(); // fills tight's in-flight credit + drop(art1); + let art2 = sealed(&publisher, b"two", "text/plain").await; + publisher + .publish("t.atomic", obj(json!({})), &[("x", &art2)]) + .await + .unwrap(); // fills tight's one queue slot + drop(art2); + e.settle("two objects rooted before the overflow attempt", |s| { + s.sealed_artifacts == 2 + }) + .await; + let art3 = sealed(&publisher, b"three", "text/plain").await; + let refused = publisher + .publish("t.atomic", obj(json!({})), &[("x", &art3)]) + .await + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::Backpressure); + drop(art3); + e.settle( + "the refused publish left no trace: still exactly the first two objects", + |s| s.sealed_artifacts == 2, + ) + .await; + let m1 = within("a's first", a.next()).await.unwrap(); + assert_eq!(m1.artifact("x").unwrap().read_all().await.unwrap(), b"one"); + let m2 = within("a's second", a.next()).await.unwrap(); + assert_eq!(m2.artifact("x").unwrap().read_all().await.unwrap(), b"two"); + quiet("a never saw the refused publish", a.next()).await; +} + +/// bus-v1 section 7: "New subscriptions with replayLatest enqueue it before subsequent accepted +/// publications." A fresh `latest` subscription's replay claims its first in-flight credit +/// immediately (there is nothing else competing for it yet), so a publish accepted right after +/// subscribing must still be observed strictly after the replay, never ahead of or merged with +/// it: each keeps its own delivery. +async fn latest_replay_is_ordered_ahead_of_a_racing_publish(via: Via) { + let e = env(via).await; + let admin = e.client("admin").await; + let reader = e.client("reader").await; + admin + .declare_topic("t.replay-race", Retained::Latest) + .await + .unwrap(); + admin + .publish("t.replay-race", obj(json!({"v": "old"})), &[]) + .await + .unwrap(); + let mut sub = reader + .subscribe( + "t.replay-race", + SubscriptionConfig::latest().in_flight(1).replay(true), + ) + .await + .unwrap(); + admin + .publish("t.replay-race", obj(json!({"v": "new"})), &[]) + .await + .unwrap(); + let first = within("the replay arrives first", sub.next()) + .await + .unwrap(); + assert_eq!(first.payload()["v"], "old"); + drop(first); // the sole in-flight credit must return before the queued second value moves + let second = within("the racing publish follows, not coalesced away", sub.next()) + .await + .unwrap(); + assert_eq!(second.payload()["v"], "new"); +} + +/// bus-v1 section 7: clearing releases only the retained root; a later `replayLatest` +/// subscription must see nothing until a fresh publish, not a stale or resurrected value. +async fn cleared_topic_gives_no_replay_until_a_fresh_publish(via: Via) { + let e = env(via).await; + let admin = e.client("admin").await; + let reader = e.client("reader").await; + admin + .declare_topic("t.clear-replay", Retained::Latest) + .await + .unwrap(); + admin + .publish("t.clear-replay", obj(json!({"v": 1})), &[]) + .await + .unwrap(); + assert!(admin.clear_topic("t.clear-replay").await.unwrap()); + let mut sub = reader + .subscribe("t.clear-replay", SubscriptionConfig::bounded().replay(true)) + .await + .unwrap(); + quiet("nothing retained to replay after a clear", sub.next()).await; + admin + .publish("t.clear-replay", obj(json!({"v": 2})), &[]) + .await + .unwrap(); + let m = within("a fresh publish arrives normally", sub.next()) + .await + .unwrap(); + assert_eq!(m.payload()["v"], 2); +} + +/// bus-v1 section 8.3: "dropping the message alone does not consume the delivery while a +/// renderer/encoder still uses its artifact." `latest_coalesces_only_undelivered_values` proves +/// this for `latest` mode; delivery-credit accounting must honour it for `bounded` mode too. +async fn bounded_credit_waits_for_every_extracted_artifact(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + let reader = e.client("reader").await; + publisher + .declare_topic("t.credit-hold", Retained::None) + .await + .unwrap(); + let mut sub = reader + .subscribe( + "t.credit-hold", + SubscriptionConfig::bounded().in_flight(1).queued(4), + ) + .await + .unwrap(); + let a0 = sealed(&publisher, b"m0", "text/plain").await; + let a1 = sealed(&publisher, b"m1", "text/plain").await; + publisher + .publish("t.credit-hold", obj(json!({})), &[("x", &a0)]) + .await + .unwrap(); + publisher + .publish("t.credit-hold", obj(json!({})), &[("x", &a1)]) + .await + .unwrap(); + drop((a0, a1)); + let m0 = within("first", sub.next()).await.unwrap(); + let held = m0.artifact("x").unwrap(); + drop(m0); // the message struct is gone, but `held` still shares its delivery guard + quiet( + "credit withheld while an extracted artifact is still alive", + sub.next(), + ) + .await; + drop(held); + let m1 = within("second, only after the real release", sub.next()) + .await + .unwrap(); + assert_eq!(m1.artifact("x").unwrap().read_all().await.unwrap(), b"m1"); +} + +/// Subscription ids are serials local to the issuing connection (`sub-`), not a global +/// namespace: a second connection quoting another connection's literal id string has no route +/// to that subscription at all, so it can only ever land on (at most) its own same-numbered +/// subscription, never the owner's. `unsubscribe` reports this as `removed:false`, not an +/// error, and the owner's subscription keeps receiving messages untouched. +async fn unsubscribe_cannot_reach_another_connections_subscription_id(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + publisher + .declare_topic("t.owned", Retained::None) + .await + .unwrap(); + let mut owner_raw = e.raw_hello("owner").await; + let sub_id = { + let r = owner_raw + .call("subscribe", json!({"topic": "t.owned", "mode": "bounded", "maxQueued": 4, "maxInFlight": 4, "replayLatest": false})) + .await + .unwrap(); + r["subscriptionId"].as_str().unwrap().to_owned() + }; + let mut intruder = e.raw_hello("intruder").await; + let r = intruder + .call("unsubscribe", json!({"subscriptionId": sub_id.clone()})) + .await + .unwrap(); + assert_eq!( + r["removed"], + json!(false), + "the intruder issued no such subscription itself" + ); + + // The owner's subscription is unaffected by the intruder's attempt. + let art = sealed(&publisher, b"still-mine", "text/plain").await; + publisher + .publish("t.owned", obj(json!({})), &[("x", &art)]) + .await + .unwrap(); + drop(art); + let delivered = owner_raw.event().await; + assert_eq!(delivered.op, "topic.message"); + + let removed = owner_raw + .call("unsubscribe", json!({"subscriptionId": sub_id})) + .await + .unwrap(); + assert_eq!(removed["removed"], json!(true)); +} + +both_transports!( + duplicate_registration_by_owner_itself_is_rejected, + unpinned_call_after_incarnation_replacement_reaches_the_new_holder, + out_of_order_replies_correlate_across_concurrent_callers, + cancel_before_dispatch_releases_queued_artifact_roots, + cancel_after_dispatch_then_late_reply_with_artifact_is_not_routed, + caller_disconnect_detaches_dispatched_call_but_service_keeps_serving, + subscriber_disconnect_releases_queued_and_delivered_artifacts_but_not_retention, + bounded_overflow_rolls_back_all_artifact_roots, + latest_replay_is_ordered_ahead_of_a_racing_publish, + cleared_topic_gives_no_replay_until_a_fresh_publish, + bounded_credit_waits_for_every_extracted_artifact, + unsubscribe_cannot_reach_another_connections_subscription_id, +); diff --git a/services/flysim/crates/flybus/tests/conformance_wire.rs b/services/flysim/crates/flybus/tests/conformance_wire.rs new file mode 100644 index 0000000..06ad73d --- /dev/null +++ b/services/flysim/crates/flybus/tests/conformance_wire.rs @@ -0,0 +1,640 @@ +//! Wire and transport conformance (bus-v1 §4 and §11 acceptance test 1): bounded +//! little-endian framing, partial reads/writes, strict JSON (duplicate keys at any depth, +//! invalid UTF-8, unknown fields), malformed ids/u64s, connection negotiation, and behavioral +//! parity between the in-memory and Unix-domain transports. +//! +//! This suite speaks the raw protocol by hand rather than going through the SDK: it sends +//! frames no correct client would ever construct, so it can check what the router does with a +//! hostile or merely buggy peer, not just what a well-behaved one gets back. + +mod common; + +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; + +use common::{Via, code, env, obj, within}; +use flybus::Limits; +use flybus::wire::{Envelope, Kind, MAX_ENVELOPE_BYTES, contract_digest, read_frame, write_frame}; +use serde_json::{Map, Value, json}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf}; + +// ------------------------------------------------------------------------------------------- +// Low-level helpers: a hand-rolled connection independent of `common::Raw`, for the framing +// tests that need control over individual bytes and reads that `Raw`'s all-at-once +// `send_bytes` cannot express. + +/// Forces every read through this wrapper to surface at most one byte, so a caller reading a +/// frame through it can only succeed by looping the way [`read_frame`] does — never by +/// getting lucky with one big read. +struct Trickle { + inner: R, +} + +impl AsyncRead for Trickle { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut one = [0u8; 1]; + let mut small = ReadBuf::new(&mut one); + match Pin::new(&mut this.inner).poll_read(cx, &mut small) { + Poll::Ready(Ok(())) => { + let n = small.filled().len(); + if n > 0 { + buf.put_slice(&one[..n]); + } + Poll::Ready(Ok(())) + } + other => other, + } + } +} + +async fn send_envelope( + wr: &mut (impl AsyncWrite + Unpin), + id: &str, + op: &str, + body: Map, +) { + let env = Envelope::new(id.to_owned(), Kind::Command, op, body); + write_frame(wr, &env.encode().unwrap()).await.unwrap(); +} + +/// Writes the same frame [`send_envelope`] would, one byte at a time, flushing and yielding +/// between every byte. +async fn send_envelope_byte_by_byte( + wr: &mut (impl AsyncWrite + Unpin), + id: &str, + op: &str, + body: Map, +) { + let env = Envelope::new(id.to_owned(), Kind::Command, op, body); + let bytes = env.encode().unwrap(); + let len = (bytes.len() as u32).to_le_bytes(); + for byte in len.iter().chain(bytes.iter()) { + wr.write_all(std::slice::from_ref(byte)).await.unwrap(); + wr.flush().await.unwrap(); + tokio::task::yield_now().await; + } +} + +async fn recv_envelope(rd: &mut (impl AsyncRead + Unpin)) -> Envelope { + let bytes = within("frame", read_frame(rd)) + .await + .unwrap() + .expect("stream open"); + Envelope::decode(&bytes).unwrap() +} + +/// A hand-rolled `bus.hello`, sent as command `msg-1`. Returns the reply's `value` object. +async fn manual_hello( + rd: &mut (impl AsyncRead + Unpin), + wr: &mut (impl AsyncWrite + Unpin), + id: &str, + slow: bool, +) -> Map { + let body = obj( + json!({"clientId": id, "clientIncarnation": format!("inc-{id}"), "supportedMajors": [1]}), + ); + if slow { + send_envelope_byte_by_byte(wr, "msg-1", "bus.hello", body).await; + } else { + send_envelope(wr, "msg-1", "bus.hello", body).await; + } + let reply = recv_envelope(rd).await; + assert_eq!(reply.kind, Kind::Reply); + assert_eq!(reply.reply_to.as_deref(), Some("msg-1")); + assert_eq!( + reply.body["ok"], + json!(true), + "hello failed: {:?}", + reply.body + ); + reply.body["value"].as_object().unwrap().clone() +} + +// ------------------------------------------------------------------------------------------- +// Bounded little-endian framing (bus-v1 §4: "Reject ... zero/oversize frames. Read length +// before allocating.") + +async fn zero_length_frame_closes_the_connection(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("zerolen").await; + raw.send_prefix(0).await; + let body = raw + .closing() + .await + .expect("a zero-length frame must close the connection with a notice"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +/// The length prefix is checked, and the frame refused, before any body bytes are read. We +/// announce an oversize frame and never send its body: an implementation that read the length +/// after allocating (or tried to read the body anyway) would hang here instead of refusing +/// promptly, and `closing`'s bounded wait turns that into a loud failure rather than a stall. +async fn oversize_length_prefix_is_rejected_before_reading_body(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("oversize").await; + raw.send_prefix((MAX_ENVELOPE_BYTES as u32) + 1).await; + let body = raw + .closing() + .await + .expect("an oversize frame must be refused, not hung waiting for a body that never comes"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +/// A frame at exactly the envelope ceiling decodes and dispatches normally; the same shape one +/// byte past it is refused before any JSON parsing. +async fn frame_at_the_size_ceiling_is_accepted_one_byte_over_is_not(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("ceiling").await; + // Pad an otherwise-ordinary rpc.call to an exact byte count: measure the unpadded shape + // once, then fill the gap with a string needing no escaping, so each character costs + // exactly one byte and the target length is hit without any search. + let shape = |pad: usize| { + json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": "msg-9", "replyTo": null, "kind": "command", "op": "rpc.call", + "body": { + "callId": "call-1", "target": "no.such.service", "expectedIncarnation": null, + "method": "M", "payload": {"pad": "a".repeat(pad)}, + }, + "attachments": [], + }) + }; + let base_len = serde_json::to_vec(&shape(0)).unwrap().len(); + let exact = serde_json::to_vec(&shape(MAX_ENVELOPE_BYTES - base_len)).unwrap(); + assert_eq!(exact.len(), MAX_ENVELOPE_BYTES); + raw.send_bytes(&exact).await; + let reply = raw.reply("msg-9").await; + // Decoded and dispatched fine: an ordinary domain reply (no such service), not a refusal. + assert_eq!(code(&reply), "NO_SERVICE"); + + let mut over_raw = e.raw_hello("overceiling").await; + let over = serde_json::to_vec(&shape(MAX_ENVELOPE_BYTES - base_len + 1)).unwrap(); + assert_eq!(over.len(), MAX_ENVELOPE_BYTES + 1); + over_raw.send_bytes(&over).await; + let body = over_raw + .closing() + .await + .expect("a frame one byte over the ceiling must be refused, not parsed"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +/// bus-v1 §4 specifies "u32 little-endian" explicitly. The prefix is built by hand here, +/// independent of the library's own `to_le_bytes` call, for a frame long enough (over 255 +/// bytes) that a big-endian misreading would produce a huge, unmistakably wrong length. A +/// byte-order regression shows up as a bounded timeout below, not a silent pass. +async fn length_prefix_is_little_endian(via: Via) { + let e = env(via).await; + let (mut rd, mut wr) = tokio::io::split(e.transport().await); + manual_hello(&mut rd, &mut wr, "byteorder", false).await; + + let body = obj(json!({"name": "a".repeat(190), "retained": "none"})); + let frame_env = Envelope::new("msg-2".into(), Kind::Command, "topic.declare", body); + let bytes = frame_env.encode().unwrap(); + assert!( + bytes.len() > 255, + "the padding must force a non-trivial high byte in the length" + ); + let len = bytes.len() as u32; + let prefix = [ + (len & 0xFF) as u8, + ((len >> 8) & 0xFF) as u8, + ((len >> 16) & 0xFF) as u8, + ((len >> 24) & 0xFF) as u8, + ]; + assert_eq!( + prefix, + len.to_le_bytes(), + "sanity: the hand-built prefix matches the standard LE encoding" + ); + let mut frame = prefix.to_vec(); + frame.extend_from_slice(&bytes); + wr.write_all(&frame).await.unwrap(); + wr.flush().await.unwrap(); + + let reply = tokio::time::timeout(Duration::from_secs(5), recv_envelope(&mut rd)) + .await + .expect("the router must read the length as little-endian, not hang reinterpreting it"); + assert_eq!(reply.body["ok"], json!(true), "{:?}", reply.body); + assert_eq!(reply.body["value"]["declared"], json!(true)); +} + +// ------------------------------------------------------------------------------------------- +// Partial reads and writes (bus-v1 §4: "Handle partial reads/writes.") + +/// A stream that ends mid-frame is a clean disconnect, not a panic and not a phantom command: +/// the router just stops reading and tears the connection down, the same as for a client that +/// vanishes between frames. +async fn truncated_frame_disconnects_cleanly(via: Via) { + let e = env(via).await; + { + let mut raw = e.raw_hello("truncated").await; + // Announce a 64-byte body, then drop the connection before sending any of it. + raw.send_prefix(64).await; + } + e.settle("the truncated connection is gone", |s| s.connections == 0) + .await; +} + +/// A frame written to the router one byte at a time, with a yield between every byte, decodes +/// exactly as if it had arrived in one write. +async fn a_frame_written_one_byte_at_a_time_still_decodes(via: Via) { + let e = env(via).await; + let (mut rd, mut wr) = tokio::io::split(e.transport().await); + let value = manual_hello(&mut rd, &mut wr, "trickle-writer", true).await; + assert_eq!(value["selectedMajor"], json!(1)); + assert_eq!(value["contractDigest"], json!(contract_digest())); +} + +/// The same guarantee on the reading side: a reply read one byte at a time through +/// [`read_frame`] — the exact function both the router and the client SDK use — reassembles +/// correctly. +async fn a_reply_read_one_byte_at_a_time_still_decodes(via: Via) { + let e = env(via).await; + let (rd, mut wr) = tokio::io::split(e.transport().await); + let mut trickle = Trickle { inner: rd }; + let value = manual_hello(&mut trickle, &mut wr, "trickle-reader", false).await; + assert_eq!(value["selectedMajor"], json!(1)); +} + +// ------------------------------------------------------------------------------------------- +// Strict JSON: duplicate keys at any depth, invalid UTF-8. + +/// bus-v1 §4: "Reject duplicate JSON keys" — the same rule at the top level, nested inside +/// `body`, and nested inside an array element within it. +async fn duplicate_json_keys_are_rejected_at_every_depth(via: Via) { + let e = env(via).await; + let cases: [&[u8]; 4] = [ + br#"{"protocol":"flybus","protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"bus.hello","body":{"clientId":"x","clientIncarnation":"y","supportedMajors":[1]},"attachments":[]}"#, + br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"bus.hello","body":{"clientId":"x","clientId":"z","clientIncarnation":"y","supportedMajors":[1]},"attachments":[]}"#, + br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"rpc.call","body":{"callId":"call-1","target":"a.b","expectedIncarnation":null,"method":"M","payload":{"n":{"d":1,"d":2}}},"attachments":[]}"#, + br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"rpc.call","body":{"callId":"call-1","target":"a.b","expectedIncarnation":null,"method":"M","payload":{"items":[{"x":1,"x":2}]}},"attachments":[]}"#, + ]; + for (i, case) in cases.iter().enumerate() { + let mut raw = e.raw().await; + raw.send_bytes(case).await; + let body = raw.closing().await; + assert!( + body.is_some(), + "case {i}: duplicate keys must close the connection" + ); + assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "case {i}"); + } +} + +/// Invalid UTF-8 anywhere in the frame is refused before JSON parsing starts, whether it falls +/// inside a string value, trails a complete object, or leads the buffer. +async fn invalid_utf8_is_rejected(via: Via) { + let e = env(via).await; + let cases: [&[u8]; 3] = [ + b"{\"protocol\":\"flybus\",\"major\":1,\"minor\":0,\"id\":\"msg-1\",\"replyTo\":null,\"kind\":\"command\",\"op\":\"bus.hello\",\"body\":{\"clientId\":\"\xff\",\"clientIncarnation\":\"y\",\"supportedMajors\":[1]},\"attachments\":[]}", + b"{\"protocol\":\"flybus\",\"major\":1,\"minor\":0,\"id\":\"msg-1\",\"replyTo\":null,\"kind\":\"command\",\"op\":\"bus.hello\",\"body\":{\"clientId\":\"x\",\"clientIncarnation\":\"y\",\"supportedMajors\":[1]},\"attachments\":[]}\xff", + b"\xc0\x80{\"protocol\":\"flybus\"}", + ]; + for (i, case) in cases.iter().enumerate() { + let mut raw = e.raw().await; + raw.send_bytes(case).await; + let body = raw.closing().await; + assert!( + body.is_some(), + "case {i}: invalid UTF-8 must close the connection" + ); + assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "case {i}"); + } +} + +// ------------------------------------------------------------------------------------------- +// Unknown envelope fields: a transport-level refusal at the envelope's own level, an ordinary +// domain error inside an operation's body. + +async fn unknown_top_level_field_closes_the_connection(via: Via) { + let e = env(via).await; + let mut raw = e.raw().await; + let v = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": "msg-1", "replyTo": null, "kind": "command", "op": "bus.hello", + "body": {"clientId": "x", "clientIncarnation": "y", "supportedMajors": [1]}, + "attachments": [], + "extra": true, + }); + raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await; + let body = raw + .closing() + .await + .expect("an unknown envelope field must close the connection"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +/// A field the *operation* does not recognize is a normal `ok:false` reply, not a transport +/// violation: only the envelope's own shape is the wire's concern, so the connection stays +/// open and keeps working afterward. +async fn unknown_body_field_is_a_domain_error_not_a_disconnect(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("unknownfield").await; + let bogus = json!({ + "callId": "call-1", "target": "no.such.service", "expectedIncarnation": null, + "method": "M", "payload": {}, "bogus": true, + }); + let reply = raw.call("rpc.call", bogus).await; + assert_eq!(code(&reply), "INVALID_ENVELOPE"); + let ok = raw + .call( + "topic.declare", + json!({"name": "still.alive", "retained": "none"}), + ) + .await; + assert_eq!(code(&ok), "OK"); +} + +// ------------------------------------------------------------------------------------------- +// Malformed ids and u64s. + +/// Every envelope-level scalar the wire validates before dispatch: a malformed `id`, an +/// unparseable `kind`, and an `op` outside its `[a-z.]` alphabet. Each is refused at decode, +/// before hello state or operation semantics are consulted at all. +async fn malformed_envelope_scalars_are_rejected(via: Via) { + let e = env(via).await; + let hello_body = json!({"clientId": "x", "clientIncarnation": "y", "supportedMajors": [1]}); + let base = |id: &str, kind: &str, op: &str| { + json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": id, "replyTo": null, "kind": kind, "op": op, + "body": hello_body, "attachments": [], + }) + }; + let cases = [ + (base("", "command", "bus.hello"), "empty id"), + (base("MSG-1", "command", "bus.hello"), "uppercase id"), + ( + base("-x", "command", "bus.hello"), + "id starting with a separator", + ), + ( + base(&"a".repeat(65), "command", "bus.hello"), + "id over 64 characters", + ), + (base("msg-1", "bogus", "bus.hello"), "unknown kind"), + (base("msg-1", "command", "Bus.Hello"), "uppercase op"), + (base("msg-1", "command", "bus.hello1"), "digit in op"), + ]; + for (v, what) in cases { + let mut raw = e.raw().await; + raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await; + let body = raw.closing().await; + assert!(body.is_some(), "{what}: must close the connection"); + assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "{what}"); + } +} + +async fn non_null_reply_to_on_a_command_is_rejected(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("replyto").await; + let v = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": "msg-2", "replyTo": "msg-1", "kind": "command", "op": "topic.declare", + "body": {"name": "a.b", "retained": "none"}, "attachments": [], + }); + raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await; + let body = raw + .closing() + .await + .expect("a non-null replyTo on a command must close the connection"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +/// `msg-` is canonical, not merely `Id`-shaped: no leading zero, no missing digits, no +/// other prefix. +async fn non_canonical_command_ids_are_rejected(via: Via) { + let e = env(via).await; + for bad in ["msg-01", "msg-abc", "notmsg-1", "msg--1", "msg-"] { + let mut raw = e.raw().await; + let v = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": bad, "replyTo": null, "kind": "command", "op": "bus.hello", + "body": {"clientId": "x", "clientIncarnation": "y", "supportedMajors": [1]}, + "attachments": [], + }); + raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await; + let body = raw.closing().await; + assert!( + body.is_some(), + "{bad}: a non-canonical command id must close the connection" + ); + assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "{bad}"); + } +} + +async fn non_increasing_command_ids_are_rejected(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("nonincreasing").await; // hello already spent msg-1 + let v = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": "msg-1", "replyTo": null, "kind": "command", "op": "topic.declare", + "body": {"name": "a.b", "retained": "none"}, "attachments": [], + }); + raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await; + let body = raw + .closing() + .await + .expect("a repeated command id must close the connection"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +/// Below the envelope's own `id`, an operation's own ids (`callId`) get the same canonical-U64 +/// treatment, but as an ordinary domain reply: the connection is unharmed by a bad one. +async fn malformed_call_id_is_a_domain_error(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("badcallid").await; + for bad in ["call-01", "call-abc", "callx-1", ""] { + let reply = + raw.call("rpc.call", json!({"callId": bad, "target": "a.b", "expectedIncarnation": null, "method": "M", "payload": {}})).await; + assert_eq!(code(&reply), "INVALID_ENVELOPE", "{bad:?}"); + } +} + +/// U64-string fields (`ipc-v1.md` §2: `"0"` or `[1-9][0-9]*`, at most `u64::MAX`) reject a +/// leading zero, a non-digit, an empty string, whitespace, a decimal point and an overflow — +/// each as a domain error the connection survives. +async fn malformed_u64_fields_are_a_domain_error(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("badu64").await; + for bad in ["01", "-1", "abc", "", "18446744073709551616", " 1", "1.0"] { + let reply = raw + .call( + "artifact.allocate", + json!({"byteLength": bad, "contentType": "application/octet-stream"}), + ) + .await; + assert_eq!(code(&reply), "INVALID_ENVELOPE", "{bad:?}"); + } + let ok = raw + .call( + "artifact.allocate", + json!({"byteLength": "1", "contentType": "application/octet-stream"}), + ) + .await; + assert_eq!(code(&ok), "OK"); +} + +// ------------------------------------------------------------------------------------------- +// Negotiation (bus-v1 §4, "Connection negotiation"). + +async fn first_command_must_be_hello(via: Via) { + let e = env(via).await; + let mut raw = e.raw().await; + raw.command( + "service.register", + json!({"name": "x.y", "maxQueued": 1, "maxInFlight": 1}), + json!([]), + ) + .await; + let body = raw + .closing() + .await + .expect("a non-hello first command must close the connection"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +async fn hello_refuses_an_unsupported_major(via: Via) { + let e = env(via).await; + let mut raw = e.raw().await; + let reply = raw.call("bus.hello", json!({"clientId": "futuristic", "clientIncarnation": "inc-1", "supportedMajors": [2, 3]})).await; + assert_eq!(code(&reply), "VERSION_MISMATCH"); + assert!( + raw.recv().await.is_none(), + "the connection must close right after refusing the major" + ); +} + +async fn hello_refuses_attachments(via: Via) { + let e = env(via).await; + let mut raw = e.raw().await; + let attachments = json!([{ + "name": "a", + "ref": { + "storeId": "s", "artifactId": "a-1", "generation": "1", + "byteLength": "1", "contentType": "x", "digest": null, + }, + "ownerId": "o", + }]); + let reply = raw + .call_with( + "bus.hello", + json!({"clientId": "attacher", "clientIncarnation": "inc-1", "supportedMajors": [1]}), + attachments, + ) + .await; + assert_eq!(code(&reply), "INVALID_ENVELOPE"); +} + +async fn a_second_hello_on_the_same_connection_is_rejected(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("rehello").await; + let v = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": "msg-2", "replyTo": null, "kind": "command", "op": "bus.hello", + "body": {"clientId": "rehello", "clientIncarnation": "inc-2", "supportedMajors": [1]}, + "attachments": [], + }); + raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await; + let body = raw + .closing() + .await + .expect("a second bus.hello must close the connection"); + assert_eq!(body["code"], json!("INVALID_ENVELOPE")); +} + +async fn hello_reports_the_contract_digest_and_valid_limits(via: Via) { + let e = env(via).await; + let mut raw = e.raw().await; + let reply = raw.hello("reporter").await.unwrap(); + assert_eq!(reply["selectedMajor"], json!(1)); + assert_eq!(reply["selectedMinor"], json!(0)); + assert_eq!(reply["contractDigest"], json!(contract_digest())); + assert!(reply["routerId"].as_str().unwrap().starts_with("router-")); + assert!(reply["connectionId"].as_str().unwrap().starts_with("conn-")); + let limits = Limits::from_json(&reply["limits"]) + .expect("the router's own limits object must round-trip"); + assert_eq!(limits, Limits::default()); +} + +both_transports!( + zero_length_frame_closes_the_connection, + oversize_length_prefix_is_rejected_before_reading_body, + frame_at_the_size_ceiling_is_accepted_one_byte_over_is_not, + length_prefix_is_little_endian, + truncated_frame_disconnects_cleanly, + a_frame_written_one_byte_at_a_time_still_decodes, + a_reply_read_one_byte_at_a_time_still_decodes, + duplicate_json_keys_are_rejected_at_every_depth, + invalid_utf8_is_rejected, + unknown_top_level_field_closes_the_connection, + unknown_body_field_is_a_domain_error_not_a_disconnect, + malformed_envelope_scalars_are_rejected, + non_null_reply_to_on_a_command_is_rejected, + non_canonical_command_ids_are_rejected, + non_increasing_command_ids_are_rejected, + malformed_call_id_is_a_domain_error, + malformed_u64_fields_are_a_domain_error, + first_command_must_be_hello, + hello_refuses_an_unsupported_major, + hello_refuses_attachments, + a_second_hello_on_the_same_connection_is_rejected, + hello_reports_the_contract_digest_and_valid_limits, +); + +// ------------------------------------------------------------------------------------------- +// bus-v1 §11 acceptance test 1: "In-memory transport must pass the same tests as Unix +// sockets." Every test above already runs on both (that is what `both_transports!` is for); +// this one drives the identical raw script over both side by side in a single test, so a +// transport-specific quirk in one implementation cannot hide behind "it still passes its own +// copy of the suite." + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memory_and_unix_negotiate_the_identical_contract() { + let mem = env(Via::Memory).await; + let unix = env(Via::Unix).await; + let mut raw_mem = mem.raw().await; + let mut raw_unix = unix.raw().await; + + let hm = raw_mem.hello("parity").await.unwrap(); + let hu = raw_unix.hello("parity").await.unwrap(); + assert_eq!(hm["selectedMajor"], hu["selectedMajor"]); + assert_eq!(hm["selectedMinor"], hu["selectedMinor"]); + assert_eq!(hm["contractDigest"], hu["contractDigest"]); + assert_eq!(hm["limits"], hu["limits"]); + + let cm = raw_mem + .call( + "topic.declare", + json!({"name": "parity.topic", "retained": "latest"}), + ) + .await + .unwrap(); + let cu = raw_unix + .call( + "topic.declare", + json!({"name": "parity.topic", "retained": "latest"}), + ) + .await + .unwrap(); + assert_eq!(cm["declared"], cu["declared"]); + assert!(cm["topicIncarnation"].as_str().unwrap().starts_with("top-")); + assert!(cu["topicIncarnation"].as_str().unwrap().starts_with("top-")); + + // The identical malformed frame gets the identical transport-level refusal on both. + let mut bad_mem = mem.raw_hello("parity-bad").await; + let mut bad_unix = unix.raw_hello("parity-bad").await; + let bad_frame = br#"{"a":1,"a":2}"#; + bad_mem.send_bytes(bad_frame).await; + bad_unix.send_bytes(bad_frame).await; + let bm = bad_mem.closing().await.unwrap(); + let bu = bad_unix.closing().await.unwrap(); + assert_eq!(bm["code"], bu["code"]); +} diff --git a/services/flysim/crates/flybus/tests/integration.rs b/services/flysim/crates/flybus/tests/integration.rs new file mode 100644 index 0000000..c4c95ba --- /dev/null +++ b/services/flysim/crates/flybus/tests/integration.rs @@ -0,0 +1,272 @@ +//! bus-v1 section 11 item 6: two parallel fake agents, complete-batch environment RPC, +//! committed snapshot publication and a deliberately slow presentation consumer, all over one +//! router. Generic services only; nothing here knows what a brain or a game is. + +mod common; + +use std::io::Write; +use std::time::Duration; + +use common::{Via, env_with, obj, within}; +use flybus::{ + Client, ErrorCode, Grants, Limits, Pattern, Policy, Retained, ServiceConfig, SubscriptionConfig, +}; +use serde_json::json; + +const STEPS: u64 = 20; +const FRAME: usize = 160 * 144 * 4; + +fn grants(f: impl FnOnce(&mut Grants)) -> Grants { + let mut g = Grants::default(); + f(&mut g); + g +} + +/// An environment service: each Advance produces a frame artifact filled with the step number. +fn spawn_environment(client: Client, mut svc: flybus::Service) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + while let Some(req) = svc.next().await { + let step = req.payload()["step"].as_u64().unwrap(); + let mut w = client + .artifacts() + .allocate(FRAME as u64, "image/x-rgba") + .await + .unwrap(); + w.write_all(&vec![step as u8; FRAME]).unwrap(); + let frame = w.seal().await.unwrap(); + req.reply( + obj(json!({"step": step, "width": 160, "height": 144})), + &[("frame", &frame)], + ) + .await + .unwrap(); + } + }) +} + +/// An agent service: checks the frame it was sent and answers with a digest of it. +fn spawn_agent(name: &'static str, mut svc: flybus::Service) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + while let Some(req) = svc.next().await { + let step = req.payload()["step"].as_u64().unwrap(); + let bytes = req.artifact("frame").unwrap().read_all().await.unwrap(); + let ok = bytes.len() == FRAME && bytes.iter().all(|b| *b == step as u8); + req.reply( + obj(json!({"agent": name, "step": step, "frameOk": ok})), + &[], + ) + .await + .unwrap(); + } + }) +} + +async fn session_over_one_router(via: Via) { + let policy = Policy::closed() + .client( + "coordinator", + grants(|g| { + g.call = vec![Pattern::prefix("agent."), Pattern::exact("env.demo")]; + g.publish = vec![Pattern::prefix("session.demo.")]; + g.manage_topics = vec![Pattern::prefix("session.demo.")]; + }), + ) + .client( + "agent-a", + grants(|g| g.register = vec![Pattern::exact("agent.fly-a")]), + ) + .client( + "agent-b", + grants(|g| g.register = vec![Pattern::exact("agent.fly-b")]), + ) + .client( + "environment", + grants(|g| g.register = vec![Pattern::exact("env.demo")]), + ) + .client( + "presenter", + grants(|g| g.subscribe = vec![Pattern::prefix("session.demo.")]), + ) + .client( + "recorder", + grants(|g| g.subscribe = vec![Pattern::prefix("session.demo.")]), + ); + let e = env_with(via, Limits::default(), policy).await; + + let environment = e.client("environment").await; + let env_svc = environment + .register( + "env.demo", + ServiceConfig { + max_queued: 4, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + let env_inc = env_svc.incarnation().to_owned(); + let env_task = spawn_environment(environment.clone(), env_svc); + let a = e.client("agent-a").await; + let b = e.client("agent-b").await; + let a_svc = a + .register("agent.fly-a", ServiceConfig::default()) + .await + .unwrap(); + let b_svc = b + .register("agent.fly-b", ServiceConfig::default()) + .await + .unwrap(); + let (a_inc, b_inc) = ( + a_svc.incarnation().to_owned(), + b_svc.incarnation().to_owned(), + ); + let agents = [spawn_agent("fly-a", a_svc), spawn_agent("fly-b", b_svc)]; + + let coordinator = e.client("coordinator").await; + coordinator + .declare_topic("session.demo.snapshots", Retained::Latest) + .await + .unwrap(); + + let presenter = e.client("presenter").await; + // Observers cannot drive the environment. + let denied = presenter + .call( + "env.demo", + None, + "Environment.Advance", + obj(json!({"step": 0})), + &[], + ) + .await + .unwrap_err(); + assert_eq!(denied.code, ErrorCode::NotAuthorized); + let mut slow = presenter + .subscribe( + "session.demo.snapshots", + SubscriptionConfig::latest().in_flight(1), + ) + .await + .unwrap(); + let presenting = tokio::spawn(async move { + let mut seen = Vec::new(); + while let Some(m) = slow.next().await { + let frame = m.artifact("frame").unwrap(); + drop(m); + tokio::time::sleep(Duration::from_millis(25)).await; // a slow renderer + let bytes = frame.read_all().await.unwrap(); + let step = bytes[0] as u64; + seen.push((step, frame.reference().artifact_id.clone())); + if step == STEPS { + break; + } + } + seen + }); + let recorder = e.client("recorder").await; + let mut all = recorder + .subscribe("session.demo.snapshots", SubscriptionConfig::bounded()) + .await + .unwrap(); + let recording = tokio::spawn(async move { + let mut seq = Vec::new(); + while let Some(m) = all.next().await { + seq.push((m.topic_sequence(), m.payload()["step"].as_u64().unwrap())); + if seq.len() as u64 == STEPS { + break; + } + } + seq + }); + + for step in 1..=STEPS { + let advanced = coordinator + .call_and_wait( + "env.demo", + Some(&env_inc), + "Environment.Advance", + obj(json!({"step": step})), + &[], + ) + .await + .unwrap(); + // Forward the delivery-owned frame to both agents at once. + let frame = advanced.artifact("frame").unwrap(); + let attachments = [("frame", &frame)]; + let (ra, rb) = tokio::join!( + coordinator.call_and_wait( + "agent.fly-a", + Some(&a_inc), + "Agent.Prepare", + obj(json!({"step": step})), + &attachments + ), + coordinator.call_and_wait( + "agent.fly-b", + Some(&b_inc), + "Agent.Prepare", + obj(json!({"step": step})), + &attachments + ), + ); + for r in [ra.unwrap(), rb.unwrap()] { + assert_eq!( + ( + r.outcome()["step"].as_u64(), + r.outcome()["frameOk"].as_bool() + ), + (Some(step), Some(true)) + ); + } + let receipt = coordinator + .publish( + "session.demo.snapshots", + obj(json!({"step": step})), + &[("frame", &frame)], + ) + .await + .unwrap(); + assert_eq!(receipt.topic_sequence, step); + } + + let recorded = within("recorder", recording).await.unwrap(); + assert_eq!( + recorded, + (1..=STEPS).map(|s| (s, s)).collect::>(), + "the bounded recorder misses nothing" + ); + let presented = within("presenter", presenting).await.unwrap(); + assert_eq!( + presented.last().unwrap().0, + STEPS, + "the slow consumer ends on the latest snapshot" + ); + assert!( + presented.len() < STEPS as usize, + "the slow consumer skipped snapshots: {presented:?}" + ); + assert!(presented.windows(2).all(|w| w[0].0 < w[1].0)); + + for t in agents { + t.abort(); + } + env_task.abort(); + drop((a, b, environment, presenter, recorder)); + // Only the retained snapshot's frame is left once everyone is gone. + let s = e + .settle("session torn down", |s| { + s.calls == 0 && s.sealed_artifacts == 1 && s.owners == 0 + }) + .await; + assert_eq!(s.store_bytes, FRAME as u64); + assert!( + coordinator + .clear_topic("session.demo.snapshots") + .await + .unwrap() + ); + e.settle("retained frame collected", |s| s.artifacts == 0) + .await; +} + +both_transports!(session_over_one_router); diff --git a/services/flysim/crates/flybus/tests/perf.rs b/services/flysim/crates/flybus/tests/perf.rs new file mode 100644 index 0000000..aa2e4ff --- /dev/null +++ b/services/flysim/crates/flybus/tests/perf.rs @@ -0,0 +1,213 @@ +//! bus-v1 section 11 item 7, as a measurement rather than a gate: 640x480 RGBA frames at +//! 60 Hz over a Unix socket to three consumers (one delayed), with 1, 2 and 4 agent services +//! pinged every frame. Router and clients share this process, so CPU and RSS are the whole +//! process. Run with: +//! +//! ```text +//! cargo test --release -p flybus --test perf -- --ignored --nocapture +//! ``` + +mod common; + +use std::io::Write; +use std::time::{Duration, Instant}; + +use common::{Via, env, obj}; +use flybus::{Client, Retained, ServiceConfig, SubscriptionConfig}; +use serde_json::json; + +const W: usize = 640; +const H: usize = 480; +const HZ: u64 = 60; +const SECONDS: u64 = 2; + +fn proc_cpu_seconds() -> f64 { + // utime + stime, fields 14 and 15 of /proc/self/stat, in clock ticks (100 Hz on Linux). + let stat = std::fs::read_to_string("/proc/self/stat").unwrap_or_default(); + let after = stat.rsplit_once(')').map_or("", |(_, rest)| rest); + let f: Vec<&str> = after.split_whitespace().collect(); + let ticks = |i: usize| f.get(i).and_then(|v| v.parse::().ok()).unwrap_or(0.0); + (ticks(11) + ticks(12)) / 100.0 +} + +fn proc_status(key: &str) -> String { + let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default(); + status + .lines() + .find(|l| l.starts_with(key)) + .map_or("?".into(), |l| l[key.len()..].trim().to_owned()) +} + +fn pct(sorted: &[Duration], p: f64) -> Duration { + if sorted.is_empty() { + return Duration::ZERO; + } + sorted[((sorted.len() - 1) as f64 * p).round() as usize] +} + +async fn consumer(client: Client, delay: Duration) -> (u64, u64) { + let mut sub = client + .subscribe("world.demo.frame", SubscriptionConfig::latest()) + .await + .unwrap(); + let (mut seen, mut replaced) = (0, 0); + while let Some(m) = sub.next().await { + if m.payload().get("end").is_some() { + break; + } + replaced += m.replaced(); + let frame = m.artifact("frame").unwrap(); + drop(m); + let bytes = frame.read_all().await.unwrap(); + assert_eq!(bytes.len(), W * H * 4); + tokio::time::sleep(delay).await; + seen += 1; + } + (seen, replaced) +} + +async fn run(agents: usize) { + let e = env(Via::Unix).await; + let producer = e.client("producer").await; + producer + .declare_topic("world.demo.frame", Retained::None) + .await + .unwrap(); + let mut consumers = Vec::new(); + for (i, delay) in [0u64, 0, 40].into_iter().enumerate() { + let c = e.client(&format!("consumer-{i}")).await; + consumers.push(tokio::spawn(consumer(c, Duration::from_millis(delay)))); + } + let mut services = Vec::new(); + for k in 0..agents { + let c = e.client(&format!("agent-{k}")).await; + let mut svc = c + .register(&format!("agent.a{k}"), ServiceConfig::default()) + .await + .unwrap(); + services.push(tokio::spawn(async move { + let _c = c; + while let Some(req) = svc.next().await { + req.reply(obj(json!({})), &[]).await.unwrap(); + } + })); + } + let caller = e.client("coordinator").await; + // Let every subscription land before the first frame. + e.settle("subscribed", |s| s.subscriptions == 3).await; + + let pixels: Vec = (0..W * H * 4).map(|i| (i % 253) as u8).collect(); + let frames = HZ * SECONDS; + let period = Duration::from_nanos(1_000_000_000 / HZ); + let (mut produce, mut publish, mut rpc) = (Vec::new(), Vec::new(), Vec::new()); + let (mut peak_bytes, mut peak_roots, mut peak_queued, mut late) = (0u64, 0u64, 0usize, 0u32); + let cpu0 = proc_cpu_seconds(); + let start = Instant::now(); + for n in 0..frames { + let deadline = start + period * n as u32; + let t = Instant::now(); + let mut w = producer + .artifacts() + .allocate(pixels.len() as u64, "image/x-rgba") + .await + .unwrap(); + w.write_all(&pixels).unwrap(); + let frame = w.seal().await.unwrap(); + produce.push(t.elapsed()); + let t = Instant::now(); + producer + .publish( + "world.demo.frame", + obj(json!({"n": n})), + &[("frame", &frame)], + ) + .await + .unwrap(); + publish.push(t.elapsed()); + drop(frame); + let mut calls = Vec::new(); + for k in 0..agents { + let caller = caller.clone(); + calls.push(tokio::spawn(async move { + let t = Instant::now(); + caller + .call_and_wait(&format!("agent.a{k}"), None, "Ping", obj(json!({})), &[]) + .await + .unwrap(); + t.elapsed() + })); + } + for c in calls { + rpc.push(c.await.unwrap()); + } + let s = e.stats(); + peak_bytes = peak_bytes.max(s.store_bytes); + peak_roots = peak_roots.max(s.artifact_roots); + peak_queued = peak_queued.max(s.queued); + let next = deadline + period; + if Instant::now() > next { + late += 1; + } else { + tokio::time::sleep_until(next.into()).await; + } + } + let wall = start.elapsed().as_secs_f64(); + let cpu = proc_cpu_seconds() - cpu0; + let end = Instant::now(); + producer + .publish("world.demo.frame", obj(json!({"end": true})), &[]) + .await + .unwrap(); + let mut results = Vec::new(); + for c in consumers { + results.push(c.await.unwrap()); + } + e.settle("collected", |s| s.store_bytes == 0).await; + let collect_lag = end.elapsed(); + for s in services { + s.abort(); + } + for v in [&mut produce, &mut publish, &mut rpc] { + v.sort(); + } + let ms = |d: Duration| format!("{:.2}", d.as_secs_f64() * 1000.0); + println!("agents={agents} frames={frames} over {wall:.2}s, late frames {late}"); + println!( + " produce (allocate+write+seal copy) ms p50/p95/p99: {}/{}/{}", + ms(pct(&produce, 0.5)), + ms(pct(&produce, 0.95)), + ms(pct(&produce, 0.99)) + ); + println!( + " publish admission ms p50/p95/p99: {}/{}/{}", + ms(pct(&publish, 0.5)), + ms(pct(&publish, 0.95)), + ms(pct(&publish, 0.99)) + ); + println!( + " rpc round trip ms p50/p95/p99: {}/{}/{}", + ms(pct(&rpc, 0.5)), + ms(pct(&rpc, 0.95)), + ms(pct(&rpc, 0.99)) + ); + println!( + " process cpu {:.2} cores; VmRSS {} VmHWM {}", + cpu / wall, + proc_status("VmRSS:"), + proc_status("VmHWM:") + ); + println!( + " store peak {:.1} MB, peak roots {peak_roots}, peak queued {peak_queued}, drain+collect {:.1} ms", + peak_bytes as f64 / 1e6, + collect_lag.as_secs_f64() * 1000.0 + ); + println!(" consumers (frames seen, replaced): {results:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "measurement; run with --release -- --ignored --nocapture"] +async fn frames_at_60hz_with_three_consumers() { + for agents in [1, 2, 4] { + run(agents).await; + } +} diff --git a/services/flysim/crates/flybus/tests/pubsub.rs b/services/flysim/crates/flybus/tests/pubsub.rs new file mode 100644 index 0000000..2a29b63 --- /dev/null +++ b/services/flysim/crates/flybus/tests/pubsub.rs @@ -0,0 +1,545 @@ +//! Pub/sub: exact topics, bounded FIFO with whole-publication backpressure, latest coalescing, +//! credits, retention, incarnations and fair control delivery under a saturated subscriber. + +mod common; + +use std::time::Duration; + +use common::{Via, code, env, env_with, obj, quiet, sealed, within}; +use flybus::{ErrorCode, Limits, Mode, Policy, Retained, ServiceConfig, SubscriptionConfig}; +use serde_json::json; + +async fn bounded_fifo_and_atomic_backpressure(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + let slow = e.client("slow").await; + let fast = e.client("fast").await; + publisher + .declare_topic("session.demo.events", Retained::Latest) + .await + .unwrap(); + let mut a = slow + .subscribe( + "session.demo.events", + SubscriptionConfig::bounded().queued(2).in_flight(1), + ) + .await + .unwrap(); + let mut b = fast + .subscribe("session.demo.events", SubscriptionConfig::bounded()) + .await + .unwrap(); + let p = |n: u64| obj(json!({ "n": n })); + + let r1 = publisher + .publish("session.demo.events", p(1), &[]) + .await + .unwrap(); + assert_eq!((r1.topic_sequence, r1.subscribers, r1.replaced), (1, 2, 0)); + let m1 = within("a gets 1", a.next()).await.unwrap(); + publisher + .publish("session.demo.events", p(2), &[]) + .await + .unwrap(); + publisher + .publish("session.demo.events", p(3), &[]) + .await + .unwrap(); + // a holds 1 in flight and 2, 3 queued: the next publication would overflow it, so nobody + // gets it, the retained value does not move and no sequence number is spent. + let full = publisher + .publish("session.demo.events", p(4), &[]) + .await + .unwrap_err(); + assert_eq!(full.code, ErrorCode::Backpressure); + for n in 1..=3u64 { + let m = within("b in order", b.next()).await.unwrap(); + assert_eq!( + (m.payload()["n"].as_u64(), m.topic_sequence()), + (Some(n), n) + ); + } + quiet("b gets no partial fan-out", b.next()).await; + let mut late = fast + .subscribe( + "session.demo.events", + SubscriptionConfig::bounded().replay(true), + ) + .await + .unwrap(); + let replayed = within("replay", late.next()).await.unwrap(); + assert_eq!( + replayed.payload()["n"], + 3, + "the refused publication did not become the retained value" + ); + drop(m1); + let m2 = within("credit returned", a.next()).await.unwrap(); + assert_eq!(m2.payload()["n"], 2); + let r5 = publisher + .publish("session.demo.events", p(5), &[]) + .await + .unwrap(); + assert_eq!(r5.topic_sequence, 4); +} + +async fn latest_coalesces_only_undelivered_values(via: Via) { + let e = env(via).await; + let camera = e.client("camera").await; + let viewer = e.client("viewer").await; + camera + .declare_topic("world.demo.frame", Retained::None) + .await + .unwrap(); + let mut sub = viewer + .subscribe( + "world.demo.frame", + SubscriptionConfig::latest().in_flight(1), + ) + .await + .unwrap(); + let frames: Vec<_> = futures_join(&camera, 4).await; + let r = camera + .publish( + "world.demo.frame", + obj(json!({"n": 1})), + &[("frame", &frames[0])], + ) + .await + .unwrap(); + assert_eq!(r.replaced, 0); + let first = within("first frame", sub.next()).await.unwrap(); + let held = first.artifact("frame").unwrap(); + let mut replaced = 0; + for (i, f) in frames.iter().enumerate().skip(1) { + let r = camera + .publish( + "world.demo.frame", + obj(json!({"n": i + 1})), + &[("frame", f)], + ) + .await + .unwrap(); + replaced += r.replaced; + } + assert_eq!( + replaced, 2, + "frames 2 and 3 were replaced while undelivered" + ); + drop(frames); + // Replaced queue entries released their roots; the delivered frame and the queued one stay. + e.settle("replaced frames collected", |s| s.sealed_artifacts == 2) + .await; + assert_eq!( + held.read_all().await.unwrap(), + b"frame-0", + "delivered data is never reclaimed early" + ); + drop((first, held)); + let last = within("latest frame", sub.next()).await.unwrap(); + assert_eq!((last.topic_sequence(), last.replaced()), (4, 2)); + assert_eq!( + last.artifact("frame").unwrap().read_all().await.unwrap(), + b"frame-3" + ); + drop(last); + e.settle("all collected", |s| s.artifacts == 0 && s.owners == 0) + .await; +} + +async fn futures_join(client: &flybus::Client, n: usize) -> Vec { + let mut out = Vec::new(); + for i in 0..n { + out.push(sealed(client, format!("frame-{i}").as_bytes(), "image/x-rgba").await); + } + out +} + +async fn credits_return_only_on_consume(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + let reader = e.client("reader").await; + publisher + .declare_topic("t.credits", Retained::None) + .await + .unwrap(); + let mut sub = reader + .subscribe("t.credits", SubscriptionConfig::bounded().in_flight(2)) + .await + .unwrap(); + for n in 0..5 { + publisher + .publish("t.credits", obj(json!({"n": n})), &[]) + .await + .unwrap(); + } + let m0 = within("0", sub.next()).await.unwrap(); + let m1 = within("1", sub.next()).await.unwrap(); + quiet("third while two are held", sub.next()).await; + drop(m0); + let m2 = within("2 after a consume", sub.next()).await.unwrap(); + assert_eq!(m2.payload()["n"], 2); + quiet("fourth while two are held", sub.next()).await; + drop((m1, m2)); + assert_eq!(within("3", sub.next()).await.unwrap().payload()["n"], 3); +} + +async fn retained_replay_clear_delete_and_incarnations(via: Via) { + let e = env(via).await; + let admin = e.client("admin").await; + let reader = e.client("reader").await; + let info = admin + .declare_topic("session.demo.snapshots", Retained::Latest) + .await + .unwrap(); + assert!(info.declared); + let again = admin + .declare_topic("session.demo.snapshots", Retained::Latest) + .await + .unwrap(); + assert_eq!( + (again.declared, &again.topic_incarnation), + (false, &info.topic_incarnation) + ); + let conflict = admin + .declare_topic("session.demo.snapshots", Retained::None) + .await + .unwrap_err(); + assert_eq!(conflict.code, ErrorCode::Conflict); + + let snap = sealed(&admin, b"snapshot-1", "application/octet-stream").await; + let r = admin + .publish( + "session.demo.snapshots", + obj(json!({"step": "1"})), + &[("state", &snap)], + ) + .await + .unwrap(); + assert_eq!(r.subscribers, 0); + drop(snap); + e.settle("retention holds the only root", |s| { + s.sealed_artifacts == 1 && s.artifact_roots == 1 + }) + .await; + + let mut plain = reader + .subscribe("session.demo.snapshots", SubscriptionConfig::bounded()) + .await + .unwrap(); + let mut replay = reader + .subscribe( + "session.demo.snapshots", + SubscriptionConfig::latest().replay(true), + ) + .await + .unwrap(); + let m = within("replay", replay.next()).await.unwrap(); + assert_eq!(m.topic_sequence(), 1, "replay keeps the original sequence"); + assert_eq!( + m.artifact("state").unwrap().read_all().await.unwrap(), + b"snapshot-1" + ); + quiet("no replay without replayLatest", plain.next()).await; + + assert!(admin.clear_topic("session.demo.snapshots").await.unwrap()); + assert!(!admin.clear_topic("session.demo.snapshots").await.unwrap()); + // Clearing does not invalidate the delivery still held. + assert_eq!( + m.artifact("state").unwrap().read_all().await.unwrap(), + b"snapshot-1" + ); + drop(m); + e.settle("cleared value collected", |s| s.artifacts == 0) + .await; + + let busy = admin + .delete_topic("session.demo.snapshots") + .await + .unwrap_err(); + assert_eq!(busy.code, ErrorCode::Conflict); + admin + .publish("session.demo.snapshots", obj(json!({"step": "2"})), &[]) + .await + .unwrap(); + let old = within("before delete", plain.next()).await.unwrap(); + drop((plain, replay)); + e.settle("unsubscribed", |s| s.subscriptions == 0).await; + assert!(admin.delete_topic("session.demo.snapshots").await.unwrap()); + assert!(!admin.delete_topic("session.demo.snapshots").await.unwrap()); + let fresh = admin + .declare_topic("session.demo.snapshots", Retained::Latest) + .await + .unwrap(); + assert_ne!(fresh.topic_incarnation, info.topic_incarnation); + assert_eq!( + old.topic_incarnation(), + info.topic_incarnation, + "old deliveries keep their incarnation" + ); + let r = admin + .publish("session.demo.snapshots", obj(json!({})), &[]) + .await + .unwrap(); + assert_eq!( + r.topic_sequence, 1, + "a fresh incarnation restarts its sequence" + ); +} + +async fn zero_subscriber_publish_retains_nothing(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + p.declare_topic("t.void", Retained::None).await.unwrap(); + let art = sealed(&p, &[7u8; 4096], "application/octet-stream").await; + let r = p + .publish("t.void", obj(json!({})), &[("x", &art)]) + .await + .unwrap(); + assert_eq!(r.subscribers, 0); + drop(art); + e.settle("no owner left", |s| s.artifacts == 0 && s.store_bytes == 0) + .await; +} + +async fn unsubscribe_discards_queue_but_not_deliveries(via: Via) { + let e = env(via).await; + let p = e.client("p").await; + let r = e.client("r").await; + p.declare_topic("t.unsub", Retained::None).await.unwrap(); + let mut sub = r + .subscribe("t.unsub", SubscriptionConfig::bounded().in_flight(1)) + .await + .unwrap(); + for i in 0..3 { + let a = sealed(&p, format!("m{i}").as_bytes(), "text/plain").await; + p.publish("t.unsub", obj(json!({})), &[("m", &a)]) + .await + .unwrap(); + } + let first = within("first", sub.next()).await.unwrap(); + e.settle("three objects", |s| s.sealed_artifacts == 3).await; + drop(sub); + e.settle("queued entries released", |s| { + s.sealed_artifacts == 1 && s.subscriptions == 0 + }) + .await; + assert_eq!( + first.artifact("m").unwrap().read_all().await.unwrap(), + b"m0" + ); + drop(first); + e.settle("delivered entry released", |s| s.artifacts == 0) + .await; +} + +async fn subscription_and_topic_validation(via: Via) { + let e = env(via).await; + let c = e.client("c").await; + let missing = c + .subscribe("t.missing", SubscriptionConfig::latest()) + .await + .unwrap_err(); + assert_eq!(missing.code, ErrorCode::NoTopic); + assert_eq!( + c.publish("t.missing", obj(json!({})), &[]) + .await + .unwrap_err() + .code, + ErrorCode::NoTopic + ); + c.declare_topic("t.ok", Retained::None).await.unwrap(); + let bad = SubscriptionConfig { + mode: Mode::Latest, + max_queued: 2, + max_in_flight: 1, + replay_latest: false, + }; + assert_eq!( + c.subscribe("t.ok", bad).await.unwrap_err().code, + ErrorCode::InvalidEnvelope + ); + let over = SubscriptionConfig::latest().in_flight(3); + assert_eq!( + c.subscribe("t.ok", over).await.unwrap_err().code, + ErrorCode::QuotaExceeded + ); + let zero = SubscriptionConfig::bounded().in_flight(0); + assert_eq!( + c.subscribe("t.ok", zero).await.unwrap_err().code, + ErrorCode::InvalidEnvelope + ); + assert_eq!( + c.declare_topic("t..bad", Retained::None) + .await + .unwrap_err() + .code, + ErrorCode::InvalidEnvelope + ); + assert_eq!( + c.declare_topic("T.upper", Retained::None) + .await + .unwrap_err() + .code, + ErrorCode::InvalidEnvelope + ); + // The connection survives every refusal above. + assert!( + c.subscribe("t.ok", SubscriptionConfig::bounded()) + .await + .is_ok() + ); +} + +async fn topic_and_retention_quotas(via: Via) { + let limits = Limits { + max_topics: 2, + max_retained_bytes: 100, + ..Limits::default() + }; + let e = env_with(via, limits, Policy::open()).await; + let c = e.client("c").await; + c.declare_topic("t.a", Retained::Latest).await.unwrap(); + c.declare_topic("t.b", Retained::Latest).await.unwrap(); + assert_eq!( + c.declare_topic("t.c", Retained::None) + .await + .unwrap_err() + .code, + ErrorCode::QuotaExceeded + ); + let small = sealed(&c, &[1u8; 60], "application/octet-stream").await; + let big = sealed(&c, &[2u8; 50], "application/octet-stream").await; + c.publish("t.a", obj(json!({})), &[("x", &small)]) + .await + .unwrap(); + // Two names for one object count once. + c.publish("t.a", obj(json!({})), &[("x", &small), ("y", &small)]) + .await + .unwrap(); + assert_eq!(e.stats().retained_bytes, 60); + let over = c + .publish("t.b", obj(json!({})), &[("x", &big)]) + .await + .unwrap_err(); + assert_eq!(over.code, ErrorCode::QuotaExceeded); + assert_eq!( + e.stats().retained_bytes, + 60, + "a refused publication changes nothing" + ); + // Replacing a topic's own retained value is measured net of the old one. + c.publish("t.a", obj(json!({})), &[("x", &big)]) + .await + .unwrap(); + assert_eq!(e.stats().retained_bytes, 50); +} + +/// A subscriber that never reads its socket must not slow anyone else: publications are +/// admitted until its queue refuses them, and RPC and control traffic between other clients +/// (and to a client that merely holds its credits) keep flowing. +async fn saturated_subscriber_does_not_block_control(via: Via) { + let e = env(via).await; + let publisher = e.client("publisher").await; + let busy = e.client("busy").await; + let caller = e.client("caller").await; + publisher + .declare_topic("t.flood", Retained::None) + .await + .unwrap(); + + // 1. A raw subscriber that never reads. + let mut stuck = e.raw_hello("stuck").await; + let r = stuck + .call("subscribe", json!({"topic": "t.flood", "mode": "bounded", "maxQueued": 64, "maxInFlight": 16, "replayLatest": false})) + .await; + assert_eq!(code(&r), "OK"); + // 2. An SDK subscriber that reads but never consumes, and also serves RPC. + let _held = busy + .subscribe("t.flood", SubscriptionConfig::bounded().in_flight(16)) + .await + .unwrap(); + let mut svc = busy + .register("busy.status", ServiceConfig::default()) + .await + .unwrap(); + let server = tokio::spawn(async move { + while let Some(req) = svc.next().await { + req.reply(obj(json!({"alive": true})), &[]).await.unwrap(); + } + }); + + let blob = "x".repeat(60_000); + let mut accepted = 0; + let refused = loop { + match within( + "publish", + publisher.publish("t.flood", obj(json!({"blob": blob})), &[]), + ) + .await + { + Ok(_) => accepted += 1, + Err(err) => break err, + } + assert!(accepted < 1000, "publications were never refused"); + }; + assert_eq!(refused.code, ErrorCode::Backpressure); + assert!(accepted >= 16, "only {accepted} publications were admitted"); + for _ in 0..20 { + let res = tokio::time::timeout( + Duration::from_secs(2), + caller.call_and_wait("busy.status", None, "Status", obj(json!({})), &[]), + ) + .await + .expect("RPC to a client whose subscription is saturated") + .unwrap(); + assert_eq!(res.outcome()["alive"], true); + } + // The saturated subscriber's own control lane still answers once it reads again. + let id = stuck + .command( + "topic.declare", + json!({"name": "t.other", "retained": "none"}), + json!([]), + ) + .await; + assert_eq!(code(&stuck.reply(&id).await), "OK"); + server.abort(); +} + +async fn router_shutdown_closes_subscriptions_with_notices(via: Via) { + let e = env(via).await; + let admin = e.client("admin").await; + admin.declare_topic("t.stop", Retained::None).await.unwrap(); + let mut raw = e.raw_hello("raw").await; + let sub = raw + .call("subscribe", json!({"topic": "t.stop", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false})) + .await + .unwrap(); + e.router.shutdown(); + let closed = raw.event().await; + assert_eq!( + (closed.op.as_str(), &closed.body["subscriptionId"]), + ("subscription.closed", &sub["subscriptionId"]) + ); + assert_eq!(closed.body["reason"], "router-stopping"); + let last = raw.event().await; + assert_eq!( + (last.op.as_str(), last.body["code"].as_str()), + ("connection.closing", Some("ROUTER_LOST")) + ); + assert!(raw.recv().await.is_none()); +} + +both_transports!( + router_shutdown_closes_subscriptions_with_notices, + bounded_fifo_and_atomic_backpressure, + latest_coalesces_only_undelivered_values, + credits_return_only_on_consume, + retained_replay_clear_delete_and_incarnations, + zero_subscriber_publish_retains_nothing, + unsubscribe_discards_queue_but_not_deliveries, + subscription_and_topic_validation, + topic_and_retention_quotas, + saturated_subscriber_does_not_block_control, +); diff --git a/services/flysim/crates/flybus/tests/rpc.rs b/services/flysim/crates/flybus/tests/rpc.rs new file mode 100644 index 0000000..81b1791 --- /dev/null +++ b/services/flysim/crates/flybus/tests/rpc.rs @@ -0,0 +1,660 @@ +//! RPC: exclusive registration, pinned incarnations, request/reply, FIFO, bounds, +//! cancellation, disconnects and an endpoint-side result cache. Every test runs over both the +//! in-memory transport and a Unix socket. + +mod common; + +use std::collections::HashMap; +use std::io::Write; + +use common::{Via, code, env, env_with, obj, quiet, sealed, within}; +use flybus::{CancelState, Dispatch, ErrorCode, Grants, Limits, Pattern, Policy, ServiceConfig}; +use serde_json::json; + +async fn request_reply_roundtrip(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("example.counter", ServiceConfig::default()) + .await + .unwrap(); + let mut pending = caller + .call( + "example.counter", + Some(svc.incarnation()), + "Counter.Increment", + obj(json!({"amount": 1})), + &[], + ) + .await + .unwrap(); + assert_eq!(pending.service_incarnation(), svc.incarnation()); + let req = within("request", svc.next()).await.unwrap(); + assert_eq!(req.method(), "Counter.Increment"); + assert_eq!(req.payload()["amount"], 1); + assert_eq!(req.target(), "example.counter"); + // The router, not the body, says who called. + assert_eq!(req.caller(), &caller.info().identity); + assert!(req.reply(obj(json!({"value": 1})), &[]).await.unwrap()); + drop(req); + let res = within("result", pending.result()).await.unwrap(); + assert_eq!(res.outcome()["value"], 1); + assert_eq!(res.responder(), &server.info().identity); + assert_eq!(res.call_id(), pending.call_id()); + drop(res); + e.settle("everything consumed", |s| s.calls == 0 && s.owners == 0) + .await; +} + +async fn registration_is_exclusive_and_pinned(via: Via) { + let e = env(via).await; + let a = e.client("a").await; + let b = e.client("b").await; + let caller = e.client("caller").await; + let first = a + .register("agent.fly-a", ServiceConfig::default()) + .await + .unwrap(); + let dup = b + .register("agent.fly-a", ServiceConfig::default()) + .await + .unwrap_err(); + assert_eq!(dup.code, ErrorCode::Conflict); + let old = first.incarnation().to_owned(); + drop(first); + // Unregistration is asynchronous; wait for the name to come free. + let second = loop { + match b.register("agent.fly-a", ServiceConfig::default()).await { + Ok(s) => break s, + Err(err) => assert_eq!(err.code, ErrorCode::Conflict), + } + }; + assert_ne!(second.incarnation(), old); + let changed = caller + .call( + "agent.fly-a", + Some(&old), + "Agent.Prepare", + obj(json!({})), + &[], + ) + .await + .unwrap_err(); + assert_eq!(changed.code, ErrorCode::TargetChanged); + assert_eq!(changed.dispatch, Dispatch::NotDispatched); + let missing = caller + .call("agent.nobody", None, "Agent.Prepare", obj(json!({})), &[]) + .await + .unwrap_err(); + assert_eq!(missing.code, ErrorCode::NoService); + let over = b + .register( + "agent.big", + ServiceConfig { + max_queued: 17, + max_in_flight: 1, + }, + ) + .await + .unwrap_err(); + assert_eq!(over.code, ErrorCode::QuotaExceeded); +} + +async fn authority_is_enforced(via: Via) { + let policy = Policy::closed() + .client( + "server", + Grants { + register: vec![Pattern::exact("a.svc")], + ..Grants::default() + }, + ) + .client( + "caller", + Grants { + call: vec![Pattern::prefix("a.")], + ..Grants::default() + }, + ); + let e = env_with(via, Limits::default(), policy).await; + assert_eq!( + e.try_client("stranger").await.unwrap_err().code, + ErrorCode::NotAuthorized + ); + let server = e.client("server").await; + let caller = e.client("caller").await; + // One live connection per configured identity. + assert_eq!( + e.try_client("caller").await.unwrap_err().code, + ErrorCode::NotAuthorized + ); + assert_eq!( + server + .register("b.svc", ServiceConfig::default()) + .await + .unwrap_err() + .code, + ErrorCode::NotAuthorized + ); + let _svc = server + .register("a.svc", ServiceConfig::default()) + .await + .unwrap(); + assert_eq!( + caller + .register("a.other", ServiceConfig::default()) + .await + .unwrap_err() + .code, + ErrorCode::NotAuthorized + ); + assert_eq!( + caller + .call("b.svc", None, "M", obj(json!({})), &[]) + .await + .unwrap_err() + .code, + ErrorCode::NotAuthorized + ); + assert_eq!( + caller + .declare_topic("a.t", flybus::Retained::None) + .await + .unwrap_err() + .code, + ErrorCode::NotAuthorized + ); + assert!( + caller + .call("a.svc", None, "M", obj(json!({})), &[]) + .await + .is_ok() + ); + // A reconnect must present a new incarnation. + let mut cfg = e.config("server"); + cfg.client_incarnation = Some(server.info().identity.client_incarnation.clone()); + drop(_svc); + server.close().await; + e.settle("server gone", |s| s.connections == 1).await; + let reused = flybus::Client::connect(e.transport_as("server").await, cfg) + .await + .unwrap_err(); + assert_eq!(reused.code, ErrorCode::NotAuthorized); + assert!(e.try_client("server").await.is_ok()); +} + +async fn fifo_dispatch_and_out_of_order_completion(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("example.echo", ServiceConfig::default()) + .await + .unwrap(); + let mut calls = Vec::new(); + for i in 0..6 { + calls.push( + caller + .call("example.echo", None, "Echo", obj(json!({"i": i})), &[]) + .await + .unwrap(), + ); + } + let mut reqs = Vec::new(); + for i in 0..6 { + let r = within("request", svc.next()).await.unwrap(); + assert_eq!(r.payload()["i"], i, "first dispatch is FIFO per caller"); + reqs.push(r); + } + for r in reqs.iter().rev() { + assert!( + r.reply(obj(json!({"echo": r.payload()["i"]})), &[]) + .await + .unwrap() + ); + } + drop(reqs); + for (i, mut c) in calls.into_iter().enumerate() { + let res = within("result", c.result()).await.unwrap(); + assert_eq!(res.outcome()["echo"], i, "results correlate by call id"); + } +} + +async fn service_queue_backpressure(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register( + "example.slow", + ServiceConfig { + max_queued: 1, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + let _c1 = caller + .call("example.slow", None, "Work", obj(json!({"n": 1})), &[]) + .await + .unwrap(); + let held = within("first request", svc.next()).await.unwrap(); + let _c2 = caller + .call("example.slow", None, "Work", obj(json!({"n": 2})), &[]) + .await + .unwrap(); + let full = caller + .call("example.slow", None, "Work", obj(json!({"n": 3})), &[]) + .await + .unwrap_err(); + assert_eq!(full.code, ErrorCode::Backpressure); + assert_eq!(full.dispatch, Dispatch::NotDispatched); + // In-flight credit returns only when the request delivery is consumed. + quiet("second request while the first is held", svc.next()).await; + drop(held); + let second = within("second request", svc.next()).await.unwrap(); + assert_eq!(second.payload()["n"], 2); +} + +async fn cancellation_states(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register( + "example.worker", + ServiceConfig { + max_queued: 4, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + + let mut dispatched = caller + .call("example.worker", None, "Work", obj(json!({"n": 1})), &[]) + .await + .unwrap(); + let req1 = within("request 1", svc.next()).await.unwrap(); + let mut queued = caller + .call("example.worker", None, "Work", obj(json!({"n": 2})), &[]) + .await + .unwrap(); + + assert_eq!( + queued.cancel().await.unwrap(), + CancelState::CancelledBeforeDispatch + ); + let gone = within("cancelled result", queued.result()) + .await + .unwrap_err(); + assert_eq!( + (gone.code, gone.dispatch), + (ErrorCode::CallGone, Dispatch::NotDispatched) + ); + + assert_eq!( + dispatched.cancel().await.unwrap(), + CancelState::ExecutionUnknown + ); + let unknown = within("detached result", dispatched.result()) + .await + .unwrap_err(); + assert_eq!( + (unknown.code, unknown.dispatch), + (ErrorCode::CallGone, Dispatch::Unknown) + ); + // The handler still finishes; its reply reaches nobody and is not an error. + assert!(!req1.reply(obj(json!({"late": true})), &[]).await.unwrap()); + drop(req1); + + let mut done = caller + .call("example.worker", None, "Work", obj(json!({"n": 3})), &[]) + .await + .unwrap(); + let req3 = within("request 3", svc.next()).await.unwrap(); + assert_eq!( + req3.payload()["n"], + 3, + "the cancelled call was never dispatched" + ); + assert!(req3.reply(obj(json!({"ok": 3})), &[]).await.unwrap()); + // Wait for the result to be admitted before cancelling. + e.settle("result admitted", |s| s.calls == 1).await; + let state = done.cancel().await.unwrap(); + assert_eq!(state, CancelState::Completed); + let res = within("completed result", done.result()).await.unwrap(); + assert_eq!(res.outcome()["ok"], 3); + drop((res, req3)); + e.settle("calls retired", |s| s.calls == 0).await; + assert_eq!(done.cancel().await.unwrap(), CancelState::CallGone); +} + +async fn replies_are_single_and_independent_of_the_request_guard(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("example.once", ServiceConfig::default()) + .await + .unwrap(); + let mut pending = caller + .call("example.once", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + let responder = req.responder(); + drop(req); // consumed before replying + assert!( + responder + .reply(obj(json!({"first": true})), &[]) + .await + .unwrap() + ); + let again = responder + .reply(obj(json!({"second": true})), &[]) + .await + .unwrap_err(); + assert_eq!(again.code, ErrorCode::CallGone); + let res = within("result", pending.result()).await.unwrap(); + assert_eq!(res.outcome()["first"], true); +} + +async fn service_disconnect_fails_calls(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register( + "example.fragile", + ServiceConfig { + max_queued: 4, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + let mut c1 = caller + .call("example.fragile", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let held = within("request", svc.next()).await.unwrap(); + let mut c2 = caller + .call("example.fragile", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + // Keep the first delivery credit occupied until unregister has synchronously failed the + // queued call. Otherwise consuming it may truthfully dispatch call 2 before unregister. + drop(svc); + let r2 = within("queued call", c2.result()).await.unwrap_err(); + assert_eq!( + (r2.code, r2.dispatch), + (ErrorCode::NoService, Dispatch::NotDispatched) + ); + drop(held); + server.close().await; + let r1 = within("dispatched call", c1.result()).await.unwrap_err(); + assert_eq!( + (r1.code, r1.dispatch), + (ErrorCode::NoService, Dispatch::Dispatched) + ); + e.settle("nothing left", |s| s.calls == 0 && s.services == 0) + .await; +} + +async fn unregister_fails_queued_but_dispatched_may_reply(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register( + "example.leaving", + ServiceConfig { + max_queued: 4, + max_in_flight: 1, + }, + ) + .await + .unwrap(); + let mut c1 = caller + .call("example.leaving", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + let mut c2 = caller + .call("example.leaving", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + drop(svc); + let r2 = within("queued call", c2.result()).await.unwrap_err(); + assert_eq!( + (r2.code, r2.dispatch), + (ErrorCode::NoService, Dispatch::NotDispatched) + ); + assert!(req.reply(obj(json!({"done": true})), &[]).await.unwrap()); + assert_eq!( + within("dispatched call", c1.result()) + .await + .unwrap() + .outcome()["done"], + true + ); +} + +async fn raw_call_ids_and_forged_replies(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let mut svc = server + .register("example.raw", ServiceConfig::default()) + .await + .unwrap(); + let mut raw = e.raw_hello("rawcaller").await; + let call = |id: &str| json!({"callId": id, "target": "example.raw", "expectedIncarnation": null, "method": "M", "payload": {}}); + assert_eq!(code(&raw.call("rpc.call", call("call-5")).await), "OK"); + assert_eq!( + code(&raw.call("rpc.call", call("call-5")).await), + "INVALID_ENVELOPE" + ); + assert_eq!( + code(&raw.call("rpc.call", call("call-3")).await), + "INVALID_ENVELOPE" + ); + assert_eq!( + code(&raw.call("rpc.call", call("call-05")).await), + "INVALID_ENVELOPE" + ); + // Identity comes from the connection; a body cannot claim one. + let mut forged = call("call-6"); + forged["caller"] = json!({"clientId": "server", "clientIncarnation": "x"}); + assert_eq!( + code(&raw.call("rpc.call", forged).await), + "INVALID_ENVELOPE" + ); + assert_eq!(code(&raw.call("rpc.call", call("call-6")).await), "OK"); + let req = within("request", svc.next()).await.unwrap(); + assert_eq!(req.caller().client_id, "rawcaller"); + // A third party cannot answer someone else's request. + let mut other = e.raw_hello("intruder").await; + let r = other + .call( + "rpc.reply", + json!({"callId": req.call_id(), "requestDeliveryId": req.delivery_id(), "outcome": {}}), + ) + .await; + assert_eq!(code(&r), "OWNER_INVALID"); + assert!(req.reply(obj(json!({"real": true})), &[]).await.unwrap()); + let result = raw.event().await; + assert_eq!(result.op, "rpc.result"); + assert_eq!(result.body["outcome"]["real"], true); + assert_eq!(result.body["responder"]["clientId"], "server"); +} + +/// bus-v1 section 6: an endpoint caches Artifact handles plus payload; a domain retry with a +/// fresh call id gets fresh delivery ownership over the same immutable bytes. +async fn endpoint_cache_replays_artifact_results(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("agent.cached", ServiceConfig::default()) + .await + .unwrap(); + let service = tokio::spawn(async move { + let mut cache: HashMap = HashMap::new(); + let mut executions = 0; + while let Some(req) = svc.next().await { + if req.method() == "Cache.Evict" { + cache.clear(); + req.reply(obj(json!({})), &[]).await.unwrap(); + continue; + } + let rid = req.payload()["requestId"].as_str().unwrap().to_owned(); + if !cache.contains_key(&rid) { + executions += 1; + let bytes = format!("result of {rid}").into_bytes(); + let mut w = server + .artifacts() + .allocate(bytes.len() as u64, "text/plain") + .await + .unwrap(); + w.write_all(&bytes).unwrap(); + cache.insert(rid.clone(), w.seal().await.unwrap()); + } + let art = cache[&rid].clone(); + req.reply(obj(json!({"executions": executions})), &[("state", &art)]) + .await + .unwrap(); + } + }); + let mut ids = Vec::new(); + for _ in 0..2 { + let res = within( + "result", + caller.call_and_wait( + "agent.cached", + None, + "Agent.Prepare", + obj(json!({"requestId": "req-41"})), + &[], + ), + ) + .await + .unwrap(); + assert_eq!( + res.outcome()["executions"], + 1, + "the retry did not re-execute" + ); + let art = res.artifact("state").unwrap(); + assert_eq!(art.read_all().await.unwrap(), b"result of req-41"); + ids.push(( + res.delivery_id().to_owned(), + art.reference().artifact_id.clone(), + )); + } + assert_ne!(ids[0].0, ids[1].0, "each replay is a fresh delivery"); + assert_eq!(ids[0].1, ids[1].1, "of the same immutable object"); + e.settle("cache holds the only root", |s| { + s.sealed_artifacts == 1 && s.artifact_roots == 1 + }) + .await; + caller + .call_and_wait("agent.cached", None, "Cache.Evict", obj(json!({})), &[]) + .await + .unwrap(); + e.settle("eviction collects", |s| { + s.artifacts == 0 && s.store_bytes == 0 + }) + .await; + e.settle_files("sealed", 0).await; + service.abort(); +} + +async fn dropped_call_is_cancelled_and_late_result_consumed(via: Via) { + let e = env(via).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut svc = server + .register("example.abandon", ServiceConfig::default()) + .await + .unwrap(); + let art = sealed(&server, b"payload", "text/plain").await; + let pending = caller + .call("example.abandon", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + drop(pending); + e.settle("call detached", |s| s.calls == 1 && s.active_calls == 0) + .await; + // Detached: the reply is not routed and creates no caller-side roots. + let routed = req.reply(obj(json!({})), &[("a", &art)]).await.unwrap(); + assert!(!routed); + drop((req, art)); + e.settle("nothing retained", |s| { + s.calls == 0 && s.artifacts == 0 && s.owners == 0 + }) + .await; + + // A result that arrives after its caller stopped waiting is consumed by the reactor. + let pending = caller + .call("example.abandon", None, "Do", obj(json!({})), &[]) + .await + .unwrap(); + let req = within("request", svc.next()).await.unwrap(); + let art = sealed(&server, b"late", "text/plain").await; + req.reply(obj(json!({})), &[("a", &art)]).await.unwrap(); + drop((req, art)); + e.settle("result delivered", |s| s.calls == 1 && s.owners == 1) + .await; + drop(pending); + e.settle("result consumed", |s| { + s.calls == 0 && s.artifacts == 0 && s.owners == 0 + }) + .await; + assert_eq!(caller.control_errors(), 0); +} + +async fn active_call_limit(via: Via) { + let limits = Limits { + max_active_calls_per_client: 2, + ..Limits::default() + }; + let e = env_with(via, limits, Policy::open()).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let _svc = server + .register("example.limit", ServiceConfig::default()) + .await + .unwrap(); + let _a = caller + .call("example.limit", None, "M", obj(json!({})), &[]) + .await + .unwrap(); + let _b = caller + .call("example.limit", None, "M", obj(json!({})), &[]) + .await + .unwrap(); + let c = caller + .call("example.limit", None, "M", obj(json!({})), &[]) + .await + .unwrap_err(); + assert_eq!(c.code, ErrorCode::Backpressure); +} + +both_transports!( + request_reply_roundtrip, + registration_is_exclusive_and_pinned, + authority_is_enforced, + fifo_dispatch_and_out_of_order_completion, + service_queue_backpressure, + cancellation_states, + replies_are_single_and_independent_of_the_request_guard, + service_disconnect_fails_calls, + unregister_fails_queued_but_dispatched_may_reply, + raw_call_ids_and_forged_replies, + endpoint_cache_replays_artifact_results, + dropped_call_is_cancelled_and_late_result_consumed, + active_call_limit, +); diff --git a/services/flysim/crates/flybus/tests/sol_rereview_regressions.rs b/services/flysim/crates/flybus/tests/sol_rereview_regressions.rs new file mode 100644 index 0000000..38042c9 --- /dev/null +++ b/services/flysim/crates/flybus/tests/sol_rereview_regressions.rs @@ -0,0 +1,876 @@ +mod common; + +use std::io; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; + +use common::{Raw, Via, env, env_with, obj, sealed, within}; +use flybus::wire::parse_serial_id; +use flybus::{ + Client, ClientConfig, Dispatch, ErrorCode, Limits, Policy, Retained, Router, RouterConfig, + ServiceConfig, SubscriptionConfig, Transport, +}; +use serde_json::json; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::sync::Notify; + +struct WriteProbe { + inner: S, + state: Arc>, + first: Arc, +} + +#[derive(Default)] +struct ProbeState { + armed: bool, + released: bool, + first_written: bool, + writer: Option, + bytes: Vec, + ops: Vec, +} + +impl WriteProbe { + fn new(inner: S) -> (WriteProbe, WriteProbeHandle) { + let state = Arc::new(Mutex::new(ProbeState::default())); + let first = Arc::new(Notify::new()); + ( + WriteProbe { + inner, + state: state.clone(), + first: first.clone(), + }, + WriteProbeHandle { state, first }, + ) + } +} + +#[derive(Clone)] +struct WriteProbeHandle { + state: Arc>, + first: Arc, +} + +impl WriteProbeHandle { + fn arm(&self) { + let mut state = self.state.lock().unwrap(); + state.armed = true; + state.released = false; + state.first_written = false; + state.writer = None; + state.bytes.clear(); + state.ops.clear(); + } + + fn release(&self) { + let mut state = self.state.lock().unwrap(); + state.released = true; + if let Some(waker) = state.writer.take() { + waker.wake(); + } + } + + async fn wait_first(&self) { + loop { + let notified = self.first.notified(); + if self.state.lock().unwrap().first_written { + return; + } + tokio::time::timeout(Duration::from_secs(2), notified) + .await + .expect("router never wrote the first byte"); + } + } + + fn ops(&self) -> Vec { + self.state.lock().unwrap().ops.clone() + } +} + +impl AsyncRead for WriteProbe { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl AsyncWrite for WriteProbe { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + { + let mut state = self.state.lock().unwrap(); + if state.armed && !state.released && state.first_written { + state.writer = Some(cx.waker().clone()); + return Poll::Pending; + } + } + let limit = if self.state.lock().unwrap().armed { + 1 + } else { + buf.len() + }; + match Pin::new(&mut self.inner).poll_write(cx, &buf[..limit]) { + Poll::Ready(Ok(n)) => { + let mut state = self.state.lock().unwrap(); + state.bytes.extend_from_slice(&buf[..n]); + while state.bytes.len() >= 4 { + let len = u32::from_le_bytes(state.bytes[..4].try_into().unwrap()) as usize; + if state.bytes.len() < len + 4 { + break; + } + let frame = state.bytes.drain(..len + 4).collect::>(); + let envelope = flybus::wire::Envelope::decode(&frame[4..]).unwrap(); + state.ops.push(envelope.op); + } + if state.armed && !state.first_written && n > 0 { + state.first_written = true; + self.first.notify_waiters(); + } + Poll::Ready(Ok(n)) + } + other => other, + } + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + +struct PollHold { + inner: S, + state: Arc<(Mutex, Condvar)>, + entered: Arc, +} + +#[derive(Default)] +struct PollHoldState { + armed: bool, + entered: bool, + released: bool, +} + +#[derive(Clone)] +struct PollHoldHandle { + state: Arc<(Mutex, Condvar)>, + entered: Arc, +} + +impl PollHold { + fn new(inner: S) -> (PollHold, PollHoldHandle) { + let state = Arc::new((Mutex::new(PollHoldState::default()), Condvar::new())); + let entered = Arc::new(Notify::new()); + ( + PollHold { + inner, + state: state.clone(), + entered: entered.clone(), + }, + PollHoldHandle { state, entered }, + ) + } +} + +impl PollHoldHandle { + fn arm(&self) { + self.state.0.lock().unwrap().armed = true; + } + + async fn wait_entered(&self) { + loop { + let notified = self.entered.notified(); + if self.state.0.lock().unwrap().entered { + return; + } + tokio::time::timeout(Duration::from_secs(2), notified) + .await + .expect("transport poll_write was never entered"); + } + } + + fn release(&self) { + let mut state = self.state.0.lock().unwrap(); + state.released = true; + self.state.1.notify_all(); + } +} + +impl AsyncRead for PollHold { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl AsyncWrite for PollHold { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let should_hold = { + let mut state = self.state.0.lock().unwrap(); + if state.armed && !state.entered { + state.entered = true; + self.entered.notify_waiters(); + true + } else { + false + } + }; + if should_hold { + let mut state = self.state.0.lock().unwrap(); + while !state.released { + state = self.state.1.wait(state).unwrap(); + } + cx.waker().wake_by_ref(); + return Poll::Pending; + } + Pin::new(&mut self.inner).poll_write(cx, &buf[..1]) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + +async fn wait_for_single_delivery_owner(router: &Router) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + let stats = router.stats(); + if stats.owners == 1 && stats.artifact_roots == 1 { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "source hold was not released before teardown: {stats:?}" + ); + tokio::task::yield_now().await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropping_last_attached_responder_settles_the_call() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.abandoned", ServiceConfig::default()) + .await + .unwrap(); + let mut pending = caller + .call("example.abandoned", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + + let request = within("request", service.next()).await.unwrap(); + drop(request); + tokio::time::sleep(Duration::from_millis(100)).await; + + let stats = e.stats(); + assert_eq!(stats.reply_capabilities, 0); + assert_eq!(stats.owners, 0); + assert_eq!(stats.calls, 0, "the last reply capability was released"); + assert_eq!(stats.active_calls, 0, "the caller slot was not retired"); + let error = tokio::time::timeout(Duration::from_millis(250), pending.result()) + .await + .expect("the caller was left waiting after all reply authority was gone") + .unwrap_err(); + assert_eq!( + (error.code, error.dispatch), + (ErrorCode::CallGone, Dispatch::Dispatched) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropping_last_attached_responder_releases_request_attachments() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.abandoned-artifact", ServiceConfig::default()) + .await + .unwrap(); + let artifact = sealed(&caller, b"request", "application/octet-stream").await; + let mut pending = caller + .call( + "example.abandoned-artifact", + None, + "Work", + obj(json!({})), + &[("data", &artifact)], + ) + .await + .unwrap(); + drop(artifact); + + let request = within("artifact request", service.next()).await.unwrap(); + drop(request); + let error = within("abandoned artifact call", pending.result()) + .await + .unwrap_err(); + assert_eq!( + (error.code, error.dispatch), + (ErrorCode::CallGone, Dispatch::Dispatched) + ); + e.settle("abandoned artifact released", |stats| { + stats.calls == 0 + && stats.active_calls == 0 + && stats.reply_capabilities == 0 + && stats.owners == 0 + && stats.artifacts == 0 + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropping_last_responder_clone_settles_the_call() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.abandoned-clone", ServiceConfig::default()) + .await + .unwrap(); + let mut pending = caller + .call("example.abandoned-clone", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + + let request = within("clone request", service.next()).await.unwrap(); + let responder = request.responder(); + drop(request); + e.settle("clone retains reply capability", |stats| { + stats.calls == 1 + && stats.active_calls == 1 + && stats.reply_capabilities == 1 + && stats.owners == 0 + }) + .await; + drop(responder); + let error = within("clone release failure", pending.result()) + .await + .unwrap_err(); + assert_eq!( + (error.code, error.dispatch), + (ErrorCode::CallGone, Dispatch::Dispatched) + ); + e.settle("clone release retired call", |stats| { + stats.calls == 0 && stats.active_calls == 0 && stats.reply_capabilities == 0 + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropping_last_responder_clone_with_attachment_settles_the_call() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.abandoned-clone-artifact", ServiceConfig::default()) + .await + .unwrap(); + let artifact = sealed(&caller, b"request", "application/octet-stream").await; + let mut pending = caller + .call( + "example.abandoned-clone-artifact", + None, + "Work", + obj(json!({})), + &[("data", &artifact)], + ) + .await + .unwrap(); + drop(artifact); + + let request = within("clone artifact request", service.next()) + .await + .unwrap(); + let responder = request.responder(); + drop(request); + e.settle("clone keeps only reply capability", |stats| { + stats.calls == 1 + && stats.active_calls == 1 + && stats.reply_capabilities == 1 + && stats.owners == 0 + && stats.artifacts == 0 + }) + .await; + drop(responder); + let error = within("clone artifact release failure", pending.result()) + .await + .unwrap_err(); + assert_eq!( + (error.code, error.dispatch), + (ErrorCode::CallGone, Dispatch::Dispatched) + ); + e.settle("clone artifact release retired call", |stats| { + stats.calls == 0 && stats.active_calls == 0 && stats.reply_capabilities == 0 + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fair_topic_insertion_preserves_router_envelope_order() { + let limits = Limits { + max_control_frames: 4096, + max_control_bytes: 16 << 20, + ..Limits::default() + }; + let e = env_with(Via::Memory, limits, Policy::open()).await; + let publisher = e.client("publisher").await; + publisher + .declare_topic("t.order", Retained::None) + .await + .unwrap(); + + let mut raw = e.raw_hello("reader").await; + raw.call( + "subscribe", + json!({"topic": "t.order", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}), + ) + .await + .unwrap(); + let mut writer = raw.take_writer(); + let flood = tokio::spawn(async move { + for n in 3..3000u64 { + let envelope = json!({ + "protocol": "flybus", "major": 1, "minor": 0, + "id": format!("msg-{n}"), "replyTo": null, "kind": "command", + "op": "no.such.op", "body": {}, "attachments": [] + }); + if !writer.send(&serde_json::to_vec(&envelope).unwrap()).await { + break; + } + } + }); + + tokio::time::sleep(Duration::from_millis(200)).await; + publisher + .publish("t.order", obj(json!({"ready": true})), &[]) + .await + .unwrap(); + + let mut last = 0; + let mut saw_delivery = false; + for _ in 0..2500 { + let envelope = within("router frame", raw.recv()).await.unwrap(); + let serial = parse_serial_id("bus", &envelope.id).unwrap(); + assert!( + serial > last, + "router emitted {} after bus-{last}", + envelope.id + ); + last = serial; + saw_delivery |= envelope.op == "topic.message"; + if saw_delivery && envelope.kind == flybus::wire::Kind::Reply { + break; + } + } + assert!( + saw_delivery, + "fair scheduling never inserted the topic message" + ); + flood.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sdk_accepts_fair_topic_insertion_through_saturated_control_backlog() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.policy = Policy::open(); + config.limits.max_control_frames = 4096; + config.limits.max_control_bytes = 16 << 20; + let router = Router::new(config).unwrap(); + let publisher = Client::connect( + router.connect_in_memory_as("publisher"), + ClientConfig::new("publisher", dir.path()), + ) + .await + .unwrap(); + publisher + .declare_topic("t.sdk-order", Retained::None) + .await + .unwrap(); + + let (client_stream, router_stream) = tokio::io::duplex(64 * 1024); + let (probed, probe) = WriteProbe::new(router_stream); + router.serve_as(probed, "reader"); + let reader = Client::connect( + Transport::from_stream(client_stream), + ClientConfig::new("reader", dir.path()), + ) + .await + .unwrap(); + let mut subscription = reader + .subscribe("t.sdk-order", SubscriptionConfig::latest().in_flight(1)) + .await + .unwrap(); + probe.arm(); + + let mut backlog = Vec::new(); + let completed = Arc::new(AtomicUsize::new(0)); + for n in 0..256 { + let client = reader.clone(); + let completed = completed.clone(); + backlog.push(tokio::spawn(async move { + let result = client + .declare_topic(&format!("t.backlog-{n}"), Retained::None) + .await; + completed.fetch_add(1, Ordering::SeqCst); + result + })); + } + probe.wait_first().await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + while router.stats().topics < 257 { + assert!( + tokio::time::Instant::now() < deadline, + "control backlog was not admitted" + ); + tokio::task::yield_now().await; + } + assert_eq!( + completed.load(Ordering::SeqCst), + 0, + "writer was not blocked" + ); + publisher + .publish("t.sdk-order", obj(json!({"ready": true})), &[]) + .await + .unwrap(); + assert_eq!(router.stats().queued, 1, "topic delivery was not queued"); + + probe.release(); + let message = within("SDK fair delivery", subscription.next()) + .await + .expect("subscription closed on router envelope ordering"); + assert_eq!(message.payload()["ready"], true); + assert!( + reader.closed().is_none(), + "strict SDK rejected router output" + ); + for task in backlog { + task.await.unwrap().unwrap(); + } + let ops = probe.ops(); + let topic = ops + .iter() + .position(|op| op == "topic.message") + .expect("instrumented stream did not carry the topic delivery"); + assert!( + ops[topic + 1..].iter().any(|op| op == "topic.declare"), + "topic delivery was not fairly inserted ahead of remaining controls: {ops:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_does_not_deliver_a_frame_after_releasing_its_owner() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.policy = Policy::open(); + let router = Router::new(config).unwrap(); + let publisher = Client::connect( + router.connect_in_memory_as("publisher"), + ClientConfig::new("publisher", dir.path()), + ) + .await + .unwrap(); + publisher + .declare_topic("t.shutdown", Retained::None) + .await + .unwrap(); + + let (client_stream, router_stream) = tokio::io::duplex(64 * 1024); + let (probed, probe) = WriteProbe::new(router_stream); + router.serve_as(probed, "reader"); + let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path()); + raw.hello("reader").await.unwrap(); + raw.call( + "subscribe", + json!({"topic": "t.shutdown", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}), + ) + .await + .unwrap(); + probe.arm(); + + let artifact = sealed(&publisher, b"still-owned", "application/octet-stream").await; + publisher + .publish("t.shutdown", obj(json!({})), &[("data", &artifact)]) + .await + .unwrap(); + drop(artifact); + probe.wait_first().await; + wait_for_single_delivery_owner(&router).await; + let before = router.stats(); + assert_eq!((before.owners, before.artifact_roots), (1, 1)); + + router.shutdown(); + assert_eq!( + router.stats().owners, + 0, + "shutdown released the selected owner" + ); + assert_eq!( + router.stats().artifacts, + 0, + "shutdown collected its attachment" + ); + assert!( + within("truncated topic frame", raw.recv()).await.is_none(), + "a complete frame was appended after a partial topic delivery" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn protocol_close_cancels_partial_topic_frame_before_reclaiming_attachment() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.policy = Policy::open(); + let router = Router::new(config).unwrap(); + let publisher = Client::connect( + router.connect_in_memory_as("publisher"), + ClientConfig::new("publisher", dir.path()), + ) + .await + .unwrap(); + publisher + .declare_topic("t.close", Retained::None) + .await + .unwrap(); + let (client_stream, router_stream) = tokio::io::duplex(64 * 1024); + let (probed, probe) = WriteProbe::new(router_stream); + router.serve_as(probed, "reader"); + let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path()); + raw.hello("reader").await.unwrap(); + raw.call( + "subscribe", + json!({"topic": "t.close", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}), + ) + .await + .unwrap(); + probe.arm(); + + let artifact = sealed(&publisher, b"close", "application/octet-stream").await; + publisher + .publish("t.close", obj(json!({})), &[("data", &artifact)]) + .await + .unwrap(); + drop(artifact); + probe.wait_first().await; + wait_for_single_delivery_owner(&router).await; + + raw.next = 0; + raw.command("topic.declare", json!({}), json!([])).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + while router.stats().connections != 1 { + assert!( + tokio::time::Instant::now() < deadline, + "protocol close did not finish" + ); + tokio::task::yield_now().await; + } + assert_eq!(router.stats().owners, 0); + assert_eq!(router.stats().artifacts, 0); + assert!( + within("protocol-close truncated frame", raw.recv()) + .await + .is_none(), + "a final notice was appended after a partial frame" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn teardown_waits_for_an_active_transport_poll_before_reclaiming() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.policy = Policy::open(); + let router = Router::new(config).unwrap(); + let publisher = Client::connect( + router.connect_in_memory_as("publisher"), + ClientConfig::new("publisher", dir.path()), + ) + .await + .unwrap(); + publisher + .declare_topic("t.poll-gate", Retained::None) + .await + .unwrap(); + let (client_stream, router_stream) = tokio::io::duplex(64 * 1024); + let (held, hold) = PollHold::new(router_stream); + router.serve_as(held, "reader"); + let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path()); + raw.hello("reader").await.unwrap(); + raw.call( + "subscribe", + json!({"topic": "t.poll-gate", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}), + ) + .await + .unwrap(); + hold.arm(); + + let artifact = sealed(&publisher, b"poll", "application/octet-stream").await; + let sealed_path = router + .store_dir() + .join("sealed") + .join(&artifact.reference().artifact_id); + publisher + .publish("t.poll-gate", obj(json!({})), &[("data", &artifact)]) + .await + .unwrap(); + drop(artifact); + hold.wait_entered().await; + wait_for_single_delivery_owner(&router).await; + + let done = Arc::new(AtomicBool::new(false)); + let shutdown_done = done.clone(); + let shutdown_router = router.clone(); + let shutdown = std::thread::spawn(move || { + shutdown_router.shutdown(); + shutdown_done.store(true, Ordering::SeqCst); + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !done.load(Ordering::SeqCst), + "teardown completed while poll_write was active" + ); + assert!( + sealed_path.exists(), + "artifact was reclaimed while poll_write was active" + ); + + hold.release(); + shutdown.join().unwrap(); + assert!(done.load(Ordering::SeqCst)); + assert_eq!(router.stats().owners, 0); + assert_eq!(router.stats().artifacts, 0); + assert!(!sealed_path.exists()); + while let Some(envelope) = within("poll-gate close", raw.recv()).await { + assert_ne!( + envelope.op, "topic.message", + "delivery completed after teardown reclaimed its owner" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_cancels_partial_rpc_request_before_reclaiming_attachment() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.policy = Policy::open(); + let router = Router::new(config).unwrap(); + let caller = Client::connect( + router.connect_in_memory_as("caller"), + ClientConfig::new("caller", dir.path()), + ) + .await + .unwrap(); + let (client_stream, router_stream) = tokio::io::duplex(64 * 1024); + let (probed, probe) = WriteProbe::new(router_stream); + router.serve_as(probed, "server"); + let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path()); + raw.hello("server").await.unwrap(); + raw.call( + "service.register", + json!({"name": "example.partial-request", "maxQueued": 1, "maxInFlight": 1}), + ) + .await + .unwrap(); + probe.arm(); + + let artifact = sealed(&caller, b"request", "application/octet-stream").await; + let _pending = caller + .call( + "example.partial-request", + None, + "Work", + obj(json!({})), + &[("data", &artifact)], + ) + .await + .unwrap(); + drop(artifact); + probe.wait_first().await; + wait_for_single_delivery_owner(&router).await; + let before = router.stats(); + assert_eq!((before.owners, before.artifact_roots), (1, 1)); + + router.shutdown(); + assert_eq!(router.stats().owners, 0); + assert_eq!(router.stats().artifacts, 0); + assert!( + within("truncated request frame", raw.recv()) + .await + .is_none(), + "a complete frame was appended after a partial RPC request" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_cancels_partial_rpc_result_before_reclaiming_attachment() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.policy = Policy::open(); + let router = Router::new(config).unwrap(); + let server = Client::connect( + router.connect_in_memory_as("server"), + ClientConfig::new("server", dir.path()), + ) + .await + .unwrap(); + let mut service = server + .register("example.partial-result", ServiceConfig::default()) + .await + .unwrap(); + let (client_stream, router_stream) = tokio::io::duplex(64 * 1024); + let (probed, probe) = WriteProbe::new(router_stream); + router.serve_as(probed, "caller"); + let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path()); + raw.hello("caller").await.unwrap(); + raw.call( + "rpc.call", + json!({"callId": "call-1", "target": "example.partial-result", "expectedIncarnation": null, "method": "Work", "payload": {}}), + ) + .await + .unwrap(); + probe.arm(); + let request = within("result request", service.next()).await.unwrap(); + let artifact = sealed(&server, b"result", "application/octet-stream").await; + request + .reply(obj(json!({})), &[("data", &artifact)]) + .await + .unwrap(); + drop((artifact, request)); + probe.wait_first().await; + wait_for_single_delivery_owner(&router).await; + let before = router.stats(); + assert_eq!((before.owners, before.artifact_roots), (1, 1)); + + router.shutdown(); + assert_eq!(router.stats().owners, 0); + assert_eq!(router.stats().artifacts, 0); + assert!( + within("truncated result frame", raw.recv()).await.is_none(), + "a complete frame was appended after a partial RPC result" + ); +} diff --git a/services/flysim/crates/flybus/tests/sol_review_races.rs b/services/flysim/crates/flybus/tests/sol_review_races.rs new file mode 100644 index 0000000..5b93401 --- /dev/null +++ b/services/flysim/crates/flybus/tests/sol_review_races.rs @@ -0,0 +1,819 @@ +mod common; + +use common::{Via, env, env_with, obj, within}; +use flybus::wire::{Envelope, Kind, contract_digest, read_frame, write_frame}; +use flybus::{ + CancelState, Client, ClientConfig, Dispatch, ErrorCode, Limits, Policy, Retained, Router, + RouterConfig, ServiceConfig, SubscriptionConfig, Transport, +}; +use serde_json::{Map, Value, json}; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::sync::oneshot; + +struct WriteFailsAfterHello { + response: Vec, + read: usize, + writes: usize, +} + +impl AsyncRead for WriteFailsAfterHello { + fn poll_read( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if self.read == self.response.len() { + return Poll::Pending; + } + let n = buf.remaining().min(self.response.len() - self.read); + let end = self.read + n; + buf.put_slice(&self.response[self.read..end]); + self.read = end; + Poll::Ready(Ok(())) + } +} + +impl AsyncWrite for WriteFailsAfterHello { + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + if self.writes == 0 { + self.writes = 1; + Poll::Ready(Ok(buf.len())) + } else { + Poll::Ready(Err(io::Error::new(io::ErrorKind::BrokenPipe, "injected"))) + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +struct WriteBlocksAsReaderFails { + response: Vec, + read: usize, + hello_written: bool, + command_write_started: bool, + reader_waker: Option, + dropped: Option>, +} + +impl Drop for WriteBlocksAsReaderFails { + fn drop(&mut self) { + if let Some(dropped) = self.dropped.take() { + let _ = dropped.send(()); + } + } +} + +impl AsyncRead for WriteBlocksAsReaderFails { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if self.read < self.response.len() { + let n = buf.remaining().min(self.response.len() - self.read); + let end = self.read + n; + buf.put_slice(&self.response[self.read..end]); + self.read = end; + return Poll::Ready(Ok(())); + } + if self.command_write_started { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::ConnectionReset, + "injected reader failure", + ))); + } + self.reader_waker = Some(cx.waker().clone()); + Poll::Pending + } +} + +impl AsyncWrite for WriteBlocksAsReaderFails { + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + if !self.hello_written { + self.hello_written = true; + return Poll::Ready(Ok(buf.len())); + } + self.command_write_started = true; + if let Some(waker) = self.reader_waker.take() { + waker.wake(); + } + Poll::Pending + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn responder_survives_cancel_then_request_drop() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.race", ServiceConfig::default()) + .await + .unwrap(); + let pending = caller + .call("example.race", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let request = within("request", service.next()).await.unwrap(); + let responder = request.responder(); + + assert_eq!( + pending.cancel().await.unwrap(), + CancelState::ExecutionUnknown + ); + drop(request); + e.settle("request consumed", |s| s.owners == 0).await; + + let routed = responder.reply(obj(json!({"done": true})), &[]).await; + assert_eq!(routed, Ok(false), "a detached reply is still a valid reply"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancel_after_request_consumption_retires_correlation() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.leak", ServiceConfig::default()) + .await + .unwrap(); + let mut pending = caller + .call("example.leak", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let request = within("request", service.next()).await.unwrap(); + drop(request); + let failure = within("last responder terminal failure", pending.result()).await; + let failure = failure.unwrap_err(); + assert_eq!( + (failure.code, failure.dispatch), + (ErrorCode::CallGone, Dispatch::Dispatched) + ); + e.settle("consumed call retired", |s| { + s.calls == 0 && s.active_calls == 0 && s.owners == 0 && s.reply_capabilities == 0 + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_replay_obeys_bounded_queue_byte_quota() { + let limits = Limits { + max_queued_bytes_per_client: 1, + max_owners_per_client: 2, + reserved_owners_per_client: 1, + ..Limits::default() + }; + let e = env_with(Via::Memory, limits, Policy::open()).await; + let publisher = e.client("publisher").await; + let reader = e.client("reader").await; + publisher + .declare_topic("t.replay", Retained::Latest) + .await + .unwrap(); + publisher + .publish("t.replay", obj(json!({"large": "x".repeat(1024)})), &[]) + .await + .unwrap(); + + // Exhaust the reader's ordinary-owner allowance so replay cannot immediately dispatch. + let _hold = reader + .artifacts() + .allocate(0, "application/octet-stream") + .await + .unwrap() + .seal() + .await + .unwrap(); + let replay = reader + .subscribe("t.replay", SubscriptionConfig::bounded().replay(true)) + .await; + assert_eq!(replay.unwrap_err().code, ErrorCode::Backpressure); + assert_eq!(e.stats().subscriptions, 0, "failed replay is atomic"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn latest_replay_remains_bounded_outside_the_bounded_byte_pool() { + let limits = Limits { + max_queued_bytes_per_client: 1, + max_owners_per_client: 2, + reserved_owners_per_client: 1, + ..Limits::default() + }; + let e = env_with(Via::Memory, limits, Policy::open()).await; + let publisher = e.client("publisher").await; + let reader = e.client("reader").await; + publisher + .declare_topic("t.latest", Retained::Latest) + .await + .unwrap(); + publisher + .publish("t.latest", obj(json!({"large": "x".repeat(1024)})), &[]) + .await + .unwrap(); + let _hold = reader + .artifacts() + .allocate(0, "application/octet-stream") + .await + .unwrap() + .seal() + .await + .unwrap(); + let sub = reader + .subscribe("t.latest", SubscriptionConfig::latest().replay(true)) + .await + .unwrap(); + assert_eq!(e.stats().subscriptions, 1); + drop(sub); + e.settle("latest replay released", |s| s.subscriptions == 0) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pending_connections_are_bounded_and_hello_expires() { + let dir = tempfile::tempdir().unwrap(); + let mut config = RouterConfig::new(dir.path()); + config.limits.max_clients = 1; + config.hello_timeout = Duration::from_millis(40); + config.policy = Policy::closed() + .client("first", flybus::Grants::all()) + .client("second", flybus::Grants::all()); + let router = Router::new(config).unwrap(); + + let pending = router.connect_in_memory_as("first"); + tokio::time::timeout(Duration::from_millis(20), async { + while router.stats().connections != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(router.stats().connections, 1); + let refused = Client::connect( + router.connect_in_memory_as("second"), + ClientConfig::new("second", dir.path()), + ) + .await; + assert_eq!(refused.unwrap_err().code, ErrorCode::RouterLost); + tokio::time::sleep(Duration::from_millis(80)).await; + assert_eq!(router.stats().connections, 0, "pending Hello timed out"); + drop(pending); + + let second = Client::connect( + router.connect_in_memory_as("second"), + ClientConfig::new("second", dir.path()), + ) + .await + .unwrap(); + second.close().await; + let unbound = Client::connect( + router.connect_in_memory(), + ClientConfig::new("first", dir.path()), + ) + .await; + assert_eq!(unbound.unwrap_err().code, ErrorCode::RouterLost); +} + +fn reply_body(value: Value) -> Map { + obj(json!({"ok": true, "value": value})) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn client_rejects_wrong_version_and_operation_from_router() { + let dir = tempfile::tempdir().unwrap(); + let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024); + let server = tokio::spawn(async move { + let hello = read_frame(&mut fake_router).await.unwrap().unwrap(); + let hello = Envelope::decode(&hello).unwrap(); + let hello_reply = Envelope { + major: 1, + minor: 0, + id: "bus-1".into(), + reply_to: Some(hello.id), + kind: Kind::Reply, + op: "bus.hello".into(), + body: reply_body(json!({ + "routerId": "router-fake", + "connectionId": "conn-1", + "selectedMajor": 1, + "selectedMinor": 0, + "contractDigest": contract_digest(), + "limits": Limits::default().to_json(), + })), + attachments: Vec::new(), + }; + write_frame(&mut fake_router, &hello_reply.encode().unwrap()) + .await + .unwrap(); + + let command = read_frame(&mut fake_router).await.unwrap().unwrap(); + let command = Envelope::decode(&command).unwrap(); + let invalid_reply = Envelope { + major: 2, + minor: 0, + id: "bus-1".into(), + reply_to: Some(command.id), + kind: Kind::Reply, + op: "publish".into(), + body: reply_body(json!({ + "declared": true, + "topicIncarnation": "top-1", + })), + attachments: Vec::new(), + }; + write_frame(&mut fake_router, &invalid_reply.encode().unwrap()) + .await + .unwrap(); + }); + + let client = Client::connect( + Transport::from_stream(client_stream), + ClientConfig::new("client", dir.path()), + ) + .await + .unwrap(); + let result = client.declare_topic("t.x", Retained::None).await; + assert!( + result.is_err(), + "invalid router envelope was accepted: {result:?}" + ); + server.await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn client_strictly_validates_hello_envelope() { + for case in ["version", "operation", "id", "attachments"] { + let dir = tempfile::tempdir().unwrap(); + let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024); + let server = tokio::spawn(async move { + let hello = + Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap(); + let mut reply = Envelope { + major: 1, + minor: 0, + id: "bus-1".into(), + reply_to: Some(hello.id), + kind: Kind::Reply, + op: "bus.hello".into(), + body: reply_body(json!({ + "routerId": "router-fake", + "connectionId": "conn-1", + "selectedMajor": 1, + "selectedMinor": 0, + "contractDigest": contract_digest(), + "limits": Limits::default().to_json(), + })), + attachments: Vec::new(), + }; + match case { + "version" => reply.major = 2, + "operation" => reply.op = "publish".into(), + "id" => reply.id = "msg-1".into(), + "attachments" => reply.attachments.push(flybus::wire::Attachment { + name: "x".into(), + reference: flybus::ArtifactRef { + store_id: "store-fake".into(), + artifact_id: "a-1".into(), + generation: 1, + byte_length: 0, + content_type: "x/y".into(), + digest: None, + }, + owner_id: "own-1".into(), + }), + _ => unreachable!(), + } + write_frame(&mut fake_router, &reply.encode().unwrap()) + .await + .unwrap(); + }); + let result = Client::connect( + Transport::from_stream(client_stream), + ClientConfig::new("client", dir.path()), + ) + .await; + assert!(result.is_err(), "invalid Hello case {case} was accepted"); + server.await.unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn client_rejects_unknown_reply_fields_and_invalid_direction_rules() { + let dir = tempfile::tempdir().unwrap(); + let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024); + let server = tokio::spawn(async move { + let hello = + Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap(); + let hello_reply = Envelope { + major: 1, + minor: 0, + id: "bus-1".into(), + reply_to: Some(hello.id), + kind: Kind::Reply, + op: "bus.hello".into(), + body: reply_body(json!({ + "routerId": "router-fake", + "connectionId": "conn-1", + "selectedMajor": 1, + "selectedMinor": 0, + "contractDigest": contract_digest(), + "limits": Limits::default().to_json(), + })), + attachments: Vec::new(), + }; + write_frame(&mut fake_router, &hello_reply.encode().unwrap()) + .await + .unwrap(); + let command = + Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap(); + let invalid = Envelope { + major: 1, + minor: 0, + id: "bus-2".into(), + reply_to: Some(command.id), + kind: Kind::Reply, + op: "topic.declare".into(), + body: reply_body(json!({ + "declared": true, + "topicIncarnation": "top-1", + "extra": true, + })), + attachments: Vec::new(), + }; + write_frame(&mut fake_router, &invalid.encode().unwrap()) + .await + .unwrap(); + }); + let client = Client::connect( + Transport::from_stream(client_stream), + ClientConfig::new("client", dir.path()), + ) + .await + .unwrap(); + assert!(client.declare_topic("t.x", Retained::None).await.is_err()); + server.await.unwrap(); + + let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024); + let server = tokio::spawn(async move { + let hello = + Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap(); + let hello_reply = Envelope { + major: 1, + minor: 0, + id: "bus-1".into(), + reply_to: Some(hello.id), + kind: Kind::Reply, + op: "bus.hello".into(), + body: reply_body(json!({ + "routerId": "router-fake", + "connectionId": "conn-1", + "selectedMajor": 1, + "selectedMinor": 0, + "contractDigest": contract_digest(), + "limits": Limits::default().to_json(), + })), + attachments: Vec::new(), + }; + write_frame(&mut fake_router, &hello_reply.encode().unwrap()) + .await + .unwrap(); + let invalid_notice = Envelope { + major: 1, + minor: 0, + id: "bus-2".into(), + reply_to: Some("msg-99".into()), + kind: Kind::Notice, + op: "connection.closing".into(), + body: obj(json!({"code": "ROUTER_LOST", "message": "bad direction"})), + attachments: Vec::new(), + }; + write_frame(&mut fake_router, &invalid_notice.encode().unwrap()) + .await + .unwrap(); + }); + let client = Client::connect( + Transport::from_stream(client_stream), + ClientConfig::new("client", dir.path()), + ) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while client.closed().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + server.await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn writer_failure_terminates_reader_and_pending_work() { + let dir = tempfile::tempdir().unwrap(); + let hello_reply = Envelope { + major: 1, + minor: 0, + id: "bus-1".into(), + reply_to: Some("msg-1".into()), + kind: Kind::Reply, + op: "bus.hello".into(), + body: reply_body(json!({ + "routerId": "router-fake", + "connectionId": "conn-1", + "selectedMajor": 1, + "selectedMinor": 0, + "contractDigest": contract_digest(), + "limits": Limits::default().to_json(), + })), + attachments: Vec::new(), + } + .encode() + .unwrap(); + let mut response = (hello_reply.len() as u32).to_le_bytes().to_vec(); + response.extend(hello_reply); + let client = Client::connect( + Transport::from_stream(WriteFailsAfterHello { + response, + read: 0, + writes: 0, + }), + ClientConfig::new("client", dir.path()), + ) + .await + .unwrap(); + let result = tokio::time::timeout( + Duration::from_secs(1), + client.declare_topic("t.x", Retained::None), + ) + .await + .expect("writer failure left command pending"); + assert_eq!(result.unwrap_err().code, ErrorCode::RouterLost); + tokio::time::timeout(Duration::from_secs(1), client.close()) + .await + .expect("writer failure left close waiting"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reader_failure_cancels_blocked_write_and_drops_transport() { + let dir = tempfile::tempdir().unwrap(); + let hello_reply = Envelope { + major: 1, + minor: 0, + id: "bus-1".into(), + reply_to: Some("msg-1".into()), + kind: Kind::Reply, + op: "bus.hello".into(), + body: reply_body(json!({ + "routerId": "router-fake", + "connectionId": "conn-1", + "selectedMajor": 1, + "selectedMinor": 0, + "contractDigest": contract_digest(), + "limits": Limits::default().to_json(), + })), + attachments: Vec::new(), + } + .encode() + .unwrap(); + let mut response = (hello_reply.len() as u32).to_le_bytes().to_vec(); + response.extend(hello_reply); + let (dropped, transport_dropped) = oneshot::channel(); + let client = Client::connect( + Transport::from_stream(WriteBlocksAsReaderFails { + response, + read: 0, + hello_written: false, + command_write_started: false, + reader_waker: None, + dropped: Some(dropped), + }), + ClientConfig::new("client", dir.path()), + ) + .await + .unwrap(); + + let result = tokio::time::timeout( + Duration::from_secs(1), + client.declare_topic("t.x", Retained::None), + ) + .await + .expect("reader failure left command pending"); + assert_eq!(result.unwrap_err().code, ErrorCode::RouterLost); + tokio::time::timeout(Duration::from_secs(1), transport_dropped) + .await + .expect("reader failure left writer blocked") + .expect("transport drop signal was discarded"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rejected_call_id_still_advances_monotonic_watermark() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let _service = server + .register("example.ids", ServiceConfig::default()) + .await + .unwrap(); + let mut raw = e.raw_hello("caller").await; + + let rejected = raw + .call( + "rpc.call", + json!({"callId": "call-5", "target": "missing.service", "expectedIncarnation": null, "method": "M", "payload": {}}), + ) + .await; + assert_eq!(common::code(&rejected), "NO_SERVICE"); + let decreasing = raw + .call( + "rpc.call", + json!({"callId": "call-4", "target": "example.ids", "expectedIncarnation": null, "method": "M", "payload": {}}), + ) + .await; + assert_eq!(common::code(&decreasing), "INVALID_ENVELOPE"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unsent_oversized_call_rolls_back_its_local_slot() { + let limits = Limits { + max_active_calls_per_client: 1, + ..Limits::default() + }; + let e = env_with(Via::Memory, limits, Policy::open()).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.rollback", ServiceConfig::default()) + .await + .unwrap(); + let oversized = caller + .call( + "example.rollback", + None, + "Work", + obj(json!({"blob": "x".repeat(70_000)})), + &[], + ) + .await; + assert_eq!(oversized.unwrap_err().code, ErrorCode::InvalidEnvelope); + + let mut pending = caller + .call("example.rollback", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let request = within("request after rollback", service.next()) + .await + .unwrap(); + request.reply(obj(json!({"ok": true})), &[]).await.unwrap(); + assert_eq!(pending.result().await.unwrap().outcome()["ok"], true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn caller_disconnect_cleanup_works_before_and_after_consumption() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let mut service = server + .register("example.detach", ServiceConfig::default()) + .await + .unwrap(); + + let caller = e.client("caller-a").await; + let pending = caller + .call("example.detach", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let request = within("request a", service.next()).await.unwrap(); + let responder = request.responder(); + drop(request); + e.settle("request consumed with responder", |s| { + s.owners == 0 && s.reply_capabilities == 1 + }) + .await; + caller.close().await; + drop(pending); + assert!(!responder.reply(obj(json!({})), &[]).await.unwrap()); + drop(responder); + e.settle("retained responder retired", |s| { + s.calls == 0 && s.reply_capabilities == 0 + }) + .await; + + let caller = e.client("caller-b").await; + let pending = caller + .call("example.detach", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let request = within("request b", service.next()).await.unwrap(); + drop(request); + e.settle("capability released first", |s| s.reply_capabilities == 0) + .await; + caller.close().await; + drop(pending); + e.settle("consumed call retired on disconnect", |s| s.calls == 0) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropping_service_with_buffered_requests_retires_every_call() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let service = server + .register( + "example.buffered", + ServiceConfig { + max_queued: 4, + max_in_flight: 4, + }, + ) + .await + .unwrap(); + let mut calls = Vec::new(); + for _ in 0..4 { + calls.push( + caller + .call("example.buffered", None, "Work", obj(json!({})), &[]) + .await + .unwrap(), + ); + } + e.settle("requests buffered in SDK", |s| s.reply_capabilities == 4) + .await; + drop(service); + for call in &mut calls { + let error = within("buffered call failure", call.result()) + .await + .unwrap_err(); + assert_eq!( + (error.code, error.dispatch), + (ErrorCode::NoService, flybus::Dispatch::Dispatched) + ); + } + e.settle("buffered calls retired", |s| { + s.calls == 0 && s.reply_capabilities == 0 && s.owners == 0 + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reply_racing_cancel_has_only_the_two_contract_outcomes() { + let e = env(Via::Memory).await; + let server = e.client("server").await; + let caller = e.client("caller").await; + let mut service = server + .register("example.reply-race", ServiceConfig::default()) + .await + .unwrap(); + for _ in 0..32 { + let mut pending = caller + .call("example.reply-race", None, "Work", obj(json!({})), &[]) + .await + .unwrap(); + let request = within("racing request", service.next()).await.unwrap(); + let responder = request.responder(); + let (cancel, reply) = tokio::join!( + pending.cancel(), + responder.reply(obj(json!({"ok": true})), &[]) + ); + match (cancel.unwrap(), reply.unwrap()) { + (CancelState::Completed, true) => { + assert_eq!(pending.result().await.unwrap().outcome()["ok"], true); + } + (CancelState::ExecutionUnknown, false) => { + assert_eq!( + pending.result().await.unwrap_err().code, + ErrorCode::CallGone + ); + } + other => panic!("invalid cancel/reply race outcome: {other:?}"), + } + drop((request, responder)); + } + e.settle("race calls retired", |s| { + s.calls == 0 && s.reply_capabilities == 0 && s.owners == 0 + }) + .await; +} diff --git a/services/flysim/crates/flybus/tests/wire.rs b/services/flysim/crates/flybus/tests/wire.rs new file mode 100644 index 0000000..0b74ede --- /dev/null +++ b/services/flysim/crates/flybus/tests/wire.rs @@ -0,0 +1,355 @@ +//! Wire and connection: hello negotiation, strict frame and envelope validation, body errors +//! that keep the connection, envelope size limits and control-lane exhaustion. + +mod common; + +use std::time::Duration; + +use common::{Raw, Via, code, env, env_with, obj}; +use flybus::wire::{Kind, MAX_ENVELOPE_BYTES, contract_digest}; +use flybus::{ErrorCode, Limits, Policy, Retained}; +use serde_json::{Value, json}; + +async fn hello_negotiation(via: Via) { + let e = env(via).await; + let mut raw = e.raw().await; + let v = raw.hello("probe").await.unwrap(); + assert_eq!(v["routerId"], e.router.router_id()); + assert_eq!( + (v["selectedMajor"].as_u64(), v["selectedMinor"].as_u64()), + (Some(1), Some(0)) + ); + assert_eq!(v["contractDigest"], contract_digest()); + assert!(v["connectionId"].as_str().unwrap().starts_with("conn-")); + assert_eq!( + flybus::Limits::from_json(&v["limits"]).unwrap(), + Limits::default() + ); + let c = e.client("sdk").await; + assert_eq!(c.info().router_id, e.router.router_id()); + assert_ne!(c.info().connection_id, v["connectionId"]); +} + +async fn expect_closed(raw: &mut Raw, want: &str) { + let notice = raw + .closing() + .await + .unwrap_or_else(|| panic!("no connection.closing notice (wanted {want})")); + assert_eq!(notice["code"], want, "{notice:?}"); +} + +async fn hello_refusals(via: Via) { + let e = env_with( + via, + Limits::default(), + Policy::closed().client("known", flybus::Grants::all()), + ) + .await; + let mut raw = e.raw_as("known").await; + raw.command( + "topic.declare", + json!({"name": "t.x", "retained": "none"}), + json!([]), + ) + .await; + expect_closed(&mut raw, "INVALID_ENVELOPE").await; + + let mut raw = e.raw_as("known").await; + let r = raw + .call( + "bus.hello", + json!({"clientId": "known", "clientIncarnation": "i-1", "supportedMajors": [2, 3]}), + ) + .await; + assert_eq!(code(&r), "VERSION_MISMATCH"); + assert!( + raw.recv().await.is_none(), + "refused hello closes the connection" + ); + + let mut raw = e.raw_as("known").await; + assert_eq!(code(&raw.hello("stranger").await), "NOT_AUTHORIZED"); + assert!(raw.recv().await.is_none()); + + let mut raw = e.raw_as("known").await; + let r = raw.call("bus.hello", json!({"clientId": "known", "clientIncarnation": "i-2", "supportedMajors": [1], "admin": true})).await; + assert_eq!(code(&r), "INVALID_ENVELOPE"); + + let mut raw = e.raw_as("known").await; + raw.hello("known").await.unwrap(); + raw.command( + "bus.hello", + json!({"clientId": "known", "clientIncarnation": "i-3", "supportedMajors": [1]}), + json!([]), + ) + .await; + expect_closed(&mut raw, "INVALID_ENVELOPE").await; + e.settle("all refused connections released", |s| s.connections == 0) + .await; +} + +fn envelope(id: &str, body: Value) -> Value { + json!({"protocol": "flybus", "major": 1, "minor": 0, "id": id, "replyTo": null, "kind": "command", + "op": "publish", "body": body, "attachments": []}) +} + +async fn malformed_frames_close_the_connection(via: Via) { + let e = env(via).await; + let admin = e.client("admin").await; + admin.declare_topic("t.x", Retained::None).await.unwrap(); + let publish = |id: &str| envelope(id, json!({"topic": "t.x", "payload": {}})); + let text = |v: Value| serde_json::to_string(&v).unwrap(); + let att = |name: &str| { + json!({"name": name, "ref": {"storeId": "s", "artifactId": "a-1", "generation": "1", + "byteLength": "1", "contentType": "x", "digest": null}, "ownerId": "own-1"}) + }; + let mut too_many = publish("msg-2"); + too_many["attachments"] = Value::Array((0..33).map(|i| att(&format!("a{i}"))).collect()); + let mut duplicate_names = publish("msg-2"); + duplicate_names["attachments"] = json!([att("a"), att("a")]); + let mut unknown_field = publish("msg-2"); + unknown_field["priority"] = json!(1); + let mut from_router = publish("msg-2"); + from_router["kind"] = json!("reply"); + let mut reply_to = publish("msg-2"); + reply_to["replyTo"] = json!("msg-1"); + let mut major = publish("msg-2"); + major["major"] = json!(2); + let mut not_flybus = publish("msg-2"); + not_flybus["protocol"] = json!("flybusx"); + + let cases: Vec<(&str, Vec)> = vec![ + ("nested duplicate key", br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-2","replyTo":null,"kind":"command","op":"publish","body":{"topic":"t.x","payload":{"a":{"b":1,"b":2}}},"attachments":[]}"#.to_vec()), + ("top-level duplicate key", br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-2","id":"msg-3","replyTo":null,"kind":"command","op":"publish","body":{"topic":"t.x","payload":{}},"attachments":[]}"#.to_vec()), + ("NaN", br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-2","replyTo":null,"kind":"command","op":"publish","body":{"topic":"t.x","payload":{"v":NaN}},"attachments":[]}"#.to_vec()), + ("invalid UTF-8", [&text(publish("msg-2")).into_bytes()[..60], b"\xff\xfe", &text(publish("msg-2")).into_bytes()[62..]].concat()), + ("trailing bytes", [text(publish("msg-2")).into_bytes(), b" {}".to_vec()].concat()), + ("not an object", b"[1,2,3]".to_vec()), + ("non-canonical id", text(publish("msg-02")).into_bytes()), + ("too many attachments", text(too_many).into_bytes()), + ("duplicate attachment names", text(duplicate_names).into_bytes()), + ("unknown envelope field", text(unknown_field).into_bytes()), + ("reply kind from a client", text(from_router).into_bytes()), + ("non-null replyTo", text(reply_to).into_bytes()), + ("wrong major after hello", text(major).into_bytes()), + ("wrong protocol", text(not_flybus).into_bytes()), + ]; + for (i, (what, bytes)) in cases.into_iter().enumerate() { + let mut raw = e.raw_hello(&format!("bad-{i}")).await; + raw.send_bytes(&bytes).await; + let notice = raw + .closing() + .await + .unwrap_or_else(|| panic!("{what}: no closing notice")); + assert_eq!(notice["code"], "INVALID_ENVELOPE", "{what}"); + e.settle(what, |s| s.connections == 1).await; + } + + // Ids must increase. + let mut raw = e.raw_hello("replay").await; + raw.send_bytes(&serde_json::to_vec(&publish("msg-5")).unwrap()) + .await; + raw.send_bytes(&serde_json::to_vec(&publish("msg-5")).unwrap()) + .await; + expect_closed(&mut raw, "INVALID_ENVELOPE").await; + + // Framing: a zero length, and a length over the limit with no body behind it. + let mut raw = e.raw_hello("zero").await; + raw.send_prefix(0).await; + expect_closed(&mut raw, "INVALID_ENVELOPE").await; + let mut raw = e.raw_hello("huge").await; + raw.send_prefix(u32::MAX).await; + expect_closed(&mut raw, "INVALID_ENVELOPE").await; + // A frame cut short by the end of the stream just ends the connection. + let mut raw = e.raw_hello("short").await; + raw.send_prefix(100).await; + drop(raw); + e.settle("only the admin remains", |s| s.connections == 1) + .await; +} + +async fn body_errors_keep_the_connection(via: Via) { + let e = env(via).await; + let mut raw = e.raw_hello("raw").await; + let cases = [ + ("no.such.op", json!({}), json!([])), + ( + "topic.declare", + json!({"name": "t.a", "retained": "none", "extra": 1}), + json!([]), + ), + ( + "topic.declare", + json!({"name": "t..a", "retained": "none"}), + json!([]), + ), + ( + "topic.declare", + json!({"name": "t.a", "retained": "sometimes"}), + json!([]), + ), + ("topic.declare", json!({"name": "t.a"}), json!([])), + ( + "subscribe", + json!({"topic": "t.a", "mode": "bounded", "maxQueued": 0, "maxInFlight": 1, "replayLatest": false}), + json!([]), + ), + ( + "subscribe", + json!({"topic": "t.a", "mode": "bounded", "maxQueued": 1.0, "maxInFlight": 1, "replayLatest": false}), + json!([]), + ), + ( + "subscribe", + json!({"topic": "t.a", "mode": "bounded", "maxQueued": 65536, "maxInFlight": 1, "replayLatest": false}), + json!([]), + ), + ( + "artifact.allocate", + json!({"byteLength": 5, "contentType": "x"}), + json!([]), + ), + ( + "artifact.allocate", + json!({"byteLength": "18446744073709551616", "contentType": "x"}), + json!([]), + ), + ( + "rpc.call", + json!({"callId": "call-1", "target": "a.b", "expectedIncarnation": null, "method": "", "payload": {}}), + json!([]), + ), + ( + "rpc.call", + json!({"callId": "call-1", "target": "a.b", "expectedIncarnation": null, "method": "M", "payload": []}), + json!([]), + ), + ( + "delivery.consumed", + json!({"deliveryIds": "dlv-1"}), + json!([]), + ), + ( + "topic.declare", + json!({"name": "t.a", "retained": "none"}), + json!([{"name": "x", "ref": {"storeId": "s", "artifactId": "a-1", "generation": "1", "byteLength": "1", "contentType": "x", "digest": null}, "ownerId": "own-1"}]), + ), + ]; + for (op, body, atts) in cases { + let r = raw.call_with(op, body.clone(), atts).await; + assert_eq!( + r, + Err(("INVALID_ENVELOPE".into(), "not-dispatched".into())), + "{op} {body}" + ); + } + assert_eq!( + code( + &raw.call("topic.declare", json!({"name": "t.a", "retained": "none"})) + .await + ), + "OK" + ); + assert_eq!(e.stats().connections, 1); +} + +async fn envelope_size_limits(via: Via) { + let e = env(via).await; + // Exactly 65536 bytes is a frame the router reads (and answers); one more is not. + let sized = |id: &str, total: usize| { + let base = serde_json::to_vec( + &json!({"protocol": "flybus", "major": 1, "minor": 0, "id": id, "replyTo": null, + "kind": "command", "op": "no.such.op", "body": {"pad": ""}, "attachments": []}), + ) + .unwrap(); + let pad = "p".repeat(total - base.len()); + let bytes = serde_json::to_vec( + &json!({"protocol": "flybus", "major": 1, "minor": 0, "id": id, "replyTo": null, + "kind": "command", "op": "no.such.op", "body": {"pad": pad}, "attachments": []}), + ) + .unwrap(); + assert_eq!(bytes.len(), total); + bytes + }; + let mut raw = e.raw_hello("raw").await; + raw.send_bytes(&sized("msg-2", MAX_ENVELOPE_BYTES)).await; + assert_eq!(code(&raw.reply("msg-2").await), "INVALID_ENVELOPE"); + raw.send_bytes(&sized("msg-3", MAX_ENVELOPE_BYTES + 1)) + .await; + expect_closed(&mut raw, "INVALID_ENVELOPE").await; + + // The SDK refuses to send an oversized envelope and the connection lives on. + let c = e.client("c").await; + c.declare_topic("t.big", Retained::None).await.unwrap(); + let huge = c + .publish( + "t.big", + obj(json!({"blob": "x".repeat(MAX_ENVELOPE_BYTES)})), + &[], + ) + .await + .unwrap_err(); + assert_eq!(huge.code, ErrorCode::InvalidEnvelope); + // An envelope that fits but whose delivery (with its added ids) would not is refused at + // admission rather than truncated later. + let edge = c + .publish("t.big", obj(json!({"blob": "x".repeat(65_300)})), &[]) + .await + .unwrap_err(); + assert_eq!(edge.code, ErrorCode::InvalidEnvelope); + assert!(edge.message.contains("delivery"), "{edge}"); + assert_eq!( + c.publish("t.big", obj(json!({"blob": "x".repeat(60_000)})), &[]) + .await + .unwrap() + .topic_sequence, + 1 + ); +} + +/// A client that sends commands and never reads replies is closed once its control lane is +/// full, with a notice, instead of the router buffering without bound. +async fn control_lane_exhaustion_closes(via: Via) { + let limits = Limits { + max_control_frames: 4, + ..Limits::default() + }; + let e = env_with(via, limits, Policy::open()).await; + let mut raw = e.raw_hello("flood").await; + let mut writer = raw.take_writer(); + let flood = tokio::spawn(async move { + for i in 2..20_000u64 { + let env = json!({"protocol": "flybus", "major": 1, "minor": 0, "id": format!("msg-{i}"), "replyTo": null, + "kind": "command", "op": "no.such.op", "body": {}, "attachments": []}); + if !writer.send(&serde_json::to_vec(&env).unwrap()).await { + break; + } + } + }); + tokio::time::sleep(Duration::from_millis(300)).await; + let mut replies = 0; + let mut last = None; + while let Some(env) = raw.recv().await { + match env.kind { + Kind::Reply => replies += 1, + _ => last = Some(env), + } + } + let last = last.expect("a closing notice"); + assert_eq!( + (last.op.as_str(), last.body["code"].as_str()), + ("connection.closing", Some("QUOTA_EXCEEDED")) + ); + assert!(replies < 19_998, "the router stopped answering"); + flood.abort(); + e.settle("flooder released", |s| s.connections == 0).await; +} + +both_transports!( + hello_negotiation, + hello_refusals, + malformed_frames_close_the_connection, + body_errors_keep_the_connection, + envelope_size_limits, + control_lane_exhaustion_closes, +);