feat(flybus): add standalone IPC bus and conformance tests
Some checks failed
Some checks failed
(cherry picked from commit 95ac8d1bdbc866a7c49fad3ddba468d945267650)
This commit is contained in:
acamilo
parent
660c3cf00d
commit
724c5b0c09
29 changed files with 14493 additions and 1 deletions
12
services/flysim/Cargo.lock
generated
12
services/flysim/Cargo.lock
generated
|
|
@ -441,6 +441,18 @@ dependencies = [
|
|||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flybus"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flysim"
|
||||
version = "0.1.1"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/flybrain-core", "crates/flybrain-gb", "crates/flysim"]
|
||||
members = ["crates/flybrain-core", "crates/flybrain-gb", "crates/flybus", "crates/flysim"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.1"
|
||||
|
|
|
|||
21
services/flysim/crates/flybus/Cargo.toml
Normal file
21
services/flysim/crates/flybus/Cargo.toml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
[package]
|
||||
name = "flybus"
|
||||
version.workspace = true
|
||||
edition = "2024"
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
description = "A small local RPC and pub/sub bus with immutable file-backed artifacts."
|
||||
|
||||
[dependencies]
|
||||
# `flock`, `posix_fallocate` and `O_NOFOLLOW`; everything else is std or Tokio.
|
||||
libc = "0.2"
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
tokio = { version = "1", features = ["rt", "net", "io-util", "sync", "time", "macros"] }
|
||||
|
||||
[dev-dependencies]
|
||||
sha2 = { workspace = true }
|
||||
tempfile = "3"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }
|
||||
370
services/flysim/crates/flybus/README.md
Normal file
370
services/flysim/crates/flybus/README.md
Normal file
|
|
@ -0,0 +1,370 @@
|
|||
# flybus
|
||||
|
||||
A small local RPC and pub/sub bus with immutable file-backed artifacts, for Tokio.
|
||||
|
||||
One library, one router, one wire protocol. Messages are small JSON envelopes carrying
|
||||
metadata and artifact references. Large immutable bytes live in a file store the router
|
||||
manages. Ownership follows deliveries and explicit holds. The router moves messages and tracks
|
||||
ownership; it does not know what the messages mean.
|
||||
|
||||
This crate implements the Flybus v1 draft (session-framework design, `bus-v1`, draft 1 of
|
||||
2026-09-18), using the scalar encodings of its companion `ipc-v1` (`Id`, `U64`, `Digest`).
|
||||
Where this crate narrows or extends the draft, the difference is listed under
|
||||
[Differences from the draft](#differences-from-the-draft).
|
||||
|
||||
Nothing in the crate is specific to a game, a brain or a stream. It is a workspace member and
|
||||
no other crate depends on it yet.
|
||||
|
||||
## Layout
|
||||
|
||||
| Module | Contents |
|
||||
| --- | --- |
|
||||
| `wire` (public) | Scalars, strict JSON, `Envelope`, `ArtifactRef`, `Attachment`, `Location`, framing, `CONTRACT` and `contract_digest()` |
|
||||
| `error` (public) | `ErrorCode`, `Dispatch`, `BusError` |
|
||||
| `limits` (public) | `Limits` and its hello encoding |
|
||||
| `policy` (public) | `Policy`, `Grants`, `Pattern` |
|
||||
| `router` | `Router`, `RouterConfig`, `RouterStats`, `UnixListenerHandle`; the state machine in `router/state.rs` |
|
||||
| `client` | `Client` and every handle type |
|
||||
| `store` | The file store (router side) and location resolution (client side) |
|
||||
| `transport` | `Transport`, the `Stream` trait |
|
||||
|
||||
Dependencies: `tokio`, `serde`, `serde_json`, `sha2` and `libc` (for `flock`,
|
||||
`posix_fallocate` and `O_NOFOLLOW`), all already in the workspace lockfile.
|
||||
|
||||
## API
|
||||
|
||||
### Router
|
||||
|
||||
```rust
|
||||
let mut config = RouterConfig::new("/run/example/bus-store"); // default Limits, closed Policy
|
||||
config.policy = Policy::closed()
|
||||
.client("coordinator", Grants { call: vec![Pattern::prefix("agent.")], ..Grants::default() })
|
||||
.client("agent-a", Grants { register: vec![Pattern::exact("agent.fly-a")], ..Grants::default() });
|
||||
let router = Router::new(config)?; // creates <root>/<storeId>, cleans orphans
|
||||
let listener = router.listen_unix_as("/run/example/coordinator.sock", "coordinator").await?;
|
||||
let transport = router.connect_in_memory_as("coordinator");
|
||||
router.serve_as(any_async_read_write_stream, "coordinator");
|
||||
router.stats(); // RouterStats (logical registry counters)
|
||||
router.shutdown(); // notices, then closes every connection
|
||||
```
|
||||
|
||||
- `Router` is `Clone`; all clones are one router. `serve_as`, `connect_in_memory_as` and
|
||||
`listen_unix_as` bind the launcher's expected participant id to the transport before Hello.
|
||||
A mismatching Hello is refused before registration or routing.
|
||||
- The unbound `serve`, `connect_in_memory` and `listen_unix` entry points are only for explicit
|
||||
`Policy::open()` trusted/test deployments. Restricted policies refuse unbound transports.
|
||||
Identity in open/unbound mode is self-asserted and is not authentication.
|
||||
- `RouterConfig::hello_timeout` defaults to 5 seconds. `max_clients` counts all accepted
|
||||
connections, pending and active.
|
||||
- `router_id()`, `store_id()`, `store_root()` and `store_dir()` report the incarnation and
|
||||
paths.
|
||||
- `RouterStats` has `connections`, `services`, `topics`, `subscriptions`, `calls` (correlation
|
||||
records, detached included), `active_calls`, `artifacts`, `sealed_artifacts`,
|
||||
`artifact_roots`, `owners`, `reply_capabilities`, `queued`, `store_bytes` and
|
||||
`retained_bytes`.
|
||||
- `Limits` (defaults in brackets): `max_clients` [64], `max_services` [256], `max_topics`
|
||||
[512], `max_subscriptions_per_client` [128], `max_subscriptions` [1024],
|
||||
`max_active_calls_per_client` [64], `max_service_queued` / `max_service_in_flight` [16 / 16],
|
||||
`max_latest_in_flight` [2], `max_bounded_queued` / `max_bounded_in_flight` [64 / 16],
|
||||
`max_owners_per_client` [256], `reserved_owners_per_client` [64], `max_store_bytes`
|
||||
[512 MiB], `max_artifact_bytes` [128 MiB], `max_retained_bytes` [128 MiB],
|
||||
`max_queued_bytes_per_client` [1 MiB], `max_control_frames` / `max_control_bytes`
|
||||
[128 / 1 MiB]. `Router::new` refuses a configuration `Limits::validate` rejects.
|
||||
- `Policy::open()` explicitly enables trusted unbound transports and admits any client id with
|
||||
every grant. `Policy::closed()` admits only the
|
||||
ids added with `.client(id, grants)`. `.with_default(Some(grants))` admits unlisted ids too.
|
||||
`Grants` lists `register`, `call`, `publish`, `subscribe` and `manage_topics` (declare, clear,
|
||||
delete) as `Pattern::Any`, `Pattern::exact(name)` or `Pattern::prefix(prefix)`. An empty list
|
||||
grants nothing.
|
||||
|
||||
### Client
|
||||
|
||||
```rust
|
||||
let bus = Client::connect(transport, ClientConfig::new("coordinator", store_root)).await?;
|
||||
let bus = Client::connect_unix(socket_path, ClientConfig::new("coordinator", store_root)).await?;
|
||||
bus.info(); // SessionInfo: router_id, connection_id, identity, selected_major,
|
||||
// selected_minor, contract_digest, limits
|
||||
bus.closed(); // Some(reason) once the connection has closed
|
||||
bus.control_errors();// fire-and-forget releases/consumes/unregisters the router refused
|
||||
bus.close().await; // flush queued releases, close, wait for the reader to stop
|
||||
```
|
||||
|
||||
`ClientConfig` has `client_id`, `client_incarnation` (generated when `None`; a reconnect needs a
|
||||
new one), `store_root` (the router's) and `control_lane_capacity` [4096]. `Client` is `Clone`.
|
||||
The connection closes when the client and every handle made from it are dropped, or on
|
||||
`close()`. After that, handles are inert and the router has released what they owned.
|
||||
|
||||
#### RPC
|
||||
|
||||
```rust
|
||||
let mut svc = bus.register("agent.fly-a", ServiceConfig { max_queued: 16, max_in_flight: 16 }).await?;
|
||||
while let Some(req) = svc.next().await { // Request
|
||||
req.method(); req.payload(); req.caller(); // authenticated only on a launcher-bound transport
|
||||
let frame = req.artifact("frame")?; // shares the request's delivery guard
|
||||
req.reply(outcome_map, &[("result", &artifact)]).await?; // Ok(true) routed, Ok(false) detached
|
||||
} // dropping a Request consumes its delivery
|
||||
|
||||
let mut pending = bus.call("agent.fly-a", Some(&incarnation), "Agent.Prepare", payload, &[("frame", &frame)]).await?;
|
||||
let result = pending.result().await?; // RpcResult: outcome(), responder(), artifact()
|
||||
let state = pending.cancel().await?; // CancelState
|
||||
let result = bus.call_and_wait(service, pin, method, payload, &attachments).await?;
|
||||
```
|
||||
|
||||
- `register` is exclusive; `Service::incarnation()` is the id callers pin. Dropping the
|
||||
`Service` unregisters it.
|
||||
- `call` returns once the router has admitted the call. The `PendingCall` then yields exactly
|
||||
one terminal outcome. `result()` is cancel-safe: wrap it in `tokio::time::timeout`, then call
|
||||
it again or `cancel()`. There is no deadline parameter; the deadline belongs to the caller.
|
||||
- `cancel()` returns `CancelledBeforeDispatch`, `ExecutionUnknown`, `Completed` or `CallGone`.
|
||||
After any state except `Completed`, `result()` fails with `CALL_GONE`. The error's dispatch is
|
||||
`not-dispatched` only for `CancelledBeforeDispatch`.
|
||||
- Dropping an unfinished `PendingCall` sends a best-effort `rpc.cancel`. The reactor consumes
|
||||
any result that still arrives.
|
||||
- `Request::responder()` returns a `Responder` that can reply after the request itself has
|
||||
been dropped. Request delivery credit returns normally; a separate router-visible reply
|
||||
capability preserves correlation until the last `Request`/`Responder` is dropped or a reply
|
||||
is admitted. If the final capability is dropped while the caller is still attached, the call
|
||||
terminates with `CALL_GONE` and dispatch `dispatched` while the route remains live, or the
|
||||
existing `NO_SERVICE`/`dispatched` terminal if the route has ended. Capabilities share the
|
||||
service connection's owner bound.
|
||||
- A call fails with `call.failed` data in its `BusError`:
|
||||
- Queued calls whose service unregisters or disconnects fail with `NO_SERVICE` and dispatch
|
||||
`not-dispatched`.
|
||||
- Dispatched calls whose service disconnects without replying fail with `NO_SERVICE` and
|
||||
dispatch `dispatched`.
|
||||
- Dispatched calls whose last responder is released without replying fail with `CALL_GONE`
|
||||
and dispatch `dispatched`.
|
||||
- A lost connection fails pending calls with `ROUTER_LOST` and dispatch `unknown`.
|
||||
|
||||
#### Pub/sub
|
||||
|
||||
```rust
|
||||
bus.declare_topic("session.demo.snapshots", Retained::Latest).await?; // TopicInfo
|
||||
bus.clear_topic(name).await?; // bool: a retained value was released
|
||||
bus.delete_topic(name).await?; // bool; CONFLICT while it has subscribers
|
||||
let mut sub = bus.subscribe(name, SubscriptionConfig::latest().in_flight(1).replay(true)).await?;
|
||||
let receipt = bus.publish(name, payload, &[("frame", &frame)]).await?; // topic_sequence, subscribers, replaced
|
||||
let msg = sub.next().await; // Option<Message>; try_next() does not wait
|
||||
msg.topic_sequence(); msg.replaced(); msg.topic_incarnation(); msg.payload(); msg.artifact("frame")?;
|
||||
```
|
||||
|
||||
- `SubscriptionConfig::latest()` is 1 queued and 2 in flight. `SubscriptionConfig::bounded()`
|
||||
is 64 queued and 16 in flight. `.queued(n)`, `.in_flight(n)` and `.replay(bool)` adjust them.
|
||||
- A credit returns only when the delivery is consumed: when the `Message` and every artifact
|
||||
or `ArtifactFile` taken from it have been dropped.
|
||||
- A retained replay into a bounded subscription is admitted atomically under the same queued
|
||||
envelope-byte quota as an ordinary bounded publication. Latest replay remains outside that
|
||||
byte pool and is bounded by one slot per subscription.
|
||||
- Dropping a `Subscription` unsubscribes and discards its queue. Messages already handed out
|
||||
stay valid.
|
||||
|
||||
#### Artifacts
|
||||
|
||||
```rust
|
||||
let mut writer = bus.artifacts().allocate(len, "image/x-rgba").await?; // ArtifactWriter: io::Write
|
||||
writer.write_all(&pixels)?;
|
||||
let frame = writer.seal().await?; // or seal_with_digest(Some(sha256_hex))
|
||||
frame.reference(); // ArtifactRef
|
||||
let bytes = frame.read_all().await?; // Vec<u8>
|
||||
let file = frame.open().await?; // ArtifactFile: io::Read + io::Seek, keeps the handle
|
||||
let kept = frame.retain().await?; // an independent explicit hold
|
||||
```
|
||||
|
||||
- `Artifact` is a read-only, cloneable handle on one owner: a delivery or an explicit hold.
|
||||
Its last clone (including any `ArtifactFile`) releases that owner.
|
||||
- `ArtifactWriter` is unique. Dropping it unsealed releases the staging storage. Writes past
|
||||
the allocated length fail. Unwritten bytes read as zeros.
|
||||
- Sealing copies staging into a fresh read-only (0444) file and unlinks staging. A descriptor
|
||||
the producer kept or duplicated afterwards writes only to the unlinked staging inode. Sealing
|
||||
therefore needs quota for both copies while it runs.
|
||||
|
||||
### Errors
|
||||
|
||||
`BusError { code: ErrorCode, message, dispatch: Dispatch }`. The codes are the draft's
|
||||
`INVALID_ENVELOPE`, `VERSION_MISMATCH`, `NOT_AUTHORIZED`, `NO_SERVICE`, `TARGET_CHANGED`,
|
||||
`BACKPRESSURE`, `CALL_GONE`, `ARTIFACT_UNSEALED`, `ARTIFACT_GONE`, `OWNER_INVALID`,
|
||||
`QUOTA_EXCEEDED`, `STORE_FAILURE` and `ROUTER_LOST`, plus `CONFLICT`, `NO_TOPIC` and
|
||||
`ARTIFACT_MISMATCH`. `Dispatch` is `not-dispatched`, `dispatched` or `unknown`. Refusals
|
||||
before admission are `not-dispatched`. A command in flight when the connection is lost reports
|
||||
`unknown`.
|
||||
|
||||
## Wire summary
|
||||
|
||||
- Frames are a `u32` little-endian length followed by 1..=65,536 bytes of UTF-8 JSON. The length
|
||||
is checked before any allocation.
|
||||
- Every frame is parsed strictly:
|
||||
- Duplicate keys at any depth, invalid UTF-8, non-finite numbers and trailing bytes are
|
||||
refused. Nesting is limited to 128 levels.
|
||||
- Unknown fields are refused in the envelope, attachments, references and every management
|
||||
body. `payload` and `outcome` are opaque objects.
|
||||
- Frame or envelope errors close the connection after a `connection.closing` notice. They
|
||||
include a bad length, bad JSON, an unknown envelope field, a wrong kind, a non-null `replyTo`,
|
||||
a command id that is not canonical or does not increase, a command before `bus.hello`, a
|
||||
second hello, and a major/minor other than 1.0 after hello. A body-level error (unknown op,
|
||||
bad field, out-of-range value, attachments on a management op) gets an `INVALID_ENVELOPE`
|
||||
reply and the connection stays up.
|
||||
- The SDK applies symmetric checks to router frames: exact 1.0 version, strictly increasing
|
||||
canonical `bus-<U64>` ids, kind/replyTo/attachment direction rules, expected reply operation,
|
||||
delivery correlations and complete strict reply, delivery and notice body parsing.
|
||||
- Ids:
|
||||
- Commands: `msg-<U64>`, strictly increasing per connection.
|
||||
- Router envelopes: `bus-<n>`. Connections: `conn-<n>`. Services: `svc-<n>`. Topics:
|
||||
`top-<n>`. Subscriptions: `sub-<n>`. Calls: `call-<U64>`, increasing per client.
|
||||
- Artifacts: `a-<n>`. Deliveries: `dlv-<n>`. Holds and writers: `own-<n>`.
|
||||
- `routerId` is `router-<16 hex>` and `storeId` is `store-<16 hex>` (same tag).
|
||||
- Delivery and hold serials have separate per-connection watermarks.
|
||||
- Notices:
|
||||
- `call.failed {callId, code, message, dispatch}`
|
||||
- `route.removed {name, serviceIncarnation, reason}`
|
||||
- `subscription.closed {subscriptionId, topic, topicIncarnation, reason}`
|
||||
- `connection.closing {code, message}`
|
||||
- The hello reply's `limits` object gives counts as JSON integers and byte sizes as U64 strings.
|
||||
Its fields: `maxEnvelopeBytes`, `maxAttachments`, `maxBatch`, `maxClients`, `maxServices`,
|
||||
`maxTopics`, `maxSubscriptionsPerClient`, `maxSubscriptions`, `maxActiveCallsPerClient`,
|
||||
`maxServiceQueued`, `maxServiceInFlight`, `maxLatestInFlight`, `maxBoundedQueued`,
|
||||
`maxBoundedInFlight`, `maxOwnersPerClient`, `reservedOwnersPerClient`, `maxControlFrames`,
|
||||
`maxStoreBytes`, `maxArtifactBytes`, `maxRetainedBytes`, `maxQueuedBytesPerClient` and
|
||||
`maxControlBytes`.
|
||||
- `contractDigest` is the SHA-256 of `wire::CONTRACT`, a text listing of every operation,
|
||||
delivery and notice shape. The client refuses a router whose digest differs from its own.
|
||||
|
||||
## Semantics worth knowing
|
||||
|
||||
- **Admission is all or nothing.** `rpc.call`, `rpc.reply` and `publish` validate every
|
||||
attachment and every bound before changing any state. A refused publication creates no
|
||||
delivery, does not move the retained value and spends no topic sequence number. A refused
|
||||
syntactically valid call id advances the issued-id watermark even when semantic admission is
|
||||
refused, while creating no call, delivery or root. Reuse or decrease is rejected.
|
||||
- **Ownership.** Each sealed artifact has a root count. Roots are held by queued deliveries,
|
||||
delivered-but-unconsumed deliveries, explicit holds and a retained topic value. A delivery
|
||||
that names one artifact twice holds one root. Fan-out adds roots and never copies bytes. When
|
||||
the last root goes, the artifact leaves the quota and its file is unlinked, outside the router
|
||||
lock. A process that still has the file open keeps its pages until it closes it.
|
||||
- **Owner budget.** `max_owners_per_client` counts deliveries handed to the client and not
|
||||
consumed, explicit holds and writers. Topic deliveries, holds and writers may use all but
|
||||
`reserved_owners_per_client`; the reserve is for RPC requests and results. A topic delivery
|
||||
over budget waits in its queue. A hold or allocation over budget fails with `QUOTA_EXCEEDED`.
|
||||
Request reply capabilities consume the same finite bound independently of request delivery
|
||||
credit, preventing retained responders from growing detached correlation state without limit.
|
||||
- **Lanes.** The router sends each client its replies and notices first, then RPC requests and
|
||||
results, then topic messages. Topic messages still get one frame after 16 consecutive
|
||||
higher-priority frames. Router envelope ids are assigned only after this scheduler selects
|
||||
the next item, so they increase in actual write order. Replies and notices waiting for one client are bounded by
|
||||
`max_control_frames` / `max_control_bytes`; exceeding either closes that client with a
|
||||
`connection.closing` notice. The client sends consumes, releases and cancels ahead of
|
||||
ordinary commands and batches up to 64 ids per command. A full client control lane closes the
|
||||
connection rather than drop a release.
|
||||
- **No waiting on readers.** A client that stops reading stalls only its own writer task. The
|
||||
router keeps admitting until that client's queues refuse, then returns `BACKPRESSURE` to
|
||||
publishers of bounded topics. Connection teardown is synchronized with each synchronous
|
||||
`poll_write`/`poll_flush` call without holding a mutex across an await. Teardown either observes
|
||||
a complete frame before reclaiming its delivery owner, or marks a partial frame canceled before
|
||||
cleanup and appends no final notice to the truncated stream. At a frame boundary, normal final
|
||||
notices are still attempted.
|
||||
- **RPC.** First dispatch is FIFO per service, which includes per caller. A call is marked
|
||||
dispatched when its bytes are about to be written. Replies correlate by call id and may
|
||||
arrive in any order. A second reply to one call fails with `CALL_GONE`. A reply to a detached
|
||||
call returns `routed:false` and creates no roots. Unregistering a service fails its queued
|
||||
calls; dispatched calls can still be answered while a `Request` or `Responder` retains reply
|
||||
capability. Both cancel/consume orderings retire cleanly.
|
||||
- **Seals** copy (and hash, when a digest was given) on Tokio's blocking pool, alongside the
|
||||
connection's reader, so a large copy does not hold up that client's releases. If the writer is released or disconnects
|
||||
mid-seal, the copy is discarded and the artifact is never published.
|
||||
|
||||
## Differences from the draft
|
||||
|
||||
1. **Extra error codes.** `CONFLICT` covers a duplicate registration, a conflicting topic
|
||||
redeclaration and deleting a topic that has subscribers. `NO_TOPIC` covers publishing or
|
||||
subscribing to an undeclared topic. `ARTIFACT_MISMATCH` covers a seal whose length or digest
|
||||
is wrong, and a reference that disagrees with the artifact it names.
|
||||
2. **Topics must be declared.** Publish and subscribe fail with `NO_TOPIC` otherwise.
|
||||
`topic.delete` of an unknown topic returns `deleted:false`. `topic.clear` of an unknown
|
||||
topic returns `NO_TOPIC`.
|
||||
3. **When notices are sent.**
|
||||
- `subscription.closed` is sent only when the router shuts down, because `topic.delete`
|
||||
requires no subscribers and nothing else closes a subscription.
|
||||
- `route.removed` goes to callers with queued or dispatched calls on the removed
|
||||
registration, not to every client.
|
||||
- `connection.closing` is an extra notice that precedes every router-initiated close.
|
||||
4. **Byte budgets.**
|
||||
- `max_queued_bytes_per_client` counts only `bounded` subscriptions. A `latest` slot is bounded
|
||||
by subscription count times envelope size.
|
||||
- `max_retained_bytes` (not in the draft's table) counts the artifact bytes pinned by
|
||||
retained values, once per topic.
|
||||
5. **Delivery size.** Admission computes the delivery's size with the router-added ids at their
|
||||
longest. If that would exceed 65,536 bytes it refuses with `INVALID_ENVELOPE`, so an inbound
|
||||
envelope near the limit can be refused even though it fits.
|
||||
6. **Identity.** One live connection per client id. The router remembers each client id's last
|
||||
incarnation and refuses its reuse. Launcher-bound `*_as` endpoints authenticate the Hello id;
|
||||
trusted open/unbound endpoints do not. With an open policy this is one small record per
|
||||
distinct client id ever seen.
|
||||
7. **Seal reply.** The reply's `ownerId` is the writer's own id, now an explicit hold.
|
||||
8. **No `budget` argument on calls, and no router executable.** Timeouts are the caller's
|
||||
(`tokio::time::timeout` plus `cancel`). The draft's executable is optional; embed `Router`.
|
||||
9. **Wire strictness.** Management bodies reject unknown fields. After hello, envelopes must
|
||||
carry `minor: 0`.
|
||||
10. **Reply capability release.** The SDK sends `rpc.responder.release {callId,
|
||||
requestDeliveryId} -> {released}` when the last local reply capability is dropped. This
|
||||
keeps request consumption independent from late-reply correlation while bounding that
|
||||
correlation under the service connection's owner limit. For an attached dispatched call,
|
||||
final release atomically retires the correlation and caller slot and emits `call.failed`
|
||||
with dispatch `dispatched`: `CALL_GONE` while the route remains live, or `NO_SERVICE` after
|
||||
route loss.
|
||||
|
||||
## Limitations
|
||||
|
||||
- **One host, one user.** Clients must run as the router's user and see the same store root.
|
||||
The store directory is 0700, staging files 0600, sealed files 0444 and the socket 0600. Bus
|
||||
authority (grants, connection-scoped owner ids, location grants) is enforced on bus
|
||||
operations, not on the filesystem: a process running as the same user can read, mutate,
|
||||
replace, chmod or unlink store files directly and can deny service. Mode 0600 authenticates
|
||||
only the shared OS user, not a Flybus participant.
|
||||
- **No memory maps.** Artifacts are read through `std::fs::File` (`ArtifactFile`) or
|
||||
`read_all()`; there is no mmap API. `ArtifactFile` reads are blocking I/O.
|
||||
- **Stats are logical.** `RouterStats` counts registry entries. Unlinked files that are still
|
||||
open, client memory and OS pages are not in it.
|
||||
- **Fairness is modest.** Fairness between clients is Tokio's scheduling plus a yield after
|
||||
every 32 commands a connection sends. Within a connection, subscriptions and services are
|
||||
served round robin. There is no weighting.
|
||||
- **Orphan cleanup needs a restart.** Store directories left by a stopped router are removed
|
||||
only when a new router starts on the same root. A directory counts as orphaned when it carries
|
||||
the store marker and its `flock` is free.
|
||||
- **Rust only.** There are no other language bindings.
|
||||
- **Two perf runs.** `tests/perf.rs` (ignored by default) measured 640x480 RGBA frames at
|
||||
60 Hz over a Unix socket to three latest-mode consumers, one delayed 40 ms per frame. It ran
|
||||
twice, on a laptop under WSL, release build, router and clients in one process. Allocate,
|
||||
write and seal took p50 1.2 to 1.3 ms and p99 1.5 to 2.0 ms per 1.2 MB frame. Publish
|
||||
admission took p50 0.2 ms. The RPC round trip took p50 0.24 / 0.29 to 0.31 / 0.50 ms with
|
||||
1 / 2 / 4 agents. The whole process used 0.18 to 0.25 cores and about 15 MB RSS. The store
|
||||
peaked at 3.7 MB. These are two runs, not capacity data.
|
||||
|
||||
## Tests
|
||||
|
||||
```text
|
||||
cargo test -p flybus # unit + integration
|
||||
cargo test --release -p flybus --test perf -- --ignored --nocapture # the measurement above
|
||||
cargo run -p flybus --example demo # counter RPC, observer, held frame
|
||||
```
|
||||
|
||||
Every integration test runs twice, once over the in-memory transport and once over a Unix
|
||||
socket, through the same router code:
|
||||
|
||||
- `tests/wire.rs`: hello negotiation and refusals, malformed frames of every kind, body errors
|
||||
that keep the connection, the exact 65,536-byte limit and control-lane exhaustion.
|
||||
- `tests/rpc.rs`: exclusive and pinned registration, authority, FIFO dispatch with
|
||||
out-of-order completion, backpressure, all four cancel states, single replies, service
|
||||
disconnect and unregister, forged replies and call ids, an endpoint cache replaying an
|
||||
artifact result, and abandoned calls.
|
||||
- `tests/sol_review_races.rs`: launcher-bound identity and Hello resource bounds, detached RPC
|
||||
orderings, responder/service teardown, reply/cancel races, replay admission, call-id
|
||||
watermarking, strict client router validation, unsent rollback and shared task shutdown.
|
||||
- `tests/pubsub.rs`: bounded FIFO with atomic backpressure, latest coalescing, credits,
|
||||
retention and incarnations, zero-subscriber publication, unsubscribe, validation, quotas, a
|
||||
saturated subscriber that does not block RPC, and shutdown notices.
|
||||
- `tests/artifacts.rs`: allocate, seal and read; unsealed use; immutability against live
|
||||
writable descriptors; length and digest checks; quotas; one physical object shared by
|
||||
fan-out and collected after the last consumer; extracted artifacts and holds; atomic
|
||||
admission; release watermarks; connection-scoped owners; abandoned futures; disconnects; and
|
||||
router restarts.
|
||||
- `tests/integration.rs`: two agents called in parallel with a forwarded frame, an environment
|
||||
service, committed snapshot publication, a slow latest consumer and a bounded recorder.
|
||||
99
services/flysim/crates/flybus/examples/demo.rs
Normal file
99
services/flysim/crates/flybus/examples/demo.rs
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
//! A counter RPC, a pub/sub observer and a frame artifact held past its message, in one
|
||||
//! process over the in-memory transport (bus-v1 section 11).
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run -p flybus --example demo
|
||||
//! ```
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
use flybus::{
|
||||
Client, ClientConfig, Policy, Retained, Router, RouterConfig, ServiceConfig, SubscriptionConfig,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
fn obj(v: Value) -> Map<String, Value> {
|
||||
v.as_object().cloned().unwrap_or_default()
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let root = std::env::temp_dir().join(format!("flybus-demo-{}", std::process::id()));
|
||||
let mut config = RouterConfig::new(&root);
|
||||
config.policy = Policy::open();
|
||||
let router = Router::new(config)?;
|
||||
let connect = |id: &str| {
|
||||
Client::connect(
|
||||
router.connect_in_memory_as(id),
|
||||
ClientConfig::new(id, &root),
|
||||
)
|
||||
};
|
||||
|
||||
// A counter service.
|
||||
let counter = connect("counter").await?;
|
||||
let mut svc = counter
|
||||
.register("example.counter", ServiceConfig::default())
|
||||
.await?;
|
||||
tokio::spawn(async move {
|
||||
let mut total = 0;
|
||||
while let Some(req) = svc.next().await {
|
||||
total += req.payload()["amount"].as_i64().unwrap_or(0);
|
||||
let _ = req.reply(obj(json!({ "total": total })), &[]).await;
|
||||
}
|
||||
});
|
||||
let app = connect("app").await?;
|
||||
for _ in 0..3 {
|
||||
let res = app
|
||||
.call_and_wait(
|
||||
"example.counter",
|
||||
None,
|
||||
"Counter.Increment",
|
||||
obj(json!({"amount": 2})),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
println!("counter total = {}", res.outcome()["total"]);
|
||||
}
|
||||
|
||||
// An observer of a frame topic.
|
||||
app.declare_topic("world.demo.frame", Retained::None)
|
||||
.await?;
|
||||
let observer = connect("observer").await?;
|
||||
let mut frames = observer
|
||||
.subscribe("world.demo.frame", SubscriptionConfig::latest())
|
||||
.await?;
|
||||
|
||||
let mut writer = app
|
||||
.artifacts()
|
||||
.allocate(160 * 144 * 4, "image/x-rgba")
|
||||
.await?;
|
||||
writer.write_all(&vec![0x7f; 160 * 144 * 4])?;
|
||||
let frame = writer.seal().await?;
|
||||
let receipt = app
|
||||
.publish(
|
||||
"world.demo.frame",
|
||||
obj(json!({"width": 160, "height": 144})),
|
||||
&[("frame", &frame)],
|
||||
)
|
||||
.await?;
|
||||
println!(
|
||||
"published sequence {} to {} subscriber(s)",
|
||||
receipt.topic_sequence, receipt.subscribers
|
||||
);
|
||||
drop(frame);
|
||||
|
||||
let message = frames.next().await.ok_or("subscription closed")?;
|
||||
let image = message.artifact("frame")?;
|
||||
drop(message); // the extracted handle still owns the delivery
|
||||
let bytes = image.read_all().await?;
|
||||
println!(
|
||||
"read {} bytes after the message was dropped; router: {:?}",
|
||||
bytes.len(),
|
||||
router.stats()
|
||||
);
|
||||
drop(image); // the last handle: the delivery is consumed and the frame collected
|
||||
|
||||
router.shutdown();
|
||||
std::fs::remove_dir_all(&root)?;
|
||||
Ok(())
|
||||
}
|
||||
677
services/flysim/crates/flybus/src/client/handles.rs
Normal file
677
services/flysim/crates/flybus/src/client/handles.rs
Normal file
|
|
@ -0,0 +1,677 @@
|
|||
//! RAII handles: artifacts, writers, deliveries, services, subscriptions and pending calls.
|
||||
//!
|
||||
//! Every owner the router tracks for this client is behind one `OwnerGuard`. Its last clone
|
||||
//! dropping queues `delivery.consumed` (for a delivery) or `artifact.release` (for a hold or
|
||||
//! writer). A message, the artifacts extracted from it and any open files share the message's
|
||||
//! guard, so a delivery is consumed only when all of them are gone.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
use super::reactor::{CallSlot, ClientConn, Control, Extra, Hook, OutCommand};
|
||||
use crate::error::{BusError, Dispatch, ErrorCode};
|
||||
use crate::store::{open_read, resolve};
|
||||
use crate::wire::{ArtifactRef, Attachment, Fields, GENERATION, Identity, Location};
|
||||
|
||||
pub(crate) struct OwnerGuard {
|
||||
pub id: String,
|
||||
pub delivery: bool,
|
||||
pub conn: Arc<ClientConn>,
|
||||
}
|
||||
|
||||
impl Drop for OwnerGuard {
|
||||
fn drop(&mut self) {
|
||||
let id = std::mem::take(&mut self.id);
|
||||
let item = if self.delivery {
|
||||
Control::Consumed(id)
|
||||
} else {
|
||||
Control::Release(id)
|
||||
};
|
||||
self.conn.shared.push_control(item);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn attachment_list(
|
||||
list: &[(&str, &Artifact)],
|
||||
) -> (Vec<Attachment>, Vec<Arc<OwnerGuard>>) {
|
||||
let atts = list
|
||||
.iter()
|
||||
.map(|(name, a)| Attachment {
|
||||
name: (*name).to_owned(),
|
||||
reference: a.reference.as_ref().clone(),
|
||||
owner_id: a.owner.id.clone(),
|
||||
})
|
||||
.collect();
|
||||
let keep = list.iter().map(|(_, a)| a.owner.clone()).collect();
|
||||
(atts, keep)
|
||||
}
|
||||
|
||||
fn find(
|
||||
list: &[(String, ArtifactRef)],
|
||||
guard: &Arc<OwnerGuard>,
|
||||
name: &str,
|
||||
) -> Result<Artifact, BusError> {
|
||||
list.iter()
|
||||
.find(|(n, _)| n == name)
|
||||
.map(|(_, r)| Artifact {
|
||||
reference: Arc::new(r.clone()),
|
||||
owner: guard.clone(),
|
||||
})
|
||||
.ok_or_else(|| BusError::invalid(format!("no attachment named {name:?}")))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Artifacts
|
||||
|
||||
/// A read-only, cloneable handle on an immutable artifact. While any clone (or a file opened
|
||||
/// from it) lives, the owner it came from keeps the bytes alive.
|
||||
#[derive(Clone)]
|
||||
pub struct Artifact {
|
||||
pub(crate) reference: Arc<ArtifactRef>,
|
||||
pub(crate) owner: Arc<OwnerGuard>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Artifact {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Artifact")
|
||||
.field("reference", &self.reference)
|
||||
.field("owner", &self.owner.id)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Artifact {
|
||||
pub fn reference(&self) -> &ArtifactRef {
|
||||
&self.reference
|
||||
}
|
||||
|
||||
/// The router-side owner this handle rides on: a delivery id or an explicit hold id.
|
||||
pub fn owner_id(&self) -> &str {
|
||||
&self.owner.id
|
||||
}
|
||||
|
||||
/// Opens the sealed bytes read-only. The file keeps this handle (and so its owner) alive.
|
||||
pub async fn open(&self) -> Result<ArtifactFile, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("ref".into(), self.reference.to_json());
|
||||
body.insert("ownerId".into(), self.owner.id.clone().into());
|
||||
let shared = &self.owner.conn.shared;
|
||||
let reply = shared
|
||||
.command(
|
||||
"artifact.open",
|
||||
body,
|
||||
Vec::new(),
|
||||
vec![self.owner.clone()],
|
||||
Hook::None,
|
||||
)
|
||||
.await?;
|
||||
let loc = Location::from_json(Fields::of(&reply.value, "reply").value("readLocation")?)?;
|
||||
if loc.store_id != self.reference.store_id {
|
||||
return Err(BusError::new(
|
||||
ErrorCode::ArtifactGone,
|
||||
"read location names another store",
|
||||
));
|
||||
}
|
||||
let path = resolve(&shared.store_root, &loc.store_id, &loc)?;
|
||||
let file = open_read(&path)
|
||||
.map_err(|e| BusError::new(ErrorCode::StoreFailure, format!("open: {e}")))?;
|
||||
let len = file
|
||||
.metadata()
|
||||
.map_err(|e| BusError::new(ErrorCode::StoreFailure, e.to_string()))?
|
||||
.len();
|
||||
if len != self.reference.byte_length {
|
||||
return Err(BusError::new(
|
||||
ErrorCode::ArtifactMismatch,
|
||||
"sealed file length disagrees with the reference",
|
||||
));
|
||||
}
|
||||
Ok(ArtifactFile {
|
||||
file,
|
||||
_artifact: self.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads the whole artifact (on the blocking pool).
|
||||
pub async fn read_all(&self) -> Result<Vec<u8>, BusError> {
|
||||
let mut file = self.open().await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut out = Vec::with_capacity(file.len() as usize);
|
||||
file.read_to_end(&mut out).map(|_| out)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| BusError::new(ErrorCode::StoreFailure, e.to_string()))?
|
||||
.map_err(|e| BusError::new(ErrorCode::StoreFailure, e.to_string()))
|
||||
}
|
||||
|
||||
/// Creates an independent explicit hold, so the bytes outlive this handle's delivery.
|
||||
pub async fn retain(&self) -> Result<Artifact, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("ref".into(), self.reference.to_json());
|
||||
body.insert("ownerId".into(), self.owner.id.clone().into());
|
||||
let shared = &self.owner.conn.shared;
|
||||
let reply = shared
|
||||
.command(
|
||||
"artifact.retain",
|
||||
body,
|
||||
Vec::new(),
|
||||
vec![self.owner.clone()],
|
||||
Hook::Owner,
|
||||
)
|
||||
.await?;
|
||||
match reply.extra {
|
||||
Extra::Owner(owner) => Ok(Artifact {
|
||||
reference: self.reference.clone(),
|
||||
owner,
|
||||
}),
|
||||
_ => Err(BusError::lost("retain reply without an owner")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An open, read-only sealed file. Holds its [`Artifact`].
|
||||
pub struct ArtifactFile {
|
||||
file: File,
|
||||
_artifact: Artifact,
|
||||
}
|
||||
|
||||
impl ArtifactFile {
|
||||
pub fn len(&self) -> u64 {
|
||||
self._artifact.reference.byte_length
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
pub fn artifact(&self) -> &Artifact {
|
||||
&self._artifact
|
||||
}
|
||||
}
|
||||
|
||||
impl Read for ArtifactFile {
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
self.file.read(buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for ArtifactFile {
|
||||
fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
|
||||
self.file.seek(pos)
|
||||
}
|
||||
}
|
||||
|
||||
/// The unique writer of a freshly allocated artifact. Not cloneable. Dropping it unsealed
|
||||
/// releases the staging storage; [`ArtifactWriter::seal`] consumes it.
|
||||
pub struct ArtifactWriter {
|
||||
pub(crate) file: Option<File>,
|
||||
pub(crate) owner: Arc<OwnerGuard>,
|
||||
pub(crate) artifact_id: String,
|
||||
pub(crate) byte_length: u64,
|
||||
pub(crate) written: u64,
|
||||
}
|
||||
|
||||
impl ArtifactWriter {
|
||||
pub fn artifact_id(&self) -> &str {
|
||||
&self.artifact_id
|
||||
}
|
||||
|
||||
pub fn byte_length(&self) -> u64 {
|
||||
self.byte_length
|
||||
}
|
||||
|
||||
/// Closes the writable handle and seals, returning the immutable artifact on an explicit
|
||||
/// hold. Bytes not written read as zeros: the staging file is preallocated.
|
||||
pub async fn seal(self) -> Result<Artifact, BusError> {
|
||||
self.seal_with_digest(None).await
|
||||
}
|
||||
|
||||
/// As [`seal`](Self::seal), and the router refuses the seal unless the content's SHA-256
|
||||
/// (lowercase hex) equals `digest`.
|
||||
pub async fn seal_with_digest(mut self, digest: Option<String>) -> Result<Artifact, BusError> {
|
||||
drop(self.file.take());
|
||||
let mut body = Map::new();
|
||||
body.insert("artifactId".into(), self.artifact_id.clone().into());
|
||||
body.insert("generation".into(), GENERATION.to_string().into());
|
||||
body.insert("ownerId".into(), self.owner.id.clone().into());
|
||||
body.insert("digest".into(), digest.map_or(Value::Null, Value::String));
|
||||
let shared = &self.owner.conn.shared;
|
||||
let reply = shared
|
||||
.command(
|
||||
"artifact.seal",
|
||||
body,
|
||||
Vec::new(),
|
||||
vec![self.owner.clone()],
|
||||
Hook::None,
|
||||
)
|
||||
.await?;
|
||||
let reference = ArtifactRef::from_json(Fields::of(&reply.value, "reply").value("ref")?)?;
|
||||
Ok(Artifact {
|
||||
reference: Arc::new(reference),
|
||||
owner: self.owner.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for ArtifactWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let room = self.byte_length - self.written;
|
||||
if buf.len() as u64 > room {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"write past the allocated length",
|
||||
));
|
||||
}
|
||||
let file = self
|
||||
.file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("writer is closed"))?;
|
||||
let n = file.write(buf)?;
|
||||
self.written += n as u64;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.as_mut().map_or(Ok(()), |f| f.flush())
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Deliveries
|
||||
|
||||
/// One `topic.message` delivery. Dropping it, and every artifact taken from it, consumes the
|
||||
/// delivery and returns its credit.
|
||||
pub struct Message {
|
||||
pub(crate) guard: Arc<OwnerGuard>,
|
||||
pub(crate) subscription_id: String,
|
||||
pub(crate) topic: String,
|
||||
pub(crate) topic_incarnation: String,
|
||||
pub(crate) topic_sequence: u64,
|
||||
pub(crate) replaced: u64,
|
||||
pub(crate) payload: Map<String, Value>,
|
||||
pub(crate) attachments: Vec<(String, ArtifactRef)>,
|
||||
}
|
||||
|
||||
impl Message {
|
||||
pub fn delivery_id(&self) -> &str {
|
||||
&self.guard.id
|
||||
}
|
||||
pub fn subscription_id(&self) -> &str {
|
||||
&self.subscription_id
|
||||
}
|
||||
pub fn topic(&self) -> &str {
|
||||
&self.topic
|
||||
}
|
||||
pub fn topic_incarnation(&self) -> &str {
|
||||
&self.topic_incarnation
|
||||
}
|
||||
pub fn topic_sequence(&self) -> u64 {
|
||||
self.topic_sequence
|
||||
}
|
||||
/// Undelivered messages coalesced into this one since the previous delivery.
|
||||
pub fn replaced(&self) -> u64 {
|
||||
self.replaced
|
||||
}
|
||||
pub fn payload(&self) -> &Map<String, Value> {
|
||||
&self.payload
|
||||
}
|
||||
pub fn attachment_names(&self) -> impl Iterator<Item = &str> {
|
||||
self.attachments.iter().map(|(n, _)| n.as_str())
|
||||
}
|
||||
/// An artifact handle sharing this delivery's guard.
|
||||
pub fn artifact(&self, name: &str) -> Result<Artifact, BusError> {
|
||||
find(&self.attachments, &self.guard, name)
|
||||
}
|
||||
}
|
||||
|
||||
/// An incoming `rpc.request`. Dropping it (and its artifacts) consumes the request delivery;
|
||||
/// that is independent of replying.
|
||||
pub struct Request {
|
||||
pub(crate) guard: Arc<OwnerGuard>,
|
||||
pub(crate) reply_guard: Arc<ReplyGuard>,
|
||||
pub(crate) call_id: String,
|
||||
pub(crate) caller: Identity,
|
||||
pub(crate) target: String,
|
||||
pub(crate) service_incarnation: String,
|
||||
pub(crate) method: String,
|
||||
pub(crate) payload: Map<String, Value>,
|
||||
pub(crate) attachments: Vec<(String, ArtifactRef)>,
|
||||
}
|
||||
|
||||
impl Request {
|
||||
pub fn delivery_id(&self) -> &str {
|
||||
&self.guard.id
|
||||
}
|
||||
pub fn call_id(&self) -> &str {
|
||||
&self.call_id
|
||||
}
|
||||
/// The caller identity supplied by the router. It is authenticated only when the router
|
||||
/// accepted this connection through a launcher-bound `*_as` transport entry point.
|
||||
pub fn caller(&self) -> &Identity {
|
||||
&self.caller
|
||||
}
|
||||
pub fn target(&self) -> &str {
|
||||
&self.target
|
||||
}
|
||||
pub fn service_incarnation(&self) -> &str {
|
||||
&self.service_incarnation
|
||||
}
|
||||
pub fn method(&self) -> &str {
|
||||
&self.method
|
||||
}
|
||||
pub fn payload(&self) -> &Map<String, Value> {
|
||||
&self.payload
|
||||
}
|
||||
pub fn artifact(&self, name: &str) -> Result<Artifact, BusError> {
|
||||
find(&self.attachments, &self.guard, name)
|
||||
}
|
||||
/// A reply capability that keeps bounded router correlation alive independently of the
|
||||
/// request delivery credit.
|
||||
pub fn responder(&self) -> Responder {
|
||||
Responder {
|
||||
guard: self.reply_guard.clone(),
|
||||
call_id: self.call_id.clone(),
|
||||
request_delivery_id: self.guard.id.clone(),
|
||||
}
|
||||
}
|
||||
/// Replies. `Ok(true)` if routed to the caller, `Ok(false)` if the caller had detached.
|
||||
pub async fn reply(
|
||||
&self,
|
||||
outcome: Map<String, Value>,
|
||||
attachments: &[(&str, &Artifact)],
|
||||
) -> Result<bool, BusError> {
|
||||
self.responder().reply(outcome, attachments).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Replies to one request, whether or not the request delivery is still held.
|
||||
#[derive(Clone)]
|
||||
pub struct Responder {
|
||||
guard: Arc<ReplyGuard>,
|
||||
call_id: String,
|
||||
request_delivery_id: String,
|
||||
}
|
||||
|
||||
impl Responder {
|
||||
pub async fn reply(
|
||||
&self,
|
||||
outcome: Map<String, Value>,
|
||||
attachments: &[(&str, &Artifact)],
|
||||
) -> Result<bool, BusError> {
|
||||
let (atts, keep) = attachment_list(attachments);
|
||||
let mut body = Map::new();
|
||||
body.insert("callId".into(), self.call_id.clone().into());
|
||||
body.insert(
|
||||
"requestDeliveryId".into(),
|
||||
self.request_delivery_id.clone().into(),
|
||||
);
|
||||
body.insert("outcome".into(), Value::Object(outcome));
|
||||
let reply = self
|
||||
.guard
|
||||
.conn
|
||||
.shared
|
||||
.command("rpc.reply", body, atts, keep, Hook::None)
|
||||
.await?;
|
||||
Ok(Fields::of(&reply.value, "reply").boolean("routed")?)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ReplyGuard {
|
||||
pub conn: Arc<ClientConn>,
|
||||
pub call_id: String,
|
||||
pub request_delivery_id: String,
|
||||
}
|
||||
|
||||
impl Drop for ReplyGuard {
|
||||
fn drop(&mut self) {
|
||||
self.conn.shared.push_control(Control::ResponderReleased {
|
||||
call_id: self.call_id.clone(),
|
||||
request_delivery_id: self.request_delivery_id.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// A terminal `rpc.result`. Dropping it (and its artifacts) consumes the result delivery.
|
||||
pub struct RpcResult {
|
||||
pub(crate) guard: Arc<OwnerGuard>,
|
||||
pub(crate) call_id: String,
|
||||
pub(crate) responder: Identity,
|
||||
pub(crate) service_incarnation: String,
|
||||
pub(crate) outcome: Map<String, Value>,
|
||||
pub(crate) attachments: Vec<(String, ArtifactRef)>,
|
||||
}
|
||||
|
||||
impl RpcResult {
|
||||
pub fn delivery_id(&self) -> &str {
|
||||
&self.guard.id
|
||||
}
|
||||
pub fn call_id(&self) -> &str {
|
||||
&self.call_id
|
||||
}
|
||||
pub fn responder(&self) -> &Identity {
|
||||
&self.responder
|
||||
}
|
||||
pub fn service_incarnation(&self) -> &str {
|
||||
&self.service_incarnation
|
||||
}
|
||||
pub fn outcome(&self) -> &Map<String, Value> {
|
||||
&self.outcome
|
||||
}
|
||||
pub fn artifact(&self, name: &str) -> Result<Artifact, BusError> {
|
||||
find(&self.attachments, &self.guard, name)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Services, subscriptions, calls
|
||||
|
||||
pub(crate) struct ServiceGuard {
|
||||
pub conn: Arc<ClientConn>,
|
||||
pub incarnation: String,
|
||||
}
|
||||
|
||||
/// A registered service endpoint. Dropping it unregisters.
|
||||
pub struct Service {
|
||||
pub(crate) name: String,
|
||||
pub(crate) rx: mpsc::UnboundedReceiver<Request>,
|
||||
pub(crate) guard: ServiceGuard,
|
||||
}
|
||||
|
||||
impl Service {
|
||||
pub fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
pub fn incarnation(&self) -> &str {
|
||||
&self.guard.incarnation
|
||||
}
|
||||
/// The next request, or `None` once the connection is closed.
|
||||
pub async fn next(&mut self) -> Option<Request> {
|
||||
self.rx.recv().await
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Service {
|
||||
fn drop(&mut self) {
|
||||
let shared = &self.guard.conn.shared;
|
||||
let tx = shared.lock().services.remove(&self.guard.incarnation);
|
||||
drop(tx);
|
||||
// Route removal must precede abandoning buffered requests, whose last responder drops
|
||||
// queue responder-release controls. This preserves NO_SERVICE as their terminal cause.
|
||||
shared.push_control(Control::Unregister {
|
||||
name: self.name.clone(),
|
||||
service_incarnation: self.guard.incarnation.clone(),
|
||||
});
|
||||
self.rx.close();
|
||||
while let Ok(request) = self.rx.try_recv() {
|
||||
drop(request);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct SubscriptionGuard {
|
||||
pub conn: Arc<ClientConn>,
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
/// An exact-topic subscription. Dropping it unsubscribes; messages already handed out stay
|
||||
/// valid.
|
||||
pub struct Subscription {
|
||||
pub(crate) rx: mpsc::UnboundedReceiver<Message>,
|
||||
pub(crate) guard: SubscriptionGuard,
|
||||
pub(crate) topic_incarnation: String,
|
||||
}
|
||||
|
||||
impl Subscription {
|
||||
pub fn id(&self) -> &str {
|
||||
&self.guard.id
|
||||
}
|
||||
pub fn topic_incarnation(&self) -> &str {
|
||||
&self.topic_incarnation
|
||||
}
|
||||
/// The next message, or `None` once the subscription or connection is closed.
|
||||
pub async fn next(&mut self) -> Option<Message> {
|
||||
self.rx.recv().await
|
||||
}
|
||||
/// A message already delivered to this client, without waiting.
|
||||
pub fn try_next(&mut self) -> Option<Message> {
|
||||
self.rx.try_recv().ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Subscription {
|
||||
fn drop(&mut self) {
|
||||
let shared = &self.guard.conn.shared;
|
||||
let tx = shared.lock().subscriptions.remove(&self.guard.id);
|
||||
drop(tx);
|
||||
let mut body = Map::new();
|
||||
body.insert("subscriptionId".into(), self.guard.id.clone().into());
|
||||
let cmd = OutCommand {
|
||||
op: "unsubscribe",
|
||||
body,
|
||||
attachments: Vec::new(),
|
||||
respond: None,
|
||||
hook: Hook::None,
|
||||
keep: Vec::new(),
|
||||
};
|
||||
let _ = shared.enqueue(cmd);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct CallGuard {
|
||||
pub conn: Arc<ClientConn>,
|
||||
pub call_id: String,
|
||||
pub done: bool,
|
||||
}
|
||||
|
||||
impl Drop for CallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.done {
|
||||
return;
|
||||
}
|
||||
let shared = &self.conn.shared;
|
||||
let waiting = {
|
||||
let mut st = shared.lock();
|
||||
match st.calls.remove(&self.call_id) {
|
||||
Some(slot @ CallSlot::Waiting(_)) => {
|
||||
st.calls.insert(self.call_id.clone(), CallSlot::Abandoned);
|
||||
Some(slot)
|
||||
}
|
||||
other => {
|
||||
if let Some(o) = other {
|
||||
st.calls.insert(self.call_id.clone(), o);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
if waiting.is_some() {
|
||||
// Best effort; a result that still arrives is consumed by the reactor.
|
||||
shared.push_control(Control::Cancel(self.call_id.clone(), None));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! debug_fields {
|
||||
($ty:ident, |$s:ident| { $($name:literal => $val:expr),* $(,)? }) => {
|
||||
impl std::fmt::Debug for $ty {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let $s = self;
|
||||
f.debug_struct(stringify!($ty))$(.field($name, &$val))*.finish()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
debug_fields!(ArtifactFile, |s| { "artifact" => s._artifact });
|
||||
debug_fields!(ArtifactWriter, |s| { "artifact_id" => s.artifact_id, "byte_length" => s.byte_length, "owner" => s.owner.id });
|
||||
debug_fields!(Message, |s| {
|
||||
"delivery_id" => s.guard.id, "topic" => s.topic, "topic_sequence" => s.topic_sequence, "replaced" => s.replaced,
|
||||
});
|
||||
debug_fields!(Request, |s| { "delivery_id" => s.guard.id, "call_id" => s.call_id, "method" => s.method });
|
||||
debug_fields!(RpcResult, |s| { "delivery_id" => s.guard.id, "call_id" => s.call_id });
|
||||
debug_fields!(Responder, |s| { "call_id" => s.call_id, "request_delivery_id" => s.request_delivery_id });
|
||||
debug_fields!(Service, |s| { "name" => s.name, "incarnation" => s.guard.incarnation });
|
||||
debug_fields!(Subscription, |s| { "id" => s.guard.id, "topic_incarnation" => s.topic_incarnation });
|
||||
debug_fields!(PendingCall, |s| { "call_id" => s.guard.call_id, "service_incarnation" => s.service_incarnation });
|
||||
|
||||
/// The router's answer to `rpc.cancel`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum CancelState {
|
||||
CancelledBeforeDispatch,
|
||||
ExecutionUnknown,
|
||||
Completed,
|
||||
CallGone,
|
||||
}
|
||||
|
||||
/// An admitted call. [`result`](Self::result) waits for the terminal result. Dropping it
|
||||
/// unfinished sends a best-effort `rpc.cancel` and consumes any result that still arrives.
|
||||
pub struct PendingCall {
|
||||
pub(crate) guard: CallGuard,
|
||||
pub(crate) rx: oneshot::Receiver<Result<RpcResult, BusError>>,
|
||||
pub(crate) service_incarnation: String,
|
||||
}
|
||||
|
||||
impl PendingCall {
|
||||
pub fn call_id(&self) -> &str {
|
||||
&self.guard.call_id
|
||||
}
|
||||
|
||||
pub fn service_incarnation(&self) -> &str {
|
||||
&self.service_incarnation
|
||||
}
|
||||
|
||||
/// Waits for the result. Cancel-safe: wrap it in a timeout and call it again, or cancel.
|
||||
pub async fn result(&mut self) -> Result<RpcResult, BusError> {
|
||||
if self.guard.done {
|
||||
return Err(BusError::new(ErrorCode::CallGone, "result already taken"));
|
||||
}
|
||||
let r = (&mut self.rx)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(BusError::lost("connection closed")));
|
||||
self.guard.done = true;
|
||||
r
|
||||
}
|
||||
|
||||
/// Asks the router to cancel. Only `CancelledBeforeDispatch` establishes that the handler
|
||||
/// never ran. After any state but `Completed`, [`result`](Self::result) fails with
|
||||
/// `CALL_GONE`.
|
||||
pub async fn cancel(&self) -> Result<CancelState, BusError> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let shared = &self.guard.conn.shared;
|
||||
shared.push_control(Control::Cancel(self.guard.call_id.clone(), Some(tx)));
|
||||
let reply = rx
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(BusError::lost("connection closed")))?;
|
||||
match Fields::of(&reply.value, "reply").string("state")? {
|
||||
"cancelled-before-dispatch" => Ok(CancelState::CancelledBeforeDispatch),
|
||||
"execution-unknown" => Ok(CancelState::ExecutionUnknown),
|
||||
"completed" => Ok(CancelState::Completed),
|
||||
"call-gone" => Ok(CancelState::CallGone),
|
||||
other => Err(BusError::lost(format!("unknown cancel state {other:?}"))
|
||||
.with_dispatch(Dispatch::Unknown)),
|
||||
}
|
||||
}
|
||||
}
|
||||
598
services/flysim/crates/flybus/src/client/mod.rs
Normal file
598
services/flysim/crates/flybus/src/client/mod.rs
Normal file
|
|
@ -0,0 +1,598 @@
|
|||
//! The client SDK: one connection for RPC, pub/sub and artifacts.
|
||||
|
||||
mod handles;
|
||||
mod reactor;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
use handles::attachment_list;
|
||||
pub use handles::{
|
||||
Artifact, ArtifactFile, ArtifactWriter, CancelState, Message, PendingCall, Request, Responder,
|
||||
RpcResult, Service, Subscription,
|
||||
};
|
||||
use reactor::{CallSlot, ClientConn, Extra, Hook, OutCommand, Shared};
|
||||
|
||||
use crate::error::{BusError, Dispatch, ErrorCode};
|
||||
use crate::limits::Limits;
|
||||
use crate::store::{open_write, resolve};
|
||||
use crate::transport::Transport;
|
||||
use crate::wire::{
|
||||
Envelope, Fields, Identity, Kind, Location, MAJOR, contract_digest, hex, is_content_type,
|
||||
is_id, parse_serial_id, read_frame, serial_id, write_frame,
|
||||
};
|
||||
|
||||
/// How a participant connects.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ClientConfig {
|
||||
/// The configured participant identity the launcher's policy knows.
|
||||
pub client_id: String,
|
||||
/// This SDK client's lifetime; generated when `None`. Reconnecting needs a new one.
|
||||
pub client_incarnation: Option<String>,
|
||||
/// The router's store root, as configured for the router.
|
||||
pub store_root: PathBuf,
|
||||
/// Queued consumes, releases and cancels before the client gives up on the connection.
|
||||
pub control_lane_capacity: usize,
|
||||
}
|
||||
|
||||
impl ClientConfig {
|
||||
pub fn new(client_id: &str, store_root: impl Into<PathBuf>) -> ClientConfig {
|
||||
ClientConfig {
|
||||
client_id: client_id.to_owned(),
|
||||
client_incarnation: None,
|
||||
store_root: store_root.into(),
|
||||
control_lane_capacity: 4096,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What `bus.hello` negotiated.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SessionInfo {
|
||||
pub router_id: String,
|
||||
pub connection_id: String,
|
||||
pub identity: Identity,
|
||||
pub selected_major: u64,
|
||||
pub selected_minor: u64,
|
||||
pub contract_digest: String,
|
||||
pub limits: Limits,
|
||||
}
|
||||
|
||||
/// `publish` admission: counts, not consumption.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct PublishReceipt {
|
||||
pub topic_sequence: u64,
|
||||
pub subscribers: u64,
|
||||
pub replaced: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TopicInfo {
|
||||
/// False when an identical declaration already existed.
|
||||
pub declared: bool,
|
||||
pub topic_incarnation: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Retained {
|
||||
None,
|
||||
Latest,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
/// One replaceable queued value; `max_queued` is always 1.
|
||||
Latest,
|
||||
/// FIFO; a full queue refuses the whole publication.
|
||||
Bounded,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct SubscriptionConfig {
|
||||
pub mode: Mode,
|
||||
pub max_queued: u32,
|
||||
pub max_in_flight: u32,
|
||||
pub replay_latest: bool,
|
||||
}
|
||||
|
||||
impl SubscriptionConfig {
|
||||
/// `latest`, 1 queued, 2 in flight.
|
||||
pub fn latest() -> SubscriptionConfig {
|
||||
SubscriptionConfig {
|
||||
mode: Mode::Latest,
|
||||
max_queued: 1,
|
||||
max_in_flight: 2,
|
||||
replay_latest: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// `bounded`, 64 queued, 16 in flight.
|
||||
pub fn bounded() -> SubscriptionConfig {
|
||||
SubscriptionConfig {
|
||||
mode: Mode::Bounded,
|
||||
max_queued: 64,
|
||||
max_in_flight: 16,
|
||||
replay_latest: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn queued(mut self, n: u32) -> SubscriptionConfig {
|
||||
self.max_queued = n;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn in_flight(mut self, n: u32) -> SubscriptionConfig {
|
||||
self.max_in_flight = n;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn replay(mut self, replay: bool) -> SubscriptionConfig {
|
||||
self.replay_latest = replay;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ServiceConfig {
|
||||
pub max_queued: u32,
|
||||
pub max_in_flight: u32,
|
||||
}
|
||||
|
||||
impl Default for ServiceConfig {
|
||||
fn default() -> ServiceConfig {
|
||||
ServiceConfig {
|
||||
max_queued: 16,
|
||||
max_in_flight: 16,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A connected participant. Cheap to clone; the connection closes when the client, and every
|
||||
/// handle made from it, is dropped, or on [`close`](Client::close).
|
||||
#[derive(Clone)]
|
||||
pub struct Client {
|
||||
conn: Arc<ClientConn>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Client {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Client").field("info", self.info()).finish()
|
||||
}
|
||||
}
|
||||
|
||||
fn fresh_incarnation() -> String {
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_nanos());
|
||||
let mut h = Sha256::new();
|
||||
h.update(std::process::id().to_le_bytes());
|
||||
h.update(nanos.to_le_bytes());
|
||||
h.update(COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes());
|
||||
format!("inc-{}", hex(&h.finalize()[..8]))
|
||||
}
|
||||
|
||||
fn field<'a>(m: &'a Map<String, Value>) -> Fields<'a> {
|
||||
Fields::of(m, "reply")
|
||||
}
|
||||
|
||||
impl Client {
|
||||
/// Connects over a transport and negotiates `bus.hello`. Must run inside a Tokio runtime.
|
||||
pub async fn connect(transport: Transport, config: ClientConfig) -> Result<Client, BusError> {
|
||||
if !is_id(&config.client_id) {
|
||||
return Err(BusError::invalid("client id is not a valid id"));
|
||||
}
|
||||
let incarnation = config
|
||||
.client_incarnation
|
||||
.clone()
|
||||
.unwrap_or_else(fresh_incarnation);
|
||||
if !is_id(&incarnation) {
|
||||
return Err(BusError::invalid("client incarnation is not a valid id"));
|
||||
}
|
||||
let (mut rd, mut wr) = tokio::io::split(transport);
|
||||
let mut body = Map::new();
|
||||
body.insert("clientId".into(), config.client_id.clone().into());
|
||||
body.insert("clientIncarnation".into(), incarnation.clone().into());
|
||||
body.insert("supportedMajors".into(), Value::Array(vec![MAJOR.into()]));
|
||||
let hello = Envelope::new(serial_id("msg", 1), Kind::Command, "bus.hello", body);
|
||||
let lost = |e: String| BusError::lost(format!("hello: {e}"));
|
||||
write_frame(&mut wr, &hello.encode()?)
|
||||
.await
|
||||
.map_err(|e| lost(e.to_string()))?;
|
||||
let bytes = read_frame(&mut rd)
|
||||
.await
|
||||
.map_err(|e| lost(e.to_string()))?
|
||||
.ok_or_else(|| lost("connection closed".into()))?;
|
||||
let env = Envelope::decode(&bytes).map_err(|e| lost(e.0))?;
|
||||
let hello_router_serial = parse_serial_id("bus", &env.id)
|
||||
.filter(|n| *n > 0)
|
||||
.ok_or_else(|| lost("hello reply id is not canonical bus-<U64>".into()))?;
|
||||
if env.major != MAJOR
|
||||
|| env.minor != crate::wire::MINOR
|
||||
|| env.kind != Kind::Reply
|
||||
|| env.reply_to.as_deref() != Some("msg-1")
|
||||
|| env.op != "bus.hello"
|
||||
|| !env.attachments.is_empty()
|
||||
{
|
||||
return Err(lost("expected the hello reply".into()));
|
||||
}
|
||||
let mut f = Fields::of(&env.body, "reply");
|
||||
if !f.boolean("ok")? {
|
||||
let e = f.object("error")?;
|
||||
let mut g = Fields::of(e, "error");
|
||||
let code = ErrorCode::parse(g.string("code")?)
|
||||
.ok_or_else(|| BusError::lost("hello error has unknown code"))?;
|
||||
let message = g.string("message")?.to_owned();
|
||||
let dispatch = Dispatch::parse(g.string("dispatch")?)
|
||||
.ok_or_else(|| BusError::lost("hello error has unknown dispatch"))?;
|
||||
g.finish()?;
|
||||
f.finish()?;
|
||||
return Err(BusError {
|
||||
code,
|
||||
message,
|
||||
dispatch,
|
||||
});
|
||||
}
|
||||
let value = f.object("value")?;
|
||||
let mut v = Fields::of(value, "hello");
|
||||
let info = SessionInfo {
|
||||
router_id: v.id("routerId")?,
|
||||
connection_id: v.id("connectionId")?,
|
||||
identity: Identity {
|
||||
client_id: config.client_id.clone(),
|
||||
client_incarnation: incarnation,
|
||||
},
|
||||
selected_major: v.int("selectedMajor", 0, u64::MAX)?,
|
||||
selected_minor: v.int("selectedMinor", 0, u64::MAX)?,
|
||||
contract_digest: v.string("contractDigest")?.to_owned(),
|
||||
limits: Limits::from_json(v.value("limits")?)?,
|
||||
};
|
||||
v.finish()?;
|
||||
f.finish()?;
|
||||
if info.selected_major != MAJOR
|
||||
|| info.selected_minor != crate::wire::MINOR
|
||||
|| info.contract_digest != contract_digest()
|
||||
{
|
||||
return Err(BusError::new(
|
||||
ErrorCode::VersionMismatch,
|
||||
"router speaks a different contract",
|
||||
));
|
||||
}
|
||||
let shared = Arc::new(Shared::new(
|
||||
info,
|
||||
config.store_root,
|
||||
config.control_lane_capacity.max(1),
|
||||
1,
|
||||
hello_router_serial,
|
||||
));
|
||||
let conn = Arc::new(ClientConn {
|
||||
shared: shared.clone(),
|
||||
});
|
||||
tokio::spawn(reactor::read_loop(
|
||||
shared.clone(),
|
||||
Arc::downgrade(&conn),
|
||||
rd,
|
||||
));
|
||||
tokio::spawn(reactor::write_loop(shared, wr));
|
||||
Ok(Client { conn })
|
||||
}
|
||||
|
||||
/// Connects to a router's Unix-domain socket.
|
||||
pub async fn connect_unix(
|
||||
path: impl AsRef<Path>,
|
||||
config: ClientConfig,
|
||||
) -> Result<Client, BusError> {
|
||||
let t = Transport::unix(path)
|
||||
.await
|
||||
.map_err(|e| BusError::lost(format!("connect: {e}")))?;
|
||||
Client::connect(t, config).await
|
||||
}
|
||||
|
||||
fn shared(&self) -> &Shared {
|
||||
&self.conn.shared
|
||||
}
|
||||
|
||||
pub fn info(&self) -> &SessionInfo {
|
||||
&self.shared().info
|
||||
}
|
||||
|
||||
/// Why the connection closed, once it has.
|
||||
pub fn closed(&self) -> Option<BusError> {
|
||||
self.shared().lock().closed.clone()
|
||||
}
|
||||
|
||||
/// Replies to fire-and-forget releases, consumes and unregisters that came back as
|
||||
/// errors. Nonzero means the client and router disagreed about ownership.
|
||||
pub fn control_errors(&self) -> u64 {
|
||||
self.shared().lock().control_errors
|
||||
}
|
||||
|
||||
/// Flushes queued releases, closes the connection and waits until the reader has stopped.
|
||||
/// Handles that outlive this become inert; the router releases what they owned.
|
||||
pub async fn close(self) {
|
||||
let shared = self.conn.shared.clone();
|
||||
shared.begin_shutdown();
|
||||
let mut done = shared.done.subscribe();
|
||||
drop(self);
|
||||
let _ = done.wait_for(|v| *v).await;
|
||||
}
|
||||
|
||||
// ---- services and RPC
|
||||
|
||||
/// Registers an exclusive service endpoint.
|
||||
pub async fn register(&self, name: &str, config: ServiceConfig) -> Result<Service, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("name".into(), name.into());
|
||||
body.insert("maxQueued".into(), config.max_queued.into());
|
||||
body.insert("maxInFlight".into(), config.max_in_flight.into());
|
||||
let reply = self
|
||||
.shared()
|
||||
.command(
|
||||
"service.register",
|
||||
body,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Hook::Register,
|
||||
)
|
||||
.await?;
|
||||
match reply.extra {
|
||||
Extra::Service(guard, rx) => Ok(Service {
|
||||
name: name.to_owned(),
|
||||
rx,
|
||||
guard,
|
||||
}),
|
||||
_ => Err(BusError::lost("register reply without a service")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Calls `method` on `service`, pinned to `expected_incarnation` when given. Returns once
|
||||
/// the router has admitted the call; the result arrives through the [`PendingCall`].
|
||||
pub async fn call(
|
||||
&self,
|
||||
service: &str,
|
||||
expected_incarnation: Option<&str>,
|
||||
method: &str,
|
||||
payload: Map<String, Value>,
|
||||
attachments: &[(&str, &Artifact)],
|
||||
) -> Result<PendingCall, BusError> {
|
||||
let (atts, keep) = attachment_list(attachments);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let (reply_tx, reply_rx) = oneshot::channel();
|
||||
let shared = self.shared();
|
||||
// The call id, its result slot and its queue position are fixed together, so call ids
|
||||
// reach the router in increasing order.
|
||||
let rejected = {
|
||||
let mut st = shared.lock();
|
||||
if let Some(e) = &st.closed {
|
||||
Some((e.clone(), keep))
|
||||
} else {
|
||||
st.next_call += 1;
|
||||
let call_id = serial_id("call", st.next_call);
|
||||
let mut body = Map::new();
|
||||
body.insert("callId".into(), call_id.clone().into());
|
||||
body.insert("target".into(), service.into());
|
||||
body.insert(
|
||||
"expectedIncarnation".into(),
|
||||
expected_incarnation.map_or(Value::Null, Value::from),
|
||||
);
|
||||
body.insert("method".into(), method.into());
|
||||
body.insert("payload".into(), Value::Object(payload));
|
||||
st.calls
|
||||
.insert(call_id.clone(), CallSlot::Waiting(Some(tx)));
|
||||
st.ordinary.push_back(OutCommand {
|
||||
op: "rpc.call",
|
||||
body,
|
||||
attachments: atts,
|
||||
respond: Some(reply_tx),
|
||||
hook: Hook::Call(call_id),
|
||||
keep,
|
||||
});
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some((e, keep)) = rejected {
|
||||
drop(keep);
|
||||
return Err(e);
|
||||
}
|
||||
shared.wake();
|
||||
let reply = reply_rx
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(BusError::lost("connection closed")))?;
|
||||
let service_incarnation = field(&reply.value).id("serviceIncarnation")?;
|
||||
match reply.extra {
|
||||
Extra::Call(guard) => Ok(PendingCall {
|
||||
guard,
|
||||
rx,
|
||||
service_incarnation,
|
||||
}),
|
||||
_ => Err(BusError::lost("call reply without a call handle")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Admits a call and waits for its result.
|
||||
pub async fn call_and_wait(
|
||||
&self,
|
||||
service: &str,
|
||||
expected_incarnation: Option<&str>,
|
||||
method: &str,
|
||||
payload: Map<String, Value>,
|
||||
attachments: &[(&str, &Artifact)],
|
||||
) -> Result<RpcResult, BusError> {
|
||||
let mut pending = self
|
||||
.call(service, expected_incarnation, method, payload, attachments)
|
||||
.await?;
|
||||
pending.result().await
|
||||
}
|
||||
|
||||
// ---- topics
|
||||
|
||||
pub async fn declare_topic(
|
||||
&self,
|
||||
name: &str,
|
||||
retained: Retained,
|
||||
) -> Result<TopicInfo, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("name".into(), name.into());
|
||||
body.insert(
|
||||
"retained".into(),
|
||||
if retained == Retained::Latest {
|
||||
"latest"
|
||||
} else {
|
||||
"none"
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let reply = self
|
||||
.shared()
|
||||
.command("topic.declare", body, Vec::new(), Vec::new(), Hook::None)
|
||||
.await?;
|
||||
let mut f = field(&reply.value);
|
||||
Ok(TopicInfo {
|
||||
declared: f.boolean("declared")?,
|
||||
topic_incarnation: f.id("topicIncarnation")?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Releases the retained value. `false` if there was none.
|
||||
pub async fn clear_topic(&self, name: &str) -> Result<bool, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("name".into(), name.into());
|
||||
let reply = self
|
||||
.shared()
|
||||
.command("topic.clear", body, Vec::new(), Vec::new(), Hook::None)
|
||||
.await?;
|
||||
Ok(field(&reply.value).boolean("cleared")?)
|
||||
}
|
||||
|
||||
/// Deletes a topic with no subscribers. `false` if it did not exist.
|
||||
pub async fn delete_topic(&self, name: &str) -> Result<bool, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("name".into(), name.into());
|
||||
let reply = self
|
||||
.shared()
|
||||
.command("topic.delete", body, Vec::new(), Vec::new(), Hook::None)
|
||||
.await?;
|
||||
Ok(field(&reply.value).boolean("deleted")?)
|
||||
}
|
||||
|
||||
pub async fn subscribe(
|
||||
&self,
|
||||
topic: &str,
|
||||
config: SubscriptionConfig,
|
||||
) -> Result<Subscription, BusError> {
|
||||
let mut body = Map::new();
|
||||
body.insert("topic".into(), topic.into());
|
||||
body.insert(
|
||||
"mode".into(),
|
||||
if config.mode == Mode::Latest {
|
||||
"latest"
|
||||
} else {
|
||||
"bounded"
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
body.insert("maxQueued".into(), config.max_queued.into());
|
||||
body.insert("maxInFlight".into(), config.max_in_flight.into());
|
||||
body.insert("replayLatest".into(), config.replay_latest.into());
|
||||
let reply = self
|
||||
.shared()
|
||||
.command("subscribe", body, Vec::new(), Vec::new(), Hook::Subscribe)
|
||||
.await?;
|
||||
let topic_incarnation = field(&reply.value).id("topicIncarnation")?;
|
||||
match reply.extra {
|
||||
Extra::Subscription(guard, rx) => Ok(Subscription {
|
||||
rx,
|
||||
guard,
|
||||
topic_incarnation,
|
||||
}),
|
||||
_ => Err(BusError::lost("subscribe reply without a subscription")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes. The attachments' owners are held until the router has answered.
|
||||
pub async fn publish(
|
||||
&self,
|
||||
topic: &str,
|
||||
payload: Map<String, Value>,
|
||||
attachments: &[(&str, &Artifact)],
|
||||
) -> Result<PublishReceipt, BusError> {
|
||||
let (atts, keep) = attachment_list(attachments);
|
||||
let mut body = Map::new();
|
||||
body.insert("topic".into(), topic.into());
|
||||
body.insert("payload".into(), Value::Object(payload));
|
||||
let reply = self
|
||||
.shared()
|
||||
.command("publish", body, atts, keep, Hook::None)
|
||||
.await?;
|
||||
let mut f = field(&reply.value);
|
||||
Ok(PublishReceipt {
|
||||
topic_sequence: f.u64_string("topicSequence")?,
|
||||
subscribers: f.u64_string("subscribers")?,
|
||||
replaced: f.u64_string("replaced")?,
|
||||
})
|
||||
}
|
||||
|
||||
// ---- artifacts
|
||||
|
||||
pub fn artifacts(&self) -> Artifacts<'_> {
|
||||
Artifacts { client: self }
|
||||
}
|
||||
}
|
||||
|
||||
/// The artifact half of the client API.
|
||||
pub struct Artifacts<'a> {
|
||||
client: &'a Client,
|
||||
}
|
||||
|
||||
impl Artifacts<'_> {
|
||||
/// Reserves `byte_length` bytes of private staging storage and opens it for writing.
|
||||
pub async fn allocate(
|
||||
&self,
|
||||
byte_length: u64,
|
||||
content_type: &str,
|
||||
) -> Result<ArtifactWriter, BusError> {
|
||||
if !is_content_type(content_type) {
|
||||
return Err(BusError::invalid(
|
||||
"contentType must be 1..=127 printable ASCII characters",
|
||||
));
|
||||
}
|
||||
let shared = self.client.shared();
|
||||
let mut body = Map::new();
|
||||
body.insert("byteLength".into(), byte_length.to_string().into());
|
||||
body.insert("contentType".into(), content_type.into());
|
||||
let reply = shared
|
||||
.command(
|
||||
"artifact.allocate",
|
||||
body,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Hook::Owner,
|
||||
)
|
||||
.await?;
|
||||
let Extra::Owner(owner) = reply.extra else {
|
||||
return Err(BusError::lost("allocate reply without an owner"));
|
||||
};
|
||||
let mut f = field(&reply.value);
|
||||
let artifact_id = f.id("artifactId")?;
|
||||
if parse_serial_id("a", &artifact_id).is_none() {
|
||||
return Err(BusError::lost("router issued a malformed artifact id"));
|
||||
}
|
||||
let loc = Location::from_json(f.value("writeLocation")?)?;
|
||||
let path = resolve(&shared.store_root, &loc.store_id, &loc)?;
|
||||
let file = open_write(&path)
|
||||
.map_err(|e| BusError::new(ErrorCode::StoreFailure, format!("staging: {e}")))?;
|
||||
Ok(ArtifactWriter {
|
||||
file: Some(file),
|
||||
owner,
|
||||
artifact_id,
|
||||
byte_length,
|
||||
written: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
992
services/flysim/crates/flybus/src/client/reactor.rs
Normal file
992
services/flysim/crates/flybus/src/client/reactor.rs
Normal file
|
|
@ -0,0 +1,992 @@
|
|||
//! The client's connection reactor: one reader task, one writer task, and the shared state
|
||||
//! they and every handle use.
|
||||
//!
|
||||
//! Lanes: the writer sends queued control items (consumes, releases, cancels) before ordinary
|
||||
//! commands, so dropping handles is never stuck behind a burst of publishes. Command ids are
|
||||
//! assigned when a command is written, which keeps them strictly increasing on the wire
|
||||
//! whatever order the lanes interleave in.
|
||||
//!
|
||||
//! Locking rule: nothing that can own a guard is dropped while the state mutex is held, because
|
||||
//! guard drops lock it. Values are taken out under the lock and dropped after it is released.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use tokio::io::{AsyncWriteExt, ReadHalf, WriteHalf};
|
||||
use tokio::sync::{Notify, mpsc, oneshot, watch};
|
||||
|
||||
use super::SessionInfo;
|
||||
use super::handles::{
|
||||
CallGuard, Message, OwnerGuard, ReplyGuard, Request, RpcResult, ServiceGuard, SubscriptionGuard,
|
||||
};
|
||||
use crate::error::{BusError, Dispatch, ErrorCode};
|
||||
use crate::transport::Transport;
|
||||
use crate::wire::{
|
||||
ArtifactRef, Attachment, Envelope, Fields, GENERATION, Identity, Kind, MAJOR, MAX_BATCH, MINOR,
|
||||
WireError, parse_serial_id, parse_u64, read_frame, serial_id, write_frame,
|
||||
};
|
||||
|
||||
pub(crate) type Responder = oneshot::Sender<Result<Reply, BusError>>;
|
||||
|
||||
/// A successful reply, plus whatever handle the reactor built from it.
|
||||
pub(crate) struct Reply {
|
||||
pub value: Map<String, Value>,
|
||||
pub extra: Extra,
|
||||
}
|
||||
|
||||
pub(crate) enum Extra {
|
||||
None,
|
||||
Owner(Arc<OwnerGuard>),
|
||||
Service(ServiceGuard, mpsc::UnboundedReceiver<Request>),
|
||||
Subscription(SubscriptionGuard, mpsc::UnboundedReceiver<Message>),
|
||||
Call(CallGuard),
|
||||
}
|
||||
|
||||
/// What the reactor does with a reply before handing it over. Handles are built here, inside
|
||||
/// the reactor, so a caller that abandoned its future cannot leak an owner it never saw: the
|
||||
/// handle drops with the undelivered reply and releases itself.
|
||||
pub(crate) enum Hook {
|
||||
None,
|
||||
/// `value.ownerId` is a new explicit hold.
|
||||
Owner,
|
||||
Register,
|
||||
Subscribe,
|
||||
Call(String),
|
||||
Cancel(String),
|
||||
}
|
||||
|
||||
pub(crate) struct OutCommand {
|
||||
pub op: &'static str,
|
||||
pub body: Map<String, Value>,
|
||||
pub attachments: Vec<Attachment>,
|
||||
pub respond: Option<Responder>,
|
||||
pub hook: Hook,
|
||||
/// Source owners held until the router has answered (bus-v1 section 8.2).
|
||||
pub keep: Vec<Arc<OwnerGuard>>,
|
||||
}
|
||||
|
||||
pub(crate) enum Control {
|
||||
Consumed(String),
|
||||
Release(String),
|
||||
Cancel(String, Option<Responder>),
|
||||
Unregister {
|
||||
name: String,
|
||||
service_incarnation: String,
|
||||
},
|
||||
ResponderReleased {
|
||||
call_id: String,
|
||||
request_delivery_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) enum CallSlot {
|
||||
Waiting(Option<oneshot::Sender<Result<RpcResult, BusError>>>),
|
||||
Abandoned,
|
||||
}
|
||||
|
||||
struct Pending {
|
||||
op: &'static str,
|
||||
respond: Option<Responder>,
|
||||
hook: Hook,
|
||||
_keep: Vec<Arc<OwnerGuard>>,
|
||||
}
|
||||
|
||||
pub(crate) struct ClientState {
|
||||
pub closed: Option<BusError>,
|
||||
pub shutting_down: bool,
|
||||
pub control: VecDeque<Control>,
|
||||
pub ordinary: VecDeque<OutCommand>,
|
||||
next_msg: u64,
|
||||
last_router: u64,
|
||||
last_delivery: u64,
|
||||
pending: HashMap<u64, Pending>,
|
||||
pub services: HashMap<String, mpsc::UnboundedSender<Request>>,
|
||||
pub subscriptions: HashMap<String, mpsc::UnboundedSender<Message>>,
|
||||
pub calls: HashMap<String, CallSlot>,
|
||||
pub next_call: u64,
|
||||
pub control_errors: u64,
|
||||
}
|
||||
|
||||
pub(crate) struct Shared {
|
||||
pub info: SessionInfo,
|
||||
pub store_root: PathBuf,
|
||||
state: Mutex<ClientState>,
|
||||
wake: Notify,
|
||||
control_capacity: usize,
|
||||
pub done: watch::Sender<bool>,
|
||||
shutdown: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
/// The connection's lifetime token. Every public handle holds one; the connection closes when
|
||||
/// the last is dropped (or on [`crate::Client::close`]).
|
||||
pub(crate) struct ClientConn {
|
||||
pub shared: Arc<Shared>,
|
||||
}
|
||||
|
||||
impl Drop for ClientConn {
|
||||
fn drop(&mut self) {
|
||||
self.shared.begin_shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
pub(crate) fn new(
|
||||
info: SessionInfo,
|
||||
store_root: PathBuf,
|
||||
control_capacity: usize,
|
||||
next_msg: u64,
|
||||
last_router: u64,
|
||||
) -> Shared {
|
||||
Shared {
|
||||
info,
|
||||
store_root,
|
||||
state: Mutex::new(ClientState {
|
||||
closed: None,
|
||||
shutting_down: false,
|
||||
control: VecDeque::new(),
|
||||
ordinary: VecDeque::new(),
|
||||
next_msg,
|
||||
last_router,
|
||||
last_delivery: 0,
|
||||
pending: HashMap::new(),
|
||||
services: HashMap::new(),
|
||||
subscriptions: HashMap::new(),
|
||||
calls: HashMap::new(),
|
||||
next_call: 0,
|
||||
control_errors: 0,
|
||||
}),
|
||||
wake: Notify::new(),
|
||||
control_capacity,
|
||||
done: watch::Sender::new(false),
|
||||
shutdown: watch::Sender::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lock(&self) -> MutexGuard<'_, ClientState> {
|
||||
self.state.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
pub(crate) fn wake(&self) {
|
||||
self.wake.notify_one();
|
||||
}
|
||||
|
||||
pub(crate) fn begin_shutdown(&self) {
|
||||
self.lock().shutting_down = true;
|
||||
self.wake.notify_one();
|
||||
}
|
||||
|
||||
/// Queues a control item. A full control lane closes the connection rather than lose a
|
||||
/// release (bus-v1 section 8.3).
|
||||
pub(crate) fn push_control(&self, item: Control) {
|
||||
let overflow = {
|
||||
let mut st = self.lock();
|
||||
if st.closed.is_some() {
|
||||
Some(item)
|
||||
} else if st.control.len() >= self.control_capacity {
|
||||
st.closed = Some(BusError::lost("client control lane exhausted"));
|
||||
Some(item)
|
||||
} else {
|
||||
st.control.push_back(item);
|
||||
None
|
||||
}
|
||||
};
|
||||
self.wake.notify_one();
|
||||
drop(overflow);
|
||||
}
|
||||
|
||||
/// Queues an ordinary command, or hands it back with the reason it cannot be sent.
|
||||
pub(crate) fn enqueue(&self, cmd: OutCommand) -> Result<(), BusError> {
|
||||
let rejected = {
|
||||
let mut st = self.lock();
|
||||
match &st.closed {
|
||||
Some(e) => Some((e.clone().with_dispatch(Dispatch::NotDispatched), cmd)),
|
||||
None => {
|
||||
st.ordinary.push_back(cmd);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
self.wake.notify_one();
|
||||
match rejected {
|
||||
Some((e, cmd)) => {
|
||||
drop(cmd);
|
||||
Err(e)
|
||||
}
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queues a command and waits for its reply.
|
||||
pub(crate) async fn command(
|
||||
&self,
|
||||
op: &'static str,
|
||||
body: Map<String, Value>,
|
||||
attachments: Vec<Attachment>,
|
||||
keep: Vec<Arc<OwnerGuard>>,
|
||||
hook: Hook,
|
||||
) -> Result<Reply, BusError> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.enqueue(OutCommand {
|
||||
op,
|
||||
body,
|
||||
attachments,
|
||||
respond: Some(tx),
|
||||
hook,
|
||||
keep,
|
||||
})?;
|
||||
rx.await
|
||||
.unwrap_or_else(|_| Err(BusError::lost("connection closed")))
|
||||
}
|
||||
|
||||
/// Fails everything outstanding. Values are dropped after the lock is released.
|
||||
fn fail_all(&self, reason: BusError) {
|
||||
let (pending, calls, services, subscriptions, control, ordinary, reason) = {
|
||||
let mut st = self.lock();
|
||||
let reason = st.closed.get_or_insert(reason).clone();
|
||||
(
|
||||
std::mem::take(&mut st.pending),
|
||||
std::mem::take(&mut st.calls),
|
||||
std::mem::take(&mut st.services),
|
||||
std::mem::take(&mut st.subscriptions),
|
||||
std::mem::take(&mut st.control),
|
||||
std::mem::take(&mut st.ordinary),
|
||||
reason,
|
||||
)
|
||||
};
|
||||
for (_, p) in pending {
|
||||
if let Some(tx) = p.respond {
|
||||
let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::Unknown)));
|
||||
}
|
||||
}
|
||||
for (_, slot) in calls {
|
||||
if let CallSlot::Waiting(Some(tx)) = slot {
|
||||
let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::Unknown)));
|
||||
}
|
||||
}
|
||||
for cmd in ordinary {
|
||||
if let Some(tx) = cmd.respond {
|
||||
let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::NotDispatched)));
|
||||
}
|
||||
}
|
||||
for item in control {
|
||||
if let Control::Cancel(_, Some(tx)) = item {
|
||||
let _ = tx.send(Err(reason.clone().with_dispatch(Dispatch::NotDispatched)));
|
||||
}
|
||||
}
|
||||
drop((services, subscriptions));
|
||||
self.done.send_replace(true);
|
||||
}
|
||||
|
||||
fn terminate(&self, reason: BusError) {
|
||||
self.fail_all(reason);
|
||||
self.shutdown.send_replace(true);
|
||||
self.wake.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Writer
|
||||
|
||||
enum Next {
|
||||
Send(Vec<u8>),
|
||||
Failed(OutCommand, BusError),
|
||||
Idle,
|
||||
Exit,
|
||||
}
|
||||
|
||||
fn take_batch(control: &mut VecDeque<Control>) -> Option<OutCommand> {
|
||||
let first = control.pop_front()?;
|
||||
let (op, key, mut ids) = match first {
|
||||
Control::Cancel(call_id, respond) => {
|
||||
let mut body = Map::new();
|
||||
body.insert("callId".into(), call_id.clone().into());
|
||||
let hook = Hook::Cancel(call_id);
|
||||
return Some(OutCommand {
|
||||
op: "rpc.cancel",
|
||||
body,
|
||||
attachments: Vec::new(),
|
||||
respond,
|
||||
hook,
|
||||
keep: Vec::new(),
|
||||
});
|
||||
}
|
||||
Control::ResponderReleased {
|
||||
call_id,
|
||||
request_delivery_id,
|
||||
} => {
|
||||
let mut body = Map::new();
|
||||
body.insert("callId".into(), call_id.into());
|
||||
body.insert("requestDeliveryId".into(), request_delivery_id.into());
|
||||
return Some(OutCommand {
|
||||
op: "rpc.responder.release",
|
||||
body,
|
||||
attachments: Vec::new(),
|
||||
respond: None,
|
||||
hook: Hook::None,
|
||||
keep: Vec::new(),
|
||||
});
|
||||
}
|
||||
Control::Unregister {
|
||||
name,
|
||||
service_incarnation,
|
||||
} => {
|
||||
let mut body = Map::new();
|
||||
body.insert("name".into(), name.into());
|
||||
body.insert("serviceIncarnation".into(), service_incarnation.into());
|
||||
return Some(OutCommand {
|
||||
op: "service.unregister",
|
||||
body,
|
||||
attachments: Vec::new(),
|
||||
respond: None,
|
||||
hook: Hook::None,
|
||||
keep: Vec::new(),
|
||||
});
|
||||
}
|
||||
Control::Consumed(id) => ("delivery.consumed", "deliveryIds", vec![id]),
|
||||
Control::Release(id) => ("artifact.release", "ownerIds", vec![id]),
|
||||
};
|
||||
while ids.len() < MAX_BATCH {
|
||||
match (control.front(), op) {
|
||||
(Some(Control::Consumed(_)), "delivery.consumed")
|
||||
| (Some(Control::Release(_)), "artifact.release") => match control.pop_front() {
|
||||
Some(Control::Consumed(id) | Control::Release(id)) => ids.push(id),
|
||||
_ => unreachable!("front was checked"),
|
||||
},
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let mut body = Map::new();
|
||||
body.insert(
|
||||
key.into(),
|
||||
Value::Array(ids.into_iter().map(Value::String).collect()),
|
||||
);
|
||||
Some(OutCommand {
|
||||
op,
|
||||
body,
|
||||
attachments: Vec::new(),
|
||||
respond: None,
|
||||
hook: Hook::None,
|
||||
keep: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn next_outgoing(shared: &Shared) -> Next {
|
||||
let mut st = shared.lock();
|
||||
if st.closed.is_some() {
|
||||
return Next::Exit;
|
||||
}
|
||||
let cmd = match take_batch(&mut st.control).or_else(|| st.ordinary.pop_front()) {
|
||||
Some(cmd) => cmd,
|
||||
None if st.shutting_down => {
|
||||
st.closed = Some(BusError::lost("client closed"));
|
||||
return Next::Exit;
|
||||
}
|
||||
None => return Next::Idle,
|
||||
};
|
||||
st.next_msg += 1;
|
||||
let n = st.next_msg;
|
||||
let mut env = Envelope::new(serial_id("msg", n), Kind::Command, cmd.op, cmd.body.clone());
|
||||
env.attachments = cmd.attachments.clone();
|
||||
match env.encode() {
|
||||
Ok(bytes) => {
|
||||
st.pending.insert(
|
||||
n,
|
||||
Pending {
|
||||
op: cmd.op,
|
||||
respond: cmd.respond,
|
||||
hook: cmd.hook,
|
||||
_keep: cmd.keep,
|
||||
},
|
||||
);
|
||||
Next::Send(bytes)
|
||||
}
|
||||
Err(e) => Next::Failed(cmd, e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn write_loop(shared: Arc<Shared>, mut wr: WriteHalf<Transport>) {
|
||||
let mut shutdown = shared.shutdown.subscribe();
|
||||
loop {
|
||||
match next_outgoing(&shared) {
|
||||
Next::Send(bytes) => {
|
||||
let result = tokio::select! {
|
||||
result = write_frame(&mut wr, &bytes) => result,
|
||||
_ = shutdown.wait_for(|v| *v) => break,
|
||||
};
|
||||
if result.is_err() {
|
||||
shared.terminate(BusError::lost("write to router failed"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
Next::Failed(cmd, e) => {
|
||||
let slot = match &cmd.hook {
|
||||
Hook::Call(call_id) => shared.lock().calls.remove(call_id),
|
||||
_ => None,
|
||||
};
|
||||
drop(slot);
|
||||
if let Some(tx) = cmd.respond {
|
||||
let _ = tx.send(Err(e));
|
||||
}
|
||||
}
|
||||
Next::Idle => {
|
||||
tokio::select! {
|
||||
_ = shared.wake.notified() => {}
|
||||
_ = shutdown.wait_for(|v| *v) => break,
|
||||
}
|
||||
}
|
||||
Next::Exit => break,
|
||||
}
|
||||
}
|
||||
tokio::select! {
|
||||
_ = wr.shutdown() => {}
|
||||
_ = shutdown.wait_for(|v| *v) => {}
|
||||
_ = tokio::time::sleep(Duration::from_secs(1)) => {}
|
||||
}
|
||||
let reason = shared
|
||||
.lock()
|
||||
.closed
|
||||
.clone()
|
||||
.unwrap_or_else(|| BusError::lost("client writer stopped"));
|
||||
shared.terminate(reason);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Reader
|
||||
|
||||
pub(crate) async fn read_loop(
|
||||
shared: Arc<Shared>,
|
||||
conn: Weak<ClientConn>,
|
||||
mut rd: ReadHalf<Transport>,
|
||||
) {
|
||||
let mut shutdown = shared.shutdown.subscribe();
|
||||
let reason = loop {
|
||||
let frame = tokio::select! {
|
||||
frame = read_frame(&mut rd) => frame,
|
||||
_ = shutdown.wait_for(|v| *v) => return,
|
||||
};
|
||||
let bytes = match frame {
|
||||
Ok(Some(b)) => b,
|
||||
Ok(None) => break BusError::lost("router closed the connection"),
|
||||
Err(e) => break BusError::lost(format!("router connection failed: {e}")),
|
||||
};
|
||||
let env = match Envelope::decode(&bytes) {
|
||||
Ok(env) => env,
|
||||
Err(e) => break BusError::lost(format!("router sent an invalid envelope: {e}")),
|
||||
};
|
||||
if let Err(e) = on_envelope(&shared, &conn, env) {
|
||||
break BusError::lost(format!("router protocol violation: {e}"));
|
||||
}
|
||||
};
|
||||
shared.terminate(reason);
|
||||
}
|
||||
|
||||
fn on_envelope(
|
||||
shared: &Arc<Shared>,
|
||||
conn: &Weak<ClientConn>,
|
||||
env: Envelope,
|
||||
) -> Result<(), WireError> {
|
||||
if env.major != MAJOR || env.minor != MINOR {
|
||||
return Err(WireError(format!(
|
||||
"router envelope version is not {MAJOR}.{MINOR}"
|
||||
)));
|
||||
}
|
||||
let router_serial = parse_serial_id("bus", &env.id)
|
||||
.filter(|n| *n > 0)
|
||||
.ok_or_else(|| WireError("router envelope id is not canonical bus-<U64>".into()))?;
|
||||
{
|
||||
let mut st = shared.lock();
|
||||
if router_serial <= st.last_router {
|
||||
return Err(WireError(
|
||||
"router envelope ids must strictly increase".into(),
|
||||
));
|
||||
}
|
||||
st.last_router = router_serial;
|
||||
}
|
||||
match env.kind {
|
||||
Kind::Reply => {
|
||||
if !env.attachments.is_empty() {
|
||||
return Err(WireError("router replies cannot carry attachments".into()));
|
||||
}
|
||||
let n = env
|
||||
.reply_to
|
||||
.as_deref()
|
||||
.and_then(|r| parse_serial_id("msg", r));
|
||||
let n = n.ok_or_else(|| WireError("reply without a command id".into()))?;
|
||||
let pending = {
|
||||
let mut st = shared.lock();
|
||||
let Some(pending) = st.pending.get(&n) else {
|
||||
return Err(WireError(format!("reply to unknown command msg-{n}")));
|
||||
};
|
||||
if env.op != pending.op {
|
||||
return Err(WireError(format!(
|
||||
"reply operation {:?} does not match {:?}",
|
||||
env.op, pending.op
|
||||
)));
|
||||
}
|
||||
st.pending.remove(&n)
|
||||
};
|
||||
let pending =
|
||||
pending.ok_or_else(|| WireError(format!("reply to unknown command msg-{n}")))?;
|
||||
let result = parse_reply(&env.body)?;
|
||||
if let Ok(value) = &result {
|
||||
validate_reply_value(pending.op, value)?;
|
||||
}
|
||||
complete(shared, conn, pending, result)
|
||||
}
|
||||
Kind::Delivery => {
|
||||
if env.reply_to.is_some() {
|
||||
return Err(WireError("deliveries must have null replyTo".into()));
|
||||
}
|
||||
on_delivery(shared, conn, env)
|
||||
}
|
||||
Kind::Notice => {
|
||||
if env.reply_to.is_some() || !env.attachments.is_empty() {
|
||||
return Err(WireError(
|
||||
"notices must have null replyTo and no attachments".into(),
|
||||
));
|
||||
}
|
||||
on_notice(shared, env)
|
||||
}
|
||||
Kind::Command => Err(WireError("routers do not send commands".into())),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_reply(
|
||||
body: &Map<String, Value>,
|
||||
) -> Result<Result<Map<String, Value>, BusError>, WireError> {
|
||||
let mut f = Fields::of(body, "reply");
|
||||
if f.boolean("ok")? {
|
||||
let value = f.object("value")?.clone();
|
||||
f.finish()?;
|
||||
return Ok(Ok(value));
|
||||
}
|
||||
let e = parse_error(f.object("error")?)?;
|
||||
f.finish()?;
|
||||
Ok(Err(e))
|
||||
}
|
||||
|
||||
fn parse_error(m: &Map<String, Value>) -> Result<BusError, WireError> {
|
||||
let mut g = Fields::of(m, "error");
|
||||
let code = ErrorCode::parse(g.string("code")?)
|
||||
.ok_or_else(|| WireError("unknown error code".into()))?;
|
||||
let message = g.string("message")?.to_owned();
|
||||
let dispatch = Dispatch::parse(g.string("dispatch")?)
|
||||
.ok_or_else(|| WireError("unknown dispatch".into()))?;
|
||||
g.finish()?;
|
||||
Ok(BusError {
|
||||
code,
|
||||
message,
|
||||
dispatch,
|
||||
})
|
||||
}
|
||||
|
||||
fn serial_field(f: &mut Fields<'_>, key: &'static str, prefix: &str) -> Result<String, WireError> {
|
||||
let id = f.id(key)?;
|
||||
parse_serial_id(prefix, &id)
|
||||
.ok_or_else(|| WireError(format!("{key} is not canonical {prefix}-<U64>")))?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
fn validate_reply_value(op: &str, value: &Map<String, Value>) -> Result<(), WireError> {
|
||||
let mut f = Fields::of(value, "reply value");
|
||||
match op {
|
||||
"service.register" => {
|
||||
serial_field(&mut f, "serviceIncarnation", "svc")?;
|
||||
}
|
||||
"service.unregister" => {
|
||||
f.boolean("removed")?;
|
||||
}
|
||||
"rpc.call" => {
|
||||
if !f.boolean("accepted")? {
|
||||
return Err(WireError(
|
||||
"an admitted rpc.call must say accepted:true".into(),
|
||||
));
|
||||
}
|
||||
serial_field(&mut f, "serviceIncarnation", "svc")?;
|
||||
}
|
||||
"rpc.reply" => {
|
||||
f.boolean("routed")?;
|
||||
}
|
||||
"rpc.responder.release" => {
|
||||
f.boolean("released")?;
|
||||
}
|
||||
"rpc.cancel" => match f.string("state")? {
|
||||
"cancelled-before-dispatch" | "execution-unknown" | "completed" | "call-gone" => {}
|
||||
_ => return Err(WireError("unknown rpc.cancel state".into())),
|
||||
},
|
||||
"topic.declare" => {
|
||||
f.boolean("declared")?;
|
||||
serial_field(&mut f, "topicIncarnation", "top")?;
|
||||
}
|
||||
"topic.clear" => {
|
||||
f.boolean("cleared")?;
|
||||
}
|
||||
"topic.delete" => {
|
||||
f.boolean("deleted")?;
|
||||
}
|
||||
"subscribe" => {
|
||||
serial_field(&mut f, "subscriptionId", "sub")?;
|
||||
serial_field(&mut f, "topicIncarnation", "top")?;
|
||||
}
|
||||
"unsubscribe" => {
|
||||
f.boolean("removed")?;
|
||||
}
|
||||
"publish" => {
|
||||
f.u64_string("topicSequence")?;
|
||||
f.u64_string("subscribers")?;
|
||||
f.u64_string("replaced")?;
|
||||
}
|
||||
"delivery.consumed" | "artifact.release" => {
|
||||
f.u64_string("released")?;
|
||||
}
|
||||
"artifact.allocate" => {
|
||||
serial_field(&mut f, "artifactId", "a")?;
|
||||
if f.u64_string("generation")? != GENERATION {
|
||||
return Err(WireError("unsupported artifact generation".into()));
|
||||
}
|
||||
serial_field(&mut f, "ownerId", "own")?;
|
||||
crate::wire::Location::from_json(f.value("writeLocation")?)?;
|
||||
}
|
||||
"artifact.seal" => {
|
||||
ArtifactRef::from_json(f.value("ref")?)?;
|
||||
serial_field(&mut f, "ownerId", "own")?;
|
||||
}
|
||||
"artifact.open" => {
|
||||
crate::wire::Location::from_json(f.value("readLocation")?)?;
|
||||
}
|
||||
"artifact.retain" => {
|
||||
serial_field(&mut f, "ownerId", "own")?;
|
||||
}
|
||||
_ => return Err(WireError(format!("reply for unknown operation {op:?}"))),
|
||||
}
|
||||
f.finish()
|
||||
}
|
||||
|
||||
fn owner_guard(conn: &Weak<ClientConn>, id: String, delivery: bool) -> Option<Arc<OwnerGuard>> {
|
||||
Some(Arc::new(OwnerGuard {
|
||||
id,
|
||||
delivery,
|
||||
conn: conn.upgrade()?,
|
||||
}))
|
||||
}
|
||||
|
||||
fn complete(
|
||||
shared: &Arc<Shared>,
|
||||
conn: &Weak<ClientConn>,
|
||||
pending: Pending,
|
||||
result: Result<Map<String, Value>, BusError>,
|
||||
) -> Result<(), WireError> {
|
||||
let Pending {
|
||||
respond,
|
||||
hook,
|
||||
_keep,
|
||||
..
|
||||
} = pending;
|
||||
let reply = match (hook, result) {
|
||||
(Hook::Call(call_id), Err(e)) => {
|
||||
let slot = shared.lock().calls.remove(&call_id);
|
||||
drop(slot);
|
||||
Err(e)
|
||||
}
|
||||
(_, Err(e)) => Err(e),
|
||||
(Hook::None, Ok(value)) => Ok(Reply {
|
||||
value,
|
||||
extra: Extra::None,
|
||||
}),
|
||||
(Hook::Owner, Ok(value)) => {
|
||||
let id = Fields::of(&value, "reply").id("ownerId")?;
|
||||
match owner_guard(conn, id, false) {
|
||||
Some(g) => Ok(Reply {
|
||||
value,
|
||||
extra: Extra::Owner(g),
|
||||
}),
|
||||
None => Err(BusError::lost("client closed")),
|
||||
}
|
||||
}
|
||||
(Hook::Register, Ok(value)) => {
|
||||
let inc = Fields::of(&value, "reply").id("serviceIncarnation")?;
|
||||
match conn.upgrade() {
|
||||
Some(conn) => {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
shared.lock().services.insert(inc.clone(), tx);
|
||||
let guard = ServiceGuard {
|
||||
conn,
|
||||
incarnation: inc,
|
||||
};
|
||||
Ok(Reply {
|
||||
value,
|
||||
extra: Extra::Service(guard, rx),
|
||||
})
|
||||
}
|
||||
None => Err(BusError::lost("client closed")),
|
||||
}
|
||||
}
|
||||
(Hook::Subscribe, Ok(value)) => {
|
||||
let id = Fields::of(&value, "reply").id("subscriptionId")?;
|
||||
match conn.upgrade() {
|
||||
Some(conn) => {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
shared.lock().subscriptions.insert(id.clone(), tx);
|
||||
let guard = SubscriptionGuard { conn, id };
|
||||
Ok(Reply {
|
||||
value,
|
||||
extra: Extra::Subscription(guard, rx),
|
||||
})
|
||||
}
|
||||
None => Err(BusError::lost("client closed")),
|
||||
}
|
||||
}
|
||||
(Hook::Call(call_id), Ok(value)) => match conn.upgrade() {
|
||||
Some(conn) => Ok(Reply {
|
||||
value,
|
||||
extra: Extra::Call(CallGuard {
|
||||
conn,
|
||||
call_id,
|
||||
done: false,
|
||||
}),
|
||||
}),
|
||||
None => Err(BusError::lost("client closed")),
|
||||
},
|
||||
(Hook::Cancel(call_id), Ok(value)) => {
|
||||
let state = Fields::of(&value, "reply").string("state")?.to_owned();
|
||||
if state != "completed" {
|
||||
// No result will follow: resolve whoever still waits for one.
|
||||
let slot = shared.lock().calls.remove(&call_id);
|
||||
if let Some(CallSlot::Waiting(Some(tx))) = slot {
|
||||
let dispatch = if state == "cancelled-before-dispatch" {
|
||||
Dispatch::NotDispatched
|
||||
} else {
|
||||
Dispatch::Unknown
|
||||
};
|
||||
let e = BusError::new(ErrorCode::CallGone, format!("cancelled: {state}"))
|
||||
.with_dispatch(dispatch);
|
||||
let _ = tx.send(Err(e));
|
||||
}
|
||||
}
|
||||
Ok(Reply {
|
||||
value,
|
||||
extra: Extra::None,
|
||||
})
|
||||
}
|
||||
};
|
||||
match respond {
|
||||
Some(tx) => {
|
||||
// If the caller is gone the reply, and any handle in it, drops here and releases.
|
||||
let _ = tx.send(reply);
|
||||
}
|
||||
None => {
|
||||
if reply.is_err() {
|
||||
shared.lock().control_errors += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn attachments(env: &Envelope, delivery_id: &str) -> Result<Vec<(String, ArtifactRef)>, WireError> {
|
||||
env.attachments
|
||||
.iter()
|
||||
.map(|a| {
|
||||
if a.owner_id != delivery_id {
|
||||
return Err(WireError(
|
||||
"delivery attachment owner is not the delivery".into(),
|
||||
));
|
||||
}
|
||||
Ok((a.name.clone(), a.reference.clone()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn u64_field(f: &mut Fields<'_>, key: &'static str) -> Result<u64, WireError> {
|
||||
let s = f.string(key)?;
|
||||
parse_u64(s).ok_or_else(|| WireError(format!("{key} is not a U64")))
|
||||
}
|
||||
|
||||
fn on_delivery(
|
||||
shared: &Arc<Shared>,
|
||||
conn: &Weak<ClientConn>,
|
||||
env: Envelope,
|
||||
) -> Result<(), WireError> {
|
||||
let mut f = Fields::of(&env.body, "delivery");
|
||||
let delivery_id = f.id("deliveryId")?;
|
||||
let delivery_serial = parse_serial_id("dlv", &delivery_id)
|
||||
.filter(|n| *n > 0)
|
||||
.ok_or_else(|| WireError("delivery id is not canonical dlv-<U64>".into()))?;
|
||||
let atts = attachments(&env, &delivery_id)?;
|
||||
// Without a live connection handle there is nobody to hand this to.
|
||||
let Some(guard) = owner_guard(conn, delivery_id.clone(), true) else {
|
||||
return Ok(());
|
||||
};
|
||||
match env.op.as_str() {
|
||||
"rpc.request" => {
|
||||
let call_id = f.id("callId")?;
|
||||
if parse_serial_id("call", &call_id).is_none() {
|
||||
return Err(WireError("rpc.request callId is not call-<U64>".into()));
|
||||
}
|
||||
let caller = Identity::from_json(f.value("caller")?)?;
|
||||
let target = f.name("target")?;
|
||||
let service_incarnation = f.id("serviceIncarnation")?;
|
||||
if parse_serial_id("svc", &service_incarnation).is_none() {
|
||||
return Err(WireError("service incarnation is not svc-<U64>".into()));
|
||||
}
|
||||
let method = f.method("method")?;
|
||||
let payload = f.object("payload")?.clone();
|
||||
f.finish()?;
|
||||
accept_delivery_serial(shared, delivery_serial)?;
|
||||
let req = Request {
|
||||
reply_guard: Arc::new(ReplyGuard {
|
||||
conn: guard.conn.clone(),
|
||||
call_id: call_id.clone(),
|
||||
request_delivery_id: delivery_id.clone(),
|
||||
}),
|
||||
guard,
|
||||
call_id,
|
||||
caller,
|
||||
target,
|
||||
service_incarnation,
|
||||
method,
|
||||
payload,
|
||||
attachments: atts,
|
||||
};
|
||||
let tx = shared
|
||||
.lock()
|
||||
.services
|
||||
.get(&req.service_incarnation)
|
||||
.cloned();
|
||||
if let Some(tx) = tx {
|
||||
let _ = tx.send(req);
|
||||
}
|
||||
}
|
||||
"rpc.result" => {
|
||||
let call_id = f.id("callId")?;
|
||||
if parse_serial_id("call", &call_id).is_none() {
|
||||
return Err(WireError("rpc.result callId is not call-<U64>".into()));
|
||||
}
|
||||
let responder = Identity::from_json(f.value("responder")?)?;
|
||||
let service_incarnation = f.id("serviceIncarnation")?;
|
||||
if parse_serial_id("svc", &service_incarnation).is_none() {
|
||||
return Err(WireError("service incarnation is not svc-<U64>".into()));
|
||||
}
|
||||
let outcome = f.object("outcome")?.clone();
|
||||
f.finish()?;
|
||||
accept_delivery_serial(shared, delivery_serial)?;
|
||||
let res = RpcResult {
|
||||
guard,
|
||||
call_id,
|
||||
responder,
|
||||
service_incarnation,
|
||||
outcome,
|
||||
attachments: atts,
|
||||
};
|
||||
let slot = shared.lock().calls.remove(&res.call_id);
|
||||
match slot {
|
||||
Some(CallSlot::Waiting(Some(tx))) => {
|
||||
let _ = tx.send(Ok(res));
|
||||
}
|
||||
// Abandoned or unknown: dropping it consumes the delivery.
|
||||
other => drop((other, res)),
|
||||
}
|
||||
}
|
||||
"topic.message" => {
|
||||
let subscription_id = f.id("subscriptionId")?;
|
||||
if parse_serial_id("sub", &subscription_id).is_none() {
|
||||
return Err(WireError("subscription id is not sub-<U64>".into()));
|
||||
}
|
||||
let topic = f.name("topic")?;
|
||||
let topic_incarnation = f.id("topicIncarnation")?;
|
||||
if parse_serial_id("top", &topic_incarnation).is_none() {
|
||||
return Err(WireError("topic incarnation is not top-<U64>".into()));
|
||||
}
|
||||
let topic_sequence = u64_field(&mut f, "topicSequence")?;
|
||||
let replaced = u64_field(&mut f, "replaced")?;
|
||||
let payload = f.object("payload")?.clone();
|
||||
f.finish()?;
|
||||
accept_delivery_serial(shared, delivery_serial)?;
|
||||
let msg = Message {
|
||||
guard,
|
||||
subscription_id,
|
||||
topic,
|
||||
topic_incarnation,
|
||||
topic_sequence,
|
||||
replaced,
|
||||
payload,
|
||||
attachments: atts,
|
||||
};
|
||||
let tx = shared
|
||||
.lock()
|
||||
.subscriptions
|
||||
.get(&msg.subscription_id)
|
||||
.cloned();
|
||||
if let Some(tx) = tx {
|
||||
let _ = tx.send(msg);
|
||||
}
|
||||
}
|
||||
other => return Err(WireError(format!("unknown delivery {other:?}"))),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn accept_delivery_serial(shared: &Shared, serial: u64) -> Result<(), WireError> {
|
||||
let mut st = shared.lock();
|
||||
if serial <= st.last_delivery {
|
||||
return Err(WireError("delivery ids must strictly increase".into()));
|
||||
}
|
||||
st.last_delivery = serial;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_notice(shared: &Arc<Shared>, env: Envelope) -> Result<(), WireError> {
|
||||
let mut f = Fields::of(&env.body, "notice");
|
||||
match env.op.as_str() {
|
||||
"call.failed" => {
|
||||
let call_id = f.id("callId")?;
|
||||
if parse_serial_id("call", &call_id).is_none() {
|
||||
return Err(WireError("call.failed callId is not call-<U64>".into()));
|
||||
}
|
||||
let code = ErrorCode::parse(f.string("code")?)
|
||||
.ok_or_else(|| WireError("unknown error code".into()))?;
|
||||
let message = f.string("message")?.to_owned();
|
||||
let dispatch = Dispatch::parse(f.string("dispatch")?)
|
||||
.ok_or_else(|| WireError("unknown dispatch".into()))?;
|
||||
f.finish()?;
|
||||
let slot = shared.lock().calls.remove(&call_id);
|
||||
if let Some(CallSlot::Waiting(Some(tx))) = slot {
|
||||
let _ = tx.send(Err(BusError {
|
||||
code,
|
||||
message,
|
||||
dispatch,
|
||||
}));
|
||||
}
|
||||
}
|
||||
"route.removed" => {
|
||||
f.name("name")?;
|
||||
serial_field(&mut f, "serviceIncarnation", "svc")?;
|
||||
f.string("reason")?;
|
||||
f.finish()?;
|
||||
}
|
||||
"subscription.closed" => {
|
||||
let id = serial_field(&mut f, "subscriptionId", "sub")?;
|
||||
f.name("topic")?;
|
||||
serial_field(&mut f, "topicIncarnation", "top")?;
|
||||
f.string("reason")?;
|
||||
f.finish()?;
|
||||
let tx = shared.lock().subscriptions.remove(&id);
|
||||
drop(tx);
|
||||
}
|
||||
"connection.closing" => {
|
||||
let code = ErrorCode::parse(f.string("code")?)
|
||||
.ok_or_else(|| WireError("unknown error code".into()))?;
|
||||
let message = f.string("message")?.to_owned();
|
||||
f.finish()?;
|
||||
let mut st = shared.lock();
|
||||
if st.closed.is_none() {
|
||||
st.closed = Some(BusError::new(code, message).with_dispatch(Dispatch::Unknown));
|
||||
}
|
||||
}
|
||||
other => return Err(WireError(format!("unknown notice {other:?}"))),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
154
services/flysim/crates/flybus/src/error.rs
Normal file
154
services/flysim/crates/flybus/src/error.rs
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
//! Transport error codes and the error value every fallible call returns.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
/// Transport error codes (bus-v1 section 9, plus the three this crate adds; see the README).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum ErrorCode {
|
||||
InvalidEnvelope,
|
||||
VersionMismatch,
|
||||
NotAuthorized,
|
||||
NoService,
|
||||
TargetChanged,
|
||||
Backpressure,
|
||||
CallGone,
|
||||
ArtifactUnsealed,
|
||||
ArtifactGone,
|
||||
OwnerInvalid,
|
||||
QuotaExceeded,
|
||||
StoreFailure,
|
||||
RouterLost,
|
||||
/// Addition: a name is already registered, a redeclaration disagrees, or a topic still has
|
||||
/// subscribers.
|
||||
Conflict,
|
||||
/// Addition: publish or subscribe named a topic nobody declared.
|
||||
NoTopic,
|
||||
/// Addition: a seal found the wrong length or digest, or a reference disagrees with the
|
||||
/// artifact it names.
|
||||
ArtifactMismatch,
|
||||
}
|
||||
|
||||
impl ErrorCode {
|
||||
pub const ALL: [ErrorCode; 16] = [
|
||||
ErrorCode::InvalidEnvelope,
|
||||
ErrorCode::VersionMismatch,
|
||||
ErrorCode::NotAuthorized,
|
||||
ErrorCode::NoService,
|
||||
ErrorCode::TargetChanged,
|
||||
ErrorCode::Backpressure,
|
||||
ErrorCode::CallGone,
|
||||
ErrorCode::ArtifactUnsealed,
|
||||
ErrorCode::ArtifactGone,
|
||||
ErrorCode::OwnerInvalid,
|
||||
ErrorCode::QuotaExceeded,
|
||||
ErrorCode::StoreFailure,
|
||||
ErrorCode::RouterLost,
|
||||
ErrorCode::Conflict,
|
||||
ErrorCode::NoTopic,
|
||||
ErrorCode::ArtifactMismatch,
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ErrorCode::InvalidEnvelope => "INVALID_ENVELOPE",
|
||||
ErrorCode::VersionMismatch => "VERSION_MISMATCH",
|
||||
ErrorCode::NotAuthorized => "NOT_AUTHORIZED",
|
||||
ErrorCode::NoService => "NO_SERVICE",
|
||||
ErrorCode::TargetChanged => "TARGET_CHANGED",
|
||||
ErrorCode::Backpressure => "BACKPRESSURE",
|
||||
ErrorCode::CallGone => "CALL_GONE",
|
||||
ErrorCode::ArtifactUnsealed => "ARTIFACT_UNSEALED",
|
||||
ErrorCode::ArtifactGone => "ARTIFACT_GONE",
|
||||
ErrorCode::OwnerInvalid => "OWNER_INVALID",
|
||||
ErrorCode::QuotaExceeded => "QUOTA_EXCEEDED",
|
||||
ErrorCode::StoreFailure => "STORE_FAILURE",
|
||||
ErrorCode::RouterLost => "ROUTER_LOST",
|
||||
ErrorCode::Conflict => "CONFLICT",
|
||||
ErrorCode::NoTopic => "NO_TOPIC",
|
||||
ErrorCode::ArtifactMismatch => "ARTIFACT_MISMATCH",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<ErrorCode> {
|
||||
ErrorCode::ALL.into_iter().find(|c| c.as_str() == s)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ErrorCode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the operation may have reached its target (bus-v1 section 5).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum Dispatch {
|
||||
NotDispatched,
|
||||
Dispatched,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl Dispatch {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Dispatch::NotDispatched => "not-dispatched",
|
||||
Dispatch::Dispatched => "dispatched",
|
||||
Dispatch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<Dispatch> {
|
||||
match s {
|
||||
"not-dispatched" => Some(Dispatch::NotDispatched),
|
||||
"dispatched" => Some(Dispatch::Dispatched),
|
||||
"unknown" => Some(Dispatch::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A transport error: code, a short message and the dispatch certainty.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct BusError {
|
||||
pub code: ErrorCode,
|
||||
pub message: String,
|
||||
pub dispatch: Dispatch,
|
||||
}
|
||||
|
||||
impl BusError {
|
||||
/// An error raised before anything could have been dispatched.
|
||||
pub fn new(code: ErrorCode, message: impl Into<String>) -> BusError {
|
||||
BusError {
|
||||
code,
|
||||
message: message.into(),
|
||||
dispatch: Dispatch::NotDispatched,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_dispatch(mut self, dispatch: Dispatch) -> BusError {
|
||||
self.dispatch = dispatch;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn invalid(message: impl Into<String>) -> BusError {
|
||||
BusError::new(ErrorCode::InvalidEnvelope, message)
|
||||
}
|
||||
|
||||
pub(crate) fn lost(message: impl Into<String>) -> BusError {
|
||||
BusError::new(ErrorCode::RouterLost, message).with_dispatch(Dispatch::Unknown)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for BusError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{} ({}): {}",
|
||||
self.code,
|
||||
self.dispatch.as_str(),
|
||||
self.message
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BusError {}
|
||||
37
services/flysim/crates/flybus/src/lib.rs
Normal file
37
services/flysim/crates/flybus/src/lib.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
//! `flybus`: a small local RPC and pub/sub bus with immutable file-backed artifacts.
|
||||
//!
|
||||
//! One library, one router, one wire protocol. Small JSON messages carry metadata and
|
||||
//! artifact references; large immutable bytes live in a file store the router manages, and
|
||||
//! ownership follows deliveries and explicit holds. The router moves messages and tracks
|
||||
//! ownership; it knows nothing about what the messages mean.
|
||||
//!
|
||||
//! ```text
|
||||
//! Client ── Transport (in-memory pipe or Unix socket) ── Router ── State (one mutex)
|
||||
//! │ │
|
||||
//! └── Artifact / ArtifactWriter ── files ──────────── Store (<root>/<storeId>/...)
|
||||
//! ```
|
||||
//!
|
||||
//! The draft contract this implements is bus-v1 (the session-framework design); the README
|
||||
//! lists the API, the limits and every place the implementation narrows or extends the draft.
|
||||
|
||||
pub mod error;
|
||||
pub mod limits;
|
||||
pub mod policy;
|
||||
pub mod wire;
|
||||
|
||||
mod client;
|
||||
mod router;
|
||||
mod store;
|
||||
mod transport;
|
||||
|
||||
pub use client::{
|
||||
Artifact, ArtifactFile, ArtifactWriter, Artifacts, CancelState, Client, ClientConfig, Message,
|
||||
Mode, PendingCall, PublishReceipt, Request, Responder, Retained, RpcResult, Service,
|
||||
ServiceConfig, SessionInfo, Subscription, SubscriptionConfig, TopicInfo,
|
||||
};
|
||||
pub use error::{BusError, Dispatch, ErrorCode};
|
||||
pub use limits::Limits;
|
||||
pub use policy::{Grants, Pattern, Policy};
|
||||
pub use router::{Router, RouterConfig, RouterStats, UnixListenerHandle};
|
||||
pub use transport::{Stream, Transport};
|
||||
pub use wire::{ArtifactRef, Identity};
|
||||
183
services/flysim/crates/flybus/src/limits.rs
Normal file
183
services/flysim/crates/flybus/src/limits.rs
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
//! Router limits (bus-v1 section 9). The defaults are the draft's prototype starting point,
|
||||
//! not capacity data.
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::wire::{Fields, MAX_ATTACHMENTS, MAX_BATCH, MAX_CREDIT, MAX_ENVELOPE_BYTES, WireError};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Limits {
|
||||
pub max_clients: usize,
|
||||
pub max_services: usize,
|
||||
pub max_topics: usize,
|
||||
pub max_subscriptions_per_client: usize,
|
||||
pub max_subscriptions: usize,
|
||||
/// Calls a client may have admitted and not yet finished (result consumed, failed or
|
||||
/// cancelled before dispatch). A call detached by a post-dispatch cancel no longer counts.
|
||||
pub max_active_calls_per_client: usize,
|
||||
/// Upper bounds on what `service.register` may ask for.
|
||||
pub max_service_queued: u64,
|
||||
pub max_service_in_flight: u64,
|
||||
/// Upper bound on a `latest` subscription's credits; its queue is always exactly 1.
|
||||
pub max_latest_in_flight: u64,
|
||||
pub max_bounded_queued: u64,
|
||||
pub max_bounded_in_flight: u64,
|
||||
/// Owners (deliveries handed to the client and not consumed, explicit holds and staging
|
||||
/// writers) a connection may have at once.
|
||||
pub max_owners_per_client: usize,
|
||||
/// Of those, how many only RPC request/result deliveries may use. Topic deliveries, holds
|
||||
/// and writers are refused (holds, writers) or wait (deliveries) beyond
|
||||
/// `max_owners_per_client - reserved_owners_per_client`.
|
||||
pub reserved_owners_per_client: usize,
|
||||
/// Bytes of staging, sealing copies and sealed objects the store may hold.
|
||||
pub max_store_bytes: u64,
|
||||
pub max_artifact_bytes: u64,
|
||||
/// Artifact bytes pinned by retained topic values, counted once per topic.
|
||||
pub max_retained_bytes: u64,
|
||||
/// Envelope bytes queued for a client's `bounded` subscriptions and not yet delivered.
|
||||
pub max_queued_bytes_per_client: usize,
|
||||
/// Replies and notices waiting to be written to one client. Exceeding either closes it.
|
||||
pub max_control_frames: usize,
|
||||
pub max_control_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for Limits {
|
||||
fn default() -> Limits {
|
||||
Limits {
|
||||
max_clients: 64,
|
||||
max_services: 256,
|
||||
max_topics: 512,
|
||||
max_subscriptions_per_client: 128,
|
||||
max_subscriptions: 1024,
|
||||
max_active_calls_per_client: 64,
|
||||
max_service_queued: 16,
|
||||
max_service_in_flight: 16,
|
||||
max_latest_in_flight: 2,
|
||||
max_bounded_queued: 64,
|
||||
max_bounded_in_flight: 16,
|
||||
max_owners_per_client: 256,
|
||||
reserved_owners_per_client: 64,
|
||||
max_store_bytes: 512 << 20,
|
||||
max_artifact_bytes: 128 << 20,
|
||||
max_retained_bytes: 128 << 20,
|
||||
max_queued_bytes_per_client: 1 << 20,
|
||||
max_control_frames: 128,
|
||||
max_control_bytes: 1 << 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Limits {
|
||||
/// Refuses a configuration the router could not honour.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
let credits = [
|
||||
("max_service_queued", self.max_service_queued),
|
||||
("max_service_in_flight", self.max_service_in_flight),
|
||||
("max_latest_in_flight", self.max_latest_in_flight),
|
||||
("max_bounded_queued", self.max_bounded_queued),
|
||||
("max_bounded_in_flight", self.max_bounded_in_flight),
|
||||
];
|
||||
for (name, v) in credits {
|
||||
if !(1..=MAX_CREDIT).contains(&v) {
|
||||
return Err(format!("{name} must be in 1..={MAX_CREDIT}"));
|
||||
}
|
||||
}
|
||||
if self.reserved_owners_per_client >= self.max_owners_per_client {
|
||||
return Err("reserved_owners_per_client must be below max_owners_per_client".into());
|
||||
}
|
||||
if self.max_artifact_bytes > self.max_store_bytes {
|
||||
return Err("max_artifact_bytes must not exceed max_store_bytes".into());
|
||||
}
|
||||
if self.max_control_frames == 0 || self.max_control_bytes < MAX_ENVELOPE_BYTES {
|
||||
return Err("the control lane must hold at least one full envelope".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The `limits` object in the hello reply. Counts are JSON integers; byte sizes are U64
|
||||
/// strings.
|
||||
pub fn to_json(&self) -> Value {
|
||||
let mut m = Map::new();
|
||||
let mut n = |k: &str, v: u64| {
|
||||
m.insert(k.into(), Value::from(v));
|
||||
};
|
||||
n("maxEnvelopeBytes", MAX_ENVELOPE_BYTES as u64);
|
||||
n("maxAttachments", MAX_ATTACHMENTS as u64);
|
||||
n("maxBatch", MAX_BATCH as u64);
|
||||
n("maxClients", self.max_clients as u64);
|
||||
n("maxServices", self.max_services as u64);
|
||||
n("maxTopics", self.max_topics as u64);
|
||||
n(
|
||||
"maxSubscriptionsPerClient",
|
||||
self.max_subscriptions_per_client as u64,
|
||||
);
|
||||
n("maxSubscriptions", self.max_subscriptions as u64);
|
||||
n(
|
||||
"maxActiveCallsPerClient",
|
||||
self.max_active_calls_per_client as u64,
|
||||
);
|
||||
n("maxServiceQueued", self.max_service_queued);
|
||||
n("maxServiceInFlight", self.max_service_in_flight);
|
||||
n("maxLatestInFlight", self.max_latest_in_flight);
|
||||
n("maxBoundedQueued", self.max_bounded_queued);
|
||||
n("maxBoundedInFlight", self.max_bounded_in_flight);
|
||||
n("maxOwnersPerClient", self.max_owners_per_client as u64);
|
||||
n(
|
||||
"reservedOwnersPerClient",
|
||||
self.reserved_owners_per_client as u64,
|
||||
);
|
||||
n("maxControlFrames", self.max_control_frames as u64);
|
||||
let mut s = |k: &str, v: u64| {
|
||||
m.insert(k.into(), Value::from(v.to_string()));
|
||||
};
|
||||
s("maxStoreBytes", self.max_store_bytes);
|
||||
s("maxArtifactBytes", self.max_artifact_bytes);
|
||||
s("maxRetainedBytes", self.max_retained_bytes);
|
||||
s(
|
||||
"maxQueuedBytesPerClient",
|
||||
self.max_queued_bytes_per_client as u64,
|
||||
);
|
||||
s("maxControlBytes", self.max_control_bytes as u64);
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
/// Reads the hello reply's `limits` object back.
|
||||
pub fn from_json(v: &Value) -> Result<Limits, WireError> {
|
||||
let mut f = Fields::new(v, "limits")?;
|
||||
let big = u64::MAX;
|
||||
for (k, want) in [
|
||||
("maxEnvelopeBytes", MAX_ENVELOPE_BYTES as u64),
|
||||
("maxAttachments", MAX_ATTACHMENTS as u64),
|
||||
("maxBatch", MAX_BATCH as u64),
|
||||
] {
|
||||
if f.int(k, 0, big)? != want {
|
||||
return Err(WireError(format!(
|
||||
"limits: {k} disagrees with this implementation"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let limits = Limits {
|
||||
max_clients: f.int("maxClients", 0, big)? as usize,
|
||||
max_services: f.int("maxServices", 0, big)? as usize,
|
||||
max_topics: f.int("maxTopics", 0, big)? as usize,
|
||||
max_subscriptions_per_client: f.int("maxSubscriptionsPerClient", 0, big)? as usize,
|
||||
max_subscriptions: f.int("maxSubscriptions", 0, big)? as usize,
|
||||
max_active_calls_per_client: f.int("maxActiveCallsPerClient", 0, big)? as usize,
|
||||
max_service_queued: f.int("maxServiceQueued", 1, MAX_CREDIT)?,
|
||||
max_service_in_flight: f.int("maxServiceInFlight", 1, MAX_CREDIT)?,
|
||||
max_latest_in_flight: f.int("maxLatestInFlight", 1, MAX_CREDIT)?,
|
||||
max_bounded_queued: f.int("maxBoundedQueued", 1, MAX_CREDIT)?,
|
||||
max_bounded_in_flight: f.int("maxBoundedInFlight", 1, MAX_CREDIT)?,
|
||||
max_owners_per_client: f.int("maxOwnersPerClient", 0, big)? as usize,
|
||||
reserved_owners_per_client: f.int("reservedOwnersPerClient", 0, big)? as usize,
|
||||
max_control_frames: f.int("maxControlFrames", 0, big)? as usize,
|
||||
max_store_bytes: f.u64_string("maxStoreBytes")?,
|
||||
max_artifact_bytes: f.u64_string("maxArtifactBytes")?,
|
||||
max_retained_bytes: f.u64_string("maxRetainedBytes")?,
|
||||
max_queued_bytes_per_client: f.u64_string("maxQueuedBytesPerClient")? as usize,
|
||||
max_control_bytes: f.u64_string("maxControlBytes")? as usize,
|
||||
};
|
||||
f.finish()?;
|
||||
Ok(limits)
|
||||
}
|
||||
}
|
||||
108
services/flysim/crates/flybus/src/policy.rs
Normal file
108
services/flysim/crates/flybus/src/policy.rs
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
//! Launcher-provided access policy (bus-v1 section 3): which configured participants may
|
||||
//! connect, and what each may register, call, publish, subscribe to and manage. Naming a
|
||||
//! target is not authority to control it.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// A set of service or topic names.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Pattern {
|
||||
Any,
|
||||
Exact(String),
|
||||
/// Every name that starts with this string, e.g. `"session.demo."`.
|
||||
Prefix(String),
|
||||
}
|
||||
|
||||
impl Pattern {
|
||||
pub fn exact(name: &str) -> Pattern {
|
||||
Pattern::Exact(name.to_owned())
|
||||
}
|
||||
|
||||
pub fn prefix(prefix: &str) -> Pattern {
|
||||
Pattern::Prefix(prefix.to_owned())
|
||||
}
|
||||
|
||||
pub fn matches(&self, name: &str) -> bool {
|
||||
match self {
|
||||
Pattern::Any => true,
|
||||
Pattern::Exact(n) => n == name,
|
||||
Pattern::Prefix(p) => name.starts_with(p.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What one participant may do. Empty lists grant nothing.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Grants {
|
||||
/// Service names it may register.
|
||||
pub register: Vec<Pattern>,
|
||||
/// Service names it may call.
|
||||
pub call: Vec<Pattern>,
|
||||
pub publish: Vec<Pattern>,
|
||||
pub subscribe: Vec<Pattern>,
|
||||
/// Topic names it may declare, clear and delete.
|
||||
pub manage_topics: Vec<Pattern>,
|
||||
}
|
||||
|
||||
impl Grants {
|
||||
pub fn all() -> Grants {
|
||||
let any = vec![Pattern::Any];
|
||||
Grants {
|
||||
register: any.clone(),
|
||||
call: any.clone(),
|
||||
publish: any.clone(),
|
||||
subscribe: any.clone(),
|
||||
manage_topics: any,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn allows(list: &[Pattern], name: &str) -> bool {
|
||||
list.iter().any(|p| p.matches(name))
|
||||
}
|
||||
}
|
||||
|
||||
/// Who may connect and with what grants.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Policy {
|
||||
clients: HashMap<String, Grants>,
|
||||
default: Option<Grants>,
|
||||
trusted_unbound: bool,
|
||||
}
|
||||
|
||||
impl Policy {
|
||||
/// Admits any client id with every grant. For tests and single-purpose local deployments.
|
||||
pub fn open() -> Policy {
|
||||
Policy {
|
||||
clients: HashMap::new(),
|
||||
default: Some(Grants::all()),
|
||||
trusted_unbound: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Admits only the clients added with [`Policy::client`].
|
||||
pub fn closed() -> Policy {
|
||||
Policy::default()
|
||||
}
|
||||
|
||||
pub fn client(mut self, client_id: &str, grants: Grants) -> Policy {
|
||||
self.clients.insert(client_id.to_owned(), grants);
|
||||
self
|
||||
}
|
||||
|
||||
/// Grants for clients not listed by id; `None` refuses them at hello.
|
||||
pub fn with_default(mut self, grants: Option<Grants>) -> Policy {
|
||||
self.default = grants;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn grants_for(&self, client_id: &str) -> Option<Grants> {
|
||||
self.clients
|
||||
.get(client_id)
|
||||
.cloned()
|
||||
.or_else(|| self.default.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn permits_unbound_transport(&self) -> bool {
|
||||
self.trusted_unbound
|
||||
}
|
||||
}
|
||||
502
services/flysim/crates/flybus/src/router/mod.rs
Normal file
502
services/flysim/crates/flybus/src/router/mod.rs
Normal file
|
|
@ -0,0 +1,502 @@
|
|||
//! The embeddable router: connection tasks around the [`state`] machine.
|
||||
//!
|
||||
//! Each connection has a reader task, which decodes one command at a time and applies it under
|
||||
//! the state lock, and a writer task, which pulls its next frame from the state and writes it
|
||||
//! with the lock released. Staging creation and seal copies run on the blocking pool; seals
|
||||
//! run beside the reader, so a large copy never holds up that client's releases. The router
|
||||
//! never waits on a subscriber: a slow reader only stalls its own writer task.
|
||||
|
||||
mod state;
|
||||
|
||||
use std::io;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
|
||||
use tokio::net::UnixListener;
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
pub use state::RouterStats;
|
||||
use state::{ConnKey, ConnSignals, NextFrame, Outcome, State};
|
||||
|
||||
use crate::limits::Limits;
|
||||
use crate::policy::Policy;
|
||||
use crate::store::{SealFailure, Store};
|
||||
use crate::transport::{Stream, Transport};
|
||||
use crate::wire::{Envelope, FrameError, contract_digest, hex, read_frame, write_frame};
|
||||
|
||||
/// How the launcher configures a router.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RouterConfig {
|
||||
/// Directory under which the store creates its per-incarnation directory. Put it on tmpfs
|
||||
/// for transient media. Clients must be given the same root.
|
||||
pub store_root: PathBuf,
|
||||
pub limits: Limits,
|
||||
pub policy: Policy,
|
||||
/// Maximum time an accepted transport may remain pending before completing `bus.hello`.
|
||||
pub hello_timeout: Duration,
|
||||
}
|
||||
|
||||
impl RouterConfig {
|
||||
/// Default limits and a closed policy: add clients with [`Policy::client`].
|
||||
pub fn new(store_root: impl Into<PathBuf>) -> RouterConfig {
|
||||
RouterConfig {
|
||||
store_root: store_root.into(),
|
||||
limits: Limits::default(),
|
||||
policy: Policy::closed(),
|
||||
hello_timeout: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
state: Mutex<State>,
|
||||
store: Store,
|
||||
store_root: PathBuf,
|
||||
router_id: String,
|
||||
store_id: String,
|
||||
stop: watch::Sender<bool>,
|
||||
hello_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
/// Runs `f` under the state lock, then unlinks whatever it released, outside the lock.
|
||||
fn with_state<R>(&self, f: impl FnOnce(&mut State) -> R) -> R {
|
||||
let mut st = self.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let r = f(&mut st);
|
||||
let unlinks = st.take_unlinks();
|
||||
drop(st);
|
||||
for rel in unlinks {
|
||||
self.store.remove(&rel);
|
||||
}
|
||||
r
|
||||
}
|
||||
}
|
||||
|
||||
/// A router and its artifact store. Cheap to clone; every clone is the same router.
|
||||
#[derive(Clone)]
|
||||
pub struct Router {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
fn fresh_tag() -> String {
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_nanos());
|
||||
let mut h = Sha256::new();
|
||||
h.update(std::process::id().to_le_bytes());
|
||||
h.update(nanos.to_le_bytes());
|
||||
h.update(COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes());
|
||||
hex(&h.finalize()[..8])
|
||||
}
|
||||
|
||||
impl Router {
|
||||
/// Creates the store directory (deleting orphans of stopped routers under the same root)
|
||||
/// and a router with a fresh `routerId`/`storeId`.
|
||||
pub fn new(config: RouterConfig) -> io::Result<Router> {
|
||||
config.limits.validate().map_err(io::Error::other)?;
|
||||
let tag = fresh_tag();
|
||||
let router_id = format!("router-{tag}");
|
||||
let store_id = format!("store-{tag}");
|
||||
let store = Store::create(&config.store_root, &store_id)?;
|
||||
let state = State::new(
|
||||
router_id.clone(),
|
||||
store_id.clone(),
|
||||
contract_digest(),
|
||||
config.limits,
|
||||
config.policy,
|
||||
);
|
||||
Ok(Router {
|
||||
inner: Arc::new(Inner {
|
||||
state: Mutex::new(state),
|
||||
store,
|
||||
store_root: config.store_root,
|
||||
router_id,
|
||||
store_id,
|
||||
stop: watch::Sender::new(false),
|
||||
hello_timeout: config.hello_timeout,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn router_id(&self) -> &str {
|
||||
&self.inner.router_id
|
||||
}
|
||||
|
||||
pub fn store_id(&self) -> &str {
|
||||
&self.inner.store_id
|
||||
}
|
||||
|
||||
pub fn store_root(&self) -> &Path {
|
||||
&self.inner.store_root
|
||||
}
|
||||
|
||||
/// `<store_root>/<store_id>`.
|
||||
pub fn store_dir(&self) -> &Path {
|
||||
self.inner.store.dir()
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> RouterStats {
|
||||
self.inner.with_state(|s| s.stats())
|
||||
}
|
||||
|
||||
/// Serves an explicitly trusted, unbound connection. This is refused unless the router
|
||||
/// uses [`Policy::open`]; Hello identity is self-asserted in this mode.
|
||||
pub fn serve<S: Stream>(&self, stream: S) {
|
||||
tokio::spawn(run_connection(self.inner.clone(), stream, None));
|
||||
}
|
||||
|
||||
/// Serves one launcher-bound participant. Hello must name `expected_client_id`.
|
||||
pub fn serve_as<S: Stream>(&self, stream: S, expected_client_id: &str) {
|
||||
tokio::spawn(run_connection(
|
||||
self.inner.clone(),
|
||||
stream,
|
||||
Some(expected_client_id.to_owned()),
|
||||
));
|
||||
}
|
||||
|
||||
/// A connected in-memory transport for a client in this process.
|
||||
pub fn connect_in_memory(&self) -> Transport {
|
||||
let (client, router) = Transport::pair();
|
||||
self.serve(router);
|
||||
client
|
||||
}
|
||||
|
||||
/// A connected in-memory transport bound to `expected_client_id` out of band.
|
||||
pub fn connect_in_memory_as(&self, expected_client_id: &str) -> Transport {
|
||||
let (client, router) = Transport::pair();
|
||||
self.serve_as(router, expected_client_id);
|
||||
client
|
||||
}
|
||||
|
||||
/// Listens on a new Unix-domain socket (mode 0600) until the returned handle is dropped or
|
||||
/// the router shuts down.
|
||||
pub async fn listen_unix(&self, path: impl AsRef<Path>) -> io::Result<UnixListenerHandle> {
|
||||
self.listen_unix_inner(path.as_ref(), None).await
|
||||
}
|
||||
|
||||
/// Listens on a launcher-created endpoint dedicated to `expected_client_id`.
|
||||
pub async fn listen_unix_as(
|
||||
&self,
|
||||
path: impl AsRef<Path>,
|
||||
expected_client_id: &str,
|
||||
) -> io::Result<UnixListenerHandle> {
|
||||
self.listen_unix_inner(path.as_ref(), Some(expected_client_id.to_owned()))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn listen_unix_inner(
|
||||
&self,
|
||||
path: &Path,
|
||||
expected_client_id: Option<String>,
|
||||
) -> io::Result<UnixListenerHandle> {
|
||||
let path = path.to_path_buf();
|
||||
let listener = UnixListener::bind(&path)?;
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))?;
|
||||
let inner = self.inner.clone();
|
||||
let mut stop = self.inner.stop.subscribe();
|
||||
let task = tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
accepted = listener.accept() => match accepted {
|
||||
Ok((stream, _)) => {
|
||||
tokio::spawn(run_connection(inner.clone(), stream, expected_client_id.clone()));
|
||||
}
|
||||
// Out of descriptors and the like: back off rather than spin.
|
||||
Err(_) => tokio::time::sleep(Duration::from_millis(10)).await,
|
||||
},
|
||||
_ = stopped(&mut stop) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(UnixListenerHandle { path, task })
|
||||
}
|
||||
|
||||
/// Closes every connection, releases retained values and refuses new connections. A
|
||||
/// connection at a frame boundary gets final notices; one inside a partial frame is closed
|
||||
/// without appending another frame. The store directory is removed once the last connection
|
||||
/// task and seal have finished and every `Router` clone is dropped.
|
||||
pub fn shutdown(&self) {
|
||||
self.inner.stop.send_replace(true);
|
||||
self.inner.with_state(|s| s.shutdown());
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops the listener and removes its socket file when dropped.
|
||||
pub struct UnixListenerHandle {
|
||||
path: PathBuf,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl UnixListenerHandle {
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UnixListenerHandle {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves once the flag is true (or its sender is gone).
|
||||
async fn stopped(rx: &mut watch::Receiver<bool>) {
|
||||
let _ = rx.wait_for(|v| *v).await;
|
||||
}
|
||||
|
||||
async fn run_connection<S: Stream>(
|
||||
inner: Arc<Inner>,
|
||||
stream: S,
|
||||
expected_client_id: Option<String>,
|
||||
) {
|
||||
let signals = Arc::new(ConnSignals::new());
|
||||
let Some(c) = inner.with_state(|s| s.add_conn(signals.clone(), expected_client_id)) else {
|
||||
return;
|
||||
};
|
||||
let (rd, wr) = tokio::io::split(stream);
|
||||
let writer = tokio::spawn(write_loop(inner.clone(), c, signals.clone(), wr));
|
||||
read_loop(&inner, c, &signals, rd).await;
|
||||
// A no-op if the state already closed it.
|
||||
inner.with_state(|s| s.close_conn(c, []));
|
||||
let _ = writer.await;
|
||||
}
|
||||
|
||||
async fn read_loop<R: AsyncRead + Unpin>(
|
||||
inner: &Arc<Inner>,
|
||||
c: ConnKey,
|
||||
signals: &ConnSignals,
|
||||
mut rd: R,
|
||||
) {
|
||||
let mut shutdown = signals.shutdown.subscribe();
|
||||
let hello_deadline = tokio::time::sleep(inner.hello_timeout);
|
||||
tokio::pin!(hello_deadline);
|
||||
let mut negotiated = false;
|
||||
let mut handled: u32 = 0;
|
||||
loop {
|
||||
let frame = tokio::select! {
|
||||
f = read_frame(&mut rd) => f,
|
||||
_ = stopped(&mut shutdown) => return,
|
||||
_ = &mut hello_deadline, if !negotiated => {
|
||||
inner.with_state(|s| s.reject_frame(c, "bus.hello deadline expired".into()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let bytes = match frame {
|
||||
Ok(Some(b)) => b,
|
||||
Ok(None) | Err(FrameError::Io(_)) | Err(FrameError::Truncated) => return,
|
||||
Err(e) => {
|
||||
inner.with_state(|s| s.reject_frame(c, e.to_string()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let env = match Envelope::decode(&bytes) {
|
||||
Ok(env) => env,
|
||||
Err(e) => {
|
||||
inner.with_state(|s| s.reject_frame(c, e.0));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let was_hello = !negotiated && env.op == "bus.hello";
|
||||
match inner.with_state(|s| s.handle(c, env)) {
|
||||
Outcome::Done => {}
|
||||
Outcome::Close => return,
|
||||
Outcome::Allocate(job) => {
|
||||
let (i, serial, len) = (inner.clone(), job.serial, job.len);
|
||||
let created =
|
||||
tokio::task::spawn_blocking(move || i.store.create_staging(serial, len))
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(io::Error::other(e)));
|
||||
inner.with_state(|s| s.finish_allocate(job, created));
|
||||
}
|
||||
Outcome::Seal(job) => {
|
||||
let inner = inner.clone();
|
||||
tokio::spawn(async move {
|
||||
let (i, serial, len, digest) =
|
||||
(inner.clone(), job.serial, job.len, job.digest.clone());
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let r = i.store.seal(serial, len, digest.as_deref());
|
||||
// Staging is finished with either way; unlink it before anyone can
|
||||
// see the outcome.
|
||||
i.store.remove(&crate::store::staging_rel(serial));
|
||||
r
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(SealFailure::Io(io::Error::other(e))));
|
||||
inner.with_state(|s| s.finish_seal(job, result));
|
||||
});
|
||||
}
|
||||
}
|
||||
if was_hello {
|
||||
negotiated = true;
|
||||
}
|
||||
handled = handled.wrapping_add(1);
|
||||
if handled.is_multiple_of(32) {
|
||||
// Fairness: a client flooding commands yields to the others.
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a closing connection gets to write final notices and shut down the transport.
|
||||
const CLOSING_WRITE: Duration = Duration::from_secs(2);
|
||||
|
||||
enum SelectedWrite {
|
||||
Complete,
|
||||
Closing { partial: bool },
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Writes one selected frame while serializing each synchronous transport poll with teardown.
|
||||
/// The standard mutex is released after every poll and is never held across an await.
|
||||
async fn write_selected<W: AsyncWrite + Unpin>(
|
||||
wr: &mut W,
|
||||
bytes: &[u8],
|
||||
signals: &ConnSignals,
|
||||
) -> SelectedWrite {
|
||||
let mut frame = Vec::with_capacity(bytes.len() + 4);
|
||||
frame.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
frame.extend_from_slice(bytes);
|
||||
{
|
||||
let mut gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if !gate.begin_frame(frame.len()) {
|
||||
return SelectedWrite::Closing {
|
||||
partial: gate.cut_partial(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
while written < frame.len() {
|
||||
let polled = std::future::poll_fn(|cx| {
|
||||
let mut gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if gate.closing() {
|
||||
return std::task::Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::Interrupted,
|
||||
"connection closing",
|
||||
)));
|
||||
}
|
||||
match std::pin::Pin::new(&mut *wr).poll_write(cx, &frame[written..]) {
|
||||
std::task::Poll::Ready(Ok(0)) => std::task::Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::WriteZero,
|
||||
"failed to write router frame",
|
||||
))),
|
||||
std::task::Poll::Ready(Ok(n)) => {
|
||||
gate.wrote(n);
|
||||
std::task::Poll::Ready(Ok(n))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
match polled {
|
||||
Ok(n) => written += n,
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => {
|
||||
let gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner());
|
||||
return SelectedWrite::Closing {
|
||||
partial: gate.cut_partial(),
|
||||
};
|
||||
}
|
||||
Err(_) => return SelectedWrite::Failed,
|
||||
}
|
||||
}
|
||||
|
||||
let flushed = std::future::poll_fn(|cx| {
|
||||
let gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if gate.closing() {
|
||||
return std::task::Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::Interrupted,
|
||||
"connection closing",
|
||||
)));
|
||||
}
|
||||
std::pin::Pin::new(&mut *wr).poll_flush(cx)
|
||||
})
|
||||
.await;
|
||||
if let Err(e) = flushed {
|
||||
if e.kind() == io::ErrorKind::Interrupted {
|
||||
let gate = signals.write_gate.lock().unwrap_or_else(|e| e.into_inner());
|
||||
return SelectedWrite::Closing {
|
||||
partial: gate.cut_partial(),
|
||||
};
|
||||
}
|
||||
return SelectedWrite::Failed;
|
||||
}
|
||||
signals
|
||||
.write_gate
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.finish_frame();
|
||||
SelectedWrite::Complete
|
||||
}
|
||||
|
||||
async fn write_loop<W: AsyncWrite + Unpin>(
|
||||
inner: Arc<Inner>,
|
||||
c: ConnKey,
|
||||
signals: Arc<ConnSignals>,
|
||||
mut wr: W,
|
||||
) {
|
||||
let mut shutdown = signals.shutdown.subscribe();
|
||||
// False once a frame was cut short: nothing more may be written on this stream.
|
||||
let mut aligned = true;
|
||||
loop {
|
||||
match inner.with_state(|s| s.next_frame(c)) {
|
||||
NextFrame::Frame(bytes) => {
|
||||
let write = write_selected(&mut wr, &bytes, &signals);
|
||||
tokio::pin!(write);
|
||||
let selected = tokio::select! {
|
||||
result = &mut write => result,
|
||||
_ = stopped(&mut shutdown) => {
|
||||
let gate = signals
|
||||
.write_gate
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
SelectedWrite::Closing { partial: gate.cut_partial() }
|
||||
}
|
||||
};
|
||||
match selected {
|
||||
SelectedWrite::Complete => {}
|
||||
SelectedWrite::Closing { partial } => {
|
||||
aligned = !partial;
|
||||
break;
|
||||
}
|
||||
SelectedWrite::Failed => {
|
||||
aligned = false;
|
||||
inner.with_state(|s| s.close_conn(c, []));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
NextFrame::Idle => {
|
||||
tokio::select! {
|
||||
_ = signals.wake.notified() => {}
|
||||
_ = stopped(&mut shutdown) => {}
|
||||
}
|
||||
}
|
||||
NextFrame::Gone => break,
|
||||
}
|
||||
}
|
||||
let finals = std::mem::take(
|
||||
&mut *signals
|
||||
.final_frames
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner()),
|
||||
);
|
||||
if aligned {
|
||||
let _ = tokio::time::timeout(CLOSING_WRITE, async {
|
||||
for f in &finals {
|
||||
if write_frame(&mut wr, f).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
let _ = tokio::time::timeout(CLOSING_WRITE, wr.shutdown()).await;
|
||||
}
|
||||
2606
services/flysim/crates/flybus/src/router/state.rs
Normal file
2606
services/flysim/crates/flybus/src/router/state.rs
Normal file
File diff suppressed because it is too large
Load diff
366
services/flysim/crates/flybus/src/store.rs
Normal file
366
services/flysim/crates/flybus/src/store.rs
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
//! The file-backed artifact store (bus-v1 section 8).
|
||||
//!
|
||||
//! Layout under the configured root, one directory per store incarnation:
|
||||
//!
|
||||
//! ```text
|
||||
//! <root>/<storeId>/.flybus-store marker: this directory belongs to a flybus store
|
||||
//! <root>/<storeId>/.lock flock()ed by the live router for its whole life
|
||||
//! <root>/<storeId>/staging/a-<n> producer-writable staging file, preallocated
|
||||
//! <root>/<storeId>/sealed/a-<n> immutable copy, mode 0444, never rewritten
|
||||
//! ```
|
||||
//!
|
||||
//! Sealing copies staging into a fresh inode rather than renaming it, so a writable handle a
|
||||
//! producer kept (or duplicated) after sealing reaches only the unlinked staging inode, never
|
||||
//! the sealed bytes. A store directory whose lock nobody holds is an orphan of a stopped
|
||||
//! router and is deleted when the next router starts on the same root.
|
||||
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::error::{BusError, ErrorCode};
|
||||
use crate::wire::{Location, hex};
|
||||
|
||||
const MARKER: &str = ".flybus-store";
|
||||
const LOCK: &str = ".lock";
|
||||
|
||||
pub(crate) fn staging_rel(serial: u64) -> String {
|
||||
format!("staging/a-{serial}")
|
||||
}
|
||||
|
||||
pub(crate) fn sealed_rel(serial: u64) -> String {
|
||||
format!("sealed/a-{serial}")
|
||||
}
|
||||
|
||||
pub(crate) enum SealFailure {
|
||||
Mismatch(String),
|
||||
Io(io::Error),
|
||||
}
|
||||
|
||||
impl From<io::Error> for SealFailure {
|
||||
fn from(e: io::Error) -> SealFailure {
|
||||
SealFailure::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Store {
|
||||
dir: PathBuf,
|
||||
// Held open for the flock; closing it releases the lock.
|
||||
_lock: File,
|
||||
}
|
||||
|
||||
fn try_lock(file: &File) -> bool {
|
||||
// SAFETY: flock on a valid, owned descriptor.
|
||||
unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) == 0 }
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Creates `<root>/<store_id>`, first deleting orphaned store directories under `root`.
|
||||
pub(crate) fn create(root: &Path, store_id: &str) -> io::Result<Store> {
|
||||
fs::DirBuilder::new()
|
||||
.recursive(true)
|
||||
.mode(0o700)
|
||||
.create(root)?;
|
||||
clean_orphans(root)?;
|
||||
let dir = root.join(store_id);
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
builder.mode(0o700);
|
||||
builder.create(&dir)?;
|
||||
builder.create(dir.join("staging"))?;
|
||||
builder.create(dir.join("sealed"))?;
|
||||
File::create(dir.join(MARKER))?;
|
||||
let lock = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(dir.join(LOCK))?;
|
||||
if !try_lock(&lock) {
|
||||
return Err(io::Error::other("could not lock a fresh store directory"));
|
||||
}
|
||||
Ok(Store { dir, _lock: lock })
|
||||
}
|
||||
|
||||
pub(crate) fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
pub(crate) fn path(&self, rel: &str) -> PathBuf {
|
||||
self.dir.join(rel)
|
||||
}
|
||||
|
||||
/// Creates the staging file and reserves its blocks, so a full disk fails here and not in
|
||||
/// the producer's write.
|
||||
pub(crate) fn create_staging(&self, serial: u64, len: u64) -> io::Result<()> {
|
||||
let path = self.path(&staging_rel(serial));
|
||||
let file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(&path)?;
|
||||
let reserve = || -> io::Result<()> {
|
||||
if len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let off_len = libc::off_t::try_from(len)
|
||||
.map_err(|_| io::Error::other("length overflows off_t"))?;
|
||||
// SAFETY: posix_fallocate on a valid descriptor opened for writing.
|
||||
let rc = unsafe { libc::posix_fallocate(file.as_raw_fd(), 0, off_len) };
|
||||
match rc {
|
||||
0 => Ok(()),
|
||||
libc::EOPNOTSUPP | libc::EINVAL => file.set_len(len),
|
||||
e => Err(io::Error::from_raw_os_error(e)),
|
||||
}
|
||||
};
|
||||
let result = reserve();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&path);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Copies exactly `len` staging bytes into a fresh sealed file, checking the length and,
|
||||
/// when given, the SHA-256. On failure the partial sealed file is removed; the staging file
|
||||
/// is left for the caller either way.
|
||||
pub(crate) fn seal(
|
||||
&self,
|
||||
serial: u64,
|
||||
len: u64,
|
||||
digest: Option<&str>,
|
||||
) -> Result<(), SealFailure> {
|
||||
let mut src = OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(self.path(&staging_rel(serial)))?;
|
||||
let dst_path = self.path(&sealed_rel(serial));
|
||||
let mut dst = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(&dst_path)?;
|
||||
let result = copy_exact(&mut src, &mut dst, len, digest).and_then(|()| {
|
||||
dst.set_permissions(fs::Permissions::from_mode(0o444))?;
|
||||
Ok(())
|
||||
});
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&dst_path);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Unlinks a store file; a missing file is not an error.
|
||||
pub(crate) fn remove(&self, rel: &str) {
|
||||
let _ = fs::remove_file(self.path(rel));
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Store {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.dir);
|
||||
}
|
||||
}
|
||||
|
||||
fn read_retry(r: &mut File, buf: &mut [u8]) -> io::Result<usize> {
|
||||
loop {
|
||||
match r.read(buf) {
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
|
||||
other => return other,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn copy_exact(
|
||||
src: &mut File,
|
||||
dst: &mut File,
|
||||
len: u64,
|
||||
digest: Option<&str>,
|
||||
) -> Result<(), SealFailure> {
|
||||
let mut hasher = digest.map(|_| Sha256::new());
|
||||
let mut buf = vec![0u8; 256 * 1024];
|
||||
let mut remaining = len;
|
||||
while remaining > 0 {
|
||||
let want = remaining.min(buf.len() as u64) as usize;
|
||||
let n = read_retry(src, &mut buf[..want])?;
|
||||
if n == 0 {
|
||||
return Err(SealFailure::Mismatch(format!(
|
||||
"staging holds {} of {len} declared bytes",
|
||||
len - remaining
|
||||
)));
|
||||
}
|
||||
if let Some(h) = hasher.as_mut() {
|
||||
h.update(&buf[..n]);
|
||||
}
|
||||
dst.write_all(&buf[..n])?;
|
||||
remaining -= n as u64;
|
||||
}
|
||||
if read_retry(src, &mut buf[..1])? != 0 {
|
||||
return Err(SealFailure::Mismatch(format!(
|
||||
"staging holds more than the declared {len} bytes"
|
||||
)));
|
||||
}
|
||||
if let (Some(h), Some(want)) = (hasher, digest) {
|
||||
let got = hex(&h.finalize());
|
||||
if got != want {
|
||||
return Err(SealFailure::Mismatch(format!(
|
||||
"digest mismatch: content hashes to {got}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Deletes store directories under `root` that carry the marker and whose lock is free.
|
||||
/// Directories without the marker are never touched.
|
||||
fn clean_orphans(root: &Path) -> io::Result<()> {
|
||||
for entry in fs::read_dir(root)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
if !entry.file_type()?.is_dir() || !path.join(MARKER).is_file() {
|
||||
continue;
|
||||
}
|
||||
let stale = match OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(path.join(LOCK))
|
||||
{
|
||||
Ok(lock) => try_lock(&lock),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => true,
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
if stale {
|
||||
fs::remove_dir_all(&path)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Client side
|
||||
|
||||
/// Resolves a location grant beneath `<root>/<store_id>`. Absolute paths, `..`, anything but
|
||||
/// plain `[a-z0-9._-]` components, and symlinks that lead outside the store are refused.
|
||||
pub(crate) fn resolve(root: &Path, store_id: &str, loc: &Location) -> Result<PathBuf, BusError> {
|
||||
if loc.store_id != store_id {
|
||||
return Err(BusError::new(
|
||||
ErrorCode::ArtifactGone,
|
||||
"location names another store incarnation",
|
||||
));
|
||||
}
|
||||
let refuse =
|
||||
|why: &str| BusError::new(ErrorCode::StoreFailure, format!("location refused: {why}"));
|
||||
let rel = Path::new(&loc.relative_path);
|
||||
if loc.relative_path.is_empty() || rel.is_absolute() {
|
||||
return Err(refuse("not a relative path"));
|
||||
}
|
||||
for c in rel.components() {
|
||||
match c {
|
||||
Component::Normal(s) => {
|
||||
let s = s.to_str().unwrap_or("");
|
||||
if s.is_empty()
|
||||
|| !s.bytes().all(|b| {
|
||||
b.is_ascii_lowercase()
|
||||
|| b.is_ascii_digit()
|
||||
|| matches!(b, b'.' | b'_' | b'-')
|
||||
})
|
||||
{
|
||||
return Err(refuse("unexpected path component"));
|
||||
}
|
||||
}
|
||||
_ => return Err(refuse("parent, root or current-directory component")),
|
||||
}
|
||||
}
|
||||
let base = root.join(&loc.store_id);
|
||||
let base = base
|
||||
.canonicalize()
|
||||
.map_err(|e| refuse(&format!("store directory: {e}")))?;
|
||||
let full = base.join(rel).canonicalize().map_err(|e| match e.kind() {
|
||||
io::ErrorKind::NotFound => BusError::new(ErrorCode::ArtifactGone, "artifact file is gone"),
|
||||
_ => refuse(&e.to_string()),
|
||||
})?;
|
||||
if !full.starts_with(&base) {
|
||||
return Err(refuse("escapes the store"));
|
||||
}
|
||||
Ok(full)
|
||||
}
|
||||
|
||||
pub(crate) fn open_read(path: &Path) -> io::Result<File> {
|
||||
OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(path)
|
||||
}
|
||||
|
||||
pub(crate) fn open_write(path: &Path) -> io::Result<File> {
|
||||
OpenOptions::new()
|
||||
.write(true)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn loc(p: &str) -> Location {
|
||||
Location {
|
||||
store_id: "store-x".into(),
|
||||
relative_path: p.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_refuses_escapes() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store = root.path().join("store-x");
|
||||
fs::create_dir_all(store.join("sealed")).unwrap();
|
||||
fs::write(store.join("sealed/a-1"), b"ok").unwrap();
|
||||
fs::write(root.path().join("secret"), b"no").unwrap();
|
||||
std::os::unix::fs::symlink(root.path().join("secret"), store.join("sealed/a-2")).unwrap();
|
||||
assert!(resolve(root.path(), "store-x", &loc("sealed/a-1")).is_ok());
|
||||
for bad in [
|
||||
"",
|
||||
"/etc/passwd",
|
||||
"../secret",
|
||||
"sealed/../../secret",
|
||||
"./sealed/a-1",
|
||||
"sealed/A-1",
|
||||
"sealed/a-2",
|
||||
] {
|
||||
assert!(
|
||||
resolve(root.path(), "store-x", &loc(bad)).is_err(),
|
||||
"{bad:?}"
|
||||
);
|
||||
}
|
||||
let other = Location {
|
||||
store_id: "store-y".into(),
|
||||
relative_path: "sealed/a-1".into(),
|
||||
};
|
||||
assert_eq!(
|
||||
resolve(root.path(), "store-x", &other).unwrap_err().code,
|
||||
ErrorCode::ArtifactGone
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphans_are_removed_and_live_stores_kept() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let live = Store::create(root.path(), "store-live").unwrap();
|
||||
let orphan = root.path().join("store-dead");
|
||||
fs::create_dir_all(orphan.join("sealed")).unwrap();
|
||||
fs::write(orphan.join(MARKER), b"").unwrap();
|
||||
fs::write(orphan.join(LOCK), b"").unwrap();
|
||||
let unrelated = root.path().join("keep-me");
|
||||
fs::create_dir_all(&unrelated).unwrap();
|
||||
let second = Store::create(root.path(), "store-next").unwrap();
|
||||
assert!(!orphan.exists());
|
||||
assert!(live.dir().exists() && second.dir().exists() && unrelated.exists());
|
||||
drop(live);
|
||||
assert!(!root.path().join("store-live").exists());
|
||||
}
|
||||
}
|
||||
67
services/flysim/crates/flybus/src/transport.rs
Normal file
67
services/flysim/crates/flybus/src/transport.rs
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
//! Byte-stream transports. Both carry the same framed protocol to the same router code: the
|
||||
//! in-memory transport is a Tokio duplex pipe, the other a Unix-domain stream socket.
|
||||
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
/// Anything a connection can run over.
|
||||
pub trait Stream: AsyncRead + AsyncWrite + Send + Unpin + 'static {}
|
||||
|
||||
impl<T: AsyncRead + AsyncWrite + Send + Unpin + 'static> Stream for T {}
|
||||
|
||||
/// One end of a connection to a router.
|
||||
pub struct Transport {
|
||||
inner: Box<dyn Stream>,
|
||||
}
|
||||
|
||||
impl Transport {
|
||||
pub fn from_stream<S: Stream>(stream: S) -> Transport {
|
||||
Transport {
|
||||
inner: Box::new(stream),
|
||||
}
|
||||
}
|
||||
|
||||
/// Connects to a router's Unix-domain socket.
|
||||
pub async fn unix(path: impl AsRef<Path>) -> io::Result<Transport> {
|
||||
Ok(Transport::from_stream(UnixStream::connect(path).await?))
|
||||
}
|
||||
|
||||
/// An in-memory pipe pair: one end for a router, the other for a client.
|
||||
pub(crate) fn pair() -> (Transport, Transport) {
|
||||
let (a, b) = tokio::io::duplex(64 * 1024);
|
||||
(Transport::from_stream(a), Transport::from_stream(b))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for Transport {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut *self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for Transport {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
Pin::new(&mut *self.inner).poll_write(cx, buf)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut *self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut *self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
816
services/flysim/crates/flybus/src/wire.rs
Normal file
816
services/flysim/crates/flybus/src/wire.rs
Normal file
|
|
@ -0,0 +1,816 @@
|
|||
//! The wire: scalar encodings, strict JSON, the envelope and its framing (bus-v1 section 4).
|
||||
//!
|
||||
//! A frame is a `u32` little-endian byte count followed by that many bytes of UTF-8 JSON. The
|
||||
//! JSON is parsed strictly: duplicate keys at any depth, invalid UTF-8, non-finite numbers,
|
||||
//! trailing bytes and unknown envelope fields are all refused. Both ends use this module, so
|
||||
//! the router and the client agree byte for byte on what is valid.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fmt;
|
||||
|
||||
use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor};
|
||||
use serde_json::{Map, Value};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
|
||||
use crate::error::BusError;
|
||||
|
||||
pub const PROTOCOL: &str = "flybus";
|
||||
pub const MAJOR: u64 = 1;
|
||||
pub const MINOR: u64 = 0;
|
||||
/// Largest JSON envelope, in bytes, not counting the four-byte length prefix.
|
||||
pub const MAX_ENVELOPE_BYTES: usize = 65_536;
|
||||
pub const MAX_ATTACHMENTS: usize = 32;
|
||||
/// Release and consume batches carry 1..=64 ids.
|
||||
pub const MAX_BATCH: usize = 64;
|
||||
/// Queue and credit requests are integers in 1..=65535.
|
||||
pub const MAX_CREDIT: u64 = 65_535;
|
||||
pub const MAX_NAME_LEN: usize = 192;
|
||||
pub const MAX_METHOD_LEN: usize = 128;
|
||||
pub const MAX_CONTENT_TYPE_LEN: usize = 127;
|
||||
/// The one artifact generation v1 issues; ids and inodes are never reused.
|
||||
pub const GENERATION: u64 = 1;
|
||||
|
||||
/// The operations, delivery and notice shapes this implementation speaks. `contractDigest` is
|
||||
/// the SHA-256 of this text, so any change to it changes the digest a client sees in hello.
|
||||
pub const CONTRACT: &str = "flybus 1.0
|
||||
frame: u32le length, 1..=65536 bytes of strict UTF-8 JSON
|
||||
envelope: protocol major minor id replyTo kind op body attachments
|
||||
attachment: name ref ownerId
|
||||
ref: storeId artifactId generation byteLength contentType digest
|
||||
reply: ok value | ok error{code message dispatch}
|
||||
bus.hello: clientId clientIncarnation supportedMajors -> routerId connectionId selectedMajor selectedMinor contractDigest limits
|
||||
service.register: name maxQueued maxInFlight -> serviceIncarnation
|
||||
service.unregister: name serviceIncarnation -> removed
|
||||
rpc.call: callId target expectedIncarnation method payload -> accepted serviceIncarnation
|
||||
rpc.reply: callId requestDeliveryId outcome -> routed
|
||||
rpc.responder.release: callId requestDeliveryId -> released; final attached release emits call.failed dispatched (CALL_GONE, or NO_SERVICE after route loss)
|
||||
rpc.cancel: callId -> state(cancelled-before-dispatch|execution-unknown|completed|call-gone)
|
||||
topic.declare: name retained(none|latest) -> declared topicIncarnation
|
||||
topic.clear: name -> cleared
|
||||
topic.delete: name -> deleted
|
||||
subscribe: topic mode(latest|bounded) maxQueued maxInFlight replayLatest -> subscriptionId topicIncarnation
|
||||
unsubscribe: subscriptionId -> removed
|
||||
publish: topic payload -> topicSequence subscribers replaced
|
||||
delivery.consumed: deliveryIds -> released
|
||||
artifact.allocate: byteLength contentType -> artifactId generation ownerId writeLocation
|
||||
artifact.seal: artifactId generation ownerId digest -> ref ownerId
|
||||
artifact.open: ref ownerId -> readLocation
|
||||
artifact.retain: ref ownerId -> ownerId
|
||||
artifact.release: ownerIds -> released
|
||||
delivery rpc.request: deliveryId callId caller target serviceIncarnation method payload
|
||||
delivery rpc.result: deliveryId callId responder serviceIncarnation outcome
|
||||
delivery topic.message: deliveryId subscriptionId topic topicIncarnation topicSequence replaced payload
|
||||
notice call.failed: callId code message dispatch
|
||||
notice route.removed: name serviceIncarnation reason
|
||||
notice subscription.closed: subscriptionId topic topicIncarnation reason
|
||||
notice connection.closing: code message
|
||||
";
|
||||
|
||||
/// SHA-256 of [`CONTRACT`], lowercase hex.
|
||||
pub fn contract_digest() -> String {
|
||||
hex(&Sha256::digest(CONTRACT.as_bytes()))
|
||||
}
|
||||
|
||||
pub(crate) fn hex(bytes: &[u8]) -> String {
|
||||
let mut s = String::with_capacity(bytes.len() * 2);
|
||||
for b in bytes {
|
||||
s.push_str(&format!("{b:02x}"));
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// A wire-level validation failure. Always maps to `INVALID_ENVELOPE`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct WireError(pub String);
|
||||
|
||||
impl fmt::Display for WireError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for WireError {}
|
||||
|
||||
impl From<WireError> for BusError {
|
||||
fn from(e: WireError) -> BusError {
|
||||
BusError::invalid(e.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn err<T>(message: impl Into<String>) -> Result<T, WireError> {
|
||||
Err(WireError(message.into()))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Scalars
|
||||
|
||||
/// `Id`: `^[a-z0-9][a-z0-9._-]{0,63}$`.
|
||||
pub fn is_id(s: &str) -> bool {
|
||||
let b = s.as_bytes();
|
||||
!b.is_empty()
|
||||
&& b.len() <= 64
|
||||
&& (b[0].is_ascii_lowercase() || b[0].is_ascii_digit())
|
||||
&& b.iter().all(|&c| {
|
||||
c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, b'.' | b'_' | b'-')
|
||||
})
|
||||
}
|
||||
|
||||
/// `U64`: `"0"` or `[1-9][0-9]*`, at most `u64::MAX`.
|
||||
pub fn parse_u64(s: &str) -> Option<u64> {
|
||||
let b = s.as_bytes();
|
||||
if b.is_empty() || b.len() > 20 || (b.len() > 1 && b[0] == b'0') {
|
||||
return None;
|
||||
}
|
||||
let mut n: u64 = 0;
|
||||
for &c in b {
|
||||
if !c.is_ascii_digit() {
|
||||
return None;
|
||||
}
|
||||
n = n.checked_mul(10)?.checked_add(u64::from(c - b'0'))?;
|
||||
}
|
||||
Some(n)
|
||||
}
|
||||
|
||||
/// `Digest`: 64 lowercase hexadecimal digits.
|
||||
pub fn is_digest(s: &str) -> bool {
|
||||
s.len() == 64
|
||||
&& s.bytes()
|
||||
.all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c))
|
||||
}
|
||||
|
||||
/// Service and topic names: 1..=192 of `[a-z0-9._-]`, no empty dot-separated segment.
|
||||
pub fn is_name(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.len() <= MAX_NAME_LEN
|
||||
&& s.bytes().all(|c| {
|
||||
c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, b'.' | b'_' | b'-')
|
||||
})
|
||||
&& s.split('.').all(|seg| !seg.is_empty())
|
||||
}
|
||||
|
||||
/// RPC method: 1..=128 printable ASCII characters.
|
||||
pub fn is_method(s: &str) -> bool {
|
||||
!s.is_empty() && s.len() <= MAX_METHOD_LEN && s.bytes().all(|c| (0x20..=0x7e).contains(&c))
|
||||
}
|
||||
|
||||
/// Content type: 1..=127 printable ASCII characters.
|
||||
pub fn is_content_type(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.len() <= MAX_CONTENT_TYPE_LEN
|
||||
&& s.bytes().all(|c| (0x20..=0x7e).contains(&c))
|
||||
}
|
||||
|
||||
/// Operation names: 1..=64 of `[a-z.]`.
|
||||
pub fn is_op(s: &str) -> bool {
|
||||
!s.is_empty() && s.len() <= 64 && s.bytes().all(|c| c.is_ascii_lowercase() || c == b'.')
|
||||
}
|
||||
|
||||
/// `<prefix>-<U64>`, the canonical form of every serial-numbered id on the bus.
|
||||
pub fn serial_id(prefix: &str, n: u64) -> String {
|
||||
format!("{prefix}-{n}")
|
||||
}
|
||||
|
||||
/// Parses `<prefix>-<U64>`; `None` unless canonical.
|
||||
pub fn parse_serial_id(prefix: &str, s: &str) -> Option<u64> {
|
||||
s.strip_prefix(prefix)?
|
||||
.strip_prefix('-')
|
||||
.and_then(parse_u64)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Strict JSON
|
||||
|
||||
/// Parses one JSON value, refusing duplicate object keys at any depth, invalid UTF-8,
|
||||
/// non-finite numbers and trailing data. Nesting depth is bounded by serde_json's recursion
|
||||
/// limit (128).
|
||||
pub fn parse_json_strict(bytes: &[u8]) -> Result<Value, WireError> {
|
||||
if std::str::from_utf8(bytes).is_err() {
|
||||
return err("invalid UTF-8");
|
||||
}
|
||||
let mut de = serde_json::Deserializer::from_slice(bytes);
|
||||
let value = StrictSeed
|
||||
.deserialize(&mut de)
|
||||
.map_err(|e| WireError(format!("invalid JSON: {e}")))?;
|
||||
de.end()
|
||||
.map_err(|e| WireError(format!("invalid JSON: {e}")))?;
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
struct StrictSeed;
|
||||
|
||||
impl<'de> DeserializeSeed<'de> for StrictSeed {
|
||||
type Value = Value;
|
||||
|
||||
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Value, D::Error> {
|
||||
d.deserialize_any(StrictVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
struct StrictVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for StrictVisitor {
|
||||
type Value = Value;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("a JSON value")
|
||||
}
|
||||
|
||||
fn visit_bool<E>(self, v: bool) -> Result<Value, E> {
|
||||
Ok(Value::Bool(v))
|
||||
}
|
||||
|
||||
fn visit_i64<E>(self, v: i64) -> Result<Value, E> {
|
||||
Ok(Value::from(v))
|
||||
}
|
||||
|
||||
fn visit_u64<E>(self, v: u64) -> Result<Value, E> {
|
||||
Ok(Value::from(v))
|
||||
}
|
||||
|
||||
fn visit_f64<E: de::Error>(self, v: f64) -> Result<Value, E> {
|
||||
serde_json::Number::from_f64(v)
|
||||
.map(Value::Number)
|
||||
.ok_or_else(|| E::custom("non-finite number"))
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, v: &str) -> Result<Value, E> {
|
||||
Ok(Value::String(v.to_owned()))
|
||||
}
|
||||
|
||||
fn visit_string<E>(self, v: String) -> Result<Value, E> {
|
||||
Ok(Value::String(v))
|
||||
}
|
||||
|
||||
fn visit_unit<E>(self) -> Result<Value, E> {
|
||||
Ok(Value::Null)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Value, A::Error> {
|
||||
let mut out = Vec::new();
|
||||
while let Some(v) = seq.next_element_seed(StrictSeed)? {
|
||||
out.push(v);
|
||||
}
|
||||
Ok(Value::Array(out))
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Value, A::Error> {
|
||||
let mut out = Map::new();
|
||||
while let Some(key) = map.next_key::<String>()? {
|
||||
if out.contains_key(&key) {
|
||||
return Err(de::Error::custom(format!("duplicate key {key:?}")));
|
||||
}
|
||||
let v = map.next_value_seed(StrictSeed)?;
|
||||
out.insert(key, v);
|
||||
}
|
||||
Ok(Value::Object(out))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Exact-field object reader
|
||||
|
||||
/// Reads the fields of one JSON object, then refuses any it did not read.
|
||||
pub struct Fields<'a> {
|
||||
map: &'a Map<String, Value>,
|
||||
seen: HashSet<&'static str>,
|
||||
what: &'static str,
|
||||
}
|
||||
|
||||
impl<'a> Fields<'a> {
|
||||
pub fn new(v: &'a Value, what: &'static str) -> Result<Fields<'a>, WireError> {
|
||||
match v {
|
||||
Value::Object(map) => Ok(Fields::of(map, what)),
|
||||
_ => err(format!("{what} must be an object")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn of(map: &'a Map<String, Value>, what: &'static str) -> Fields<'a> {
|
||||
Fields {
|
||||
map,
|
||||
seen: HashSet::new(),
|
||||
what,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn value(&mut self, key: &'static str) -> Result<&'a Value, WireError> {
|
||||
self.seen.insert(key);
|
||||
match self.map.get(key) {
|
||||
Some(v) => Ok(v),
|
||||
None => err(format!("{}: missing field {key:?}", self.what)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn string(&mut self, key: &'static str) -> Result<&'a str, WireError> {
|
||||
let what = self.what;
|
||||
self.value(key)?
|
||||
.as_str()
|
||||
.ok_or_else(|| WireError(format!("{what}: {key} must be a string")))
|
||||
}
|
||||
|
||||
fn checked(
|
||||
&mut self,
|
||||
key: &'static str,
|
||||
ok: fn(&str) -> bool,
|
||||
kind: &str,
|
||||
) -> Result<String, WireError> {
|
||||
let s = self.string(key)?;
|
||||
if ok(s) {
|
||||
Ok(s.to_owned())
|
||||
} else {
|
||||
err(format!("{}: {key} is not a valid {kind}", self.what))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn id(&mut self, key: &'static str) -> Result<String, WireError> {
|
||||
self.checked(key, is_id, "id")
|
||||
}
|
||||
|
||||
pub fn nullable_id(&mut self, key: &'static str) -> Result<Option<String>, WireError> {
|
||||
match self.value(key)? {
|
||||
Value::Null => Ok(None),
|
||||
_ => self.id(key).map(Some),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn name(&mut self, key: &'static str) -> Result<String, WireError> {
|
||||
self.checked(key, is_name, "name")
|
||||
}
|
||||
|
||||
pub fn method(&mut self, key: &'static str) -> Result<String, WireError> {
|
||||
self.checked(key, is_method, "method")
|
||||
}
|
||||
|
||||
pub fn u64_string(&mut self, key: &'static str) -> Result<u64, WireError> {
|
||||
let s = self.string(key)?;
|
||||
parse_u64(s).ok_or_else(|| {
|
||||
WireError(format!(
|
||||
"{}: {key} is not a canonical U64 string",
|
||||
self.what
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// A JSON integer in `lo..=hi`.
|
||||
pub fn int(&mut self, key: &'static str, lo: u64, hi: u64) -> Result<u64, WireError> {
|
||||
let what = self.what;
|
||||
match self.value(key)?.as_u64() {
|
||||
Some(n) if (lo..=hi).contains(&n) => Ok(n),
|
||||
_ => err(format!("{what}: {key} must be an integer in {lo}..={hi}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn boolean(&mut self, key: &'static str) -> Result<bool, WireError> {
|
||||
let what = self.what;
|
||||
self.value(key)?
|
||||
.as_bool()
|
||||
.ok_or_else(|| WireError(format!("{what}: {key} must be a boolean")))
|
||||
}
|
||||
|
||||
pub fn object(&mut self, key: &'static str) -> Result<&'a Map<String, Value>, WireError> {
|
||||
let what = self.what;
|
||||
self.value(key)?
|
||||
.as_object()
|
||||
.ok_or_else(|| WireError(format!("{what}: {key} must be an object")))
|
||||
}
|
||||
|
||||
pub fn array(
|
||||
&mut self,
|
||||
key: &'static str,
|
||||
lo: usize,
|
||||
hi: usize,
|
||||
) -> Result<&'a Vec<Value>, WireError> {
|
||||
let what = self.what;
|
||||
match self.value(key)?.as_array() {
|
||||
Some(a) if (lo..=hi).contains(&a.len()) => Ok(a),
|
||||
_ => err(format!(
|
||||
"{what}: {key} must be an array of {lo}..={hi} items"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses fields that were not read.
|
||||
pub fn finish(self) -> Result<(), WireError> {
|
||||
if let Some(extra) = self.map.keys().find(|k| !self.seen.contains(k.as_str())) {
|
||||
return err(format!("{}: unknown field {extra:?}", self.what));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An array of ids, each `<prefix>-<U64>`, 1..=64 of them.
|
||||
pub fn id_batch(v: &[Value], what: &str) -> Result<Vec<String>, WireError> {
|
||||
let mut out = Vec::with_capacity(v.len());
|
||||
for item in v {
|
||||
match item.as_str() {
|
||||
Some(s) if is_id(s) => out.push(s.to_owned()),
|
||||
_ => return err(format!("{what}: every entry must be an id")),
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Shared structures
|
||||
|
||||
/// An immutable artifact's identity. Not an address and not authority to read.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct ArtifactRef {
|
||||
pub store_id: String,
|
||||
pub artifact_id: String,
|
||||
pub generation: u64,
|
||||
pub byte_length: u64,
|
||||
pub content_type: String,
|
||||
pub digest: Option<String>,
|
||||
}
|
||||
|
||||
impl ArtifactRef {
|
||||
pub fn to_json(&self) -> Value {
|
||||
let mut m = Map::new();
|
||||
m.insert("storeId".into(), self.store_id.clone().into());
|
||||
m.insert("artifactId".into(), self.artifact_id.clone().into());
|
||||
m.insert("generation".into(), self.generation.to_string().into());
|
||||
m.insert("byteLength".into(), self.byte_length.to_string().into());
|
||||
m.insert("contentType".into(), self.content_type.clone().into());
|
||||
m.insert(
|
||||
"digest".into(),
|
||||
self.digest.clone().map_or(Value::Null, Value::String),
|
||||
);
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
pub fn from_json(v: &Value) -> Result<ArtifactRef, WireError> {
|
||||
let mut f = Fields::new(v, "ref")?;
|
||||
let store_id = f.id("storeId")?;
|
||||
let artifact_id = f.id("artifactId")?;
|
||||
let generation = f.u64_string("generation")?;
|
||||
let byte_length = f.u64_string("byteLength")?;
|
||||
let content_type = f.string("contentType")?;
|
||||
if !is_content_type(content_type) {
|
||||
return err("ref: contentType must be 1..=127 printable ASCII characters");
|
||||
}
|
||||
let digest = match f.value("digest")? {
|
||||
Value::Null => None,
|
||||
Value::String(s) if is_digest(s) => Some(s.clone()),
|
||||
_ => return err("ref: digest must be null or 64 lowercase hex digits"),
|
||||
};
|
||||
f.finish()?;
|
||||
Ok(ArtifactRef {
|
||||
store_id,
|
||||
artifact_id,
|
||||
generation,
|
||||
byte_length,
|
||||
content_type: content_type.to_owned(),
|
||||
digest,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// One attachment entry: an application name, the reference and the sender's owner token.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Attachment {
|
||||
pub name: String,
|
||||
pub reference: ArtifactRef,
|
||||
pub owner_id: String,
|
||||
}
|
||||
|
||||
impl Attachment {
|
||||
pub fn to_json(&self) -> Value {
|
||||
let mut m = Map::new();
|
||||
m.insert("name".into(), self.name.clone().into());
|
||||
m.insert("ref".into(), self.reference.to_json());
|
||||
m.insert("ownerId".into(), self.owner_id.clone().into());
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
pub fn from_json(v: &Value) -> Result<Attachment, WireError> {
|
||||
let mut f = Fields::new(v, "attachment")?;
|
||||
let name = f.id("name")?;
|
||||
let reference = ArtifactRef::from_json(f.value("ref")?)?;
|
||||
let owner_id = f.id("ownerId")?;
|
||||
f.finish()?;
|
||||
Ok(Attachment {
|
||||
name,
|
||||
reference,
|
||||
owner_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A store location grant: a path relative to `<store root>/<storeId>`. SDK-private.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Location {
|
||||
pub store_id: String,
|
||||
pub relative_path: String,
|
||||
}
|
||||
|
||||
impl Location {
|
||||
pub fn to_json(&self) -> Value {
|
||||
let mut m = Map::new();
|
||||
m.insert("storeId".into(), self.store_id.clone().into());
|
||||
m.insert("relativePath".into(), self.relative_path.clone().into());
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
pub fn from_json(v: &Value) -> Result<Location, WireError> {
|
||||
let mut f = Fields::new(v, "location")?;
|
||||
let store_id = f.id("storeId")?;
|
||||
let relative_path = f.string("relativePath")?.to_owned();
|
||||
f.finish()?;
|
||||
Ok(Location {
|
||||
store_id,
|
||||
relative_path,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A participant identity as the router reports it on deliveries. It is authenticated only
|
||||
/// when the connection was accepted through a launcher-bound transport entry point.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct Identity {
|
||||
pub client_id: String,
|
||||
pub client_incarnation: String,
|
||||
}
|
||||
|
||||
impl Identity {
|
||||
pub fn to_json(&self) -> Value {
|
||||
let mut m = Map::new();
|
||||
m.insert("clientId".into(), self.client_id.clone().into());
|
||||
m.insert(
|
||||
"clientIncarnation".into(),
|
||||
self.client_incarnation.clone().into(),
|
||||
);
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
pub fn from_json(v: &Value) -> Result<Identity, WireError> {
|
||||
let mut f = Fields::new(v, "identity")?;
|
||||
let client_id = f.id("clientId")?;
|
||||
let client_incarnation = f.id("clientIncarnation")?;
|
||||
f.finish()?;
|
||||
Ok(Identity {
|
||||
client_id,
|
||||
client_incarnation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Envelope
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Command,
|
||||
Reply,
|
||||
Delivery,
|
||||
Notice,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Kind::Command => "command",
|
||||
Kind::Reply => "reply",
|
||||
Kind::Delivery => "delivery",
|
||||
Kind::Notice => "notice",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(s: &str) -> Option<Kind> {
|
||||
match s {
|
||||
"command" => Some(Kind::Command),
|
||||
"reply" => Some(Kind::Reply),
|
||||
"delivery" => Some(Kind::Delivery),
|
||||
"notice" => Some(Kind::Notice),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Envelope {
|
||||
pub major: u64,
|
||||
pub minor: u64,
|
||||
pub id: String,
|
||||
pub reply_to: Option<String>,
|
||||
pub kind: Kind,
|
||||
pub op: String,
|
||||
pub body: Map<String, Value>,
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl Envelope {
|
||||
pub fn new(id: String, kind: Kind, op: &str, body: Map<String, Value>) -> Envelope {
|
||||
Envelope {
|
||||
major: MAJOR,
|
||||
minor: MINOR,
|
||||
id,
|
||||
reply_to: None,
|
||||
kind,
|
||||
op: op.to_owned(),
|
||||
body,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes and validates one frame's bytes.
|
||||
pub fn decode(bytes: &[u8]) -> Result<Envelope, WireError> {
|
||||
if bytes.is_empty() {
|
||||
return err("empty envelope");
|
||||
}
|
||||
if bytes.len() > MAX_ENVELOPE_BYTES {
|
||||
return err(format!(
|
||||
"envelope of {} bytes exceeds {MAX_ENVELOPE_BYTES}",
|
||||
bytes.len()
|
||||
));
|
||||
}
|
||||
let v = parse_json_strict(bytes)?;
|
||||
let mut f = Fields::new(&v, "envelope")?;
|
||||
if f.string("protocol")? != PROTOCOL {
|
||||
return err("envelope: protocol must be \"flybus\"");
|
||||
}
|
||||
let major = f.int("major", 0, MAX_CREDIT)?;
|
||||
let minor = f.int("minor", 0, MAX_CREDIT)?;
|
||||
let id = f.id("id")?;
|
||||
let reply_to = f.nullable_id("replyTo")?;
|
||||
let kind = Kind::parse(f.string("kind")?)
|
||||
.ok_or_else(|| WireError("envelope: unknown kind".into()))?;
|
||||
let op = f.string("op")?;
|
||||
if !is_op(op) {
|
||||
return err("envelope: op must be 1..=64 of [a-z.]");
|
||||
}
|
||||
let op = op.to_owned();
|
||||
let body = f.object("body")?.clone();
|
||||
let raw = f.array("attachments", 0, MAX_ATTACHMENTS)?;
|
||||
let mut attachments = Vec::with_capacity(raw.len());
|
||||
let mut names = HashSet::new();
|
||||
for a in raw {
|
||||
let a = Attachment::from_json(a)?;
|
||||
if !names.insert(a.name.clone()) {
|
||||
return err(format!("envelope: duplicate attachment name {:?}", a.name));
|
||||
}
|
||||
attachments.push(a);
|
||||
}
|
||||
f.finish()?;
|
||||
Ok(Envelope {
|
||||
major,
|
||||
minor,
|
||||
id,
|
||||
reply_to,
|
||||
kind,
|
||||
op,
|
||||
body,
|
||||
attachments,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn to_value(&self) -> Value {
|
||||
let mut m = Map::new();
|
||||
m.insert("protocol".into(), PROTOCOL.into());
|
||||
m.insert("major".into(), self.major.into());
|
||||
m.insert("minor".into(), self.minor.into());
|
||||
m.insert("id".into(), self.id.clone().into());
|
||||
m.insert(
|
||||
"replyTo".into(),
|
||||
self.reply_to.clone().map_or(Value::Null, Value::String),
|
||||
);
|
||||
m.insert("kind".into(), self.kind.as_str().into());
|
||||
m.insert("op".into(), self.op.clone().into());
|
||||
m.insert("body".into(), Value::Object(self.body.clone()));
|
||||
m.insert(
|
||||
"attachments".into(),
|
||||
Value::Array(self.attachments.iter().map(Attachment::to_json).collect()),
|
||||
);
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
/// Serializes, refusing anything over [`MAX_ENVELOPE_BYTES`].
|
||||
pub fn encode(&self) -> Result<Vec<u8>, WireError> {
|
||||
let bytes = serde_json::to_vec(&self.to_value()).map_err(|e| WireError(e.to_string()))?;
|
||||
if bytes.len() > MAX_ENVELOPE_BYTES {
|
||||
return err(format!(
|
||||
"envelope of {} bytes exceeds {MAX_ENVELOPE_BYTES}",
|
||||
bytes.len()
|
||||
));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Framing
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum FrameError {
|
||||
Io(std::io::Error),
|
||||
/// A zero length prefix.
|
||||
Empty,
|
||||
/// The length prefix exceeds the limit; nothing was allocated for it.
|
||||
TooLarge(u32),
|
||||
/// End of stream inside a frame.
|
||||
Truncated,
|
||||
}
|
||||
|
||||
impl fmt::Display for FrameError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
FrameError::Io(e) => write!(f, "I/O error: {e}"),
|
||||
FrameError::Empty => f.write_str("zero-length frame"),
|
||||
FrameError::TooLarge(n) => write!(f, "frame of {n} bytes exceeds {MAX_ENVELOPE_BYTES}"),
|
||||
FrameError::Truncated => f.write_str("stream ended inside a frame"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FrameError {}
|
||||
|
||||
/// Reads one frame. `Ok(None)` is a clean end of stream at a frame boundary. The length is
|
||||
/// checked against [`MAX_ENVELOPE_BYTES`] before any buffer is allocated.
|
||||
pub async fn read_frame<R: AsyncRead + Unpin>(r: &mut R) -> Result<Option<Vec<u8>>, FrameError> {
|
||||
let mut len = [0u8; 4];
|
||||
let mut got = 0;
|
||||
while got < 4 {
|
||||
let n = r.read(&mut len[got..]).await.map_err(FrameError::Io)?;
|
||||
if n == 0 {
|
||||
return if got == 0 {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(FrameError::Truncated)
|
||||
};
|
||||
}
|
||||
got += n;
|
||||
}
|
||||
let len = u32::from_le_bytes(len);
|
||||
if len == 0 {
|
||||
return Err(FrameError::Empty);
|
||||
}
|
||||
if len as usize > MAX_ENVELOPE_BYTES {
|
||||
return Err(FrameError::TooLarge(len));
|
||||
}
|
||||
let mut buf = vec![0u8; len as usize];
|
||||
r.read_exact(&mut buf).await.map_err(|e| match e.kind() {
|
||||
std::io::ErrorKind::UnexpectedEof => FrameError::Truncated,
|
||||
_ => FrameError::Io(e),
|
||||
})?;
|
||||
Ok(Some(buf))
|
||||
}
|
||||
|
||||
/// Writes one frame: the length prefix and the bytes, as one buffer.
|
||||
pub async fn write_frame<W: AsyncWrite + Unpin>(w: &mut W, bytes: &[u8]) -> std::io::Result<()> {
|
||||
debug_assert!(!bytes.is_empty() && bytes.len() <= MAX_ENVELOPE_BYTES);
|
||||
let mut buf = Vec::with_capacity(bytes.len() + 4);
|
||||
buf.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
buf.extend_from_slice(bytes);
|
||||
w.write_all(&buf).await?;
|
||||
w.flush().await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scalars() {
|
||||
assert!(is_id("a") && is_id("msg-7") && is_id("0.a_b-c"));
|
||||
assert!(!is_id("") && !is_id("-a") && !is_id("A") && !is_id(&"a".repeat(65)));
|
||||
assert!(is_id(&"a".repeat(64)));
|
||||
assert_eq!(parse_u64("0"), Some(0));
|
||||
assert_eq!(parse_u64("18446744073709551615"), Some(u64::MAX));
|
||||
for bad in ["", "01", "18446744073709551616", "-1", "1.0", "+1", " 1"] {
|
||||
assert_eq!(parse_u64(bad), None, "{bad:?}");
|
||||
}
|
||||
assert!(is_name("session.demo.snapshots") && is_name(&"a".repeat(192)));
|
||||
for bad in ["", ".a", "a.", "a..b", "A", "a/b", &"a".repeat(193)] {
|
||||
assert!(!is_name(bad), "{bad:?}");
|
||||
}
|
||||
assert!(is_digest(&"0f".repeat(32)) && !is_digest(&"0F".repeat(32)) && !is_digest("00"));
|
||||
assert_eq!(parse_serial_id("msg", "msg-12"), Some(12));
|
||||
assert_eq!(parse_serial_id("msg", "msg-012"), None);
|
||||
assert_eq!(parse_serial_id("msg", "msgx-1"), None);
|
||||
assert!(is_method("Counter.Increment") && !is_method("a\u{7f}") && !is_method(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_json() {
|
||||
assert!(parse_json_strict(br#"{"a":{"b":[{"c":1,"d":2}]}}"#).is_ok());
|
||||
for bad in [
|
||||
&br#"{"a":1,"a":2}"#[..],
|
||||
br#"{"a":{"b":[{"c":1,"c":2}]}}"#,
|
||||
br#"[{"x":1},{"y":{"z":1,"z":1}}]"#,
|
||||
br#"{"a":NaN}"#,
|
||||
br#"{"a":Infinity}"#,
|
||||
br#"{"a":1e400}"#,
|
||||
br#"{"a":"\ud800"}"#,
|
||||
br#"{"a":1} x"#,
|
||||
b"{\"a\":\"\xff\"}",
|
||||
] {
|
||||
assert!(
|
||||
parse_json_strict(bad).is_err(),
|
||||
"{:?}",
|
||||
String::from_utf8_lossy(bad)
|
||||
);
|
||||
}
|
||||
let deep = format!("{}{}", "[".repeat(200), "]".repeat(200));
|
||||
assert!(parse_json_strict(deep.as_bytes()).is_err());
|
||||
}
|
||||
}
|
||||
704
services/flysim/crates/flybus/tests/artifacts.rs
Normal file
704
services/flysim/crates/flybus/tests/artifacts.rs
Normal file
|
|
@ -0,0 +1,704 @@
|
|||
//! Artifacts: allocate/seal/read, immutability against live writable handles, ownership
|
||||
//! through fan-out and retention, quotas, atomic admission, watermarked releases, abandoned
|
||||
//! futures, disconnects and router restarts.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{Via, code, env, env_with, obj, quiet, sealed, within};
|
||||
use flybus::{
|
||||
ErrorCode, Limits, Policy, Retained, Router, RouterConfig, ServiceConfig, SubscriptionConfig,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
fn sha(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes)
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn allocate_write_seal_read(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
let data: Vec<u8> = (0..100_000u32).map(|i| (i % 251) as u8).collect();
|
||||
let mut w = c
|
||||
.artifacts()
|
||||
.allocate(data.len() as u64, "application/octet-stream")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
w.write_all(&[0; 100_001]).is_err(),
|
||||
"writes past the allocation are refused"
|
||||
);
|
||||
w.write_all(&data).unwrap();
|
||||
assert_eq!(e.files("staging"), 1);
|
||||
let art = w.seal_with_digest(Some(sha(&data))).await.unwrap();
|
||||
let r = art.reference();
|
||||
assert_eq!(
|
||||
(r.generation, r.byte_length, r.digest.clone()),
|
||||
(1, 100_000, Some(sha(&data)))
|
||||
);
|
||||
assert_eq!(r.store_id, e.router.store_id());
|
||||
assert_eq!(art.read_all().await.unwrap(), data);
|
||||
assert_eq!((e.files("staging"), e.files("sealed")), (0, 1));
|
||||
let meta = std::fs::metadata(e.router.store_dir().join("sealed").join(&r.artifact_id)).unwrap();
|
||||
assert_eq!(
|
||||
meta.permissions().mode() & 0o222,
|
||||
0,
|
||||
"sealed files are read-only"
|
||||
);
|
||||
|
||||
let empty = c
|
||||
.artifacts()
|
||||
.allocate(0, "text/plain")
|
||||
.await
|
||||
.unwrap()
|
||||
.seal()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(empty.read_all().await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
c.artifacts().allocate(1, "").await.unwrap_err().code,
|
||||
ErrorCode::InvalidEnvelope
|
||||
);
|
||||
drop((art, empty));
|
||||
e.settle("collected", |s| s.artifacts == 0 && s.store_bytes == 0)
|
||||
.await;
|
||||
e.settle_files("sealed", 0).await;
|
||||
}
|
||||
|
||||
async fn unsealed_artifacts_cannot_be_used(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call("topic.declare", json!({"name": "t.x", "retained": "none"}))
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
let (a, _) = raw.allocate(8).await;
|
||||
let store = a["writeLocation"]["storeId"].clone();
|
||||
let reference = json!({"storeId": store, "artifactId": a["artifactId"], "generation": "1", "byteLength": "8",
|
||||
"contentType": "application/octet-stream", "digest": null});
|
||||
let att = json!([{"name": "x", "ref": reference, "ownerId": a["ownerId"]}]);
|
||||
let r = raw
|
||||
.call_with("publish", json!({"topic": "t.x", "payload": {}}), att)
|
||||
.await;
|
||||
assert_eq!(code(&r), "ARTIFACT_UNSEALED");
|
||||
let r = raw
|
||||
.call(
|
||||
"artifact.open",
|
||||
json!({"ref": reference, "ownerId": a["ownerId"]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&r), "ARTIFACT_UNSEALED");
|
||||
let r = raw
|
||||
.call(
|
||||
"artifact.retain",
|
||||
json!({"ref": reference, "ownerId": a["ownerId"]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&r), "ARTIFACT_UNSEALED");
|
||||
assert_eq!(code(&raw.seal(&a, Value::Null).await), "OK");
|
||||
assert_eq!(
|
||||
code(&raw.seal(&a, Value::Null).await),
|
||||
"OWNER_INVALID",
|
||||
"a writer seals once"
|
||||
);
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call(
|
||||
"artifact.allocate",
|
||||
json!({"byteLength": "01", "contentType": "x"})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
}
|
||||
|
||||
/// A producer that kept (or duplicated) its writable descriptor cannot change sealed bytes.
|
||||
async fn seal_is_immune_to_live_writable_handles(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
let (a, path) = raw.allocate(8).await;
|
||||
let mut kept = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
|
||||
kept.write_all(b"AAAAAAAA").unwrap();
|
||||
let mut dup = kept.try_clone().unwrap();
|
||||
let sealed = raw.seal(&a, json!(sha(b"AAAAAAAA"))).await.unwrap();
|
||||
// Both descriptors still write, into an inode the store no longer uses.
|
||||
dup.seek(SeekFrom::Start(0)).unwrap();
|
||||
let _ = dup.write_all(b"BBBBBBBB");
|
||||
let _ = kept.write_all(b"CCCC");
|
||||
assert!(!path.exists(), "staging is unlinked after sealing");
|
||||
let open = raw
|
||||
.call(
|
||||
"artifact.open",
|
||||
json!({"ref": sealed["ref"], "ownerId": sealed["ownerId"]}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let read_path = raw.path(&open["readLocation"]);
|
||||
assert_eq!(std::fs::read(&read_path).unwrap(), b"AAAAAAAA");
|
||||
assert_eq!(
|
||||
std::fs::metadata(&read_path).unwrap().permissions().mode() & 0o222,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
async fn seal_checks_length_and_digest(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
let (short, path) = raw.allocate(4).await;
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&path)
|
||||
.unwrap()
|
||||
.set_len(2)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
code(&raw.seal(&short, Value::Null).await),
|
||||
"ARTIFACT_MISMATCH"
|
||||
);
|
||||
let (long, path) = raw.allocate(4).await;
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&path)
|
||||
.unwrap()
|
||||
.set_len(6)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
code(&raw.seal(&long, Value::Null).await),
|
||||
"ARTIFACT_MISMATCH"
|
||||
);
|
||||
let (bad, _) = raw.allocate(4).await;
|
||||
assert_eq!(
|
||||
code(&raw.seal(&bad, json!("XYZ")).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
assert_eq!(
|
||||
code(&raw.seal(&bad, json!("0".repeat(64))).await),
|
||||
"ARTIFACT_MISMATCH"
|
||||
);
|
||||
// Failed seals clean up both the staging file and the copy.
|
||||
let s = e.settle("failed seals cleaned", |s| s.artifacts == 0).await;
|
||||
assert_eq!((s.store_bytes, s.owners), (0, 0));
|
||||
e.settle_files("staging", 0).await;
|
||||
e.settle_files("sealed", 0).await;
|
||||
|
||||
let c = e.client("c").await;
|
||||
let mut w = c.artifacts().allocate(3, "text/plain").await.unwrap();
|
||||
w.write_all(b"abc").unwrap();
|
||||
let err = w.seal_with_digest(Some(sha(b"abd"))).await.unwrap_err();
|
||||
assert_eq!(err.code, ErrorCode::ArtifactMismatch);
|
||||
e.settle("sdk seal cleaned", |s| {
|
||||
s.artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn quotas_are_enforced(via: Via) {
|
||||
let limits = Limits {
|
||||
max_store_bytes: 1000,
|
||||
max_artifact_bytes: 600,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(via, limits, Policy::open()).await;
|
||||
let c = e.client("c").await;
|
||||
assert_eq!(
|
||||
c.artifacts().allocate(601, "x/y").await.unwrap_err().code,
|
||||
ErrorCode::QuotaExceeded
|
||||
);
|
||||
let w = c.artifacts().allocate(600, "x/y").await.unwrap();
|
||||
assert_eq!(
|
||||
c.artifacts().allocate(401, "x/y").await.unwrap_err().code,
|
||||
ErrorCode::QuotaExceeded
|
||||
);
|
||||
// Sealing copies, so it needs room for both; failing releases the staging object.
|
||||
assert_eq!(w.seal().await.unwrap_err().code, ErrorCode::QuotaExceeded);
|
||||
e.settle("staging released", |s| s.store_bytes == 0 && s.owners == 0)
|
||||
.await;
|
||||
|
||||
let limits = Limits {
|
||||
max_store_bytes: 1200,
|
||||
max_artifact_bytes: 600,
|
||||
max_owners_per_client: 4,
|
||||
reserved_owners_per_client: 1,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(via, limits, Policy::open()).await;
|
||||
let c = e.client("c").await;
|
||||
let art = c
|
||||
.artifacts()
|
||||
.allocate(600, "x/y")
|
||||
.await
|
||||
.unwrap()
|
||||
.seal()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(e.stats().store_bytes, 600);
|
||||
let _w1 = c.artifacts().allocate(1, "x/y").await.unwrap();
|
||||
let _w2 = c.artifacts().allocate(1, "x/y").await.unwrap();
|
||||
assert_eq!(
|
||||
c.artifacts().allocate(1, "x/y").await.unwrap_err().code,
|
||||
ErrorCode::QuotaExceeded
|
||||
);
|
||||
assert_eq!(
|
||||
art.retain().await.unwrap_err().code,
|
||||
ErrorCode::QuotaExceeded
|
||||
);
|
||||
}
|
||||
|
||||
async fn fan_out_shares_one_object_and_the_last_consumer_collects(via: Via) {
|
||||
let e = env(via).await;
|
||||
let camera = e.client("camera").await;
|
||||
camera
|
||||
.declare_topic("world.demo.frame", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut viewers = Vec::new();
|
||||
for i in 0..3 {
|
||||
let c = e.client(&format!("viewer-{i}")).await;
|
||||
let s = c
|
||||
.subscribe("world.demo.frame", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
viewers.push((c, s));
|
||||
}
|
||||
let frame: Vec<u8> = (0..640 * 480 * 4).map(|i| (i % 256) as u8).collect();
|
||||
let art = sealed(&camera, &frame, "image/x-rgba").await;
|
||||
let r = camera
|
||||
.publish(
|
||||
"world.demo.frame",
|
||||
obj(json!({"width": 640, "height": 480})),
|
||||
&[("frame", &art)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.subscribers, 3);
|
||||
drop(art);
|
||||
let mut messages = Vec::new();
|
||||
for (_, s) in viewers.iter_mut() {
|
||||
let m = within("frame", s.next()).await.unwrap();
|
||||
assert_eq!(
|
||||
m.artifact("frame").unwrap().read_all().await.unwrap(),
|
||||
frame
|
||||
);
|
||||
messages.push(m);
|
||||
}
|
||||
let s = e
|
||||
.settle("three deliveries of one object", |s| s.artifact_roots == 3)
|
||||
.await;
|
||||
assert_eq!((s.sealed_artifacts, s.store_bytes), (1, frame.len() as u64));
|
||||
assert_eq!(e.files("sealed"), 1, "fan-out never copies bytes");
|
||||
let path = e.router.store_dir().join("sealed").join(
|
||||
&messages[0]
|
||||
.artifact("frame")
|
||||
.unwrap()
|
||||
.reference()
|
||||
.artifact_id,
|
||||
);
|
||||
for (i, m) in messages.drain(..).enumerate() {
|
||||
drop(m);
|
||||
let left = 2 - i as u64;
|
||||
e.settle("one root per consumer", |s| s.artifact_roots == left)
|
||||
.await;
|
||||
if left > 0 {
|
||||
assert!(
|
||||
path.exists(),
|
||||
"collected while {left} consumers still hold it"
|
||||
);
|
||||
}
|
||||
}
|
||||
e.settle_files("sealed", 0).await;
|
||||
e.settle("collected", |s| s.artifacts == 0 && s.store_bytes == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn extracted_artifacts_outlive_their_message(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
let r = e.client("r").await;
|
||||
p.declare_topic("t.hold", Retained::None).await.unwrap();
|
||||
let mut sub = r
|
||||
.subscribe("t.hold", SubscriptionConfig::bounded().in_flight(1))
|
||||
.await
|
||||
.unwrap();
|
||||
for i in 0..3 {
|
||||
let a = sealed(&p, format!("f{i}").as_bytes(), "text/plain").await;
|
||||
p.publish("t.hold", obj(json!({})), &[("f", &a)])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let m = within("first", sub.next()).await.unwrap();
|
||||
let image = m.artifact("f").unwrap();
|
||||
drop(m);
|
||||
// The extracted handle still owns the delivery, so the credit is not back yet.
|
||||
quiet("credit held by the extracted artifact", sub.next()).await;
|
||||
assert_eq!(image.read_all().await.unwrap(), b"f0");
|
||||
let file = image.open().await.unwrap();
|
||||
drop(image);
|
||||
quiet("credit held by the open file", sub.next()).await;
|
||||
drop(file);
|
||||
let m = within("second", sub.next()).await.unwrap();
|
||||
// An explicit hold outlives the delivery and does not keep its credit.
|
||||
let kept = m.artifact("f").unwrap().retain().await.unwrap();
|
||||
drop(m);
|
||||
let third = within("third", sub.next()).await.unwrap();
|
||||
assert_eq!(kept.read_all().await.unwrap(), b"f1");
|
||||
drop((third, kept));
|
||||
e.settle("collected", |s| s.artifacts == 0 && s.owners == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn failed_admission_is_atomic(via: Via) {
|
||||
let e = env(via).await;
|
||||
let reader = e.client("reader").await;
|
||||
let server = e.client("server").await;
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call(
|
||||
"topic.declare",
|
||||
json!({"name": "t.atomic", "retained": "latest"})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
let mut sub = reader
|
||||
.subscribe("t.atomic", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut svc = server
|
||||
.register("example.atomic", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let (a, path) = raw.allocate(3).await;
|
||||
std::fs::write(&path, b"abc").unwrap();
|
||||
let s = raw.seal(&a, Value::Null).await.unwrap();
|
||||
let good = |name: &str| json!({"name": name, "ref": s["ref"], "ownerId": s["ownerId"]});
|
||||
let before = e.stats();
|
||||
let mut wrong_len = s["ref"].clone();
|
||||
wrong_len["byteLength"] = json!("4");
|
||||
let mut other_store = s["ref"].clone();
|
||||
other_store["storeId"] = json!("store-0000000000000000");
|
||||
let cases = [
|
||||
(
|
||||
json!([good("a"), {"name": "b", "ref": s["ref"], "ownerId": "own-7"}]),
|
||||
"OWNER_INVALID",
|
||||
),
|
||||
(
|
||||
json!([good("a"), {"name": "b", "ref": wrong_len, "ownerId": s["ownerId"]}]),
|
||||
"ARTIFACT_MISMATCH",
|
||||
),
|
||||
(
|
||||
json!([good("a"), {"name": "b", "ref": other_store, "ownerId": s["ownerId"]}]),
|
||||
"ARTIFACT_GONE",
|
||||
),
|
||||
];
|
||||
for (i, (atts, want)) in cases.iter().enumerate() {
|
||||
let r = raw
|
||||
.call_with(
|
||||
"publish",
|
||||
json!({"topic": "t.atomic", "payload": {}}),
|
||||
atts.clone(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&r), *want);
|
||||
let call = json!({"callId": format!("call-{}", i + 1), "target": "example.atomic", "expectedIncarnation": null, "method": "M", "payload": {}});
|
||||
let r = raw.call_with("rpc.call", call, atts.clone()).await;
|
||||
assert_eq!(
|
||||
code(&r),
|
||||
*want,
|
||||
"semantic refusal advances correlation history but preserves the refusal reason"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
e.stats(),
|
||||
before,
|
||||
"refused admissions leave no roots, calls or retained values"
|
||||
);
|
||||
quiet("no partial delivery", sub.next()).await;
|
||||
quiet("no partial request", svc.next()).await;
|
||||
// Two names for one object: both are delivered, one root is held.
|
||||
let r = raw
|
||||
.call_with(
|
||||
"publish",
|
||||
json!({"topic": "t.atomic", "payload": {}}),
|
||||
json!([good("a"), good("b")]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r["topicSequence"], "1",
|
||||
"refused publications spent no sequence number"
|
||||
);
|
||||
let m = within("delivery", sub.next()).await.unwrap();
|
||||
assert_eq!(m.attachment_names().collect::<Vec<_>>(), ["a", "b"]);
|
||||
assert_eq!(m.artifact("b").unwrap().read_all().await.unwrap(), b"abc");
|
||||
// Hold, retained value, delivery: three roots, not four.
|
||||
e.settle("one root per holder", |s| s.artifact_roots == 3)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn release_ids_are_watermarked(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
let (a1, _) = raw.allocate(4).await;
|
||||
let (a2, _) = raw.allocate(4).await;
|
||||
let rel = |ids: Value| json!({ "ownerIds": ids });
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call("artifact.release", rel(json!([a2["ownerId"], "own-99"])))
|
||||
.await
|
||||
),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
e.stats().artifacts,
|
||||
2,
|
||||
"a batch with a future id releases nothing"
|
||||
);
|
||||
assert_eq!(
|
||||
raw.call("artifact.release", rel(json!([a1["ownerId"]])))
|
||||
.await
|
||||
.unwrap()["released"],
|
||||
"1"
|
||||
);
|
||||
assert_eq!(
|
||||
raw.call("artifact.release", rel(json!([a1["ownerId"]])))
|
||||
.await
|
||||
.unwrap()["released"],
|
||||
"0"
|
||||
);
|
||||
assert_eq!(
|
||||
code(&raw.call("artifact.release", rel(json!(["dlv-1"]))).await),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
code(&raw.call("artifact.release", rel(json!([]))).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
let many: Vec<String> = (1..=65).map(|i| format!("own-{i}")).collect();
|
||||
assert_eq!(
|
||||
code(&raw.call("artifact.release", rel(json!(many))).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
let consumed = |ids: Value| json!({ "deliveryIds": ids });
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call("delivery.consumed", consumed(json!(["dlv-3"])))
|
||||
.await
|
||||
),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call("delivery.consumed", consumed(json!([a2["ownerId"]])))
|
||||
.await
|
||||
),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
raw.call(
|
||||
"artifact.release",
|
||||
rel(json!([a2["ownerId"], a2["ownerId"]]))
|
||||
)
|
||||
.await
|
||||
.unwrap()["released"],
|
||||
"1"
|
||||
);
|
||||
e.settle("all released", |s| s.artifacts == 0 && s.store_bytes == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn owners_are_scoped_to_their_connection(via: Via) {
|
||||
let e = env(via).await;
|
||||
let owner = e.client("owner").await;
|
||||
let art = sealed(&owner, b"private", "text/plain").await;
|
||||
let reference = art.reference().to_json();
|
||||
let mut thief = e.raw_hello("thief").await;
|
||||
let open = |owner_id: &str| json!({"ref": reference, "ownerId": owner_id});
|
||||
// Naming the victim's owner id means nothing on another connection.
|
||||
assert_eq!(
|
||||
code(&thief.call("artifact.open", open(art.owner_id())).await),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
let (mine, _) = thief.allocate(1).await;
|
||||
let my_owner = mine["ownerId"].as_str().unwrap().to_owned();
|
||||
assert_eq!(
|
||||
code(&thief.call("artifact.open", open(&my_owner)).await),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
code(&thief.call("artifact.retain", open(&my_owner)).await),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
code(
|
||||
&thief
|
||||
.call("artifact.release", json!({"ownerIds": [art.owner_id()]}))
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
assert_eq!(
|
||||
art.read_all().await.unwrap(),
|
||||
b"private",
|
||||
"another connection cannot release it"
|
||||
);
|
||||
}
|
||||
|
||||
async fn abandoned_futures_do_not_leak_owners(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
// Each future is polled once (its command is sent) and then dropped.
|
||||
let _ = tokio::time::timeout(Duration::ZERO, c.artifacts().allocate(1000, "x/y")).await;
|
||||
e.settle("abandoned allocation released", |s| {
|
||||
s.artifacts == 0 && s.owners == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
e.settle_files("staging", 0).await;
|
||||
|
||||
let mut w = c.artifacts().allocate(4, "x/y").await.unwrap();
|
||||
w.write_all(b"seal").unwrap();
|
||||
let _ = tokio::time::timeout(Duration::ZERO, w.seal()).await;
|
||||
e.settle("abandoned seal released", |s| {
|
||||
s.artifacts == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
|
||||
let art = sealed(&c, b"keep", "x/y").await;
|
||||
let _ = tokio::time::timeout(Duration::ZERO, art.retain()).await;
|
||||
e.settle("abandoned retain released", |s| s.owners == 1)
|
||||
.await;
|
||||
drop(art);
|
||||
e.settle("all released", |s| s.artifacts == 0 && s.owners == 0)
|
||||
.await;
|
||||
assert_eq!(c.control_errors(), 0);
|
||||
}
|
||||
|
||||
async fn writer_drop_releases_staging(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
let mut w = c.artifacts().allocate(1000, "x/y").await.unwrap();
|
||||
w.write_all(&[1; 10]).unwrap();
|
||||
assert_eq!(e.files("staging"), 1);
|
||||
drop(w);
|
||||
e.settle("staging released", |s| {
|
||||
s.artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
e.settle_files("staging", 0).await;
|
||||
}
|
||||
|
||||
async fn disconnect_releases_all_but_retained(via: Via) {
|
||||
let e = env(via).await;
|
||||
let producer = e.client("producer").await;
|
||||
let reader = e.client("reader").await;
|
||||
producer
|
||||
.declare_topic("t.keep", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = reader
|
||||
.subscribe("t.keep", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let a1 = sealed(&producer, b"retained", "x/y").await;
|
||||
let a2 = sealed(&producer, b"held", "x/y").await;
|
||||
let w3 = producer.artifacts().allocate(10, "x/y").await.unwrap();
|
||||
producer
|
||||
.publish("t.keep", obj(json!({})), &[("a", &a1)])
|
||||
.await
|
||||
.unwrap();
|
||||
let m = within("delivery", sub.next()).await.unwrap();
|
||||
// Close both while handles are still alive; the router releases what they owned.
|
||||
producer.close().await;
|
||||
reader.close().await;
|
||||
let s = e
|
||||
.settle("connections gone", |s| s.connections == 0 && s.owners == 0)
|
||||
.await;
|
||||
assert_eq!(
|
||||
(s.artifacts, s.artifact_roots, s.store_bytes),
|
||||
(1, 1, 8),
|
||||
"only the retained value survives"
|
||||
);
|
||||
// Handles that outlived their connection are inert.
|
||||
assert_eq!(a2.read_all().await.unwrap_err().code, ErrorCode::RouterLost);
|
||||
drop((a1, a2, w3, m, sub));
|
||||
let admin = e.client("admin").await;
|
||||
assert!(admin.clear_topic("t.keep").await.unwrap());
|
||||
e.settle("retained released", |s| {
|
||||
s.artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn router_restart_invalidates_old_handles(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
c.declare_topic("t.r", Retained::None).await.unwrap();
|
||||
let mut sub = c
|
||||
.subscribe("t.r", SubscriptionConfig::latest())
|
||||
.await
|
||||
.unwrap();
|
||||
let art = sealed(&c, b"old", "x/y").await;
|
||||
e.router.shutdown();
|
||||
assert!(within("subscription closed", sub.next()).await.is_none());
|
||||
let err = art.read_all().await.unwrap_err();
|
||||
assert_eq!(err.code, ErrorCode::RouterLost);
|
||||
assert!(c.closed().is_some());
|
||||
assert_eq!(
|
||||
e.try_client("late").await.unwrap_err().code,
|
||||
ErrorCode::RouterLost
|
||||
);
|
||||
|
||||
// A restarted router on the same root is a new store incarnation.
|
||||
let mut cfg = RouterConfig::new(e.router.store_root());
|
||||
cfg.policy = Policy::open();
|
||||
let second = Router::new(cfg).unwrap();
|
||||
assert_ne!(second.store_id(), e.router.store_id());
|
||||
let mut raw = common::Raw::over(second.connect_in_memory(), second.store_root());
|
||||
raw.hello("new").await.unwrap();
|
||||
let (mine, _) = raw.allocate(1).await;
|
||||
// An old reference is refused even alongside an owner that is live on the new router.
|
||||
let r = raw
|
||||
.call(
|
||||
"artifact.open",
|
||||
json!({"ref": art.reference().to_json(), "ownerId": mine["ownerId"]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&r), "ARTIFACT_GONE");
|
||||
let client = flybus::Client::connect(
|
||||
second.connect_in_memory(),
|
||||
flybus::ClientConfig::new("sdk", second.store_root()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let fresh = sealed(&client, b"new", "x/y").await;
|
||||
assert_eq!(fresh.read_all().await.unwrap(), b"new");
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
allocate_write_seal_read,
|
||||
unsealed_artifacts_cannot_be_used,
|
||||
seal_is_immune_to_live_writable_handles,
|
||||
seal_checks_length_and_digest,
|
||||
quotas_are_enforced,
|
||||
fan_out_shares_one_object_and_the_last_consumer_collects,
|
||||
extracted_artifacts_outlive_their_message,
|
||||
failed_admission_is_atomic,
|
||||
release_ids_are_watermarked,
|
||||
owners_are_scoped_to_their_connection,
|
||||
abandoned_futures_do_not_leak_owners,
|
||||
writer_drop_releases_staging,
|
||||
disconnect_releases_all_but_retained,
|
||||
router_restart_invalidates_old_handles,
|
||||
);
|
||||
416
services/flysim/crates/flybus/tests/common/mod.rs
Normal file
416
services/flysim/crates/flybus/tests/common/mod.rs
Normal file
|
|
@ -0,0 +1,416 @@
|
|||
//! Shared harness: a router on a temporary store, reached over either transport, plus a raw
|
||||
//! protocol client for adversarial frames the SDK would never send.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use flybus::wire::{Envelope, Kind, Location, read_frame};
|
||||
use flybus::{
|
||||
Artifact, Client, ClientConfig, Limits, Policy, Router, RouterConfig, RouterStats, Transport,
|
||||
UnixListenerHandle,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::io::{AsyncWriteExt, ReadHalf, WriteHalf};
|
||||
|
||||
pub const WAIT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Generates one test per transport from an `async fn name(via: Via)`.
|
||||
#[macro_export]
|
||||
macro_rules! both_transports {
|
||||
($($name:ident),* $(,)?) => {
|
||||
mod in_memory {
|
||||
$(
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn $name() {
|
||||
super::$name($crate::common::Via::Memory).await
|
||||
}
|
||||
)*
|
||||
}
|
||||
mod unix_socket {
|
||||
$(
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn $name() {
|
||||
super::$name($crate::common::Via::Unix).await
|
||||
}
|
||||
)*
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Via {
|
||||
Memory,
|
||||
Unix,
|
||||
}
|
||||
|
||||
pub struct Env {
|
||||
pub router: Router,
|
||||
pub via: Via,
|
||||
pub dir: tempfile::TempDir,
|
||||
listeners: Mutex<Vec<UnixListenerHandle>>,
|
||||
next_socket: AtomicU64,
|
||||
}
|
||||
|
||||
pub async fn env(via: Via) -> Env {
|
||||
env_with(via, Limits::default(), Policy::open()).await
|
||||
}
|
||||
|
||||
pub async fn env_with(via: Via, limits: Limits, policy: Policy) -> Env {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path().join("store"));
|
||||
config.limits = limits;
|
||||
config.policy = policy;
|
||||
let router = Router::new(config).unwrap();
|
||||
Env {
|
||||
router,
|
||||
via,
|
||||
dir,
|
||||
listeners: Mutex::new(Vec::new()),
|
||||
next_socket: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
impl Env {
|
||||
pub async fn transport(&self) -> Transport {
|
||||
match self.via {
|
||||
Via::Memory => self.router.connect_in_memory(),
|
||||
Via::Unix => {
|
||||
let n = self.next_socket.fetch_add(1, Ordering::Relaxed);
|
||||
let path = self.dir.path().join(format!("unbound-{n}.sock"));
|
||||
let listener = self.router.listen_unix(&path).await.unwrap();
|
||||
let transport = Transport::unix(&path).await.unwrap();
|
||||
self.listeners.lock().unwrap().push(listener);
|
||||
transport
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn transport_as(&self, id: &str) -> Transport {
|
||||
self.try_transport_as(id).await.unwrap()
|
||||
}
|
||||
|
||||
async fn try_transport_as(&self, id: &str) -> std::io::Result<Transport> {
|
||||
match self.via {
|
||||
Via::Memory => Ok(self.router.connect_in_memory_as(id)),
|
||||
Via::Unix => {
|
||||
let n = self.next_socket.fetch_add(1, Ordering::Relaxed);
|
||||
let path = self.dir.path().join(format!("bound-{n}.sock"));
|
||||
let listener = self.router.listen_unix_as(&path, id).await?;
|
||||
let transport = Transport::unix(&path).await?;
|
||||
self.listeners.lock().unwrap().push(listener);
|
||||
Ok(transport)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn config(&self, id: &str) -> ClientConfig {
|
||||
ClientConfig::new(id, self.router.store_root())
|
||||
}
|
||||
|
||||
pub async fn try_client(&self, id: &str) -> Result<Client, flybus::BusError> {
|
||||
let transport = self.try_transport_as(id).await.map_err(|e| {
|
||||
flybus::BusError::new(flybus::ErrorCode::RouterLost, format!("connect: {e}"))
|
||||
})?;
|
||||
Client::connect(transport, self.config(id)).await
|
||||
}
|
||||
|
||||
pub async fn client(&self, id: &str) -> Client {
|
||||
self.try_client(id).await.unwrap()
|
||||
}
|
||||
|
||||
pub async fn raw(&self) -> Raw {
|
||||
Raw::over(self.transport().await, self.router.store_root())
|
||||
}
|
||||
|
||||
pub async fn raw_as(&self, id: &str) -> Raw {
|
||||
Raw::over(self.transport_as(id).await, self.router.store_root())
|
||||
}
|
||||
|
||||
pub async fn raw_hello(&self, id: &str) -> Raw {
|
||||
let mut raw = self.raw_as(id).await;
|
||||
raw.hello(id).await.unwrap();
|
||||
raw
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> RouterStats {
|
||||
self.router.stats()
|
||||
}
|
||||
|
||||
/// Polls the router until `ok` holds; panics with the last stats after [`WAIT`].
|
||||
pub async fn settle(&self, what: &str, ok: impl Fn(&RouterStats) -> bool) -> RouterStats {
|
||||
let deadline = tokio::time::Instant::now() + WAIT;
|
||||
loop {
|
||||
let s = self.stats();
|
||||
if ok(&s) {
|
||||
return s;
|
||||
}
|
||||
if tokio::time::Instant::now() > deadline {
|
||||
panic!("{what}: router never settled: {s:?}");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Files currently in the store's `sealed/` or `staging/` directory.
|
||||
pub fn files(&self, sub: &str) -> usize {
|
||||
std::fs::read_dir(self.router.store_dir().join(sub))
|
||||
.map(|d| d.count())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Waits for the file count to reach `n`. Unlinks follow the registry update, outside the
|
||||
/// router lock, so a file can briefly outlive its entry.
|
||||
pub async fn settle_files(&self, sub: &str, n: usize) {
|
||||
let deadline = tokio::time::Instant::now() + WAIT;
|
||||
while self.files(sub) != n {
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"{sub}: {} files, wanted {n}",
|
||||
self.files(sub)
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn obj(v: Value) -> Map<String, Value> {
|
||||
match v {
|
||||
Value::Object(m) => m,
|
||||
_ => panic!("not an object"),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn within<T>(what: &str, f: impl Future<Output = T>) -> T {
|
||||
match tokio::time::timeout(WAIT, f).await {
|
||||
Ok(v) => v,
|
||||
Err(_) => panic!("{what}: timed out"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Asserts nothing arrives for a short while.
|
||||
pub async fn quiet<T: std::fmt::Debug>(what: &str, f: impl Future<Output = Option<T>>) {
|
||||
if let Ok(Some(v)) = tokio::time::timeout(Duration::from_millis(150), f).await {
|
||||
panic!("{what}: unexpected {v:?}");
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn sealed(client: &Client, bytes: &[u8], content_type: &str) -> Artifact {
|
||||
use std::io::Write;
|
||||
let mut w = client
|
||||
.artifacts()
|
||||
.allocate(bytes.len() as u64, content_type)
|
||||
.await
|
||||
.unwrap();
|
||||
w.write_all(bytes).unwrap();
|
||||
w.seal().await.unwrap()
|
||||
}
|
||||
|
||||
/// The write half of a [`Raw`], for a task that floods the router.
|
||||
pub struct RawWriter {
|
||||
wr: WriteHalf<Transport>,
|
||||
}
|
||||
|
||||
impl RawWriter {
|
||||
/// Sends one frame; false once the router has gone.
|
||||
pub async fn send(&mut self, bytes: &[u8]) -> bool {
|
||||
let mut buf = (bytes.len() as u32).to_le_bytes().to_vec();
|
||||
buf.extend_from_slice(bytes);
|
||||
self.wr.write_all(&buf).await.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// A client speaking the wire protocol by hand.
|
||||
pub struct Raw {
|
||||
rd: ReadHalf<Transport>,
|
||||
wr: Option<WriteHalf<Transport>>,
|
||||
pub next: u64,
|
||||
pub stash: Vec<Envelope>,
|
||||
store_root: PathBuf,
|
||||
}
|
||||
|
||||
pub type Reply = Result<Map<String, Value>, (String, String)>;
|
||||
|
||||
impl Raw {
|
||||
pub fn over(transport: Transport, store_root: &Path) -> Raw {
|
||||
let (rd, wr) = tokio::io::split(transport);
|
||||
Raw {
|
||||
rd,
|
||||
wr: Some(wr),
|
||||
next: 0,
|
||||
stash: Vec::new(),
|
||||
store_root: store_root.to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take_writer(&mut self) -> RawWriter {
|
||||
RawWriter {
|
||||
wr: self.wr.take().expect("writer already taken"),
|
||||
}
|
||||
}
|
||||
|
||||
fn wr(&mut self) -> &mut WriteHalf<Transport> {
|
||||
self.wr.as_mut().expect("writer was taken")
|
||||
}
|
||||
|
||||
pub async fn send_bytes(&mut self, bytes: &[u8]) {
|
||||
let mut buf = (bytes.len() as u32).to_le_bytes().to_vec();
|
||||
buf.extend_from_slice(bytes);
|
||||
let _ = self.wr().write_all(&buf).await;
|
||||
let _ = self.wr().flush().await;
|
||||
}
|
||||
|
||||
pub async fn send_prefix(&mut self, len: u32) {
|
||||
let _ = self.wr().write_all(&len.to_le_bytes()).await;
|
||||
let _ = self.wr().flush().await;
|
||||
}
|
||||
|
||||
/// Sends a command with the next id; returns the id.
|
||||
pub async fn command(&mut self, op: &str, body: Value, attachments: Value) -> String {
|
||||
self.next += 1;
|
||||
let id = format!("msg-{}", self.next);
|
||||
let env = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": id, "replyTo": null, "kind": "command", "op": op,
|
||||
"body": body, "attachments": attachments,
|
||||
});
|
||||
self.send_bytes(&serde_json::to_vec(&env).unwrap()).await;
|
||||
id
|
||||
}
|
||||
|
||||
/// The next envelope, `None` at end of stream.
|
||||
pub async fn recv(&mut self) -> Option<Envelope> {
|
||||
if !self.stash.is_empty() {
|
||||
return Some(self.stash.remove(0));
|
||||
}
|
||||
self.read().await
|
||||
}
|
||||
|
||||
async fn read(&mut self) -> Option<Envelope> {
|
||||
match within("raw read", read_frame(&mut self.rd)).await {
|
||||
Ok(Some(bytes)) => Some(Envelope::decode(&bytes).expect("router frames are valid")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Waits for the reply to `id`, stashing anything else.
|
||||
pub async fn reply(&mut self, id: &str) -> Reply {
|
||||
if let Some(i) = self
|
||||
.stash
|
||||
.iter()
|
||||
.position(|e| e.reply_to.as_deref() == Some(id))
|
||||
{
|
||||
return parse_reply(self.stash.remove(i));
|
||||
}
|
||||
loop {
|
||||
let env = self
|
||||
.read()
|
||||
.await
|
||||
.unwrap_or_else(|| panic!("closed while waiting for {id}"));
|
||||
if env.reply_to.as_deref() == Some(id) {
|
||||
return parse_reply(env);
|
||||
}
|
||||
self.stash.push(env);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn call(&mut self, op: &str, body: Value) -> Reply {
|
||||
let id = self.command(op, body, json!([])).await;
|
||||
self.reply(&id).await
|
||||
}
|
||||
|
||||
pub async fn call_with(&mut self, op: &str, body: Value, attachments: Value) -> Reply {
|
||||
let id = self.command(op, body, attachments).await;
|
||||
self.reply(&id).await
|
||||
}
|
||||
|
||||
pub async fn hello(&mut self, id: &str) -> Reply {
|
||||
self.call(
|
||||
"bus.hello",
|
||||
json!({"clientId": id, "clientIncarnation": "inc-raw", "supportedMajors": [1]}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Reads to end of stream and returns the last `connection.closing` notice, if any.
|
||||
pub async fn closing(&mut self) -> Option<Map<String, Value>> {
|
||||
let mut last = None;
|
||||
while let Some(env) = self.recv().await {
|
||||
if env.kind == Kind::Notice && env.op == "connection.closing" {
|
||||
last = Some(env.body);
|
||||
}
|
||||
}
|
||||
last
|
||||
}
|
||||
|
||||
/// The next delivery or notice, from the stash first.
|
||||
pub async fn event(&mut self) -> Envelope {
|
||||
if let Some(i) = self.stash.iter().position(|e| e.kind != Kind::Reply) {
|
||||
return self.stash.remove(i);
|
||||
}
|
||||
loop {
|
||||
let env = self
|
||||
.read()
|
||||
.await
|
||||
.expect("closed while waiting for an event");
|
||||
if env.kind != Kind::Reply {
|
||||
return env;
|
||||
}
|
||||
self.stash.push(env);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn path(&self, loc: &Value) -> PathBuf {
|
||||
let loc = Location::from_json(loc).unwrap();
|
||||
self.store_root.join(loc.store_id).join(loc.relative_path)
|
||||
}
|
||||
|
||||
/// Allocates and writes an artifact by hand; returns (ref-less allocate value, staging path).
|
||||
pub async fn allocate(&mut self, len: u64) -> (Map<String, Value>, PathBuf) {
|
||||
let v = self
|
||||
.call(
|
||||
"artifact.allocate",
|
||||
json!({"byteLength": len.to_string(), "contentType": "application/octet-stream"}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let path = self.path(&v["writeLocation"]);
|
||||
(v, path)
|
||||
}
|
||||
|
||||
pub async fn seal(&mut self, alloc: &Map<String, Value>, digest: Value) -> Reply {
|
||||
self.call(
|
||||
"artifact.seal",
|
||||
json!({"artifactId": alloc["artifactId"], "generation": "1", "ownerId": alloc["ownerId"], "digest": digest}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_reply(env: Envelope) -> Reply {
|
||||
assert_eq!(env.kind, Kind::Reply);
|
||||
if env.body["ok"] == json!(true) {
|
||||
Ok(env.body["value"].as_object().unwrap().clone())
|
||||
} else {
|
||||
let e = &env.body["error"];
|
||||
Err((
|
||||
e["code"].as_str().unwrap().to_owned(),
|
||||
e["dispatch"].as_str().unwrap().to_owned(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn code(r: &Reply) -> &str {
|
||||
match r {
|
||||
Ok(_) => "OK",
|
||||
Err((c, _)) => c,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn store_path(root: &Path, store_id: &str, rel: &str) -> PathBuf {
|
||||
root.join(store_id).join(rel)
|
||||
}
|
||||
788
services/flysim/crates/flybus/tests/conformance_artifacts.rs
Normal file
788
services/flysim/crates/flybus/tests/conformance_artifacts.rs
Normal file
|
|
@ -0,0 +1,788 @@
|
|||
//! Black-box conformance tests for artifact ownership (bus-v1 section 8): allocate/seal/open,
|
||||
//! immutability past a stale writable handle, ownership surviving message drop, explicit
|
||||
//! retain/release, final-owner collection, root accounting across queued/latest/retained
|
||||
//! delivery, forward-before-release ordering, admission rollback, disconnect cleanup, and
|
||||
//! rejection of stale or forged references and owners. Every test runs over both transports.
|
||||
//!
|
||||
//! These tests use only the public `flybus` API (plus the `common::Raw` protocol harness for
|
||||
//! adversarial frames no well-behaved SDK client would send) and generic bytes over synthetic
|
||||
//! messages; nothing here depends on any application on top of the bus.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
use common::{Via, code, env, env_with, obj, sealed, within};
|
||||
use flybus::{ErrorCode, Limits, Policy, Retained, ServiceConfig, SubscriptionConfig};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Allocate / seal / open
|
||||
|
||||
async fn allocate_write_seal_open_roundtrip_and_mismatches(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
|
||||
let mut w = c.artifacts().allocate(5, "text/plain").await.unwrap();
|
||||
assert_eq!(w.byte_length(), 5);
|
||||
w.write_all(b"hello").unwrap();
|
||||
assert!(
|
||||
w.write(b"!").is_err(),
|
||||
"a write past the declared length must fail locally"
|
||||
);
|
||||
let art = w.seal().await.unwrap();
|
||||
assert_eq!(art.reference().byte_length, 5);
|
||||
assert_eq!(art.reference().content_type, "text/plain");
|
||||
assert_eq!(art.read_all().await.unwrap(), b"hello");
|
||||
drop(art);
|
||||
e.settle("sealed object collected", |s| s.sealed_artifacts == 0)
|
||||
.await;
|
||||
|
||||
// Sealing declares more bytes than were actually staged: rejected, not silently truncated.
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
let (alloc, path) = raw.allocate(10).await;
|
||||
std::fs::write(&path, b"short").unwrap();
|
||||
assert_eq!(
|
||||
code(&raw.seal(&alloc, Value::Null).await),
|
||||
"ARTIFACT_MISMATCH"
|
||||
);
|
||||
|
||||
// A syntactically valid but wrong digest is refused too.
|
||||
let (alloc2, path2) = raw.allocate(4).await;
|
||||
std::fs::write(&path2, b"data").unwrap();
|
||||
assert_eq!(
|
||||
code(&raw.seal(&alloc2, json!("0".repeat(64))).await),
|
||||
"ARTIFACT_MISMATCH"
|
||||
);
|
||||
}
|
||||
|
||||
/// Sealing copies staging into a fresh inode; a producer's writable handle that survives the
|
||||
/// seal (or a duplicate of it) can only reach the abandoned staging file, never the sealed
|
||||
/// bytes (bus-v1 section 8.1 and the store module's own contract).
|
||||
async fn seal_is_immutable_despite_a_stale_writable_handle(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("producer").await;
|
||||
let (alloc, path) = raw.allocate(7).await;
|
||||
let mut stale = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
|
||||
stale.write_all(b"before1").unwrap();
|
||||
stale.flush().unwrap();
|
||||
|
||||
let sealed_reply = raw.seal(&alloc, Value::Null).await.unwrap();
|
||||
// Tamper through the handle only after the router has answered the seal: the sealed copy
|
||||
// already exists and the staging file is already unlinked.
|
||||
stale.write_all(b"AFTER!!").unwrap();
|
||||
|
||||
let open_reply = raw
|
||||
.call(
|
||||
"artifact.open",
|
||||
json!({"ref": sealed_reply["ref"], "ownerId": sealed_reply["ownerId"]}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let sealed_path = raw.path(&open_reply["readLocation"]);
|
||||
assert_eq!(
|
||||
std::fs::read(sealed_path).unwrap(),
|
||||
b"before1",
|
||||
"a post-seal write must never reach the sealed copy"
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Ownership surviving drop, explicit retain/release, final-owner collection
|
||||
|
||||
/// The doc's own illustrative snippet (bus-v1 section 2): `drop(message)` alone must not
|
||||
/// consume the delivery while an artifact extracted from it is still held.
|
||||
async fn extracted_artifact_outlives_the_message_it_came_from(via: Via) {
|
||||
let e = env(via).await;
|
||||
let camera = e.client("camera").await;
|
||||
let viewer = e.client("viewer").await;
|
||||
camera
|
||||
.declare_topic("world.demo.frame", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = viewer
|
||||
.subscribe("world.demo.frame", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let frame = sealed(&camera, b"pixels", "image/x-rgba").await;
|
||||
camera
|
||||
.publish("world.demo.frame", obj(json!({})), &[("frame", &frame)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(frame);
|
||||
|
||||
let message = within("frame delivered", sub.next()).await.unwrap();
|
||||
let image = message.artifact("frame").unwrap();
|
||||
drop(message);
|
||||
e.settle("the delivery still owns its artifact", |s| {
|
||||
s.owners == 1 && s.sealed_artifacts == 1
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
image.read_all().await.unwrap(),
|
||||
b"pixels",
|
||||
"the surviving handle still reads fine"
|
||||
);
|
||||
drop(image);
|
||||
e.settle("dropping the last handle finally consumes it", |s| {
|
||||
s.owners == 0 && s.sealed_artifacts == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn explicit_retain_outlives_the_original_hold(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
let original = sealed(&c, b"payload", "application/octet-stream").await;
|
||||
let retained = original.retain().await.unwrap();
|
||||
assert_eq!(
|
||||
retained.reference().artifact_id,
|
||||
original.reference().artifact_id
|
||||
);
|
||||
assert_ne!(
|
||||
retained.owner_id(),
|
||||
original.owner_id(),
|
||||
"retain creates an independent owner"
|
||||
);
|
||||
|
||||
drop(original);
|
||||
e.settle("the retained hold keeps it sealed", |s| {
|
||||
s.sealed_artifacts == 1 && s.artifact_roots == 1
|
||||
})
|
||||
.await;
|
||||
assert_eq!(retained.read_all().await.unwrap(), b"payload");
|
||||
drop(retained);
|
||||
e.settle("collected once the last owner drops", |s| {
|
||||
s.sealed_artifacts == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn collection_waits_for_every_retained_owner(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
let a = sealed(&c, b"payload", "application/octet-stream").await;
|
||||
let h1 = a.retain().await.unwrap();
|
||||
let h2 = a.retain().await.unwrap();
|
||||
drop(a);
|
||||
e.settle("two independent holds remain", |s| {
|
||||
s.owners == 2 && s.sealed_artifacts == 1
|
||||
})
|
||||
.await;
|
||||
drop(h1);
|
||||
e.settle("one hold remains", |s| {
|
||||
s.owners == 1 && s.sealed_artifacts == 1
|
||||
})
|
||||
.await;
|
||||
drop(h2);
|
||||
e.settle("the final owner's drop collects it", |s| {
|
||||
s.owners == 0 && s.sealed_artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
assert_eq!(e.files("sealed"), 0);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Root accounting across queued, latest and retained delivery (bus-v1 section 8.2)
|
||||
|
||||
/// A message queued behind a saturated credit already holds a root at admission time, before
|
||||
/// it is ever dispatched to the client; dispatch transfers that root into an owner, it does
|
||||
/// not add a second one.
|
||||
async fn queued_deliveries_hold_roots_before_dispatch(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
let r = e.client("r").await;
|
||||
p.declare_topic("t.queued", Retained::None).await.unwrap();
|
||||
let mut sub = r
|
||||
.subscribe(
|
||||
"t.queued",
|
||||
SubscriptionConfig::bounded().in_flight(1).queued(4),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let a1 = sealed(&p, b"m1", "application/octet-stream").await;
|
||||
let a2 = sealed(&p, b"m2", "application/octet-stream").await;
|
||||
p.publish("t.queued", obj(json!({})), &[("x", &a1)])
|
||||
.await
|
||||
.unwrap();
|
||||
p.publish("t.queued", obj(json!({})), &[("x", &a2)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop((a1, a2));
|
||||
e.settle("one dispatched owner, two roots", |s| {
|
||||
s.artifact_roots == 2 && s.owners == 1 && s.sealed_artifacts == 2
|
||||
})
|
||||
.await;
|
||||
|
||||
let held = within("dispatched message", sub.next()).await.unwrap();
|
||||
assert_eq!(held.artifact("x").unwrap().read_all().await.unwrap(), b"m1");
|
||||
drop(held);
|
||||
let second = within("previously queued message", sub.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
second.artifact("x").unwrap().read_all().await.unwrap(),
|
||||
b"m2"
|
||||
);
|
||||
e.settle("dispatch moved the root, it did not add one", |s| {
|
||||
s.artifact_roots == 1 && s.owners == 1
|
||||
})
|
||||
.await;
|
||||
drop(second);
|
||||
e.settle("fully collected", |s| {
|
||||
s.artifact_roots == 0 && s.owners == 0 && s.sealed_artifacts == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// `latest` never queues more than one undelivered value; coalescing releases the replaced
|
||||
/// root immediately, and the already-dispatched value stays independent of it.
|
||||
async fn latest_mode_holds_at_most_two_roots_delivered_plus_queued(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
let r = e.client("r").await;
|
||||
p.declare_topic("t.latest", Retained::None).await.unwrap();
|
||||
let mut sub = r
|
||||
.subscribe("t.latest", SubscriptionConfig::latest().in_flight(1))
|
||||
.await
|
||||
.unwrap();
|
||||
for i in 0..3u8 {
|
||||
let a = sealed(&p, &[i], "application/octet-stream").await;
|
||||
p.publish("t.latest", obj(json!({})), &[("x", &a)])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
// i=0 was already dispatched (an owner); i=1 was coalesced away by i=2 while still queued.
|
||||
e.settle(
|
||||
"a delivered root plus one coalesced-survivor root, no more",
|
||||
|s| s.artifact_roots == 2 && s.owners == 1 && s.sealed_artifacts == 2,
|
||||
)
|
||||
.await;
|
||||
|
||||
let first = within("delivered", sub.next()).await.unwrap();
|
||||
assert_eq!(first.artifact("x").unwrap().read_all().await.unwrap(), &[0]);
|
||||
drop(first);
|
||||
let last = within("coalesced survivor", sub.next()).await.unwrap();
|
||||
assert_eq!(last.artifact("x").unwrap().read_all().await.unwrap(), &[2]);
|
||||
drop(last);
|
||||
e.settle("fully collected", |s| {
|
||||
s.artifact_roots == 0 && s.owners == 0 && s.sealed_artifacts == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// A retained topic value is a root the router itself keeps; it has no per-connection owner,
|
||||
/// so nothing about a subscriber connecting, reading or disconnecting can move it.
|
||||
async fn retained_topic_value_holds_a_root_independent_of_subscribers(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
p.declare_topic("t.retained", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
let a = sealed(&p, b"snapshot", "application/octet-stream").await;
|
||||
let r = p
|
||||
.publish("t.retained", obj(json!({})), &[("x", &a)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.subscribers, 0);
|
||||
drop(a);
|
||||
e.settle("a retained root with no owner", |s| {
|
||||
s.artifact_roots == 1 && s.owners == 0 && s.sealed_artifacts == 1
|
||||
})
|
||||
.await;
|
||||
assert!(p.clear_topic("t.retained").await.unwrap());
|
||||
e.settle("clearing releases the retained root", |s| {
|
||||
s.artifact_roots == 0 && s.sealed_artifacts == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Forward before release (bus-v1 section 8.2: destination roots before source release)
|
||||
|
||||
/// A forward or reply must use a still-live source owner; once a delivery is released its
|
||||
/// artifacts cannot source a new admission, but forwarding *before* releasing is exactly what
|
||||
/// keeps bytes alive once the original delivery finally goes.
|
||||
async fn forward_requires_the_source_owner_still_live(via: Via) {
|
||||
let e = env(via).await;
|
||||
let caller = e.client("caller").await;
|
||||
let consumer = e.client("consumer").await;
|
||||
consumer
|
||||
.declare_topic("t.relay", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = consumer
|
||||
.subscribe("t.relay", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut server = e.raw_hello("server").await;
|
||||
assert_eq!(
|
||||
code(
|
||||
&server
|
||||
.call(
|
||||
"service.register",
|
||||
json!({"name": "agent.relay", "maxQueued": 4, "maxInFlight": 4})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
|
||||
// Round 1: release the request delivery, then try to forward using its stale owner.
|
||||
let art1 = sealed(&caller, b"payload-1", "application/octet-stream").await;
|
||||
let _c1 = caller
|
||||
.call(
|
||||
"agent.relay",
|
||||
None,
|
||||
"Relay",
|
||||
obj(json!({})),
|
||||
&[("in", &art1)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let req1 = within("request 1", server.event()).await;
|
||||
let delivery1 = req1.body["deliveryId"].as_str().unwrap().to_owned();
|
||||
let att1 = req1.attachments[0].clone();
|
||||
assert_eq!(
|
||||
code(
|
||||
&server
|
||||
.call("delivery.consumed", json!({"deliveryIds": [delivery1]}))
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
e.settle("the request delivery is released", |s| s.owners == 1)
|
||||
.await; // only art1's own hold remains
|
||||
let refused = server
|
||||
.call_with(
|
||||
"publish",
|
||||
json!({"topic": "t.relay", "payload": {}}),
|
||||
json!([{"name": "out", "ref": att1.reference.to_json(), "ownerId": att1.owner_id}]),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
code(&refused),
|
||||
"OWNER_INVALID",
|
||||
"a released delivery cannot source a forward"
|
||||
);
|
||||
drop(art1);
|
||||
|
||||
// Round 2: forward while the request delivery is still live, then release it.
|
||||
let art2 = sealed(&caller, b"payload-2", "application/octet-stream").await;
|
||||
let _c2 = caller
|
||||
.call(
|
||||
"agent.relay",
|
||||
None,
|
||||
"Relay",
|
||||
obj(json!({})),
|
||||
&[("in", &art2)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let req2 = within("request 2", server.event()).await;
|
||||
let delivery2 = req2.body["deliveryId"].as_str().unwrap().to_owned();
|
||||
let att2 = req2.attachments[0].clone();
|
||||
let forwarded = server
|
||||
.call_with(
|
||||
"publish",
|
||||
json!({"topic": "t.relay", "payload": {}}),
|
||||
json!([{"name": "out", "ref": att2.reference.to_json(), "ownerId": att2.owner_id}]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(forwarded["subscribers"], "1");
|
||||
assert_eq!(
|
||||
code(
|
||||
&server
|
||||
.call("delivery.consumed", json!({"deliveryIds": [delivery2]}))
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
drop(art2);
|
||||
|
||||
let msg = within("relayed", sub.next()).await.unwrap();
|
||||
assert_eq!(
|
||||
msg.artifact("out").unwrap().read_all().await.unwrap(),
|
||||
b"payload-2"
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Failed admission rollback (bus-v1 section 6/7: rejection establishes no root)
|
||||
|
||||
async fn rejected_publish_creates_no_roots(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
let r = e.client("r").await;
|
||||
p.declare_topic("t.rollback", Retained::None).await.unwrap();
|
||||
let mut sub = r
|
||||
.subscribe(
|
||||
"t.rollback",
|
||||
SubscriptionConfig::bounded().queued(1).in_flight(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
p.publish("t.rollback", obj(json!({})), &[]).await.unwrap();
|
||||
let held = within("first delivered", sub.next()).await.unwrap(); // consumes the one in-flight credit
|
||||
p.publish("t.rollback", obj(json!({})), &[]).await.unwrap(); // fills the one queue slot
|
||||
|
||||
let art = sealed(&p, b"payload", "application/octet-stream").await;
|
||||
let before = e.stats();
|
||||
let refused = p
|
||||
.publish("t.rollback", obj(json!({})), &[("x", &art)])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(refused.code, ErrorCode::Backpressure);
|
||||
assert_eq!(
|
||||
e.stats(),
|
||||
before,
|
||||
"a refused publish must not touch any existing root, owner or byte count"
|
||||
);
|
||||
|
||||
drop(held);
|
||||
p.declare_topic("t.rollback.ok", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let receipt = p
|
||||
.publish("t.rollback.ok", obj(json!({})), &[("x", &art)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
receipt.subscribers, 0,
|
||||
"the untouched artifact is still usable after the rejection"
|
||||
);
|
||||
}
|
||||
|
||||
async fn rejected_call_creates_no_roots(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"agent.rollback",
|
||||
ServiceConfig {
|
||||
max_queued: 1,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _c1 = caller
|
||||
.call("agent.rollback", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let dispatched = within("dispatched", svc.next()).await.unwrap();
|
||||
let _c2 = caller
|
||||
.call("agent.rollback", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap(); // fills the queue
|
||||
|
||||
let art = sealed(&caller, b"payload", "application/octet-stream").await;
|
||||
let before = e.stats();
|
||||
let refused = caller
|
||||
.call(
|
||||
"agent.rollback",
|
||||
None,
|
||||
"Work",
|
||||
obj(json!({})),
|
||||
&[("x", &art)],
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(refused.code, ErrorCode::Backpressure);
|
||||
assert_eq!(
|
||||
e.stats(),
|
||||
before,
|
||||
"a refused call must not touch any existing root, owner or byte count"
|
||||
);
|
||||
drop(dispatched);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Disconnect cleanup (bus-v1 section 8.4)
|
||||
|
||||
/// An abrupt disconnect (no `artifact.release` ever sent) must still abandon an unsealed
|
||||
/// writer's staging reservation.
|
||||
async fn disconnect_abandons_an_unsealed_writer(via: Via) {
|
||||
let e = env(via).await;
|
||||
{
|
||||
let mut raw = e.raw_hello("producer").await;
|
||||
let (_alloc, path) = raw.allocate(64).await;
|
||||
std::fs::write(&path, [7u8; 64]).unwrap();
|
||||
e.settle("writer visible", |s| {
|
||||
s.artifacts == 1 && s.store_bytes == 64
|
||||
})
|
||||
.await;
|
||||
// `raw` drops here without ever releasing anything.
|
||||
}
|
||||
e.settle("disconnect abandoned the writer", |s| {
|
||||
s.artifacts == 0 && s.store_bytes == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
assert_eq!(e.files("staging"), 0);
|
||||
}
|
||||
|
||||
/// An abrupt disconnect must release an explicit hold too, when it was the object's only root.
|
||||
async fn disconnect_releases_an_explicit_hold(via: Via) {
|
||||
let e = env(via).await;
|
||||
{
|
||||
let mut raw = e.raw_hello("producer").await;
|
||||
let (alloc, path) = raw.allocate(5).await;
|
||||
std::fs::write(&path, b"hello").unwrap();
|
||||
let reply = raw.seal(&alloc, Value::Null).await.unwrap();
|
||||
assert!(reply.contains_key("ref"));
|
||||
}
|
||||
e.settle("disconnect released the only hold", |s| {
|
||||
s.sealed_artifacts == 0 && s.owners == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
assert_eq!(e.files("sealed"), 0);
|
||||
}
|
||||
|
||||
/// A vanished subscriber must give up both a delivery it already holds and one still queued
|
||||
/// behind it, never issued to the wire.
|
||||
async fn disconnect_releases_queued_and_dispatched_deliveries(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
p.declare_topic("t.gone", Retained::None).await.unwrap();
|
||||
{
|
||||
let mut raw = e.raw_hello("subscriber").await;
|
||||
assert_eq!(
|
||||
code(&raw.call("subscribe", json!({"topic": "t.gone", "mode": "bounded", "maxQueued": 4, "maxInFlight": 1, "replayLatest": false})).await),
|
||||
"OK"
|
||||
);
|
||||
let a1 = sealed(&p, b"m1", "application/octet-stream").await;
|
||||
let a2 = sealed(&p, b"m2", "application/octet-stream").await;
|
||||
p.publish("t.gone", obj(json!({})), &[("x", &a1)])
|
||||
.await
|
||||
.unwrap();
|
||||
p.publish("t.gone", obj(json!({})), &[("x", &a2)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop((a1, a2));
|
||||
let delivered = within("dispatched delivery", raw.event()).await;
|
||||
assert_eq!(delivered.op, "topic.message");
|
||||
e.settle("one dispatched, one still queued", |s| {
|
||||
s.artifact_roots == 2 && s.sealed_artifacts == 2
|
||||
})
|
||||
.await;
|
||||
// `raw` drops here without ever consuming the dispatched delivery.
|
||||
}
|
||||
e.settle("disconnect released both", |s| {
|
||||
s.artifact_roots == 0 && s.owners == 0 && s.sealed_artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Stale references, stale generations and forged/borrowed owners
|
||||
|
||||
async fn stale_store_incarnation_and_generation_are_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("c").await;
|
||||
let (alloc, path) = raw.allocate(4).await;
|
||||
std::fs::write(&path, b"data").unwrap();
|
||||
let sealed_reply = raw.seal(&alloc, Value::Null).await.unwrap();
|
||||
|
||||
let mut bad_store_ref = sealed_reply["ref"].clone();
|
||||
bad_store_ref["storeId"] = json!("store-does-not-exist");
|
||||
let r = raw
|
||||
.call(
|
||||
"artifact.open",
|
||||
json!({"ref": bad_store_ref, "ownerId": sealed_reply["ownerId"]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
code(&r),
|
||||
"ARTIFACT_GONE",
|
||||
"a reference naming another store incarnation is stale"
|
||||
);
|
||||
|
||||
let (alloc2, path2) = raw.allocate(4).await;
|
||||
std::fs::write(&path2, b"more!").unwrap();
|
||||
let bad_generation = raw
|
||||
.call("artifact.seal", json!({"artifactId": alloc2["artifactId"], "generation": "2", "ownerId": alloc2["ownerId"], "digest": Value::Null}))
|
||||
.await;
|
||||
assert_eq!(
|
||||
code(&bad_generation),
|
||||
"ARTIFACT_GONE",
|
||||
"v1 has exactly one generation"
|
||||
);
|
||||
}
|
||||
|
||||
async fn owner_ids_are_scoped_to_their_connection(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut a = e.raw_hello("a").await;
|
||||
let mut b = e.raw_hello("b").await;
|
||||
let (alloc, path) = a.allocate(4).await;
|
||||
std::fs::write(&path, b"data").unwrap();
|
||||
let sealed_reply = a.seal(&alloc, Value::Null).await.unwrap();
|
||||
let (reference, owner_id) = (sealed_reply["ref"].clone(), sealed_reply["ownerId"].clone());
|
||||
|
||||
// `b` was never issued this owner id; a real owner id from another connection is just as
|
||||
// invalid as one that was never issued at all.
|
||||
assert_eq!(
|
||||
code(
|
||||
&b.call(
|
||||
"artifact.open",
|
||||
json!({"ref": reference, "ownerId": owner_id})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
assert_eq!(
|
||||
code(
|
||||
&b.call(
|
||||
"artifact.retain",
|
||||
json!({"ref": reference, "ownerId": owner_id})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
|
||||
// The writer's own owner id cannot open before sealing.
|
||||
let (alloc2, path2) = a.allocate(4).await;
|
||||
std::fs::write(&path2, b"data").unwrap();
|
||||
let unsealed_ref = json!({
|
||||
"storeId": alloc2["writeLocation"]["storeId"], "artifactId": alloc2["artifactId"], "generation": "1",
|
||||
"byteLength": "4", "contentType": "application/octet-stream", "digest": Value::Null,
|
||||
});
|
||||
assert_eq!(
|
||||
code(
|
||||
&a.call(
|
||||
"artifact.open",
|
||||
json!({"ref": unsealed_ref, "ownerId": alloc2["ownerId"]})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"ARTIFACT_UNSEALED"
|
||||
);
|
||||
|
||||
// A malformed owner id is refused outright, not treated as an unknown serial.
|
||||
assert_eq!(
|
||||
code(
|
||||
&a.call(
|
||||
"artifact.open",
|
||||
json!({"ref": reference, "ownerId": "not-an-owner"})
|
||||
)
|
||||
.await
|
||||
),
|
||||
"OWNER_INVALID"
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Bounds, quotas and path containment (bus-v1 section 4/9)
|
||||
|
||||
async fn artifact_bounds_and_owner_budget_are_enforced(via: Via) {
|
||||
let limits = Limits {
|
||||
max_artifact_bytes: 100,
|
||||
max_store_bytes: 160,
|
||||
max_owners_per_client: 4,
|
||||
reserved_owners_per_client: 1,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(via, limits, Policy::open()).await;
|
||||
let c = e.client("c").await;
|
||||
|
||||
let too_big = c
|
||||
.artifacts()
|
||||
.allocate(101, "application/octet-stream")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
too_big.code,
|
||||
ErrorCode::QuotaExceeded,
|
||||
"byteLength exceeds the per-object limit"
|
||||
);
|
||||
|
||||
let a = sealed(&c, &[0u8; 80], "application/octet-stream").await;
|
||||
let before = e.stats();
|
||||
let no_room = c
|
||||
.artifacts()
|
||||
.allocate(90, "application/octet-stream")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(no_room.code, ErrorCode::QuotaExceeded, "the store is full");
|
||||
assert_eq!(
|
||||
e.stats(),
|
||||
before,
|
||||
"a rejected allocation must not charge the store"
|
||||
);
|
||||
|
||||
// Owner budget: 4 total minus 1 reserved leaves room for 3 ordinary owners; `a` itself is
|
||||
// already one of them, so exactly two more retains fit.
|
||||
let h1 = a.retain().await.unwrap();
|
||||
let h2 = a.retain().await.unwrap();
|
||||
let over = a.retain().await.unwrap_err();
|
||||
assert_eq!(
|
||||
over.code,
|
||||
ErrorCode::QuotaExceeded,
|
||||
"owner budget exhausted"
|
||||
);
|
||||
drop((h1, h2, a));
|
||||
}
|
||||
|
||||
/// Every location the router issues stays a plain relative path under the store; the router
|
||||
/// never hands the client anything to escape with (store.rs enforces this on the resolving
|
||||
/// side; this checks the issuing side never even offers an unsafe shape).
|
||||
async fn issued_locations_are_relative_and_contained(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("c").await;
|
||||
let (alloc, path) = raw.allocate(4).await;
|
||||
let assert_safe = |loc: &Value| {
|
||||
assert_eq!(loc["storeId"].as_str().unwrap(), e.router.store_id());
|
||||
let rel = loc["relativePath"].as_str().unwrap();
|
||||
assert!(
|
||||
!rel.starts_with('/') && !rel.contains(".."),
|
||||
"{rel:?} escapes the store"
|
||||
);
|
||||
assert!(
|
||||
rel.bytes().all(|b| b.is_ascii_lowercase()
|
||||
|| b.is_ascii_digit()
|
||||
|| matches!(b, b'.' | b'_' | b'-' | b'/')),
|
||||
"{rel:?} has an unexpected character"
|
||||
);
|
||||
};
|
||||
assert_safe(&alloc["writeLocation"]);
|
||||
|
||||
std::fs::write(&path, b"data").unwrap();
|
||||
let sealed_reply = raw.seal(&alloc, Value::Null).await.unwrap();
|
||||
let open_reply = raw
|
||||
.call(
|
||||
"artifact.open",
|
||||
json!({"ref": sealed_reply["ref"], "ownerId": sealed_reply["ownerId"]}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_safe(&open_reply["readLocation"]);
|
||||
assert_eq!(
|
||||
std::fs::read(raw.path(&open_reply["readLocation"])).unwrap(),
|
||||
b"data"
|
||||
);
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
allocate_write_seal_open_roundtrip_and_mismatches,
|
||||
seal_is_immutable_despite_a_stale_writable_handle,
|
||||
extracted_artifact_outlives_the_message_it_came_from,
|
||||
explicit_retain_outlives_the_original_hold,
|
||||
collection_waits_for_every_retained_owner,
|
||||
queued_deliveries_hold_roots_before_dispatch,
|
||||
latest_mode_holds_at_most_two_roots_delivered_plus_queued,
|
||||
retained_topic_value_holds_a_root_independent_of_subscribers,
|
||||
forward_requires_the_source_owner_still_live,
|
||||
rejected_publish_creates_no_roots,
|
||||
rejected_call_creates_no_roots,
|
||||
disconnect_abandons_an_unsealed_writer,
|
||||
disconnect_releases_an_explicit_hold,
|
||||
disconnect_releases_queued_and_dispatched_deliveries,
|
||||
stale_store_incarnation_and_generation_are_rejected,
|
||||
owner_ids_are_scoped_to_their_connection,
|
||||
artifact_bounds_and_owner_budget_are_enforced,
|
||||
issued_locations_are_relative_and_contained,
|
||||
);
|
||||
596
services/flysim/crates/flybus/tests/conformance_routing.rs
Normal file
596
services/flysim/crates/flybus/tests/conformance_routing.rs
Normal file
|
|
@ -0,0 +1,596 @@
|
|||
//! Routing conformance: adversarial contract cases for bus-v1 sections 5-8, deliberately not
|
||||
//! duplicating the happy paths already covered by `rpc.rs` and `pubsub.rs`. Generic synthetic
|
||||
//! services and topics only; no application/session semantics. Every test runs over both the
|
||||
//! in-memory transport and a Unix socket.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::{Via, env, obj, quiet, sealed, within};
|
||||
use flybus::{CancelState, Dispatch, ErrorCode, Retained, ServiceConfig, SubscriptionConfig};
|
||||
use serde_json::json;
|
||||
|
||||
/// bus-v1 section 5: "One live registration owns a service name." A client re-registering a
|
||||
/// name it already owns is a duplicate too, not an idempotent no-op, and the failed attempt
|
||||
/// must not disturb the live registration.
|
||||
async fn duplicate_registration_by_owner_itself_is_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let a = e.client("a").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = a
|
||||
.register("agent.self", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = svc.incarnation().to_owned();
|
||||
let dup = a
|
||||
.register("agent.self", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(dup.code, ErrorCode::Conflict);
|
||||
assert_eq!(
|
||||
svc.incarnation(),
|
||||
incarnation,
|
||||
"the failed self-duplicate did not replace the live registration"
|
||||
);
|
||||
let mut pending = caller
|
||||
.call("agent.self", Some(&incarnation), "M", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
assert!(req.reply(obj(json!({"ok": true})), &[]).await.unwrap());
|
||||
assert_eq!(
|
||||
within("result", pending.result()).await.unwrap().outcome()["ok"],
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
/// bus-v1 section 3: "Callers pin it after discovery." An unpinned call made *after* the name
|
||||
/// changes hands must resolve to the new incarnation, never linger on the old one.
|
||||
async fn unpinned_call_after_incarnation_replacement_reaches_the_new_holder(via: Via) {
|
||||
let e = env(via).await;
|
||||
let a = e.client("a").await;
|
||||
let b = e.client("b").await;
|
||||
let caller = e.client("caller").await;
|
||||
let first = a
|
||||
.register("agent.fly", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let old = first.incarnation().to_owned();
|
||||
drop(first);
|
||||
// Unregistration is asynchronous; wait for the name to come free.
|
||||
let mut second = loop {
|
||||
match b.register("agent.fly", ServiceConfig::default()).await {
|
||||
Ok(s) => break s,
|
||||
Err(err) => assert_eq!(err.code, ErrorCode::Conflict),
|
||||
}
|
||||
};
|
||||
assert_ne!(second.incarnation(), old);
|
||||
let mut pending = caller
|
||||
.call("agent.fly", None, "M", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
pending.service_incarnation(),
|
||||
second.incarnation(),
|
||||
"unpinned discovery resolves to the current holder"
|
||||
);
|
||||
let req = within("request", second.next()).await.unwrap();
|
||||
assert_eq!(req.service_incarnation(), second.incarnation());
|
||||
assert!(
|
||||
req.reply(obj(json!({"from": "second"})), &[])
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
within("result", pending.result()).await.unwrap().outcome()["from"],
|
||||
"second"
|
||||
);
|
||||
}
|
||||
|
||||
/// bus-v1 section 6: "responses may complete out of order and correlate by callId." Two callers
|
||||
/// interleave calls to the same service and the service answers in reverse admission order;
|
||||
/// every result must reach the caller it belongs to, never a sibling's.
|
||||
async fn out_of_order_replies_correlate_across_concurrent_callers(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let alice = e.client("alice").await;
|
||||
let bob = e.client("bob").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"example.multi",
|
||||
ServiceConfig {
|
||||
max_queued: 8,
|
||||
max_in_flight: 4,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut a1 = alice
|
||||
.call(
|
||||
"example.multi",
|
||||
None,
|
||||
"M",
|
||||
obj(json!({"who": "alice", "n": 1})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut b1 = bob
|
||||
.call(
|
||||
"example.multi",
|
||||
None,
|
||||
"M",
|
||||
obj(json!({"who": "bob", "n": 1})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut a2 = alice
|
||||
.call(
|
||||
"example.multi",
|
||||
None,
|
||||
"M",
|
||||
obj(json!({"who": "alice", "n": 2})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut b2 = bob
|
||||
.call(
|
||||
"example.multi",
|
||||
None,
|
||||
"M",
|
||||
obj(json!({"who": "bob", "n": 2})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut reqs = Vec::new();
|
||||
for _ in 0..4 {
|
||||
reqs.push(within("request", svc.next()).await.unwrap());
|
||||
}
|
||||
// Reply in the reverse of admission order.
|
||||
for r in reqs.iter().rev() {
|
||||
let who = r.payload()["who"].clone();
|
||||
let n = r.payload()["n"].clone();
|
||||
assert!(
|
||||
r.reply(obj(json!({"who": who, "n": n})), &[])
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
drop(reqs);
|
||||
let ra1 = within("a1", a1.result()).await.unwrap();
|
||||
let rb1 = within("b1", b1.result()).await.unwrap();
|
||||
let ra2 = within("a2", a2.result()).await.unwrap();
|
||||
let rb2 = within("b2", b2.result()).await.unwrap();
|
||||
assert_eq!(
|
||||
(ra1.outcome()["who"].as_str(), ra1.outcome()["n"].clone()),
|
||||
(Some("alice"), json!(1))
|
||||
);
|
||||
assert_eq!(
|
||||
(rb1.outcome()["who"].as_str(), rb1.outcome()["n"].clone()),
|
||||
(Some("bob"), json!(1))
|
||||
);
|
||||
assert_eq!(
|
||||
(ra2.outcome()["who"].as_str(), ra2.outcome()["n"].clone()),
|
||||
(Some("alice"), json!(2))
|
||||
);
|
||||
assert_eq!(
|
||||
(rb2.outcome()["who"].as_str(), rb2.outcome()["n"].clone()),
|
||||
(Some("bob"), json!(2))
|
||||
);
|
||||
}
|
||||
|
||||
/// bus-v1 section 5: "Queued cancellation releases its queued artifact roots." Not exercised by
|
||||
/// `cancellation_states`, which never attaches artifacts.
|
||||
async fn cancel_before_dispatch_releases_queued_artifact_roots(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"example.holdup",
|
||||
ServiceConfig {
|
||||
max_queued: 4,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _held = caller
|
||||
.call("example.holdup", None, "Work", obj(json!({"n": 1})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req1 = within("request 1", svc.next()).await.unwrap();
|
||||
let art = sealed(&caller, b"queued-payload", "text/plain").await;
|
||||
let queued = caller
|
||||
.call(
|
||||
"example.holdup",
|
||||
None,
|
||||
"Work",
|
||||
obj(json!({"n": 2})),
|
||||
&[("x", &art)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(art); // only the queued call's own root should keep the bytes alive now
|
||||
e.settle("the queued call's attachment is rooted", |s| {
|
||||
s.artifacts == 1 && s.artifact_roots >= 1
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
queued.cancel().await.unwrap(),
|
||||
CancelState::CancelledBeforeDispatch
|
||||
);
|
||||
e.settle("cancelling before dispatch released the queued root", |s| {
|
||||
s.artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
drop(req1);
|
||||
}
|
||||
|
||||
/// bus-v1 section 6: after a post-dispatch cancel, "a later reply to a detached call returns
|
||||
/// `routed:false`, with no caller-result roots" — including when that late reply carries an
|
||||
/// artifact the caller must never see rooted.
|
||||
async fn cancel_after_dispatch_then_late_reply_with_artifact_is_not_routed(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("example.late", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut dispatched = caller
|
||||
.call("example.late", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
assert_eq!(
|
||||
dispatched.cancel().await.unwrap(),
|
||||
CancelState::ExecutionUnknown
|
||||
);
|
||||
let art = sealed(&server, b"late-with-artifact", "text/plain").await;
|
||||
let routed = req.reply(obj(json!({})), &[("a", &art)]).await.unwrap();
|
||||
assert!(
|
||||
!routed,
|
||||
"a detached call must not be routed, artifact attached or not"
|
||||
);
|
||||
drop((req, art));
|
||||
e.settle("no caller-side roots leaked from a detached reply", |s| {
|
||||
s.artifacts == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
let after = within("result of a cancelled call", dispatched.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(after.code, after.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Unknown)
|
||||
);
|
||||
}
|
||||
|
||||
/// bus-v1 section 8.4: a caller's full disconnection (not merely dropping its pending future)
|
||||
/// detaches its dispatched call, and the service itself keeps serving other callers afterward.
|
||||
async fn caller_disconnect_detaches_dispatched_call_but_service_keeps_serving(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("example.vanish", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let pending = caller
|
||||
.call("example.vanish", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
caller.close().await;
|
||||
e.settle("caller gone", |s| s.connections == 1).await;
|
||||
drop(pending); // inert now: must not panic or double-release
|
||||
let routed = req.reply(obj(json!({"late": true})), &[]).await.unwrap();
|
||||
assert!(!routed, "the vanished caller cannot receive the reply");
|
||||
drop(req);
|
||||
e.settle("everything the vanished caller held is cleaned up", |s| {
|
||||
s.calls == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
let other = e.client("other").await;
|
||||
let mut fresh = other
|
||||
.call("example.vanish", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req2 = within("second request", svc.next()).await.unwrap();
|
||||
assert!(req2.reply(obj(json!({"ok": true})), &[]).await.unwrap());
|
||||
assert_eq!(
|
||||
within("result", fresh.result()).await.unwrap().outcome()["ok"],
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
/// bus-v1 section 8.4: disconnect "release[s] that connection's active writers/explicit/delivery
|
||||
/// roots", for a subscriber holding both delivered-but-unconsumed and still-queued artifacts,
|
||||
/// while a topic's retained value (owned by the topic, not the connection) survives untouched.
|
||||
async fn subscriber_disconnect_releases_queued_and_delivered_artifacts_but_not_retention(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let reader = e.client("reader").await;
|
||||
publisher
|
||||
.declare_topic("t.gone", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = reader
|
||||
.subscribe(
|
||||
"t.gone",
|
||||
SubscriptionConfig::bounded().queued(4).in_flight(4),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for i in 0..3 {
|
||||
let a = sealed(&publisher, format!("m{i}").as_bytes(), "text/plain").await;
|
||||
publisher
|
||||
.publish("t.gone", obj(json!({})), &[("m", &a)])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let first = within("first delivered", sub.next()).await.unwrap();
|
||||
e.settle("three sealed objects, all rooted", |s| {
|
||||
s.sealed_artifacts == 3
|
||||
})
|
||||
.await;
|
||||
drop(first);
|
||||
reader.close().await;
|
||||
e.settle(
|
||||
"everything the reader held is gone; only retention remains",
|
||||
|s| s.sealed_artifacts == 1 && s.subscriptions == 0 && s.connections == 1,
|
||||
)
|
||||
.await;
|
||||
let art = sealed(&publisher, b"after-disconnect", "text/plain").await;
|
||||
publisher
|
||||
.publish("t.gone", obj(json!({})), &[("m", &art)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(art);
|
||||
e.settle("retention alone carries the new value", |s| {
|
||||
s.sealed_artifacts == 1 && s.artifact_roots == 1
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// bus-v1 section 7: "reject the whole publish; no partial fan-out or retained-latest update."
|
||||
/// Not exercised with attachments by `bounded_fifo_and_atomic_backpressure`: a refused publish
|
||||
/// must leave no artifact root anywhere, on any subscriber. `maxQueued` and `maxInFlight` are
|
||||
/// separate counters, so filling `tight`'s single in-flight credit does not yet overflow it;
|
||||
/// its one queue slot has to fill too before a third publish overflows it.
|
||||
async fn bounded_overflow_rolls_back_all_artifact_roots(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let roomy = e.client("roomy").await;
|
||||
let tight = e.client("tight").await;
|
||||
publisher
|
||||
.declare_topic("t.atomic", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut a = roomy
|
||||
.subscribe(
|
||||
"t.atomic",
|
||||
SubscriptionConfig::bounded().queued(4).in_flight(4),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _tight_sub = tight
|
||||
.subscribe(
|
||||
"t.atomic",
|
||||
SubscriptionConfig::bounded().queued(1).in_flight(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let art1 = sealed(&publisher, b"one", "text/plain").await;
|
||||
publisher
|
||||
.publish("t.atomic", obj(json!({})), &[("x", &art1)])
|
||||
.await
|
||||
.unwrap(); // fills tight's in-flight credit
|
||||
drop(art1);
|
||||
let art2 = sealed(&publisher, b"two", "text/plain").await;
|
||||
publisher
|
||||
.publish("t.atomic", obj(json!({})), &[("x", &art2)])
|
||||
.await
|
||||
.unwrap(); // fills tight's one queue slot
|
||||
drop(art2);
|
||||
e.settle("two objects rooted before the overflow attempt", |s| {
|
||||
s.sealed_artifacts == 2
|
||||
})
|
||||
.await;
|
||||
let art3 = sealed(&publisher, b"three", "text/plain").await;
|
||||
let refused = publisher
|
||||
.publish("t.atomic", obj(json!({})), &[("x", &art3)])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(refused.code, ErrorCode::Backpressure);
|
||||
drop(art3);
|
||||
e.settle(
|
||||
"the refused publish left no trace: still exactly the first two objects",
|
||||
|s| s.sealed_artifacts == 2,
|
||||
)
|
||||
.await;
|
||||
let m1 = within("a's first", a.next()).await.unwrap();
|
||||
assert_eq!(m1.artifact("x").unwrap().read_all().await.unwrap(), b"one");
|
||||
let m2 = within("a's second", a.next()).await.unwrap();
|
||||
assert_eq!(m2.artifact("x").unwrap().read_all().await.unwrap(), b"two");
|
||||
quiet("a never saw the refused publish", a.next()).await;
|
||||
}
|
||||
|
||||
/// bus-v1 section 7: "New subscriptions with replayLatest enqueue it before subsequent accepted
|
||||
/// publications." A fresh `latest` subscription's replay claims its first in-flight credit
|
||||
/// immediately (there is nothing else competing for it yet), so a publish accepted right after
|
||||
/// subscribing must still be observed strictly after the replay, never ahead of or merged with
|
||||
/// it: each keeps its own delivery.
|
||||
async fn latest_replay_is_ordered_ahead_of_a_racing_publish(via: Via) {
|
||||
let e = env(via).await;
|
||||
let admin = e.client("admin").await;
|
||||
let reader = e.client("reader").await;
|
||||
admin
|
||||
.declare_topic("t.replay-race", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.publish("t.replay-race", obj(json!({"v": "old"})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = reader
|
||||
.subscribe(
|
||||
"t.replay-race",
|
||||
SubscriptionConfig::latest().in_flight(1).replay(true),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.publish("t.replay-race", obj(json!({"v": "new"})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let first = within("the replay arrives first", sub.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.payload()["v"], "old");
|
||||
drop(first); // the sole in-flight credit must return before the queued second value moves
|
||||
let second = within("the racing publish follows, not coalesced away", sub.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second.payload()["v"], "new");
|
||||
}
|
||||
|
||||
/// bus-v1 section 7: clearing releases only the retained root; a later `replayLatest`
|
||||
/// subscription must see nothing until a fresh publish, not a stale or resurrected value.
|
||||
async fn cleared_topic_gives_no_replay_until_a_fresh_publish(via: Via) {
|
||||
let e = env(via).await;
|
||||
let admin = e.client("admin").await;
|
||||
let reader = e.client("reader").await;
|
||||
admin
|
||||
.declare_topic("t.clear-replay", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.publish("t.clear-replay", obj(json!({"v": 1})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(admin.clear_topic("t.clear-replay").await.unwrap());
|
||||
let mut sub = reader
|
||||
.subscribe("t.clear-replay", SubscriptionConfig::bounded().replay(true))
|
||||
.await
|
||||
.unwrap();
|
||||
quiet("nothing retained to replay after a clear", sub.next()).await;
|
||||
admin
|
||||
.publish("t.clear-replay", obj(json!({"v": 2})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let m = within("a fresh publish arrives normally", sub.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(m.payload()["v"], 2);
|
||||
}
|
||||
|
||||
/// bus-v1 section 8.3: "dropping the message alone does not consume the delivery while a
|
||||
/// renderer/encoder still uses its artifact." `latest_coalesces_only_undelivered_values` proves
|
||||
/// this for `latest` mode; delivery-credit accounting must honour it for `bounded` mode too.
|
||||
async fn bounded_credit_waits_for_every_extracted_artifact(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let reader = e.client("reader").await;
|
||||
publisher
|
||||
.declare_topic("t.credit-hold", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = reader
|
||||
.subscribe(
|
||||
"t.credit-hold",
|
||||
SubscriptionConfig::bounded().in_flight(1).queued(4),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let a0 = sealed(&publisher, b"m0", "text/plain").await;
|
||||
let a1 = sealed(&publisher, b"m1", "text/plain").await;
|
||||
publisher
|
||||
.publish("t.credit-hold", obj(json!({})), &[("x", &a0)])
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.publish("t.credit-hold", obj(json!({})), &[("x", &a1)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop((a0, a1));
|
||||
let m0 = within("first", sub.next()).await.unwrap();
|
||||
let held = m0.artifact("x").unwrap();
|
||||
drop(m0); // the message struct is gone, but `held` still shares its delivery guard
|
||||
quiet(
|
||||
"credit withheld while an extracted artifact is still alive",
|
||||
sub.next(),
|
||||
)
|
||||
.await;
|
||||
drop(held);
|
||||
let m1 = within("second, only after the real release", sub.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(m1.artifact("x").unwrap().read_all().await.unwrap(), b"m1");
|
||||
}
|
||||
|
||||
/// Subscription ids are serials local to the issuing connection (`sub-<n>`), not a global
|
||||
/// namespace: a second connection quoting another connection's literal id string has no route
|
||||
/// to that subscription at all, so it can only ever land on (at most) its own same-numbered
|
||||
/// subscription, never the owner's. `unsubscribe` reports this as `removed:false`, not an
|
||||
/// error, and the owner's subscription keeps receiving messages untouched.
|
||||
async fn unsubscribe_cannot_reach_another_connections_subscription_id(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
publisher
|
||||
.declare_topic("t.owned", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut owner_raw = e.raw_hello("owner").await;
|
||||
let sub_id = {
|
||||
let r = owner_raw
|
||||
.call("subscribe", json!({"topic": "t.owned", "mode": "bounded", "maxQueued": 4, "maxInFlight": 4, "replayLatest": false}))
|
||||
.await
|
||||
.unwrap();
|
||||
r["subscriptionId"].as_str().unwrap().to_owned()
|
||||
};
|
||||
let mut intruder = e.raw_hello("intruder").await;
|
||||
let r = intruder
|
||||
.call("unsubscribe", json!({"subscriptionId": sub_id.clone()}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r["removed"],
|
||||
json!(false),
|
||||
"the intruder issued no such subscription itself"
|
||||
);
|
||||
|
||||
// The owner's subscription is unaffected by the intruder's attempt.
|
||||
let art = sealed(&publisher, b"still-mine", "text/plain").await;
|
||||
publisher
|
||||
.publish("t.owned", obj(json!({})), &[("x", &art)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(art);
|
||||
let delivered = owner_raw.event().await;
|
||||
assert_eq!(delivered.op, "topic.message");
|
||||
|
||||
let removed = owner_raw
|
||||
.call("unsubscribe", json!({"subscriptionId": sub_id}))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(removed["removed"], json!(true));
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
duplicate_registration_by_owner_itself_is_rejected,
|
||||
unpinned_call_after_incarnation_replacement_reaches_the_new_holder,
|
||||
out_of_order_replies_correlate_across_concurrent_callers,
|
||||
cancel_before_dispatch_releases_queued_artifact_roots,
|
||||
cancel_after_dispatch_then_late_reply_with_artifact_is_not_routed,
|
||||
caller_disconnect_detaches_dispatched_call_but_service_keeps_serving,
|
||||
subscriber_disconnect_releases_queued_and_delivered_artifacts_but_not_retention,
|
||||
bounded_overflow_rolls_back_all_artifact_roots,
|
||||
latest_replay_is_ordered_ahead_of_a_racing_publish,
|
||||
cleared_topic_gives_no_replay_until_a_fresh_publish,
|
||||
bounded_credit_waits_for_every_extracted_artifact,
|
||||
unsubscribe_cannot_reach_another_connections_subscription_id,
|
||||
);
|
||||
640
services/flysim/crates/flybus/tests/conformance_wire.rs
Normal file
640
services/flysim/crates/flybus/tests/conformance_wire.rs
Normal file
|
|
@ -0,0 +1,640 @@
|
|||
//! Wire and transport conformance (bus-v1 §4 and §11 acceptance test 1): bounded
|
||||
//! little-endian framing, partial reads/writes, strict JSON (duplicate keys at any depth,
|
||||
//! invalid UTF-8, unknown fields), malformed ids/u64s, connection negotiation, and behavioral
|
||||
//! parity between the in-memory and Unix-domain transports.
|
||||
//!
|
||||
//! This suite speaks the raw protocol by hand rather than going through the SDK: it sends
|
||||
//! frames no correct client would ever construct, so it can check what the router does with a
|
||||
//! hostile or merely buggy peer, not just what a well-behaved one gets back.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{Via, code, env, obj, within};
|
||||
use flybus::Limits;
|
||||
use flybus::wire::{Envelope, Kind, MAX_ENVELOPE_BYTES, contract_digest, read_frame, write_frame};
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Low-level helpers: a hand-rolled connection independent of `common::Raw`, for the framing
|
||||
// tests that need control over individual bytes and reads that `Raw`'s all-at-once
|
||||
// `send_bytes` cannot express.
|
||||
|
||||
/// Forces every read through this wrapper to surface at most one byte, so a caller reading a
|
||||
/// frame through it can only succeed by looping the way [`read_frame`] does — never by
|
||||
/// getting lucky with one big read.
|
||||
struct Trickle<R> {
|
||||
inner: R,
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for Trickle<R> {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
let mut one = [0u8; 1];
|
||||
let mut small = ReadBuf::new(&mut one);
|
||||
match Pin::new(&mut this.inner).poll_read(cx, &mut small) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
let n = small.filled().len();
|
||||
if n > 0 {
|
||||
buf.put_slice(&one[..n]);
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_envelope(
|
||||
wr: &mut (impl AsyncWrite + Unpin),
|
||||
id: &str,
|
||||
op: &str,
|
||||
body: Map<String, Value>,
|
||||
) {
|
||||
let env = Envelope::new(id.to_owned(), Kind::Command, op, body);
|
||||
write_frame(wr, &env.encode().unwrap()).await.unwrap();
|
||||
}
|
||||
|
||||
/// Writes the same frame [`send_envelope`] would, one byte at a time, flushing and yielding
|
||||
/// between every byte.
|
||||
async fn send_envelope_byte_by_byte(
|
||||
wr: &mut (impl AsyncWrite + Unpin),
|
||||
id: &str,
|
||||
op: &str,
|
||||
body: Map<String, Value>,
|
||||
) {
|
||||
let env = Envelope::new(id.to_owned(), Kind::Command, op, body);
|
||||
let bytes = env.encode().unwrap();
|
||||
let len = (bytes.len() as u32).to_le_bytes();
|
||||
for byte in len.iter().chain(bytes.iter()) {
|
||||
wr.write_all(std::slice::from_ref(byte)).await.unwrap();
|
||||
wr.flush().await.unwrap();
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn recv_envelope(rd: &mut (impl AsyncRead + Unpin)) -> Envelope {
|
||||
let bytes = within("frame", read_frame(rd))
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("stream open");
|
||||
Envelope::decode(&bytes).unwrap()
|
||||
}
|
||||
|
||||
/// A hand-rolled `bus.hello`, sent as command `msg-1`. Returns the reply's `value` object.
|
||||
async fn manual_hello(
|
||||
rd: &mut (impl AsyncRead + Unpin),
|
||||
wr: &mut (impl AsyncWrite + Unpin),
|
||||
id: &str,
|
||||
slow: bool,
|
||||
) -> Map<String, Value> {
|
||||
let body = obj(
|
||||
json!({"clientId": id, "clientIncarnation": format!("inc-{id}"), "supportedMajors": [1]}),
|
||||
);
|
||||
if slow {
|
||||
send_envelope_byte_by_byte(wr, "msg-1", "bus.hello", body).await;
|
||||
} else {
|
||||
send_envelope(wr, "msg-1", "bus.hello", body).await;
|
||||
}
|
||||
let reply = recv_envelope(rd).await;
|
||||
assert_eq!(reply.kind, Kind::Reply);
|
||||
assert_eq!(reply.reply_to.as_deref(), Some("msg-1"));
|
||||
assert_eq!(
|
||||
reply.body["ok"],
|
||||
json!(true),
|
||||
"hello failed: {:?}",
|
||||
reply.body
|
||||
);
|
||||
reply.body["value"].as_object().unwrap().clone()
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Bounded little-endian framing (bus-v1 §4: "Reject ... zero/oversize frames. Read length
|
||||
// before allocating.")
|
||||
|
||||
async fn zero_length_frame_closes_the_connection(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("zerolen").await;
|
||||
raw.send_prefix(0).await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("a zero-length frame must close the connection with a notice");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
/// The length prefix is checked, and the frame refused, before any body bytes are read. We
|
||||
/// announce an oversize frame and never send its body: an implementation that read the length
|
||||
/// after allocating (or tried to read the body anyway) would hang here instead of refusing
|
||||
/// promptly, and `closing`'s bounded wait turns that into a loud failure rather than a stall.
|
||||
async fn oversize_length_prefix_is_rejected_before_reading_body(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("oversize").await;
|
||||
raw.send_prefix((MAX_ENVELOPE_BYTES as u32) + 1).await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("an oversize frame must be refused, not hung waiting for a body that never comes");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
/// A frame at exactly the envelope ceiling decodes and dispatches normally; the same shape one
|
||||
/// byte past it is refused before any JSON parsing.
|
||||
async fn frame_at_the_size_ceiling_is_accepted_one_byte_over_is_not(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("ceiling").await;
|
||||
// Pad an otherwise-ordinary rpc.call to an exact byte count: measure the unpadded shape
|
||||
// once, then fill the gap with a string needing no escaping, so each character costs
|
||||
// exactly one byte and the target length is hit without any search.
|
||||
let shape = |pad: usize| {
|
||||
json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": "msg-9", "replyTo": null, "kind": "command", "op": "rpc.call",
|
||||
"body": {
|
||||
"callId": "call-1", "target": "no.such.service", "expectedIncarnation": null,
|
||||
"method": "M", "payload": {"pad": "a".repeat(pad)},
|
||||
},
|
||||
"attachments": [],
|
||||
})
|
||||
};
|
||||
let base_len = serde_json::to_vec(&shape(0)).unwrap().len();
|
||||
let exact = serde_json::to_vec(&shape(MAX_ENVELOPE_BYTES - base_len)).unwrap();
|
||||
assert_eq!(exact.len(), MAX_ENVELOPE_BYTES);
|
||||
raw.send_bytes(&exact).await;
|
||||
let reply = raw.reply("msg-9").await;
|
||||
// Decoded and dispatched fine: an ordinary domain reply (no such service), not a refusal.
|
||||
assert_eq!(code(&reply), "NO_SERVICE");
|
||||
|
||||
let mut over_raw = e.raw_hello("overceiling").await;
|
||||
let over = serde_json::to_vec(&shape(MAX_ENVELOPE_BYTES - base_len + 1)).unwrap();
|
||||
assert_eq!(over.len(), MAX_ENVELOPE_BYTES + 1);
|
||||
over_raw.send_bytes(&over).await;
|
||||
let body = over_raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("a frame one byte over the ceiling must be refused, not parsed");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
/// bus-v1 §4 specifies "u32 little-endian" explicitly. The prefix is built by hand here,
|
||||
/// independent of the library's own `to_le_bytes` call, for a frame long enough (over 255
|
||||
/// bytes) that a big-endian misreading would produce a huge, unmistakably wrong length. A
|
||||
/// byte-order regression shows up as a bounded timeout below, not a silent pass.
|
||||
async fn length_prefix_is_little_endian(via: Via) {
|
||||
let e = env(via).await;
|
||||
let (mut rd, mut wr) = tokio::io::split(e.transport().await);
|
||||
manual_hello(&mut rd, &mut wr, "byteorder", false).await;
|
||||
|
||||
let body = obj(json!({"name": "a".repeat(190), "retained": "none"}));
|
||||
let frame_env = Envelope::new("msg-2".into(), Kind::Command, "topic.declare", body);
|
||||
let bytes = frame_env.encode().unwrap();
|
||||
assert!(
|
||||
bytes.len() > 255,
|
||||
"the padding must force a non-trivial high byte in the length"
|
||||
);
|
||||
let len = bytes.len() as u32;
|
||||
let prefix = [
|
||||
(len & 0xFF) as u8,
|
||||
((len >> 8) & 0xFF) as u8,
|
||||
((len >> 16) & 0xFF) as u8,
|
||||
((len >> 24) & 0xFF) as u8,
|
||||
];
|
||||
assert_eq!(
|
||||
prefix,
|
||||
len.to_le_bytes(),
|
||||
"sanity: the hand-built prefix matches the standard LE encoding"
|
||||
);
|
||||
let mut frame = prefix.to_vec();
|
||||
frame.extend_from_slice(&bytes);
|
||||
wr.write_all(&frame).await.unwrap();
|
||||
wr.flush().await.unwrap();
|
||||
|
||||
let reply = tokio::time::timeout(Duration::from_secs(5), recv_envelope(&mut rd))
|
||||
.await
|
||||
.expect("the router must read the length as little-endian, not hang reinterpreting it");
|
||||
assert_eq!(reply.body["ok"], json!(true), "{:?}", reply.body);
|
||||
assert_eq!(reply.body["value"]["declared"], json!(true));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Partial reads and writes (bus-v1 §4: "Handle partial reads/writes.")
|
||||
|
||||
/// A stream that ends mid-frame is a clean disconnect, not a panic and not a phantom command:
|
||||
/// the router just stops reading and tears the connection down, the same as for a client that
|
||||
/// vanishes between frames.
|
||||
async fn truncated_frame_disconnects_cleanly(via: Via) {
|
||||
let e = env(via).await;
|
||||
{
|
||||
let mut raw = e.raw_hello("truncated").await;
|
||||
// Announce a 64-byte body, then drop the connection before sending any of it.
|
||||
raw.send_prefix(64).await;
|
||||
}
|
||||
e.settle("the truncated connection is gone", |s| s.connections == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// A frame written to the router one byte at a time, with a yield between every byte, decodes
|
||||
/// exactly as if it had arrived in one write.
|
||||
async fn a_frame_written_one_byte_at_a_time_still_decodes(via: Via) {
|
||||
let e = env(via).await;
|
||||
let (mut rd, mut wr) = tokio::io::split(e.transport().await);
|
||||
let value = manual_hello(&mut rd, &mut wr, "trickle-writer", true).await;
|
||||
assert_eq!(value["selectedMajor"], json!(1));
|
||||
assert_eq!(value["contractDigest"], json!(contract_digest()));
|
||||
}
|
||||
|
||||
/// The same guarantee on the reading side: a reply read one byte at a time through
|
||||
/// [`read_frame`] — the exact function both the router and the client SDK use — reassembles
|
||||
/// correctly.
|
||||
async fn a_reply_read_one_byte_at_a_time_still_decodes(via: Via) {
|
||||
let e = env(via).await;
|
||||
let (rd, mut wr) = tokio::io::split(e.transport().await);
|
||||
let mut trickle = Trickle { inner: rd };
|
||||
let value = manual_hello(&mut trickle, &mut wr, "trickle-reader", false).await;
|
||||
assert_eq!(value["selectedMajor"], json!(1));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Strict JSON: duplicate keys at any depth, invalid UTF-8.
|
||||
|
||||
/// bus-v1 §4: "Reject duplicate JSON keys" — the same rule at the top level, nested inside
|
||||
/// `body`, and nested inside an array element within it.
|
||||
async fn duplicate_json_keys_are_rejected_at_every_depth(via: Via) {
|
||||
let e = env(via).await;
|
||||
let cases: [&[u8]; 4] = [
|
||||
br#"{"protocol":"flybus","protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"bus.hello","body":{"clientId":"x","clientIncarnation":"y","supportedMajors":[1]},"attachments":[]}"#,
|
||||
br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"bus.hello","body":{"clientId":"x","clientId":"z","clientIncarnation":"y","supportedMajors":[1]},"attachments":[]}"#,
|
||||
br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"rpc.call","body":{"callId":"call-1","target":"a.b","expectedIncarnation":null,"method":"M","payload":{"n":{"d":1,"d":2}}},"attachments":[]}"#,
|
||||
br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-1","replyTo":null,"kind":"command","op":"rpc.call","body":{"callId":"call-1","target":"a.b","expectedIncarnation":null,"method":"M","payload":{"items":[{"x":1,"x":2}]}},"attachments":[]}"#,
|
||||
];
|
||||
for (i, case) in cases.iter().enumerate() {
|
||||
let mut raw = e.raw().await;
|
||||
raw.send_bytes(case).await;
|
||||
let body = raw.closing().await;
|
||||
assert!(
|
||||
body.is_some(),
|
||||
"case {i}: duplicate keys must close the connection"
|
||||
);
|
||||
assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "case {i}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Invalid UTF-8 anywhere in the frame is refused before JSON parsing starts, whether it falls
|
||||
/// inside a string value, trails a complete object, or leads the buffer.
|
||||
async fn invalid_utf8_is_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let cases: [&[u8]; 3] = [
|
||||
b"{\"protocol\":\"flybus\",\"major\":1,\"minor\":0,\"id\":\"msg-1\",\"replyTo\":null,\"kind\":\"command\",\"op\":\"bus.hello\",\"body\":{\"clientId\":\"\xff\",\"clientIncarnation\":\"y\",\"supportedMajors\":[1]},\"attachments\":[]}",
|
||||
b"{\"protocol\":\"flybus\",\"major\":1,\"minor\":0,\"id\":\"msg-1\",\"replyTo\":null,\"kind\":\"command\",\"op\":\"bus.hello\",\"body\":{\"clientId\":\"x\",\"clientIncarnation\":\"y\",\"supportedMajors\":[1]},\"attachments\":[]}\xff",
|
||||
b"\xc0\x80{\"protocol\":\"flybus\"}",
|
||||
];
|
||||
for (i, case) in cases.iter().enumerate() {
|
||||
let mut raw = e.raw().await;
|
||||
raw.send_bytes(case).await;
|
||||
let body = raw.closing().await;
|
||||
assert!(
|
||||
body.is_some(),
|
||||
"case {i}: invalid UTF-8 must close the connection"
|
||||
);
|
||||
assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "case {i}");
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Unknown envelope fields: a transport-level refusal at the envelope's own level, an ordinary
|
||||
// domain error inside an operation's body.
|
||||
|
||||
async fn unknown_top_level_field_closes_the_connection(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw().await;
|
||||
let v = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": "msg-1", "replyTo": null, "kind": "command", "op": "bus.hello",
|
||||
"body": {"clientId": "x", "clientIncarnation": "y", "supportedMajors": [1]},
|
||||
"attachments": [],
|
||||
"extra": true,
|
||||
});
|
||||
raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("an unknown envelope field must close the connection");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
/// A field the *operation* does not recognize is a normal `ok:false` reply, not a transport
|
||||
/// violation: only the envelope's own shape is the wire's concern, so the connection stays
|
||||
/// open and keeps working afterward.
|
||||
async fn unknown_body_field_is_a_domain_error_not_a_disconnect(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("unknownfield").await;
|
||||
let bogus = json!({
|
||||
"callId": "call-1", "target": "no.such.service", "expectedIncarnation": null,
|
||||
"method": "M", "payload": {}, "bogus": true,
|
||||
});
|
||||
let reply = raw.call("rpc.call", bogus).await;
|
||||
assert_eq!(code(&reply), "INVALID_ENVELOPE");
|
||||
let ok = raw
|
||||
.call(
|
||||
"topic.declare",
|
||||
json!({"name": "still.alive", "retained": "none"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&ok), "OK");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Malformed ids and u64s.
|
||||
|
||||
/// Every envelope-level scalar the wire validates before dispatch: a malformed `id`, an
|
||||
/// unparseable `kind`, and an `op` outside its `[a-z.]` alphabet. Each is refused at decode,
|
||||
/// before hello state or operation semantics are consulted at all.
|
||||
async fn malformed_envelope_scalars_are_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let hello_body = json!({"clientId": "x", "clientIncarnation": "y", "supportedMajors": [1]});
|
||||
let base = |id: &str, kind: &str, op: &str| {
|
||||
json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": id, "replyTo": null, "kind": kind, "op": op,
|
||||
"body": hello_body, "attachments": [],
|
||||
})
|
||||
};
|
||||
let cases = [
|
||||
(base("", "command", "bus.hello"), "empty id"),
|
||||
(base("MSG-1", "command", "bus.hello"), "uppercase id"),
|
||||
(
|
||||
base("-x", "command", "bus.hello"),
|
||||
"id starting with a separator",
|
||||
),
|
||||
(
|
||||
base(&"a".repeat(65), "command", "bus.hello"),
|
||||
"id over 64 characters",
|
||||
),
|
||||
(base("msg-1", "bogus", "bus.hello"), "unknown kind"),
|
||||
(base("msg-1", "command", "Bus.Hello"), "uppercase op"),
|
||||
(base("msg-1", "command", "bus.hello1"), "digit in op"),
|
||||
];
|
||||
for (v, what) in cases {
|
||||
let mut raw = e.raw().await;
|
||||
raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await;
|
||||
let body = raw.closing().await;
|
||||
assert!(body.is_some(), "{what}: must close the connection");
|
||||
assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "{what}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn non_null_reply_to_on_a_command_is_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("replyto").await;
|
||||
let v = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": "msg-2", "replyTo": "msg-1", "kind": "command", "op": "topic.declare",
|
||||
"body": {"name": "a.b", "retained": "none"}, "attachments": [],
|
||||
});
|
||||
raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("a non-null replyTo on a command must close the connection");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
/// `msg-<U64>` is canonical, not merely `Id`-shaped: no leading zero, no missing digits, no
|
||||
/// other prefix.
|
||||
async fn non_canonical_command_ids_are_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
for bad in ["msg-01", "msg-abc", "notmsg-1", "msg--1", "msg-"] {
|
||||
let mut raw = e.raw().await;
|
||||
let v = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": bad, "replyTo": null, "kind": "command", "op": "bus.hello",
|
||||
"body": {"clientId": "x", "clientIncarnation": "y", "supportedMajors": [1]},
|
||||
"attachments": [],
|
||||
});
|
||||
raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await;
|
||||
let body = raw.closing().await;
|
||||
assert!(
|
||||
body.is_some(),
|
||||
"{bad}: a non-canonical command id must close the connection"
|
||||
);
|
||||
assert_eq!(body.unwrap()["code"], json!("INVALID_ENVELOPE"), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn non_increasing_command_ids_are_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("nonincreasing").await; // hello already spent msg-1
|
||||
let v = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": "msg-1", "replyTo": null, "kind": "command", "op": "topic.declare",
|
||||
"body": {"name": "a.b", "retained": "none"}, "attachments": [],
|
||||
});
|
||||
raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("a repeated command id must close the connection");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
/// Below the envelope's own `id`, an operation's own ids (`callId`) get the same canonical-U64
|
||||
/// treatment, but as an ordinary domain reply: the connection is unharmed by a bad one.
|
||||
async fn malformed_call_id_is_a_domain_error(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("badcallid").await;
|
||||
for bad in ["call-01", "call-abc", "callx-1", ""] {
|
||||
let reply =
|
||||
raw.call("rpc.call", json!({"callId": bad, "target": "a.b", "expectedIncarnation": null, "method": "M", "payload": {}})).await;
|
||||
assert_eq!(code(&reply), "INVALID_ENVELOPE", "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// U64-string fields (`ipc-v1.md` §2: `"0"` or `[1-9][0-9]*`, at most `u64::MAX`) reject a
|
||||
/// leading zero, a non-digit, an empty string, whitespace, a decimal point and an overflow —
|
||||
/// each as a domain error the connection survives.
|
||||
async fn malformed_u64_fields_are_a_domain_error(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("badu64").await;
|
||||
for bad in ["01", "-1", "abc", "", "18446744073709551616", " 1", "1.0"] {
|
||||
let reply = raw
|
||||
.call(
|
||||
"artifact.allocate",
|
||||
json!({"byteLength": bad, "contentType": "application/octet-stream"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&reply), "INVALID_ENVELOPE", "{bad:?}");
|
||||
}
|
||||
let ok = raw
|
||||
.call(
|
||||
"artifact.allocate",
|
||||
json!({"byteLength": "1", "contentType": "application/octet-stream"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&ok), "OK");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// Negotiation (bus-v1 §4, "Connection negotiation").
|
||||
|
||||
async fn first_command_must_be_hello(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw().await;
|
||||
raw.command(
|
||||
"service.register",
|
||||
json!({"name": "x.y", "maxQueued": 1, "maxInFlight": 1}),
|
||||
json!([]),
|
||||
)
|
||||
.await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("a non-hello first command must close the connection");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
async fn hello_refuses_an_unsupported_major(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw().await;
|
||||
let reply = raw.call("bus.hello", json!({"clientId": "futuristic", "clientIncarnation": "inc-1", "supportedMajors": [2, 3]})).await;
|
||||
assert_eq!(code(&reply), "VERSION_MISMATCH");
|
||||
assert!(
|
||||
raw.recv().await.is_none(),
|
||||
"the connection must close right after refusing the major"
|
||||
);
|
||||
}
|
||||
|
||||
async fn hello_refuses_attachments(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw().await;
|
||||
let attachments = json!([{
|
||||
"name": "a",
|
||||
"ref": {
|
||||
"storeId": "s", "artifactId": "a-1", "generation": "1",
|
||||
"byteLength": "1", "contentType": "x", "digest": null,
|
||||
},
|
||||
"ownerId": "o",
|
||||
}]);
|
||||
let reply = raw
|
||||
.call_with(
|
||||
"bus.hello",
|
||||
json!({"clientId": "attacher", "clientIncarnation": "inc-1", "supportedMajors": [1]}),
|
||||
attachments,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&reply), "INVALID_ENVELOPE");
|
||||
}
|
||||
|
||||
async fn a_second_hello_on_the_same_connection_is_rejected(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("rehello").await;
|
||||
let v = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": "msg-2", "replyTo": null, "kind": "command", "op": "bus.hello",
|
||||
"body": {"clientId": "rehello", "clientIncarnation": "inc-2", "supportedMajors": [1]},
|
||||
"attachments": [],
|
||||
});
|
||||
raw.send_bytes(&serde_json::to_vec(&v).unwrap()).await;
|
||||
let body = raw
|
||||
.closing()
|
||||
.await
|
||||
.expect("a second bus.hello must close the connection");
|
||||
assert_eq!(body["code"], json!("INVALID_ENVELOPE"));
|
||||
}
|
||||
|
||||
async fn hello_reports_the_contract_digest_and_valid_limits(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw().await;
|
||||
let reply = raw.hello("reporter").await.unwrap();
|
||||
assert_eq!(reply["selectedMajor"], json!(1));
|
||||
assert_eq!(reply["selectedMinor"], json!(0));
|
||||
assert_eq!(reply["contractDigest"], json!(contract_digest()));
|
||||
assert!(reply["routerId"].as_str().unwrap().starts_with("router-"));
|
||||
assert!(reply["connectionId"].as_str().unwrap().starts_with("conn-"));
|
||||
let limits = Limits::from_json(&reply["limits"])
|
||||
.expect("the router's own limits object must round-trip");
|
||||
assert_eq!(limits, Limits::default());
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
zero_length_frame_closes_the_connection,
|
||||
oversize_length_prefix_is_rejected_before_reading_body,
|
||||
frame_at_the_size_ceiling_is_accepted_one_byte_over_is_not,
|
||||
length_prefix_is_little_endian,
|
||||
truncated_frame_disconnects_cleanly,
|
||||
a_frame_written_one_byte_at_a_time_still_decodes,
|
||||
a_reply_read_one_byte_at_a_time_still_decodes,
|
||||
duplicate_json_keys_are_rejected_at_every_depth,
|
||||
invalid_utf8_is_rejected,
|
||||
unknown_top_level_field_closes_the_connection,
|
||||
unknown_body_field_is_a_domain_error_not_a_disconnect,
|
||||
malformed_envelope_scalars_are_rejected,
|
||||
non_null_reply_to_on_a_command_is_rejected,
|
||||
non_canonical_command_ids_are_rejected,
|
||||
non_increasing_command_ids_are_rejected,
|
||||
malformed_call_id_is_a_domain_error,
|
||||
malformed_u64_fields_are_a_domain_error,
|
||||
first_command_must_be_hello,
|
||||
hello_refuses_an_unsupported_major,
|
||||
hello_refuses_attachments,
|
||||
a_second_hello_on_the_same_connection_is_rejected,
|
||||
hello_reports_the_contract_digest_and_valid_limits,
|
||||
);
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
// bus-v1 §11 acceptance test 1: "In-memory transport must pass the same tests as Unix
|
||||
// sockets." Every test above already runs on both (that is what `both_transports!` is for);
|
||||
// this one drives the identical raw script over both side by side in a single test, so a
|
||||
// transport-specific quirk in one implementation cannot hide behind "it still passes its own
|
||||
// copy of the suite."
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn memory_and_unix_negotiate_the_identical_contract() {
|
||||
let mem = env(Via::Memory).await;
|
||||
let unix = env(Via::Unix).await;
|
||||
let mut raw_mem = mem.raw().await;
|
||||
let mut raw_unix = unix.raw().await;
|
||||
|
||||
let hm = raw_mem.hello("parity").await.unwrap();
|
||||
let hu = raw_unix.hello("parity").await.unwrap();
|
||||
assert_eq!(hm["selectedMajor"], hu["selectedMajor"]);
|
||||
assert_eq!(hm["selectedMinor"], hu["selectedMinor"]);
|
||||
assert_eq!(hm["contractDigest"], hu["contractDigest"]);
|
||||
assert_eq!(hm["limits"], hu["limits"]);
|
||||
|
||||
let cm = raw_mem
|
||||
.call(
|
||||
"topic.declare",
|
||||
json!({"name": "parity.topic", "retained": "latest"}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let cu = raw_unix
|
||||
.call(
|
||||
"topic.declare",
|
||||
json!({"name": "parity.topic", "retained": "latest"}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cm["declared"], cu["declared"]);
|
||||
assert!(cm["topicIncarnation"].as_str().unwrap().starts_with("top-"));
|
||||
assert!(cu["topicIncarnation"].as_str().unwrap().starts_with("top-"));
|
||||
|
||||
// The identical malformed frame gets the identical transport-level refusal on both.
|
||||
let mut bad_mem = mem.raw_hello("parity-bad").await;
|
||||
let mut bad_unix = unix.raw_hello("parity-bad").await;
|
||||
let bad_frame = br#"{"a":1,"a":2}"#;
|
||||
bad_mem.send_bytes(bad_frame).await;
|
||||
bad_unix.send_bytes(bad_frame).await;
|
||||
let bm = bad_mem.closing().await.unwrap();
|
||||
let bu = bad_unix.closing().await.unwrap();
|
||||
assert_eq!(bm["code"], bu["code"]);
|
||||
}
|
||||
272
services/flysim/crates/flybus/tests/integration.rs
Normal file
272
services/flysim/crates/flybus/tests/integration.rs
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
//! bus-v1 section 11 item 6: two parallel fake agents, complete-batch environment RPC,
|
||||
//! committed snapshot publication and a deliberately slow presentation consumer, all over one
|
||||
//! router. Generic services only; nothing here knows what a brain or a game is.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::io::Write;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{Via, env_with, obj, within};
|
||||
use flybus::{
|
||||
Client, ErrorCode, Grants, Limits, Pattern, Policy, Retained, ServiceConfig, SubscriptionConfig,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
const STEPS: u64 = 20;
|
||||
const FRAME: usize = 160 * 144 * 4;
|
||||
|
||||
fn grants(f: impl FnOnce(&mut Grants)) -> Grants {
|
||||
let mut g = Grants::default();
|
||||
f(&mut g);
|
||||
g
|
||||
}
|
||||
|
||||
/// An environment service: each Advance produces a frame artifact filled with the step number.
|
||||
fn spawn_environment(client: Client, mut svc: flybus::Service) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
while let Some(req) = svc.next().await {
|
||||
let step = req.payload()["step"].as_u64().unwrap();
|
||||
let mut w = client
|
||||
.artifacts()
|
||||
.allocate(FRAME as u64, "image/x-rgba")
|
||||
.await
|
||||
.unwrap();
|
||||
w.write_all(&vec![step as u8; FRAME]).unwrap();
|
||||
let frame = w.seal().await.unwrap();
|
||||
req.reply(
|
||||
obj(json!({"step": step, "width": 160, "height": 144})),
|
||||
&[("frame", &frame)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// An agent service: checks the frame it was sent and answers with a digest of it.
|
||||
fn spawn_agent(name: &'static str, mut svc: flybus::Service) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
while let Some(req) = svc.next().await {
|
||||
let step = req.payload()["step"].as_u64().unwrap();
|
||||
let bytes = req.artifact("frame").unwrap().read_all().await.unwrap();
|
||||
let ok = bytes.len() == FRAME && bytes.iter().all(|b| *b == step as u8);
|
||||
req.reply(
|
||||
obj(json!({"agent": name, "step": step, "frameOk": ok})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn session_over_one_router(via: Via) {
|
||||
let policy = Policy::closed()
|
||||
.client(
|
||||
"coordinator",
|
||||
grants(|g| {
|
||||
g.call = vec![Pattern::prefix("agent."), Pattern::exact("env.demo")];
|
||||
g.publish = vec![Pattern::prefix("session.demo.")];
|
||||
g.manage_topics = vec![Pattern::prefix("session.demo.")];
|
||||
}),
|
||||
)
|
||||
.client(
|
||||
"agent-a",
|
||||
grants(|g| g.register = vec![Pattern::exact("agent.fly-a")]),
|
||||
)
|
||||
.client(
|
||||
"agent-b",
|
||||
grants(|g| g.register = vec![Pattern::exact("agent.fly-b")]),
|
||||
)
|
||||
.client(
|
||||
"environment",
|
||||
grants(|g| g.register = vec![Pattern::exact("env.demo")]),
|
||||
)
|
||||
.client(
|
||||
"presenter",
|
||||
grants(|g| g.subscribe = vec![Pattern::prefix("session.demo.")]),
|
||||
)
|
||||
.client(
|
||||
"recorder",
|
||||
grants(|g| g.subscribe = vec![Pattern::prefix("session.demo.")]),
|
||||
);
|
||||
let e = env_with(via, Limits::default(), policy).await;
|
||||
|
||||
let environment = e.client("environment").await;
|
||||
let env_svc = environment
|
||||
.register(
|
||||
"env.demo",
|
||||
ServiceConfig {
|
||||
max_queued: 4,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let env_inc = env_svc.incarnation().to_owned();
|
||||
let env_task = spawn_environment(environment.clone(), env_svc);
|
||||
let a = e.client("agent-a").await;
|
||||
let b = e.client("agent-b").await;
|
||||
let a_svc = a
|
||||
.register("agent.fly-a", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let b_svc = b
|
||||
.register("agent.fly-b", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let (a_inc, b_inc) = (
|
||||
a_svc.incarnation().to_owned(),
|
||||
b_svc.incarnation().to_owned(),
|
||||
);
|
||||
let agents = [spawn_agent("fly-a", a_svc), spawn_agent("fly-b", b_svc)];
|
||||
|
||||
let coordinator = e.client("coordinator").await;
|
||||
coordinator
|
||||
.declare_topic("session.demo.snapshots", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let presenter = e.client("presenter").await;
|
||||
// Observers cannot drive the environment.
|
||||
let denied = presenter
|
||||
.call(
|
||||
"env.demo",
|
||||
None,
|
||||
"Environment.Advance",
|
||||
obj(json!({"step": 0})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(denied.code, ErrorCode::NotAuthorized);
|
||||
let mut slow = presenter
|
||||
.subscribe(
|
||||
"session.demo.snapshots",
|
||||
SubscriptionConfig::latest().in_flight(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let presenting = tokio::spawn(async move {
|
||||
let mut seen = Vec::new();
|
||||
while let Some(m) = slow.next().await {
|
||||
let frame = m.artifact("frame").unwrap();
|
||||
drop(m);
|
||||
tokio::time::sleep(Duration::from_millis(25)).await; // a slow renderer
|
||||
let bytes = frame.read_all().await.unwrap();
|
||||
let step = bytes[0] as u64;
|
||||
seen.push((step, frame.reference().artifact_id.clone()));
|
||||
if step == STEPS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
seen
|
||||
});
|
||||
let recorder = e.client("recorder").await;
|
||||
let mut all = recorder
|
||||
.subscribe("session.demo.snapshots", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let recording = tokio::spawn(async move {
|
||||
let mut seq = Vec::new();
|
||||
while let Some(m) = all.next().await {
|
||||
seq.push((m.topic_sequence(), m.payload()["step"].as_u64().unwrap()));
|
||||
if seq.len() as u64 == STEPS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
seq
|
||||
});
|
||||
|
||||
for step in 1..=STEPS {
|
||||
let advanced = coordinator
|
||||
.call_and_wait(
|
||||
"env.demo",
|
||||
Some(&env_inc),
|
||||
"Environment.Advance",
|
||||
obj(json!({"step": step})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Forward the delivery-owned frame to both agents at once.
|
||||
let frame = advanced.artifact("frame").unwrap();
|
||||
let attachments = [("frame", &frame)];
|
||||
let (ra, rb) = tokio::join!(
|
||||
coordinator.call_and_wait(
|
||||
"agent.fly-a",
|
||||
Some(&a_inc),
|
||||
"Agent.Prepare",
|
||||
obj(json!({"step": step})),
|
||||
&attachments
|
||||
),
|
||||
coordinator.call_and_wait(
|
||||
"agent.fly-b",
|
||||
Some(&b_inc),
|
||||
"Agent.Prepare",
|
||||
obj(json!({"step": step})),
|
||||
&attachments
|
||||
),
|
||||
);
|
||||
for r in [ra.unwrap(), rb.unwrap()] {
|
||||
assert_eq!(
|
||||
(
|
||||
r.outcome()["step"].as_u64(),
|
||||
r.outcome()["frameOk"].as_bool()
|
||||
),
|
||||
(Some(step), Some(true))
|
||||
);
|
||||
}
|
||||
let receipt = coordinator
|
||||
.publish(
|
||||
"session.demo.snapshots",
|
||||
obj(json!({"step": step})),
|
||||
&[("frame", &frame)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(receipt.topic_sequence, step);
|
||||
}
|
||||
|
||||
let recorded = within("recorder", recording).await.unwrap();
|
||||
assert_eq!(
|
||||
recorded,
|
||||
(1..=STEPS).map(|s| (s, s)).collect::<Vec<_>>(),
|
||||
"the bounded recorder misses nothing"
|
||||
);
|
||||
let presented = within("presenter", presenting).await.unwrap();
|
||||
assert_eq!(
|
||||
presented.last().unwrap().0,
|
||||
STEPS,
|
||||
"the slow consumer ends on the latest snapshot"
|
||||
);
|
||||
assert!(
|
||||
presented.len() < STEPS as usize,
|
||||
"the slow consumer skipped snapshots: {presented:?}"
|
||||
);
|
||||
assert!(presented.windows(2).all(|w| w[0].0 < w[1].0));
|
||||
|
||||
for t in agents {
|
||||
t.abort();
|
||||
}
|
||||
env_task.abort();
|
||||
drop((a, b, environment, presenter, recorder));
|
||||
// Only the retained snapshot's frame is left once everyone is gone.
|
||||
let s = e
|
||||
.settle("session torn down", |s| {
|
||||
s.calls == 0 && s.sealed_artifacts == 1 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
assert_eq!(s.store_bytes, FRAME as u64);
|
||||
assert!(
|
||||
coordinator
|
||||
.clear_topic("session.demo.snapshots")
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
e.settle("retained frame collected", |s| s.artifacts == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
both_transports!(session_over_one_router);
|
||||
213
services/flysim/crates/flybus/tests/perf.rs
Normal file
213
services/flysim/crates/flybus/tests/perf.rs
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
//! bus-v1 section 11 item 7, as a measurement rather than a gate: 640x480 RGBA frames at
|
||||
//! 60 Hz over a Unix socket to three consumers (one delayed), with 1, 2 and 4 agent services
|
||||
//! pinged every frame. Router and clients share this process, so CPU and RSS are the whole
|
||||
//! process. Run with:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo test --release -p flybus --test perf -- --ignored --nocapture
|
||||
//! ```
|
||||
|
||||
mod common;
|
||||
|
||||
use std::io::Write;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use common::{Via, env, obj};
|
||||
use flybus::{Client, Retained, ServiceConfig, SubscriptionConfig};
|
||||
use serde_json::json;
|
||||
|
||||
const W: usize = 640;
|
||||
const H: usize = 480;
|
||||
const HZ: u64 = 60;
|
||||
const SECONDS: u64 = 2;
|
||||
|
||||
fn proc_cpu_seconds() -> f64 {
|
||||
// utime + stime, fields 14 and 15 of /proc/self/stat, in clock ticks (100 Hz on Linux).
|
||||
let stat = std::fs::read_to_string("/proc/self/stat").unwrap_or_default();
|
||||
let after = stat.rsplit_once(')').map_or("", |(_, rest)| rest);
|
||||
let f: Vec<&str> = after.split_whitespace().collect();
|
||||
let ticks = |i: usize| f.get(i).and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0);
|
||||
(ticks(11) + ticks(12)) / 100.0
|
||||
}
|
||||
|
||||
fn proc_status(key: &str) -> String {
|
||||
let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
|
||||
status
|
||||
.lines()
|
||||
.find(|l| l.starts_with(key))
|
||||
.map_or("?".into(), |l| l[key.len()..].trim().to_owned())
|
||||
}
|
||||
|
||||
fn pct(sorted: &[Duration], p: f64) -> Duration {
|
||||
if sorted.is_empty() {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
sorted[((sorted.len() - 1) as f64 * p).round() as usize]
|
||||
}
|
||||
|
||||
async fn consumer(client: Client, delay: Duration) -> (u64, u64) {
|
||||
let mut sub = client
|
||||
.subscribe("world.demo.frame", SubscriptionConfig::latest())
|
||||
.await
|
||||
.unwrap();
|
||||
let (mut seen, mut replaced) = (0, 0);
|
||||
while let Some(m) = sub.next().await {
|
||||
if m.payload().get("end").is_some() {
|
||||
break;
|
||||
}
|
||||
replaced += m.replaced();
|
||||
let frame = m.artifact("frame").unwrap();
|
||||
drop(m);
|
||||
let bytes = frame.read_all().await.unwrap();
|
||||
assert_eq!(bytes.len(), W * H * 4);
|
||||
tokio::time::sleep(delay).await;
|
||||
seen += 1;
|
||||
}
|
||||
(seen, replaced)
|
||||
}
|
||||
|
||||
async fn run(agents: usize) {
|
||||
let e = env(Via::Unix).await;
|
||||
let producer = e.client("producer").await;
|
||||
producer
|
||||
.declare_topic("world.demo.frame", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut consumers = Vec::new();
|
||||
for (i, delay) in [0u64, 0, 40].into_iter().enumerate() {
|
||||
let c = e.client(&format!("consumer-{i}")).await;
|
||||
consumers.push(tokio::spawn(consumer(c, Duration::from_millis(delay))));
|
||||
}
|
||||
let mut services = Vec::new();
|
||||
for k in 0..agents {
|
||||
let c = e.client(&format!("agent-{k}")).await;
|
||||
let mut svc = c
|
||||
.register(&format!("agent.a{k}"), ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
services.push(tokio::spawn(async move {
|
||||
let _c = c;
|
||||
while let Some(req) = svc.next().await {
|
||||
req.reply(obj(json!({})), &[]).await.unwrap();
|
||||
}
|
||||
}));
|
||||
}
|
||||
let caller = e.client("coordinator").await;
|
||||
// Let every subscription land before the first frame.
|
||||
e.settle("subscribed", |s| s.subscriptions == 3).await;
|
||||
|
||||
let pixels: Vec<u8> = (0..W * H * 4).map(|i| (i % 253) as u8).collect();
|
||||
let frames = HZ * SECONDS;
|
||||
let period = Duration::from_nanos(1_000_000_000 / HZ);
|
||||
let (mut produce, mut publish, mut rpc) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let (mut peak_bytes, mut peak_roots, mut peak_queued, mut late) = (0u64, 0u64, 0usize, 0u32);
|
||||
let cpu0 = proc_cpu_seconds();
|
||||
let start = Instant::now();
|
||||
for n in 0..frames {
|
||||
let deadline = start + period * n as u32;
|
||||
let t = Instant::now();
|
||||
let mut w = producer
|
||||
.artifacts()
|
||||
.allocate(pixels.len() as u64, "image/x-rgba")
|
||||
.await
|
||||
.unwrap();
|
||||
w.write_all(&pixels).unwrap();
|
||||
let frame = w.seal().await.unwrap();
|
||||
produce.push(t.elapsed());
|
||||
let t = Instant::now();
|
||||
producer
|
||||
.publish(
|
||||
"world.demo.frame",
|
||||
obj(json!({"n": n})),
|
||||
&[("frame", &frame)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
publish.push(t.elapsed());
|
||||
drop(frame);
|
||||
let mut calls = Vec::new();
|
||||
for k in 0..agents {
|
||||
let caller = caller.clone();
|
||||
calls.push(tokio::spawn(async move {
|
||||
let t = Instant::now();
|
||||
caller
|
||||
.call_and_wait(&format!("agent.a{k}"), None, "Ping", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
t.elapsed()
|
||||
}));
|
||||
}
|
||||
for c in calls {
|
||||
rpc.push(c.await.unwrap());
|
||||
}
|
||||
let s = e.stats();
|
||||
peak_bytes = peak_bytes.max(s.store_bytes);
|
||||
peak_roots = peak_roots.max(s.artifact_roots);
|
||||
peak_queued = peak_queued.max(s.queued);
|
||||
let next = deadline + period;
|
||||
if Instant::now() > next {
|
||||
late += 1;
|
||||
} else {
|
||||
tokio::time::sleep_until(next.into()).await;
|
||||
}
|
||||
}
|
||||
let wall = start.elapsed().as_secs_f64();
|
||||
let cpu = proc_cpu_seconds() - cpu0;
|
||||
let end = Instant::now();
|
||||
producer
|
||||
.publish("world.demo.frame", obj(json!({"end": true})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let mut results = Vec::new();
|
||||
for c in consumers {
|
||||
results.push(c.await.unwrap());
|
||||
}
|
||||
e.settle("collected", |s| s.store_bytes == 0).await;
|
||||
let collect_lag = end.elapsed();
|
||||
for s in services {
|
||||
s.abort();
|
||||
}
|
||||
for v in [&mut produce, &mut publish, &mut rpc] {
|
||||
v.sort();
|
||||
}
|
||||
let ms = |d: Duration| format!("{:.2}", d.as_secs_f64() * 1000.0);
|
||||
println!("agents={agents} frames={frames} over {wall:.2}s, late frames {late}");
|
||||
println!(
|
||||
" produce (allocate+write+seal copy) ms p50/p95/p99: {}/{}/{}",
|
||||
ms(pct(&produce, 0.5)),
|
||||
ms(pct(&produce, 0.95)),
|
||||
ms(pct(&produce, 0.99))
|
||||
);
|
||||
println!(
|
||||
" publish admission ms p50/p95/p99: {}/{}/{}",
|
||||
ms(pct(&publish, 0.5)),
|
||||
ms(pct(&publish, 0.95)),
|
||||
ms(pct(&publish, 0.99))
|
||||
);
|
||||
println!(
|
||||
" rpc round trip ms p50/p95/p99: {}/{}/{}",
|
||||
ms(pct(&rpc, 0.5)),
|
||||
ms(pct(&rpc, 0.95)),
|
||||
ms(pct(&rpc, 0.99))
|
||||
);
|
||||
println!(
|
||||
" process cpu {:.2} cores; VmRSS {} VmHWM {}",
|
||||
cpu / wall,
|
||||
proc_status("VmRSS:"),
|
||||
proc_status("VmHWM:")
|
||||
);
|
||||
println!(
|
||||
" store peak {:.1} MB, peak roots {peak_roots}, peak queued {peak_queued}, drain+collect {:.1} ms",
|
||||
peak_bytes as f64 / 1e6,
|
||||
collect_lag.as_secs_f64() * 1000.0
|
||||
);
|
||||
println!(" consumers (frames seen, replaced): {results:?}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "measurement; run with --release -- --ignored --nocapture"]
|
||||
async fn frames_at_60hz_with_three_consumers() {
|
||||
for agents in [1, 2, 4] {
|
||||
run(agents).await;
|
||||
}
|
||||
}
|
||||
545
services/flysim/crates/flybus/tests/pubsub.rs
Normal file
545
services/flysim/crates/flybus/tests/pubsub.rs
Normal file
|
|
@ -0,0 +1,545 @@
|
|||
//! Pub/sub: exact topics, bounded FIFO with whole-publication backpressure, latest coalescing,
|
||||
//! credits, retention, incarnations and fair control delivery under a saturated subscriber.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{Via, code, env, env_with, obj, quiet, sealed, within};
|
||||
use flybus::{ErrorCode, Limits, Mode, Policy, Retained, ServiceConfig, SubscriptionConfig};
|
||||
use serde_json::json;
|
||||
|
||||
async fn bounded_fifo_and_atomic_backpressure(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let slow = e.client("slow").await;
|
||||
let fast = e.client("fast").await;
|
||||
publisher
|
||||
.declare_topic("session.demo.events", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut a = slow
|
||||
.subscribe(
|
||||
"session.demo.events",
|
||||
SubscriptionConfig::bounded().queued(2).in_flight(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut b = fast
|
||||
.subscribe("session.demo.events", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let p = |n: u64| obj(json!({ "n": n }));
|
||||
|
||||
let r1 = publisher
|
||||
.publish("session.demo.events", p(1), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((r1.topic_sequence, r1.subscribers, r1.replaced), (1, 2, 0));
|
||||
let m1 = within("a gets 1", a.next()).await.unwrap();
|
||||
publisher
|
||||
.publish("session.demo.events", p(2), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.publish("session.demo.events", p(3), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
// a holds 1 in flight and 2, 3 queued: the next publication would overflow it, so nobody
|
||||
// gets it, the retained value does not move and no sequence number is spent.
|
||||
let full = publisher
|
||||
.publish("session.demo.events", p(4), &[])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(full.code, ErrorCode::Backpressure);
|
||||
for n in 1..=3u64 {
|
||||
let m = within("b in order", b.next()).await.unwrap();
|
||||
assert_eq!(
|
||||
(m.payload()["n"].as_u64(), m.topic_sequence()),
|
||||
(Some(n), n)
|
||||
);
|
||||
}
|
||||
quiet("b gets no partial fan-out", b.next()).await;
|
||||
let mut late = fast
|
||||
.subscribe(
|
||||
"session.demo.events",
|
||||
SubscriptionConfig::bounded().replay(true),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let replayed = within("replay", late.next()).await.unwrap();
|
||||
assert_eq!(
|
||||
replayed.payload()["n"],
|
||||
3,
|
||||
"the refused publication did not become the retained value"
|
||||
);
|
||||
drop(m1);
|
||||
let m2 = within("credit returned", a.next()).await.unwrap();
|
||||
assert_eq!(m2.payload()["n"], 2);
|
||||
let r5 = publisher
|
||||
.publish("session.demo.events", p(5), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r5.topic_sequence, 4);
|
||||
}
|
||||
|
||||
async fn latest_coalesces_only_undelivered_values(via: Via) {
|
||||
let e = env(via).await;
|
||||
let camera = e.client("camera").await;
|
||||
let viewer = e.client("viewer").await;
|
||||
camera
|
||||
.declare_topic("world.demo.frame", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = viewer
|
||||
.subscribe(
|
||||
"world.demo.frame",
|
||||
SubscriptionConfig::latest().in_flight(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let frames: Vec<_> = futures_join(&camera, 4).await;
|
||||
let r = camera
|
||||
.publish(
|
||||
"world.demo.frame",
|
||||
obj(json!({"n": 1})),
|
||||
&[("frame", &frames[0])],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.replaced, 0);
|
||||
let first = within("first frame", sub.next()).await.unwrap();
|
||||
let held = first.artifact("frame").unwrap();
|
||||
let mut replaced = 0;
|
||||
for (i, f) in frames.iter().enumerate().skip(1) {
|
||||
let r = camera
|
||||
.publish(
|
||||
"world.demo.frame",
|
||||
obj(json!({"n": i + 1})),
|
||||
&[("frame", f)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
replaced += r.replaced;
|
||||
}
|
||||
assert_eq!(
|
||||
replaced, 2,
|
||||
"frames 2 and 3 were replaced while undelivered"
|
||||
);
|
||||
drop(frames);
|
||||
// Replaced queue entries released their roots; the delivered frame and the queued one stay.
|
||||
e.settle("replaced frames collected", |s| s.sealed_artifacts == 2)
|
||||
.await;
|
||||
assert_eq!(
|
||||
held.read_all().await.unwrap(),
|
||||
b"frame-0",
|
||||
"delivered data is never reclaimed early"
|
||||
);
|
||||
drop((first, held));
|
||||
let last = within("latest frame", sub.next()).await.unwrap();
|
||||
assert_eq!((last.topic_sequence(), last.replaced()), (4, 2));
|
||||
assert_eq!(
|
||||
last.artifact("frame").unwrap().read_all().await.unwrap(),
|
||||
b"frame-3"
|
||||
);
|
||||
drop(last);
|
||||
e.settle("all collected", |s| s.artifacts == 0 && s.owners == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn futures_join(client: &flybus::Client, n: usize) -> Vec<flybus::Artifact> {
|
||||
let mut out = Vec::new();
|
||||
for i in 0..n {
|
||||
out.push(sealed(client, format!("frame-{i}").as_bytes(), "image/x-rgba").await);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
async fn credits_return_only_on_consume(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let reader = e.client("reader").await;
|
||||
publisher
|
||||
.declare_topic("t.credits", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut sub = reader
|
||||
.subscribe("t.credits", SubscriptionConfig::bounded().in_flight(2))
|
||||
.await
|
||||
.unwrap();
|
||||
for n in 0..5 {
|
||||
publisher
|
||||
.publish("t.credits", obj(json!({"n": n})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let m0 = within("0", sub.next()).await.unwrap();
|
||||
let m1 = within("1", sub.next()).await.unwrap();
|
||||
quiet("third while two are held", sub.next()).await;
|
||||
drop(m0);
|
||||
let m2 = within("2 after a consume", sub.next()).await.unwrap();
|
||||
assert_eq!(m2.payload()["n"], 2);
|
||||
quiet("fourth while two are held", sub.next()).await;
|
||||
drop((m1, m2));
|
||||
assert_eq!(within("3", sub.next()).await.unwrap().payload()["n"], 3);
|
||||
}
|
||||
|
||||
async fn retained_replay_clear_delete_and_incarnations(via: Via) {
|
||||
let e = env(via).await;
|
||||
let admin = e.client("admin").await;
|
||||
let reader = e.client("reader").await;
|
||||
let info = admin
|
||||
.declare_topic("session.demo.snapshots", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(info.declared);
|
||||
let again = admin
|
||||
.declare_topic("session.demo.snapshots", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(again.declared, &again.topic_incarnation),
|
||||
(false, &info.topic_incarnation)
|
||||
);
|
||||
let conflict = admin
|
||||
.declare_topic("session.demo.snapshots", Retained::None)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(conflict.code, ErrorCode::Conflict);
|
||||
|
||||
let snap = sealed(&admin, b"snapshot-1", "application/octet-stream").await;
|
||||
let r = admin
|
||||
.publish(
|
||||
"session.demo.snapshots",
|
||||
obj(json!({"step": "1"})),
|
||||
&[("state", &snap)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.subscribers, 0);
|
||||
drop(snap);
|
||||
e.settle("retention holds the only root", |s| {
|
||||
s.sealed_artifacts == 1 && s.artifact_roots == 1
|
||||
})
|
||||
.await;
|
||||
|
||||
let mut plain = reader
|
||||
.subscribe("session.demo.snapshots", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut replay = reader
|
||||
.subscribe(
|
||||
"session.demo.snapshots",
|
||||
SubscriptionConfig::latest().replay(true),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let m = within("replay", replay.next()).await.unwrap();
|
||||
assert_eq!(m.topic_sequence(), 1, "replay keeps the original sequence");
|
||||
assert_eq!(
|
||||
m.artifact("state").unwrap().read_all().await.unwrap(),
|
||||
b"snapshot-1"
|
||||
);
|
||||
quiet("no replay without replayLatest", plain.next()).await;
|
||||
|
||||
assert!(admin.clear_topic("session.demo.snapshots").await.unwrap());
|
||||
assert!(!admin.clear_topic("session.demo.snapshots").await.unwrap());
|
||||
// Clearing does not invalidate the delivery still held.
|
||||
assert_eq!(
|
||||
m.artifact("state").unwrap().read_all().await.unwrap(),
|
||||
b"snapshot-1"
|
||||
);
|
||||
drop(m);
|
||||
e.settle("cleared value collected", |s| s.artifacts == 0)
|
||||
.await;
|
||||
|
||||
let busy = admin
|
||||
.delete_topic("session.demo.snapshots")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(busy.code, ErrorCode::Conflict);
|
||||
admin
|
||||
.publish("session.demo.snapshots", obj(json!({"step": "2"})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let old = within("before delete", plain.next()).await.unwrap();
|
||||
drop((plain, replay));
|
||||
e.settle("unsubscribed", |s| s.subscriptions == 0).await;
|
||||
assert!(admin.delete_topic("session.demo.snapshots").await.unwrap());
|
||||
assert!(!admin.delete_topic("session.demo.snapshots").await.unwrap());
|
||||
let fresh = admin
|
||||
.declare_topic("session.demo.snapshots", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(fresh.topic_incarnation, info.topic_incarnation);
|
||||
assert_eq!(
|
||||
old.topic_incarnation(),
|
||||
info.topic_incarnation,
|
||||
"old deliveries keep their incarnation"
|
||||
);
|
||||
let r = admin
|
||||
.publish("session.demo.snapshots", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r.topic_sequence, 1,
|
||||
"a fresh incarnation restarts its sequence"
|
||||
);
|
||||
}
|
||||
|
||||
async fn zero_subscriber_publish_retains_nothing(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
p.declare_topic("t.void", Retained::None).await.unwrap();
|
||||
let art = sealed(&p, &[7u8; 4096], "application/octet-stream").await;
|
||||
let r = p
|
||||
.publish("t.void", obj(json!({})), &[("x", &art)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.subscribers, 0);
|
||||
drop(art);
|
||||
e.settle("no owner left", |s| s.artifacts == 0 && s.store_bytes == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn unsubscribe_discards_queue_but_not_deliveries(via: Via) {
|
||||
let e = env(via).await;
|
||||
let p = e.client("p").await;
|
||||
let r = e.client("r").await;
|
||||
p.declare_topic("t.unsub", Retained::None).await.unwrap();
|
||||
let mut sub = r
|
||||
.subscribe("t.unsub", SubscriptionConfig::bounded().in_flight(1))
|
||||
.await
|
||||
.unwrap();
|
||||
for i in 0..3 {
|
||||
let a = sealed(&p, format!("m{i}").as_bytes(), "text/plain").await;
|
||||
p.publish("t.unsub", obj(json!({})), &[("m", &a)])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let first = within("first", sub.next()).await.unwrap();
|
||||
e.settle("three objects", |s| s.sealed_artifacts == 3).await;
|
||||
drop(sub);
|
||||
e.settle("queued entries released", |s| {
|
||||
s.sealed_artifacts == 1 && s.subscriptions == 0
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
first.artifact("m").unwrap().read_all().await.unwrap(),
|
||||
b"m0"
|
||||
);
|
||||
drop(first);
|
||||
e.settle("delivered entry released", |s| s.artifacts == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn subscription_and_topic_validation(via: Via) {
|
||||
let e = env(via).await;
|
||||
let c = e.client("c").await;
|
||||
let missing = c
|
||||
.subscribe("t.missing", SubscriptionConfig::latest())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(missing.code, ErrorCode::NoTopic);
|
||||
assert_eq!(
|
||||
c.publish("t.missing", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::NoTopic
|
||||
);
|
||||
c.declare_topic("t.ok", Retained::None).await.unwrap();
|
||||
let bad = SubscriptionConfig {
|
||||
mode: Mode::Latest,
|
||||
max_queued: 2,
|
||||
max_in_flight: 1,
|
||||
replay_latest: false,
|
||||
};
|
||||
assert_eq!(
|
||||
c.subscribe("t.ok", bad).await.unwrap_err().code,
|
||||
ErrorCode::InvalidEnvelope
|
||||
);
|
||||
let over = SubscriptionConfig::latest().in_flight(3);
|
||||
assert_eq!(
|
||||
c.subscribe("t.ok", over).await.unwrap_err().code,
|
||||
ErrorCode::QuotaExceeded
|
||||
);
|
||||
let zero = SubscriptionConfig::bounded().in_flight(0);
|
||||
assert_eq!(
|
||||
c.subscribe("t.ok", zero).await.unwrap_err().code,
|
||||
ErrorCode::InvalidEnvelope
|
||||
);
|
||||
assert_eq!(
|
||||
c.declare_topic("t..bad", Retained::None)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::InvalidEnvelope
|
||||
);
|
||||
assert_eq!(
|
||||
c.declare_topic("T.upper", Retained::None)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::InvalidEnvelope
|
||||
);
|
||||
// The connection survives every refusal above.
|
||||
assert!(
|
||||
c.subscribe("t.ok", SubscriptionConfig::bounded())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
async fn topic_and_retention_quotas(via: Via) {
|
||||
let limits = Limits {
|
||||
max_topics: 2,
|
||||
max_retained_bytes: 100,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(via, limits, Policy::open()).await;
|
||||
let c = e.client("c").await;
|
||||
c.declare_topic("t.a", Retained::Latest).await.unwrap();
|
||||
c.declare_topic("t.b", Retained::Latest).await.unwrap();
|
||||
assert_eq!(
|
||||
c.declare_topic("t.c", Retained::None)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::QuotaExceeded
|
||||
);
|
||||
let small = sealed(&c, &[1u8; 60], "application/octet-stream").await;
|
||||
let big = sealed(&c, &[2u8; 50], "application/octet-stream").await;
|
||||
c.publish("t.a", obj(json!({})), &[("x", &small)])
|
||||
.await
|
||||
.unwrap();
|
||||
// Two names for one object count once.
|
||||
c.publish("t.a", obj(json!({})), &[("x", &small), ("y", &small)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(e.stats().retained_bytes, 60);
|
||||
let over = c
|
||||
.publish("t.b", obj(json!({})), &[("x", &big)])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(over.code, ErrorCode::QuotaExceeded);
|
||||
assert_eq!(
|
||||
e.stats().retained_bytes,
|
||||
60,
|
||||
"a refused publication changes nothing"
|
||||
);
|
||||
// Replacing a topic's own retained value is measured net of the old one.
|
||||
c.publish("t.a", obj(json!({})), &[("x", &big)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(e.stats().retained_bytes, 50);
|
||||
}
|
||||
|
||||
/// A subscriber that never reads its socket must not slow anyone else: publications are
|
||||
/// admitted until its queue refuses them, and RPC and control traffic between other clients
|
||||
/// (and to a client that merely holds its credits) keep flowing.
|
||||
async fn saturated_subscriber_does_not_block_control(via: Via) {
|
||||
let e = env(via).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let busy = e.client("busy").await;
|
||||
let caller = e.client("caller").await;
|
||||
publisher
|
||||
.declare_topic("t.flood", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 1. A raw subscriber that never reads.
|
||||
let mut stuck = e.raw_hello("stuck").await;
|
||||
let r = stuck
|
||||
.call("subscribe", json!({"topic": "t.flood", "mode": "bounded", "maxQueued": 64, "maxInFlight": 16, "replayLatest": false}))
|
||||
.await;
|
||||
assert_eq!(code(&r), "OK");
|
||||
// 2. An SDK subscriber that reads but never consumes, and also serves RPC.
|
||||
let _held = busy
|
||||
.subscribe("t.flood", SubscriptionConfig::bounded().in_flight(16))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut svc = busy
|
||||
.register("busy.status", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
while let Some(req) = svc.next().await {
|
||||
req.reply(obj(json!({"alive": true})), &[]).await.unwrap();
|
||||
}
|
||||
});
|
||||
|
||||
let blob = "x".repeat(60_000);
|
||||
let mut accepted = 0;
|
||||
let refused = loop {
|
||||
match within(
|
||||
"publish",
|
||||
publisher.publish("t.flood", obj(json!({"blob": blob})), &[]),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => accepted += 1,
|
||||
Err(err) => break err,
|
||||
}
|
||||
assert!(accepted < 1000, "publications were never refused");
|
||||
};
|
||||
assert_eq!(refused.code, ErrorCode::Backpressure);
|
||||
assert!(accepted >= 16, "only {accepted} publications were admitted");
|
||||
for _ in 0..20 {
|
||||
let res = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
caller.call_and_wait("busy.status", None, "Status", obj(json!({})), &[]),
|
||||
)
|
||||
.await
|
||||
.expect("RPC to a client whose subscription is saturated")
|
||||
.unwrap();
|
||||
assert_eq!(res.outcome()["alive"], true);
|
||||
}
|
||||
// The saturated subscriber's own control lane still answers once it reads again.
|
||||
let id = stuck
|
||||
.command(
|
||||
"topic.declare",
|
||||
json!({"name": "t.other", "retained": "none"}),
|
||||
json!([]),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&stuck.reply(&id).await), "OK");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
async fn router_shutdown_closes_subscriptions_with_notices(via: Via) {
|
||||
let e = env(via).await;
|
||||
let admin = e.client("admin").await;
|
||||
admin.declare_topic("t.stop", Retained::None).await.unwrap();
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
let sub = raw
|
||||
.call("subscribe", json!({"topic": "t.stop", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}))
|
||||
.await
|
||||
.unwrap();
|
||||
e.router.shutdown();
|
||||
let closed = raw.event().await;
|
||||
assert_eq!(
|
||||
(closed.op.as_str(), &closed.body["subscriptionId"]),
|
||||
("subscription.closed", &sub["subscriptionId"])
|
||||
);
|
||||
assert_eq!(closed.body["reason"], "router-stopping");
|
||||
let last = raw.event().await;
|
||||
assert_eq!(
|
||||
(last.op.as_str(), last.body["code"].as_str()),
|
||||
("connection.closing", Some("ROUTER_LOST"))
|
||||
);
|
||||
assert!(raw.recv().await.is_none());
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
router_shutdown_closes_subscriptions_with_notices,
|
||||
bounded_fifo_and_atomic_backpressure,
|
||||
latest_coalesces_only_undelivered_values,
|
||||
credits_return_only_on_consume,
|
||||
retained_replay_clear_delete_and_incarnations,
|
||||
zero_subscriber_publish_retains_nothing,
|
||||
unsubscribe_discards_queue_but_not_deliveries,
|
||||
subscription_and_topic_validation,
|
||||
topic_and_retention_quotas,
|
||||
saturated_subscriber_does_not_block_control,
|
||||
);
|
||||
660
services/flysim/crates/flybus/tests/rpc.rs
Normal file
660
services/flysim/crates/flybus/tests/rpc.rs
Normal file
|
|
@ -0,0 +1,660 @@
|
|||
//! RPC: exclusive registration, pinned incarnations, request/reply, FIFO, bounds,
|
||||
//! cancellation, disconnects and an endpoint-side result cache. Every test runs over both the
|
||||
//! in-memory transport and a Unix socket.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Write;
|
||||
|
||||
use common::{Via, code, env, env_with, obj, quiet, sealed, within};
|
||||
use flybus::{CancelState, Dispatch, ErrorCode, Grants, Limits, Pattern, Policy, ServiceConfig};
|
||||
use serde_json::json;
|
||||
|
||||
async fn request_reply_roundtrip(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("example.counter", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = caller
|
||||
.call(
|
||||
"example.counter",
|
||||
Some(svc.incarnation()),
|
||||
"Counter.Increment",
|
||||
obj(json!({"amount": 1})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pending.service_incarnation(), svc.incarnation());
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
assert_eq!(req.method(), "Counter.Increment");
|
||||
assert_eq!(req.payload()["amount"], 1);
|
||||
assert_eq!(req.target(), "example.counter");
|
||||
// The router, not the body, says who called.
|
||||
assert_eq!(req.caller(), &caller.info().identity);
|
||||
assert!(req.reply(obj(json!({"value": 1})), &[]).await.unwrap());
|
||||
drop(req);
|
||||
let res = within("result", pending.result()).await.unwrap();
|
||||
assert_eq!(res.outcome()["value"], 1);
|
||||
assert_eq!(res.responder(), &server.info().identity);
|
||||
assert_eq!(res.call_id(), pending.call_id());
|
||||
drop(res);
|
||||
e.settle("everything consumed", |s| s.calls == 0 && s.owners == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn registration_is_exclusive_and_pinned(via: Via) {
|
||||
let e = env(via).await;
|
||||
let a = e.client("a").await;
|
||||
let b = e.client("b").await;
|
||||
let caller = e.client("caller").await;
|
||||
let first = a
|
||||
.register("agent.fly-a", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let dup = b
|
||||
.register("agent.fly-a", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(dup.code, ErrorCode::Conflict);
|
||||
let old = first.incarnation().to_owned();
|
||||
drop(first);
|
||||
// Unregistration is asynchronous; wait for the name to come free.
|
||||
let second = loop {
|
||||
match b.register("agent.fly-a", ServiceConfig::default()).await {
|
||||
Ok(s) => break s,
|
||||
Err(err) => assert_eq!(err.code, ErrorCode::Conflict),
|
||||
}
|
||||
};
|
||||
assert_ne!(second.incarnation(), old);
|
||||
let changed = caller
|
||||
.call(
|
||||
"agent.fly-a",
|
||||
Some(&old),
|
||||
"Agent.Prepare",
|
||||
obj(json!({})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(changed.code, ErrorCode::TargetChanged);
|
||||
assert_eq!(changed.dispatch, Dispatch::NotDispatched);
|
||||
let missing = caller
|
||||
.call("agent.nobody", None, "Agent.Prepare", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(missing.code, ErrorCode::NoService);
|
||||
let over = b
|
||||
.register(
|
||||
"agent.big",
|
||||
ServiceConfig {
|
||||
max_queued: 17,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(over.code, ErrorCode::QuotaExceeded);
|
||||
}
|
||||
|
||||
async fn authority_is_enforced(via: Via) {
|
||||
let policy = Policy::closed()
|
||||
.client(
|
||||
"server",
|
||||
Grants {
|
||||
register: vec![Pattern::exact("a.svc")],
|
||||
..Grants::default()
|
||||
},
|
||||
)
|
||||
.client(
|
||||
"caller",
|
||||
Grants {
|
||||
call: vec![Pattern::prefix("a.")],
|
||||
..Grants::default()
|
||||
},
|
||||
);
|
||||
let e = env_with(via, Limits::default(), policy).await;
|
||||
assert_eq!(
|
||||
e.try_client("stranger").await.unwrap_err().code,
|
||||
ErrorCode::NotAuthorized
|
||||
);
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
// One live connection per configured identity.
|
||||
assert_eq!(
|
||||
e.try_client("caller").await.unwrap_err().code,
|
||||
ErrorCode::NotAuthorized
|
||||
);
|
||||
assert_eq!(
|
||||
server
|
||||
.register("b.svc", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::NotAuthorized
|
||||
);
|
||||
let _svc = server
|
||||
.register("a.svc", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
caller
|
||||
.register("a.other", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::NotAuthorized
|
||||
);
|
||||
assert_eq!(
|
||||
caller
|
||||
.call("b.svc", None, "M", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::NotAuthorized
|
||||
);
|
||||
assert_eq!(
|
||||
caller
|
||||
.declare_topic("a.t", flybus::Retained::None)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
ErrorCode::NotAuthorized
|
||||
);
|
||||
assert!(
|
||||
caller
|
||||
.call("a.svc", None, "M", obj(json!({})), &[])
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
// A reconnect must present a new incarnation.
|
||||
let mut cfg = e.config("server");
|
||||
cfg.client_incarnation = Some(server.info().identity.client_incarnation.clone());
|
||||
drop(_svc);
|
||||
server.close().await;
|
||||
e.settle("server gone", |s| s.connections == 1).await;
|
||||
let reused = flybus::Client::connect(e.transport_as("server").await, cfg)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(reused.code, ErrorCode::NotAuthorized);
|
||||
assert!(e.try_client("server").await.is_ok());
|
||||
}
|
||||
|
||||
async fn fifo_dispatch_and_out_of_order_completion(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("example.echo", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut calls = Vec::new();
|
||||
for i in 0..6 {
|
||||
calls.push(
|
||||
caller
|
||||
.call("example.echo", None, "Echo", obj(json!({"i": i})), &[])
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let mut reqs = Vec::new();
|
||||
for i in 0..6 {
|
||||
let r = within("request", svc.next()).await.unwrap();
|
||||
assert_eq!(r.payload()["i"], i, "first dispatch is FIFO per caller");
|
||||
reqs.push(r);
|
||||
}
|
||||
for r in reqs.iter().rev() {
|
||||
assert!(
|
||||
r.reply(obj(json!({"echo": r.payload()["i"]})), &[])
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
drop(reqs);
|
||||
for (i, mut c) in calls.into_iter().enumerate() {
|
||||
let res = within("result", c.result()).await.unwrap();
|
||||
assert_eq!(res.outcome()["echo"], i, "results correlate by call id");
|
||||
}
|
||||
}
|
||||
|
||||
async fn service_queue_backpressure(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"example.slow",
|
||||
ServiceConfig {
|
||||
max_queued: 1,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _c1 = caller
|
||||
.call("example.slow", None, "Work", obj(json!({"n": 1})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let held = within("first request", svc.next()).await.unwrap();
|
||||
let _c2 = caller
|
||||
.call("example.slow", None, "Work", obj(json!({"n": 2})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let full = caller
|
||||
.call("example.slow", None, "Work", obj(json!({"n": 3})), &[])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(full.code, ErrorCode::Backpressure);
|
||||
assert_eq!(full.dispatch, Dispatch::NotDispatched);
|
||||
// In-flight credit returns only when the request delivery is consumed.
|
||||
quiet("second request while the first is held", svc.next()).await;
|
||||
drop(held);
|
||||
let second = within("second request", svc.next()).await.unwrap();
|
||||
assert_eq!(second.payload()["n"], 2);
|
||||
}
|
||||
|
||||
async fn cancellation_states(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"example.worker",
|
||||
ServiceConfig {
|
||||
max_queued: 4,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut dispatched = caller
|
||||
.call("example.worker", None, "Work", obj(json!({"n": 1})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req1 = within("request 1", svc.next()).await.unwrap();
|
||||
let mut queued = caller
|
||||
.call("example.worker", None, "Work", obj(json!({"n": 2})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
queued.cancel().await.unwrap(),
|
||||
CancelState::CancelledBeforeDispatch
|
||||
);
|
||||
let gone = within("cancelled result", queued.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(gone.code, gone.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::NotDispatched)
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
dispatched.cancel().await.unwrap(),
|
||||
CancelState::ExecutionUnknown
|
||||
);
|
||||
let unknown = within("detached result", dispatched.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(unknown.code, unknown.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Unknown)
|
||||
);
|
||||
// The handler still finishes; its reply reaches nobody and is not an error.
|
||||
assert!(!req1.reply(obj(json!({"late": true})), &[]).await.unwrap());
|
||||
drop(req1);
|
||||
|
||||
let mut done = caller
|
||||
.call("example.worker", None, "Work", obj(json!({"n": 3})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req3 = within("request 3", svc.next()).await.unwrap();
|
||||
assert_eq!(
|
||||
req3.payload()["n"],
|
||||
3,
|
||||
"the cancelled call was never dispatched"
|
||||
);
|
||||
assert!(req3.reply(obj(json!({"ok": 3})), &[]).await.unwrap());
|
||||
// Wait for the result to be admitted before cancelling.
|
||||
e.settle("result admitted", |s| s.calls == 1).await;
|
||||
let state = done.cancel().await.unwrap();
|
||||
assert_eq!(state, CancelState::Completed);
|
||||
let res = within("completed result", done.result()).await.unwrap();
|
||||
assert_eq!(res.outcome()["ok"], 3);
|
||||
drop((res, req3));
|
||||
e.settle("calls retired", |s| s.calls == 0).await;
|
||||
assert_eq!(done.cancel().await.unwrap(), CancelState::CallGone);
|
||||
}
|
||||
|
||||
async fn replies_are_single_and_independent_of_the_request_guard(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("example.once", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = caller
|
||||
.call("example.once", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
let responder = req.responder();
|
||||
drop(req); // consumed before replying
|
||||
assert!(
|
||||
responder
|
||||
.reply(obj(json!({"first": true})), &[])
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let again = responder
|
||||
.reply(obj(json!({"second": true})), &[])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(again.code, ErrorCode::CallGone);
|
||||
let res = within("result", pending.result()).await.unwrap();
|
||||
assert_eq!(res.outcome()["first"], true);
|
||||
}
|
||||
|
||||
async fn service_disconnect_fails_calls(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"example.fragile",
|
||||
ServiceConfig {
|
||||
max_queued: 4,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut c1 = caller
|
||||
.call("example.fragile", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let held = within("request", svc.next()).await.unwrap();
|
||||
let mut c2 = caller
|
||||
.call("example.fragile", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
// Keep the first delivery credit occupied until unregister has synchronously failed the
|
||||
// queued call. Otherwise consuming it may truthfully dispatch call 2 before unregister.
|
||||
drop(svc);
|
||||
let r2 = within("queued call", c2.result()).await.unwrap_err();
|
||||
assert_eq!(
|
||||
(r2.code, r2.dispatch),
|
||||
(ErrorCode::NoService, Dispatch::NotDispatched)
|
||||
);
|
||||
drop(held);
|
||||
server.close().await;
|
||||
let r1 = within("dispatched call", c1.result()).await.unwrap_err();
|
||||
assert_eq!(
|
||||
(r1.code, r1.dispatch),
|
||||
(ErrorCode::NoService, Dispatch::Dispatched)
|
||||
);
|
||||
e.settle("nothing left", |s| s.calls == 0 && s.services == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn unregister_fails_queued_but_dispatched_may_reply(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register(
|
||||
"example.leaving",
|
||||
ServiceConfig {
|
||||
max_queued: 4,
|
||||
max_in_flight: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut c1 = caller
|
||||
.call("example.leaving", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
let mut c2 = caller
|
||||
.call("example.leaving", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(svc);
|
||||
let r2 = within("queued call", c2.result()).await.unwrap_err();
|
||||
assert_eq!(
|
||||
(r2.code, r2.dispatch),
|
||||
(ErrorCode::NoService, Dispatch::NotDispatched)
|
||||
);
|
||||
assert!(req.reply(obj(json!({"done": true})), &[]).await.unwrap());
|
||||
assert_eq!(
|
||||
within("dispatched call", c1.result())
|
||||
.await
|
||||
.unwrap()
|
||||
.outcome()["done"],
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
async fn raw_call_ids_and_forged_replies(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let mut svc = server
|
||||
.register("example.raw", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut raw = e.raw_hello("rawcaller").await;
|
||||
let call = |id: &str| json!({"callId": id, "target": "example.raw", "expectedIncarnation": null, "method": "M", "payload": {}});
|
||||
assert_eq!(code(&raw.call("rpc.call", call("call-5")).await), "OK");
|
||||
assert_eq!(
|
||||
code(&raw.call("rpc.call", call("call-5")).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
assert_eq!(
|
||||
code(&raw.call("rpc.call", call("call-3")).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
assert_eq!(
|
||||
code(&raw.call("rpc.call", call("call-05")).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
// Identity comes from the connection; a body cannot claim one.
|
||||
let mut forged = call("call-6");
|
||||
forged["caller"] = json!({"clientId": "server", "clientIncarnation": "x"});
|
||||
assert_eq!(
|
||||
code(&raw.call("rpc.call", forged).await),
|
||||
"INVALID_ENVELOPE"
|
||||
);
|
||||
assert_eq!(code(&raw.call("rpc.call", call("call-6")).await), "OK");
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
assert_eq!(req.caller().client_id, "rawcaller");
|
||||
// A third party cannot answer someone else's request.
|
||||
let mut other = e.raw_hello("intruder").await;
|
||||
let r = other
|
||||
.call(
|
||||
"rpc.reply",
|
||||
json!({"callId": req.call_id(), "requestDeliveryId": req.delivery_id(), "outcome": {}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&r), "OWNER_INVALID");
|
||||
assert!(req.reply(obj(json!({"real": true})), &[]).await.unwrap());
|
||||
let result = raw.event().await;
|
||||
assert_eq!(result.op, "rpc.result");
|
||||
assert_eq!(result.body["outcome"]["real"], true);
|
||||
assert_eq!(result.body["responder"]["clientId"], "server");
|
||||
}
|
||||
|
||||
/// bus-v1 section 6: an endpoint caches Artifact handles plus payload; a domain retry with a
|
||||
/// fresh call id gets fresh delivery ownership over the same immutable bytes.
|
||||
async fn endpoint_cache_replays_artifact_results(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("agent.cached", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let service = tokio::spawn(async move {
|
||||
let mut cache: HashMap<String, flybus::Artifact> = HashMap::new();
|
||||
let mut executions = 0;
|
||||
while let Some(req) = svc.next().await {
|
||||
if req.method() == "Cache.Evict" {
|
||||
cache.clear();
|
||||
req.reply(obj(json!({})), &[]).await.unwrap();
|
||||
continue;
|
||||
}
|
||||
let rid = req.payload()["requestId"].as_str().unwrap().to_owned();
|
||||
if !cache.contains_key(&rid) {
|
||||
executions += 1;
|
||||
let bytes = format!("result of {rid}").into_bytes();
|
||||
let mut w = server
|
||||
.artifacts()
|
||||
.allocate(bytes.len() as u64, "text/plain")
|
||||
.await
|
||||
.unwrap();
|
||||
w.write_all(&bytes).unwrap();
|
||||
cache.insert(rid.clone(), w.seal().await.unwrap());
|
||||
}
|
||||
let art = cache[&rid].clone();
|
||||
req.reply(obj(json!({"executions": executions})), &[("state", &art)])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
let mut ids = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let res = within(
|
||||
"result",
|
||||
caller.call_and_wait(
|
||||
"agent.cached",
|
||||
None,
|
||||
"Agent.Prepare",
|
||||
obj(json!({"requestId": "req-41"})),
|
||||
&[],
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
res.outcome()["executions"],
|
||||
1,
|
||||
"the retry did not re-execute"
|
||||
);
|
||||
let art = res.artifact("state").unwrap();
|
||||
assert_eq!(art.read_all().await.unwrap(), b"result of req-41");
|
||||
ids.push((
|
||||
res.delivery_id().to_owned(),
|
||||
art.reference().artifact_id.clone(),
|
||||
));
|
||||
}
|
||||
assert_ne!(ids[0].0, ids[1].0, "each replay is a fresh delivery");
|
||||
assert_eq!(ids[0].1, ids[1].1, "of the same immutable object");
|
||||
e.settle("cache holds the only root", |s| {
|
||||
s.sealed_artifacts == 1 && s.artifact_roots == 1
|
||||
})
|
||||
.await;
|
||||
caller
|
||||
.call_and_wait("agent.cached", None, "Cache.Evict", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
e.settle("eviction collects", |s| {
|
||||
s.artifacts == 0 && s.store_bytes == 0
|
||||
})
|
||||
.await;
|
||||
e.settle_files("sealed", 0).await;
|
||||
service.abort();
|
||||
}
|
||||
|
||||
async fn dropped_call_is_cancelled_and_late_result_consumed(via: Via) {
|
||||
let e = env(via).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut svc = server
|
||||
.register("example.abandon", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let art = sealed(&server, b"payload", "text/plain").await;
|
||||
let pending = caller
|
||||
.call("example.abandon", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
drop(pending);
|
||||
e.settle("call detached", |s| s.calls == 1 && s.active_calls == 0)
|
||||
.await;
|
||||
// Detached: the reply is not routed and creates no caller-side roots.
|
||||
let routed = req.reply(obj(json!({})), &[("a", &art)]).await.unwrap();
|
||||
assert!(!routed);
|
||||
drop((req, art));
|
||||
e.settle("nothing retained", |s| {
|
||||
s.calls == 0 && s.artifacts == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
|
||||
// A result that arrives after its caller stopped waiting is consumed by the reactor.
|
||||
let pending = caller
|
||||
.call("example.abandon", None, "Do", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let req = within("request", svc.next()).await.unwrap();
|
||||
let art = sealed(&server, b"late", "text/plain").await;
|
||||
req.reply(obj(json!({})), &[("a", &art)]).await.unwrap();
|
||||
drop((req, art));
|
||||
e.settle("result delivered", |s| s.calls == 1 && s.owners == 1)
|
||||
.await;
|
||||
drop(pending);
|
||||
e.settle("result consumed", |s| {
|
||||
s.calls == 0 && s.artifacts == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
assert_eq!(caller.control_errors(), 0);
|
||||
}
|
||||
|
||||
async fn active_call_limit(via: Via) {
|
||||
let limits = Limits {
|
||||
max_active_calls_per_client: 2,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(via, limits, Policy::open()).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let _svc = server
|
||||
.register("example.limit", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let _a = caller
|
||||
.call("example.limit", None, "M", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let _b = caller
|
||||
.call("example.limit", None, "M", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let c = caller
|
||||
.call("example.limit", None, "M", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(c.code, ErrorCode::Backpressure);
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
request_reply_roundtrip,
|
||||
registration_is_exclusive_and_pinned,
|
||||
authority_is_enforced,
|
||||
fifo_dispatch_and_out_of_order_completion,
|
||||
service_queue_backpressure,
|
||||
cancellation_states,
|
||||
replies_are_single_and_independent_of_the_request_guard,
|
||||
service_disconnect_fails_calls,
|
||||
unregister_fails_queued_but_dispatched_may_reply,
|
||||
raw_call_ids_and_forged_replies,
|
||||
endpoint_cache_replays_artifact_results,
|
||||
dropped_call_is_cancelled_and_late_result_consumed,
|
||||
active_call_limit,
|
||||
);
|
||||
876
services/flysim/crates/flybus/tests/sol_rereview_regressions.rs
Normal file
876
services/flysim/crates/flybus/tests/sol_rereview_regressions.rs
Normal file
|
|
@ -0,0 +1,876 @@
|
|||
mod common;
|
||||
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::task::{Context, Poll, Waker};
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{Raw, Via, env, env_with, obj, sealed, within};
|
||||
use flybus::wire::parse_serial_id;
|
||||
use flybus::{
|
||||
Client, ClientConfig, Dispatch, ErrorCode, Limits, Policy, Retained, Router, RouterConfig,
|
||||
ServiceConfig, SubscriptionConfig, Transport,
|
||||
};
|
||||
use serde_json::json;
|
||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct WriteProbe<S> {
|
||||
inner: S,
|
||||
state: Arc<Mutex<ProbeState>>,
|
||||
first: Arc<Notify>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ProbeState {
|
||||
armed: bool,
|
||||
released: bool,
|
||||
first_written: bool,
|
||||
writer: Option<Waker>,
|
||||
bytes: Vec<u8>,
|
||||
ops: Vec<String>,
|
||||
}
|
||||
|
||||
impl<S> WriteProbe<S> {
|
||||
fn new(inner: S) -> (WriteProbe<S>, WriteProbeHandle) {
|
||||
let state = Arc::new(Mutex::new(ProbeState::default()));
|
||||
let first = Arc::new(Notify::new());
|
||||
(
|
||||
WriteProbe {
|
||||
inner,
|
||||
state: state.clone(),
|
||||
first: first.clone(),
|
||||
},
|
||||
WriteProbeHandle { state, first },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct WriteProbeHandle {
|
||||
state: Arc<Mutex<ProbeState>>,
|
||||
first: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl WriteProbeHandle {
|
||||
fn arm(&self) {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.armed = true;
|
||||
state.released = false;
|
||||
state.first_written = false;
|
||||
state.writer = None;
|
||||
state.bytes.clear();
|
||||
state.ops.clear();
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.released = true;
|
||||
if let Some(waker) = state.writer.take() {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_first(&self) {
|
||||
loop {
|
||||
let notified = self.first.notified();
|
||||
if self.state.lock().unwrap().first_written {
|
||||
return;
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(2), notified)
|
||||
.await
|
||||
.expect("router never wrote the first byte");
|
||||
}
|
||||
}
|
||||
|
||||
fn ops(&self) -> Vec<String> {
|
||||
self.state.lock().unwrap().ops.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncRead + Unpin> AsyncRead for WriteProbe<S> {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncWrite + Unpin> AsyncWrite for WriteProbe<S> {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if state.armed && !state.released && state.first_written {
|
||||
state.writer = Some(cx.waker().clone());
|
||||
return Poll::Pending;
|
||||
}
|
||||
}
|
||||
let limit = if self.state.lock().unwrap().armed {
|
||||
1
|
||||
} else {
|
||||
buf.len()
|
||||
};
|
||||
match Pin::new(&mut self.inner).poll_write(cx, &buf[..limit]) {
|
||||
Poll::Ready(Ok(n)) => {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.bytes.extend_from_slice(&buf[..n]);
|
||||
while state.bytes.len() >= 4 {
|
||||
let len = u32::from_le_bytes(state.bytes[..4].try_into().unwrap()) as usize;
|
||||
if state.bytes.len() < len + 4 {
|
||||
break;
|
||||
}
|
||||
let frame = state.bytes.drain(..len + 4).collect::<Vec<_>>();
|
||||
let envelope = flybus::wire::Envelope::decode(&frame[4..]).unwrap();
|
||||
state.ops.push(envelope.op);
|
||||
}
|
||||
if state.armed && !state.first_written && n > 0 {
|
||||
state.first_written = true;
|
||||
self.first.notify_waiters();
|
||||
}
|
||||
Poll::Ready(Ok(n))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
struct PollHold<S> {
|
||||
inner: S,
|
||||
state: Arc<(Mutex<PollHoldState>, Condvar)>,
|
||||
entered: Arc<Notify>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct PollHoldState {
|
||||
armed: bool,
|
||||
entered: bool,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PollHoldHandle {
|
||||
state: Arc<(Mutex<PollHoldState>, Condvar)>,
|
||||
entered: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl<S> PollHold<S> {
|
||||
fn new(inner: S) -> (PollHold<S>, PollHoldHandle) {
|
||||
let state = Arc::new((Mutex::new(PollHoldState::default()), Condvar::new()));
|
||||
let entered = Arc::new(Notify::new());
|
||||
(
|
||||
PollHold {
|
||||
inner,
|
||||
state: state.clone(),
|
||||
entered: entered.clone(),
|
||||
},
|
||||
PollHoldHandle { state, entered },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl PollHoldHandle {
|
||||
fn arm(&self) {
|
||||
self.state.0.lock().unwrap().armed = true;
|
||||
}
|
||||
|
||||
async fn wait_entered(&self) {
|
||||
loop {
|
||||
let notified = self.entered.notified();
|
||||
if self.state.0.lock().unwrap().entered {
|
||||
return;
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(2), notified)
|
||||
.await
|
||||
.expect("transport poll_write was never entered");
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
let mut state = self.state.0.lock().unwrap();
|
||||
state.released = true;
|
||||
self.state.1.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncRead + Unpin> AsyncRead for PollHold<S> {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: AsyncWrite + Unpin> AsyncWrite for PollHold<S> {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
let should_hold = {
|
||||
let mut state = self.state.0.lock().unwrap();
|
||||
if state.armed && !state.entered {
|
||||
state.entered = true;
|
||||
self.entered.notify_waiters();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_hold {
|
||||
let mut state = self.state.0.lock().unwrap();
|
||||
while !state.released {
|
||||
state = self.state.1.wait(state).unwrap();
|
||||
}
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_write(cx, &buf[..1])
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_single_delivery_owner(router: &Router) {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
|
||||
loop {
|
||||
let stats = router.stats();
|
||||
if stats.owners == 1 && stats.artifact_roots == 1 {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"source hold was not released before teardown: {stats:?}"
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_last_attached_responder_settles_the_call() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.abandoned", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = caller
|
||||
.call("example.abandoned", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let request = within("request", service.next()).await.unwrap();
|
||||
drop(request);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let stats = e.stats();
|
||||
assert_eq!(stats.reply_capabilities, 0);
|
||||
assert_eq!(stats.owners, 0);
|
||||
assert_eq!(stats.calls, 0, "the last reply capability was released");
|
||||
assert_eq!(stats.active_calls, 0, "the caller slot was not retired");
|
||||
let error = tokio::time::timeout(Duration::from_millis(250), pending.result())
|
||||
.await
|
||||
.expect("the caller was left waiting after all reply authority was gone")
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(error.code, error.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Dispatched)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_last_attached_responder_releases_request_attachments() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.abandoned-artifact", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let artifact = sealed(&caller, b"request", "application/octet-stream").await;
|
||||
let mut pending = caller
|
||||
.call(
|
||||
"example.abandoned-artifact",
|
||||
None,
|
||||
"Work",
|
||||
obj(json!({})),
|
||||
&[("data", &artifact)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(artifact);
|
||||
|
||||
let request = within("artifact request", service.next()).await.unwrap();
|
||||
drop(request);
|
||||
let error = within("abandoned artifact call", pending.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(error.code, error.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Dispatched)
|
||||
);
|
||||
e.settle("abandoned artifact released", |stats| {
|
||||
stats.calls == 0
|
||||
&& stats.active_calls == 0
|
||||
&& stats.reply_capabilities == 0
|
||||
&& stats.owners == 0
|
||||
&& stats.artifacts == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_last_responder_clone_settles_the_call() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.abandoned-clone", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = caller
|
||||
.call("example.abandoned-clone", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let request = within("clone request", service.next()).await.unwrap();
|
||||
let responder = request.responder();
|
||||
drop(request);
|
||||
e.settle("clone retains reply capability", |stats| {
|
||||
stats.calls == 1
|
||||
&& stats.active_calls == 1
|
||||
&& stats.reply_capabilities == 1
|
||||
&& stats.owners == 0
|
||||
})
|
||||
.await;
|
||||
drop(responder);
|
||||
let error = within("clone release failure", pending.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(error.code, error.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Dispatched)
|
||||
);
|
||||
e.settle("clone release retired call", |stats| {
|
||||
stats.calls == 0 && stats.active_calls == 0 && stats.reply_capabilities == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_last_responder_clone_with_attachment_settles_the_call() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.abandoned-clone-artifact", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let artifact = sealed(&caller, b"request", "application/octet-stream").await;
|
||||
let mut pending = caller
|
||||
.call(
|
||||
"example.abandoned-clone-artifact",
|
||||
None,
|
||||
"Work",
|
||||
obj(json!({})),
|
||||
&[("data", &artifact)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(artifact);
|
||||
|
||||
let request = within("clone artifact request", service.next())
|
||||
.await
|
||||
.unwrap();
|
||||
let responder = request.responder();
|
||||
drop(request);
|
||||
e.settle("clone keeps only reply capability", |stats| {
|
||||
stats.calls == 1
|
||||
&& stats.active_calls == 1
|
||||
&& stats.reply_capabilities == 1
|
||||
&& stats.owners == 0
|
||||
&& stats.artifacts == 0
|
||||
})
|
||||
.await;
|
||||
drop(responder);
|
||||
let error = within("clone artifact release failure", pending.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(error.code, error.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Dispatched)
|
||||
);
|
||||
e.settle("clone artifact release retired call", |stats| {
|
||||
stats.calls == 0 && stats.active_calls == 0 && stats.reply_capabilities == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn fair_topic_insertion_preserves_router_envelope_order() {
|
||||
let limits = Limits {
|
||||
max_control_frames: 4096,
|
||||
max_control_bytes: 16 << 20,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(Via::Memory, limits, Policy::open()).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
publisher
|
||||
.declare_topic("t.order", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut raw = e.raw_hello("reader").await;
|
||||
raw.call(
|
||||
"subscribe",
|
||||
json!({"topic": "t.order", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut writer = raw.take_writer();
|
||||
let flood = tokio::spawn(async move {
|
||||
for n in 3..3000u64 {
|
||||
let envelope = json!({
|
||||
"protocol": "flybus", "major": 1, "minor": 0,
|
||||
"id": format!("msg-{n}"), "replyTo": null, "kind": "command",
|
||||
"op": "no.such.op", "body": {}, "attachments": []
|
||||
});
|
||||
if !writer.send(&serde_json::to_vec(&envelope).unwrap()).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
publisher
|
||||
.publish("t.order", obj(json!({"ready": true})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut last = 0;
|
||||
let mut saw_delivery = false;
|
||||
for _ in 0..2500 {
|
||||
let envelope = within("router frame", raw.recv()).await.unwrap();
|
||||
let serial = parse_serial_id("bus", &envelope.id).unwrap();
|
||||
assert!(
|
||||
serial > last,
|
||||
"router emitted {} after bus-{last}",
|
||||
envelope.id
|
||||
);
|
||||
last = serial;
|
||||
saw_delivery |= envelope.op == "topic.message";
|
||||
if saw_delivery && envelope.kind == flybus::wire::Kind::Reply {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_delivery,
|
||||
"fair scheduling never inserted the topic message"
|
||||
);
|
||||
flood.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn sdk_accepts_fair_topic_insertion_through_saturated_control_backlog() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.policy = Policy::open();
|
||||
config.limits.max_control_frames = 4096;
|
||||
config.limits.max_control_bytes = 16 << 20;
|
||||
let router = Router::new(config).unwrap();
|
||||
let publisher = Client::connect(
|
||||
router.connect_in_memory_as("publisher"),
|
||||
ClientConfig::new("publisher", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.declare_topic("t.sdk-order", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (client_stream, router_stream) = tokio::io::duplex(64 * 1024);
|
||||
let (probed, probe) = WriteProbe::new(router_stream);
|
||||
router.serve_as(probed, "reader");
|
||||
let reader = Client::connect(
|
||||
Transport::from_stream(client_stream),
|
||||
ClientConfig::new("reader", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut subscription = reader
|
||||
.subscribe("t.sdk-order", SubscriptionConfig::latest().in_flight(1))
|
||||
.await
|
||||
.unwrap();
|
||||
probe.arm();
|
||||
|
||||
let mut backlog = Vec::new();
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
for n in 0..256 {
|
||||
let client = reader.clone();
|
||||
let completed = completed.clone();
|
||||
backlog.push(tokio::spawn(async move {
|
||||
let result = client
|
||||
.declare_topic(&format!("t.backlog-{n}"), Retained::None)
|
||||
.await;
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
result
|
||||
}));
|
||||
}
|
||||
probe.wait_first().await;
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
|
||||
while router.stats().topics < 257 {
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"control backlog was not admitted"
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(
|
||||
completed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"writer was not blocked"
|
||||
);
|
||||
publisher
|
||||
.publish("t.sdk-order", obj(json!({"ready": true})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(router.stats().queued, 1, "topic delivery was not queued");
|
||||
|
||||
probe.release();
|
||||
let message = within("SDK fair delivery", subscription.next())
|
||||
.await
|
||||
.expect("subscription closed on router envelope ordering");
|
||||
assert_eq!(message.payload()["ready"], true);
|
||||
assert!(
|
||||
reader.closed().is_none(),
|
||||
"strict SDK rejected router output"
|
||||
);
|
||||
for task in backlog {
|
||||
task.await.unwrap().unwrap();
|
||||
}
|
||||
let ops = probe.ops();
|
||||
let topic = ops
|
||||
.iter()
|
||||
.position(|op| op == "topic.message")
|
||||
.expect("instrumented stream did not carry the topic delivery");
|
||||
assert!(
|
||||
ops[topic + 1..].iter().any(|op| op == "topic.declare"),
|
||||
"topic delivery was not fairly inserted ahead of remaining controls: {ops:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn shutdown_does_not_deliver_a_frame_after_releasing_its_owner() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.policy = Policy::open();
|
||||
let router = Router::new(config).unwrap();
|
||||
let publisher = Client::connect(
|
||||
router.connect_in_memory_as("publisher"),
|
||||
ClientConfig::new("publisher", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.declare_topic("t.shutdown", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (client_stream, router_stream) = tokio::io::duplex(64 * 1024);
|
||||
let (probed, probe) = WriteProbe::new(router_stream);
|
||||
router.serve_as(probed, "reader");
|
||||
let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path());
|
||||
raw.hello("reader").await.unwrap();
|
||||
raw.call(
|
||||
"subscribe",
|
||||
json!({"topic": "t.shutdown", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
probe.arm();
|
||||
|
||||
let artifact = sealed(&publisher, b"still-owned", "application/octet-stream").await;
|
||||
publisher
|
||||
.publish("t.shutdown", obj(json!({})), &[("data", &artifact)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(artifact);
|
||||
probe.wait_first().await;
|
||||
wait_for_single_delivery_owner(&router).await;
|
||||
let before = router.stats();
|
||||
assert_eq!((before.owners, before.artifact_roots), (1, 1));
|
||||
|
||||
router.shutdown();
|
||||
assert_eq!(
|
||||
router.stats().owners,
|
||||
0,
|
||||
"shutdown released the selected owner"
|
||||
);
|
||||
assert_eq!(
|
||||
router.stats().artifacts,
|
||||
0,
|
||||
"shutdown collected its attachment"
|
||||
);
|
||||
assert!(
|
||||
within("truncated topic frame", raw.recv()).await.is_none(),
|
||||
"a complete frame was appended after a partial topic delivery"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn protocol_close_cancels_partial_topic_frame_before_reclaiming_attachment() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.policy = Policy::open();
|
||||
let router = Router::new(config).unwrap();
|
||||
let publisher = Client::connect(
|
||||
router.connect_in_memory_as("publisher"),
|
||||
ClientConfig::new("publisher", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.declare_topic("t.close", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let (client_stream, router_stream) = tokio::io::duplex(64 * 1024);
|
||||
let (probed, probe) = WriteProbe::new(router_stream);
|
||||
router.serve_as(probed, "reader");
|
||||
let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path());
|
||||
raw.hello("reader").await.unwrap();
|
||||
raw.call(
|
||||
"subscribe",
|
||||
json!({"topic": "t.close", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
probe.arm();
|
||||
|
||||
let artifact = sealed(&publisher, b"close", "application/octet-stream").await;
|
||||
publisher
|
||||
.publish("t.close", obj(json!({})), &[("data", &artifact)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(artifact);
|
||||
probe.wait_first().await;
|
||||
wait_for_single_delivery_owner(&router).await;
|
||||
|
||||
raw.next = 0;
|
||||
raw.command("topic.declare", json!({}), json!([])).await;
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
|
||||
while router.stats().connections != 1 {
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"protocol close did not finish"
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(router.stats().owners, 0);
|
||||
assert_eq!(router.stats().artifacts, 0);
|
||||
assert!(
|
||||
within("protocol-close truncated frame", raw.recv())
|
||||
.await
|
||||
.is_none(),
|
||||
"a final notice was appended after a partial frame"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn teardown_waits_for_an_active_transport_poll_before_reclaiming() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.policy = Policy::open();
|
||||
let router = Router::new(config).unwrap();
|
||||
let publisher = Client::connect(
|
||||
router.connect_in_memory_as("publisher"),
|
||||
ClientConfig::new("publisher", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.declare_topic("t.poll-gate", Retained::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let (client_stream, router_stream) = tokio::io::duplex(64 * 1024);
|
||||
let (held, hold) = PollHold::new(router_stream);
|
||||
router.serve_as(held, "reader");
|
||||
let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path());
|
||||
raw.hello("reader").await.unwrap();
|
||||
raw.call(
|
||||
"subscribe",
|
||||
json!({"topic": "t.poll-gate", "mode": "latest", "maxQueued": 1, "maxInFlight": 1, "replayLatest": false}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
hold.arm();
|
||||
|
||||
let artifact = sealed(&publisher, b"poll", "application/octet-stream").await;
|
||||
let sealed_path = router
|
||||
.store_dir()
|
||||
.join("sealed")
|
||||
.join(&artifact.reference().artifact_id);
|
||||
publisher
|
||||
.publish("t.poll-gate", obj(json!({})), &[("data", &artifact)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop(artifact);
|
||||
hold.wait_entered().await;
|
||||
wait_for_single_delivery_owner(&router).await;
|
||||
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let shutdown_done = done.clone();
|
||||
let shutdown_router = router.clone();
|
||||
let shutdown = std::thread::spawn(move || {
|
||||
shutdown_router.shutdown();
|
||||
shutdown_done.store(true, Ordering::SeqCst);
|
||||
});
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(
|
||||
!done.load(Ordering::SeqCst),
|
||||
"teardown completed while poll_write was active"
|
||||
);
|
||||
assert!(
|
||||
sealed_path.exists(),
|
||||
"artifact was reclaimed while poll_write was active"
|
||||
);
|
||||
|
||||
hold.release();
|
||||
shutdown.join().unwrap();
|
||||
assert!(done.load(Ordering::SeqCst));
|
||||
assert_eq!(router.stats().owners, 0);
|
||||
assert_eq!(router.stats().artifacts, 0);
|
||||
assert!(!sealed_path.exists());
|
||||
while let Some(envelope) = within("poll-gate close", raw.recv()).await {
|
||||
assert_ne!(
|
||||
envelope.op, "topic.message",
|
||||
"delivery completed after teardown reclaimed its owner"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn shutdown_cancels_partial_rpc_request_before_reclaiming_attachment() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.policy = Policy::open();
|
||||
let router = Router::new(config).unwrap();
|
||||
let caller = Client::connect(
|
||||
router.connect_in_memory_as("caller"),
|
||||
ClientConfig::new("caller", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (client_stream, router_stream) = tokio::io::duplex(64 * 1024);
|
||||
let (probed, probe) = WriteProbe::new(router_stream);
|
||||
router.serve_as(probed, "server");
|
||||
let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path());
|
||||
raw.hello("server").await.unwrap();
|
||||
raw.call(
|
||||
"service.register",
|
||||
json!({"name": "example.partial-request", "maxQueued": 1, "maxInFlight": 1}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
probe.arm();
|
||||
|
||||
let artifact = sealed(&caller, b"request", "application/octet-stream").await;
|
||||
let _pending = caller
|
||||
.call(
|
||||
"example.partial-request",
|
||||
None,
|
||||
"Work",
|
||||
obj(json!({})),
|
||||
&[("data", &artifact)],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(artifact);
|
||||
probe.wait_first().await;
|
||||
wait_for_single_delivery_owner(&router).await;
|
||||
let before = router.stats();
|
||||
assert_eq!((before.owners, before.artifact_roots), (1, 1));
|
||||
|
||||
router.shutdown();
|
||||
assert_eq!(router.stats().owners, 0);
|
||||
assert_eq!(router.stats().artifacts, 0);
|
||||
assert!(
|
||||
within("truncated request frame", raw.recv())
|
||||
.await
|
||||
.is_none(),
|
||||
"a complete frame was appended after a partial RPC request"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn shutdown_cancels_partial_rpc_result_before_reclaiming_attachment() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.policy = Policy::open();
|
||||
let router = Router::new(config).unwrap();
|
||||
let server = Client::connect(
|
||||
router.connect_in_memory_as("server"),
|
||||
ClientConfig::new("server", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut service = server
|
||||
.register("example.partial-result", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let (client_stream, router_stream) = tokio::io::duplex(64 * 1024);
|
||||
let (probed, probe) = WriteProbe::new(router_stream);
|
||||
router.serve_as(probed, "caller");
|
||||
let mut raw = Raw::over(Transport::from_stream(client_stream), dir.path());
|
||||
raw.hello("caller").await.unwrap();
|
||||
raw.call(
|
||||
"rpc.call",
|
||||
json!({"callId": "call-1", "target": "example.partial-result", "expectedIncarnation": null, "method": "Work", "payload": {}}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
probe.arm();
|
||||
let request = within("result request", service.next()).await.unwrap();
|
||||
let artifact = sealed(&server, b"result", "application/octet-stream").await;
|
||||
request
|
||||
.reply(obj(json!({})), &[("data", &artifact)])
|
||||
.await
|
||||
.unwrap();
|
||||
drop((artifact, request));
|
||||
probe.wait_first().await;
|
||||
wait_for_single_delivery_owner(&router).await;
|
||||
let before = router.stats();
|
||||
assert_eq!((before.owners, before.artifact_roots), (1, 1));
|
||||
|
||||
router.shutdown();
|
||||
assert_eq!(router.stats().owners, 0);
|
||||
assert_eq!(router.stats().artifacts, 0);
|
||||
assert!(
|
||||
within("truncated result frame", raw.recv()).await.is_none(),
|
||||
"a complete frame was appended after a partial RPC result"
|
||||
);
|
||||
}
|
||||
819
services/flysim/crates/flybus/tests/sol_review_races.rs
Normal file
819
services/flysim/crates/flybus/tests/sol_review_races.rs
Normal file
|
|
@ -0,0 +1,819 @@
|
|||
mod common;
|
||||
|
||||
use common::{Via, env, env_with, obj, within};
|
||||
use flybus::wire::{Envelope, Kind, contract_digest, read_frame, write_frame};
|
||||
use flybus::{
|
||||
CancelState, Client, ClientConfig, Dispatch, ErrorCode, Limits, Policy, Retained, Router,
|
||||
RouterConfig, ServiceConfig, SubscriptionConfig, Transport,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
struct WriteFailsAfterHello {
|
||||
response: Vec<u8>,
|
||||
read: usize,
|
||||
writes: usize,
|
||||
}
|
||||
|
||||
impl AsyncRead for WriteFailsAfterHello {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
_cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
if self.read == self.response.len() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
let n = buf.remaining().min(self.response.len() - self.read);
|
||||
let end = self.read + n;
|
||||
buf.put_slice(&self.response[self.read..end]);
|
||||
self.read = end;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for WriteFailsAfterHello {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
_cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
if self.writes == 0 {
|
||||
self.writes = 1;
|
||||
Poll::Ready(Ok(buf.len()))
|
||||
} else {
|
||||
Poll::Ready(Err(io::Error::new(io::ErrorKind::BrokenPipe, "injected")))
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct WriteBlocksAsReaderFails {
|
||||
response: Vec<u8>,
|
||||
read: usize,
|
||||
hello_written: bool,
|
||||
command_write_started: bool,
|
||||
reader_waker: Option<Waker>,
|
||||
dropped: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl Drop for WriteBlocksAsReaderFails {
|
||||
fn drop(&mut self) {
|
||||
if let Some(dropped) = self.dropped.take() {
|
||||
let _ = dropped.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for WriteBlocksAsReaderFails {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
if self.read < self.response.len() {
|
||||
let n = buf.remaining().min(self.response.len() - self.read);
|
||||
let end = self.read + n;
|
||||
buf.put_slice(&self.response[self.read..end]);
|
||||
self.read = end;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
if self.command_write_started {
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionReset,
|
||||
"injected reader failure",
|
||||
)));
|
||||
}
|
||||
self.reader_waker = Some(cx.waker().clone());
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for WriteBlocksAsReaderFails {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
_cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
if !self.hello_written {
|
||||
self.hello_written = true;
|
||||
return Poll::Ready(Ok(buf.len()));
|
||||
}
|
||||
self.command_write_started = true;
|
||||
if let Some(waker) = self.reader_waker.take() {
|
||||
waker.wake();
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn responder_survives_cancel_then_request_drop() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.race", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let pending = caller
|
||||
.call("example.race", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let request = within("request", service.next()).await.unwrap();
|
||||
let responder = request.responder();
|
||||
|
||||
assert_eq!(
|
||||
pending.cancel().await.unwrap(),
|
||||
CancelState::ExecutionUnknown
|
||||
);
|
||||
drop(request);
|
||||
e.settle("request consumed", |s| s.owners == 0).await;
|
||||
|
||||
let routed = responder.reply(obj(json!({"done": true})), &[]).await;
|
||||
assert_eq!(routed, Ok(false), "a detached reply is still a valid reply");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancel_after_request_consumption_retires_correlation() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.leak", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = caller
|
||||
.call("example.leak", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let request = within("request", service.next()).await.unwrap();
|
||||
drop(request);
|
||||
let failure = within("last responder terminal failure", pending.result()).await;
|
||||
let failure = failure.unwrap_err();
|
||||
assert_eq!(
|
||||
(failure.code, failure.dispatch),
|
||||
(ErrorCode::CallGone, Dispatch::Dispatched)
|
||||
);
|
||||
e.settle("consumed call retired", |s| {
|
||||
s.calls == 0 && s.active_calls == 0 && s.owners == 0 && s.reply_capabilities == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn retained_replay_obeys_bounded_queue_byte_quota() {
|
||||
let limits = Limits {
|
||||
max_queued_bytes_per_client: 1,
|
||||
max_owners_per_client: 2,
|
||||
reserved_owners_per_client: 1,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(Via::Memory, limits, Policy::open()).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let reader = e.client("reader").await;
|
||||
publisher
|
||||
.declare_topic("t.replay", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.publish("t.replay", obj(json!({"large": "x".repeat(1024)})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Exhaust the reader's ordinary-owner allowance so replay cannot immediately dispatch.
|
||||
let _hold = reader
|
||||
.artifacts()
|
||||
.allocate(0, "application/octet-stream")
|
||||
.await
|
||||
.unwrap()
|
||||
.seal()
|
||||
.await
|
||||
.unwrap();
|
||||
let replay = reader
|
||||
.subscribe("t.replay", SubscriptionConfig::bounded().replay(true))
|
||||
.await;
|
||||
assert_eq!(replay.unwrap_err().code, ErrorCode::Backpressure);
|
||||
assert_eq!(e.stats().subscriptions, 0, "failed replay is atomic");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn latest_replay_remains_bounded_outside_the_bounded_byte_pool() {
|
||||
let limits = Limits {
|
||||
max_queued_bytes_per_client: 1,
|
||||
max_owners_per_client: 2,
|
||||
reserved_owners_per_client: 1,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(Via::Memory, limits, Policy::open()).await;
|
||||
let publisher = e.client("publisher").await;
|
||||
let reader = e.client("reader").await;
|
||||
publisher
|
||||
.declare_topic("t.latest", Retained::Latest)
|
||||
.await
|
||||
.unwrap();
|
||||
publisher
|
||||
.publish("t.latest", obj(json!({"large": "x".repeat(1024)})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let _hold = reader
|
||||
.artifacts()
|
||||
.allocate(0, "application/octet-stream")
|
||||
.await
|
||||
.unwrap()
|
||||
.seal()
|
||||
.await
|
||||
.unwrap();
|
||||
let sub = reader
|
||||
.subscribe("t.latest", SubscriptionConfig::latest().replay(true))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(e.stats().subscriptions, 1);
|
||||
drop(sub);
|
||||
e.settle("latest replay released", |s| s.subscriptions == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn pending_connections_are_bounded_and_hello_expires() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = RouterConfig::new(dir.path());
|
||||
config.limits.max_clients = 1;
|
||||
config.hello_timeout = Duration::from_millis(40);
|
||||
config.policy = Policy::closed()
|
||||
.client("first", flybus::Grants::all())
|
||||
.client("second", flybus::Grants::all());
|
||||
let router = Router::new(config).unwrap();
|
||||
|
||||
let pending = router.connect_in_memory_as("first");
|
||||
tokio::time::timeout(Duration::from_millis(20), async {
|
||||
while router.stats().connections != 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(router.stats().connections, 1);
|
||||
let refused = Client::connect(
|
||||
router.connect_in_memory_as("second"),
|
||||
ClientConfig::new("second", dir.path()),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(refused.unwrap_err().code, ErrorCode::RouterLost);
|
||||
tokio::time::sleep(Duration::from_millis(80)).await;
|
||||
assert_eq!(router.stats().connections, 0, "pending Hello timed out");
|
||||
drop(pending);
|
||||
|
||||
let second = Client::connect(
|
||||
router.connect_in_memory_as("second"),
|
||||
ClientConfig::new("second", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
second.close().await;
|
||||
let unbound = Client::connect(
|
||||
router.connect_in_memory(),
|
||||
ClientConfig::new("first", dir.path()),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(unbound.unwrap_err().code, ErrorCode::RouterLost);
|
||||
}
|
||||
|
||||
fn reply_body(value: Value) -> Map<String, Value> {
|
||||
obj(json!({"ok": true, "value": value}))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn client_rejects_wrong_version_and_operation_from_router() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024);
|
||||
let server = tokio::spawn(async move {
|
||||
let hello = read_frame(&mut fake_router).await.unwrap().unwrap();
|
||||
let hello = Envelope::decode(&hello).unwrap();
|
||||
let hello_reply = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some(hello.id),
|
||||
kind: Kind::Reply,
|
||||
op: "bus.hello".into(),
|
||||
body: reply_body(json!({
|
||||
"routerId": "router-fake",
|
||||
"connectionId": "conn-1",
|
||||
"selectedMajor": 1,
|
||||
"selectedMinor": 0,
|
||||
"contractDigest": contract_digest(),
|
||||
"limits": Limits::default().to_json(),
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
write_frame(&mut fake_router, &hello_reply.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let command = read_frame(&mut fake_router).await.unwrap().unwrap();
|
||||
let command = Envelope::decode(&command).unwrap();
|
||||
let invalid_reply = Envelope {
|
||||
major: 2,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some(command.id),
|
||||
kind: Kind::Reply,
|
||||
op: "publish".into(),
|
||||
body: reply_body(json!({
|
||||
"declared": true,
|
||||
"topicIncarnation": "top-1",
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
write_frame(&mut fake_router, &invalid_reply.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let client = Client::connect(
|
||||
Transport::from_stream(client_stream),
|
||||
ClientConfig::new("client", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = client.declare_topic("t.x", Retained::None).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"invalid router envelope was accepted: {result:?}"
|
||||
);
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn client_strictly_validates_hello_envelope() {
|
||||
for case in ["version", "operation", "id", "attachments"] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024);
|
||||
let server = tokio::spawn(async move {
|
||||
let hello =
|
||||
Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap();
|
||||
let mut reply = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some(hello.id),
|
||||
kind: Kind::Reply,
|
||||
op: "bus.hello".into(),
|
||||
body: reply_body(json!({
|
||||
"routerId": "router-fake",
|
||||
"connectionId": "conn-1",
|
||||
"selectedMajor": 1,
|
||||
"selectedMinor": 0,
|
||||
"contractDigest": contract_digest(),
|
||||
"limits": Limits::default().to_json(),
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
match case {
|
||||
"version" => reply.major = 2,
|
||||
"operation" => reply.op = "publish".into(),
|
||||
"id" => reply.id = "msg-1".into(),
|
||||
"attachments" => reply.attachments.push(flybus::wire::Attachment {
|
||||
name: "x".into(),
|
||||
reference: flybus::ArtifactRef {
|
||||
store_id: "store-fake".into(),
|
||||
artifact_id: "a-1".into(),
|
||||
generation: 1,
|
||||
byte_length: 0,
|
||||
content_type: "x/y".into(),
|
||||
digest: None,
|
||||
},
|
||||
owner_id: "own-1".into(),
|
||||
}),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
write_frame(&mut fake_router, &reply.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let result = Client::connect(
|
||||
Transport::from_stream(client_stream),
|
||||
ClientConfig::new("client", dir.path()),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "invalid Hello case {case} was accepted");
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn client_rejects_unknown_reply_fields_and_invalid_direction_rules() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024);
|
||||
let server = tokio::spawn(async move {
|
||||
let hello =
|
||||
Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap();
|
||||
let hello_reply = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some(hello.id),
|
||||
kind: Kind::Reply,
|
||||
op: "bus.hello".into(),
|
||||
body: reply_body(json!({
|
||||
"routerId": "router-fake",
|
||||
"connectionId": "conn-1",
|
||||
"selectedMajor": 1,
|
||||
"selectedMinor": 0,
|
||||
"contractDigest": contract_digest(),
|
||||
"limits": Limits::default().to_json(),
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
write_frame(&mut fake_router, &hello_reply.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let command =
|
||||
Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap();
|
||||
let invalid = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-2".into(),
|
||||
reply_to: Some(command.id),
|
||||
kind: Kind::Reply,
|
||||
op: "topic.declare".into(),
|
||||
body: reply_body(json!({
|
||||
"declared": true,
|
||||
"topicIncarnation": "top-1",
|
||||
"extra": true,
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
write_frame(&mut fake_router, &invalid.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let client = Client::connect(
|
||||
Transport::from_stream(client_stream),
|
||||
ClientConfig::new("client", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(client.declare_topic("t.x", Retained::None).await.is_err());
|
||||
server.await.unwrap();
|
||||
|
||||
let (client_stream, mut fake_router) = tokio::io::duplex(64 * 1024);
|
||||
let server = tokio::spawn(async move {
|
||||
let hello =
|
||||
Envelope::decode(&read_frame(&mut fake_router).await.unwrap().unwrap()).unwrap();
|
||||
let hello_reply = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some(hello.id),
|
||||
kind: Kind::Reply,
|
||||
op: "bus.hello".into(),
|
||||
body: reply_body(json!({
|
||||
"routerId": "router-fake",
|
||||
"connectionId": "conn-1",
|
||||
"selectedMajor": 1,
|
||||
"selectedMinor": 0,
|
||||
"contractDigest": contract_digest(),
|
||||
"limits": Limits::default().to_json(),
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
write_frame(&mut fake_router, &hello_reply.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let invalid_notice = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-2".into(),
|
||||
reply_to: Some("msg-99".into()),
|
||||
kind: Kind::Notice,
|
||||
op: "connection.closing".into(),
|
||||
body: obj(json!({"code": "ROUTER_LOST", "message": "bad direction"})),
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
write_frame(&mut fake_router, &invalid_notice.encode().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let client = Client::connect(
|
||||
Transport::from_stream(client_stream),
|
||||
ClientConfig::new("client", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while client.closed().is_none() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn writer_failure_terminates_reader_and_pending_work() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let hello_reply = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some("msg-1".into()),
|
||||
kind: Kind::Reply,
|
||||
op: "bus.hello".into(),
|
||||
body: reply_body(json!({
|
||||
"routerId": "router-fake",
|
||||
"connectionId": "conn-1",
|
||||
"selectedMajor": 1,
|
||||
"selectedMinor": 0,
|
||||
"contractDigest": contract_digest(),
|
||||
"limits": Limits::default().to_json(),
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
.encode()
|
||||
.unwrap();
|
||||
let mut response = (hello_reply.len() as u32).to_le_bytes().to_vec();
|
||||
response.extend(hello_reply);
|
||||
let client = Client::connect(
|
||||
Transport::from_stream(WriteFailsAfterHello {
|
||||
response,
|
||||
read: 0,
|
||||
writes: 0,
|
||||
}),
|
||||
ClientConfig::new("client", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
client.declare_topic("t.x", Retained::None),
|
||||
)
|
||||
.await
|
||||
.expect("writer failure left command pending");
|
||||
assert_eq!(result.unwrap_err().code, ErrorCode::RouterLost);
|
||||
tokio::time::timeout(Duration::from_secs(1), client.close())
|
||||
.await
|
||||
.expect("writer failure left close waiting");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn reader_failure_cancels_blocked_write_and_drops_transport() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let hello_reply = Envelope {
|
||||
major: 1,
|
||||
minor: 0,
|
||||
id: "bus-1".into(),
|
||||
reply_to: Some("msg-1".into()),
|
||||
kind: Kind::Reply,
|
||||
op: "bus.hello".into(),
|
||||
body: reply_body(json!({
|
||||
"routerId": "router-fake",
|
||||
"connectionId": "conn-1",
|
||||
"selectedMajor": 1,
|
||||
"selectedMinor": 0,
|
||||
"contractDigest": contract_digest(),
|
||||
"limits": Limits::default().to_json(),
|
||||
})),
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
.encode()
|
||||
.unwrap();
|
||||
let mut response = (hello_reply.len() as u32).to_le_bytes().to_vec();
|
||||
response.extend(hello_reply);
|
||||
let (dropped, transport_dropped) = oneshot::channel();
|
||||
let client = Client::connect(
|
||||
Transport::from_stream(WriteBlocksAsReaderFails {
|
||||
response,
|
||||
read: 0,
|
||||
hello_written: false,
|
||||
command_write_started: false,
|
||||
reader_waker: None,
|
||||
dropped: Some(dropped),
|
||||
}),
|
||||
ClientConfig::new("client", dir.path()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
client.declare_topic("t.x", Retained::None),
|
||||
)
|
||||
.await
|
||||
.expect("reader failure left command pending");
|
||||
assert_eq!(result.unwrap_err().code, ErrorCode::RouterLost);
|
||||
tokio::time::timeout(Duration::from_secs(1), transport_dropped)
|
||||
.await
|
||||
.expect("reader failure left writer blocked")
|
||||
.expect("transport drop signal was discarded");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn rejected_call_id_still_advances_monotonic_watermark() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let _service = server
|
||||
.register("example.ids", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut raw = e.raw_hello("caller").await;
|
||||
|
||||
let rejected = raw
|
||||
.call(
|
||||
"rpc.call",
|
||||
json!({"callId": "call-5", "target": "missing.service", "expectedIncarnation": null, "method": "M", "payload": {}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(common::code(&rejected), "NO_SERVICE");
|
||||
let decreasing = raw
|
||||
.call(
|
||||
"rpc.call",
|
||||
json!({"callId": "call-4", "target": "example.ids", "expectedIncarnation": null, "method": "M", "payload": {}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(common::code(&decreasing), "INVALID_ENVELOPE");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn unsent_oversized_call_rolls_back_its_local_slot() {
|
||||
let limits = Limits {
|
||||
max_active_calls_per_client: 1,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(Via::Memory, limits, Policy::open()).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.rollback", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let oversized = caller
|
||||
.call(
|
||||
"example.rollback",
|
||||
None,
|
||||
"Work",
|
||||
obj(json!({"blob": "x".repeat(70_000)})),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
assert_eq!(oversized.unwrap_err().code, ErrorCode::InvalidEnvelope);
|
||||
|
||||
let mut pending = caller
|
||||
.call("example.rollback", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let request = within("request after rollback", service.next())
|
||||
.await
|
||||
.unwrap();
|
||||
request.reply(obj(json!({"ok": true})), &[]).await.unwrap();
|
||||
assert_eq!(pending.result().await.unwrap().outcome()["ok"], true);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn caller_disconnect_cleanup_works_before_and_after_consumption() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let mut service = server
|
||||
.register("example.detach", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let caller = e.client("caller-a").await;
|
||||
let pending = caller
|
||||
.call("example.detach", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let request = within("request a", service.next()).await.unwrap();
|
||||
let responder = request.responder();
|
||||
drop(request);
|
||||
e.settle("request consumed with responder", |s| {
|
||||
s.owners == 0 && s.reply_capabilities == 1
|
||||
})
|
||||
.await;
|
||||
caller.close().await;
|
||||
drop(pending);
|
||||
assert!(!responder.reply(obj(json!({})), &[]).await.unwrap());
|
||||
drop(responder);
|
||||
e.settle("retained responder retired", |s| {
|
||||
s.calls == 0 && s.reply_capabilities == 0
|
||||
})
|
||||
.await;
|
||||
|
||||
let caller = e.client("caller-b").await;
|
||||
let pending = caller
|
||||
.call("example.detach", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let request = within("request b", service.next()).await.unwrap();
|
||||
drop(request);
|
||||
e.settle("capability released first", |s| s.reply_capabilities == 0)
|
||||
.await;
|
||||
caller.close().await;
|
||||
drop(pending);
|
||||
e.settle("consumed call retired on disconnect", |s| s.calls == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_service_with_buffered_requests_retires_every_call() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let service = server
|
||||
.register(
|
||||
"example.buffered",
|
||||
ServiceConfig {
|
||||
max_queued: 4,
|
||||
max_in_flight: 4,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut calls = Vec::new();
|
||||
for _ in 0..4 {
|
||||
calls.push(
|
||||
caller
|
||||
.call("example.buffered", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
e.settle("requests buffered in SDK", |s| s.reply_capabilities == 4)
|
||||
.await;
|
||||
drop(service);
|
||||
for call in &mut calls {
|
||||
let error = within("buffered call failure", call.result())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
(error.code, error.dispatch),
|
||||
(ErrorCode::NoService, flybus::Dispatch::Dispatched)
|
||||
);
|
||||
}
|
||||
e.settle("buffered calls retired", |s| {
|
||||
s.calls == 0 && s.reply_capabilities == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn reply_racing_cancel_has_only_the_two_contract_outcomes() {
|
||||
let e = env(Via::Memory).await;
|
||||
let server = e.client("server").await;
|
||||
let caller = e.client("caller").await;
|
||||
let mut service = server
|
||||
.register("example.reply-race", ServiceConfig::default())
|
||||
.await
|
||||
.unwrap();
|
||||
for _ in 0..32 {
|
||||
let mut pending = caller
|
||||
.call("example.reply-race", None, "Work", obj(json!({})), &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let request = within("racing request", service.next()).await.unwrap();
|
||||
let responder = request.responder();
|
||||
let (cancel, reply) = tokio::join!(
|
||||
pending.cancel(),
|
||||
responder.reply(obj(json!({"ok": true})), &[])
|
||||
);
|
||||
match (cancel.unwrap(), reply.unwrap()) {
|
||||
(CancelState::Completed, true) => {
|
||||
assert_eq!(pending.result().await.unwrap().outcome()["ok"], true);
|
||||
}
|
||||
(CancelState::ExecutionUnknown, false) => {
|
||||
assert_eq!(
|
||||
pending.result().await.unwrap_err().code,
|
||||
ErrorCode::CallGone
|
||||
);
|
||||
}
|
||||
other => panic!("invalid cancel/reply race outcome: {other:?}"),
|
||||
}
|
||||
drop((request, responder));
|
||||
}
|
||||
e.settle("race calls retired", |s| {
|
||||
s.calls == 0 && s.reply_capabilities == 0 && s.owners == 0
|
||||
})
|
||||
.await;
|
||||
}
|
||||
355
services/flysim/crates/flybus/tests/wire.rs
Normal file
355
services/flysim/crates/flybus/tests/wire.rs
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
//! Wire and connection: hello negotiation, strict frame and envelope validation, body errors
|
||||
//! that keep the connection, envelope size limits and control-lane exhaustion.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{Raw, Via, code, env, env_with, obj};
|
||||
use flybus::wire::{Kind, MAX_ENVELOPE_BYTES, contract_digest};
|
||||
use flybus::{ErrorCode, Limits, Policy, Retained};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
async fn hello_negotiation(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw().await;
|
||||
let v = raw.hello("probe").await.unwrap();
|
||||
assert_eq!(v["routerId"], e.router.router_id());
|
||||
assert_eq!(
|
||||
(v["selectedMajor"].as_u64(), v["selectedMinor"].as_u64()),
|
||||
(Some(1), Some(0))
|
||||
);
|
||||
assert_eq!(v["contractDigest"], contract_digest());
|
||||
assert!(v["connectionId"].as_str().unwrap().starts_with("conn-"));
|
||||
assert_eq!(
|
||||
flybus::Limits::from_json(&v["limits"]).unwrap(),
|
||||
Limits::default()
|
||||
);
|
||||
let c = e.client("sdk").await;
|
||||
assert_eq!(c.info().router_id, e.router.router_id());
|
||||
assert_ne!(c.info().connection_id, v["connectionId"]);
|
||||
}
|
||||
|
||||
async fn expect_closed(raw: &mut Raw, want: &str) {
|
||||
let notice = raw
|
||||
.closing()
|
||||
.await
|
||||
.unwrap_or_else(|| panic!("no connection.closing notice (wanted {want})"));
|
||||
assert_eq!(notice["code"], want, "{notice:?}");
|
||||
}
|
||||
|
||||
async fn hello_refusals(via: Via) {
|
||||
let e = env_with(
|
||||
via,
|
||||
Limits::default(),
|
||||
Policy::closed().client("known", flybus::Grants::all()),
|
||||
)
|
||||
.await;
|
||||
let mut raw = e.raw_as("known").await;
|
||||
raw.command(
|
||||
"topic.declare",
|
||||
json!({"name": "t.x", "retained": "none"}),
|
||||
json!([]),
|
||||
)
|
||||
.await;
|
||||
expect_closed(&mut raw, "INVALID_ENVELOPE").await;
|
||||
|
||||
let mut raw = e.raw_as("known").await;
|
||||
let r = raw
|
||||
.call(
|
||||
"bus.hello",
|
||||
json!({"clientId": "known", "clientIncarnation": "i-1", "supportedMajors": [2, 3]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code(&r), "VERSION_MISMATCH");
|
||||
assert!(
|
||||
raw.recv().await.is_none(),
|
||||
"refused hello closes the connection"
|
||||
);
|
||||
|
||||
let mut raw = e.raw_as("known").await;
|
||||
assert_eq!(code(&raw.hello("stranger").await), "NOT_AUTHORIZED");
|
||||
assert!(raw.recv().await.is_none());
|
||||
|
||||
let mut raw = e.raw_as("known").await;
|
||||
let r = raw.call("bus.hello", json!({"clientId": "known", "clientIncarnation": "i-2", "supportedMajors": [1], "admin": true})).await;
|
||||
assert_eq!(code(&r), "INVALID_ENVELOPE");
|
||||
|
||||
let mut raw = e.raw_as("known").await;
|
||||
raw.hello("known").await.unwrap();
|
||||
raw.command(
|
||||
"bus.hello",
|
||||
json!({"clientId": "known", "clientIncarnation": "i-3", "supportedMajors": [1]}),
|
||||
json!([]),
|
||||
)
|
||||
.await;
|
||||
expect_closed(&mut raw, "INVALID_ENVELOPE").await;
|
||||
e.settle("all refused connections released", |s| s.connections == 0)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn envelope(id: &str, body: Value) -> Value {
|
||||
json!({"protocol": "flybus", "major": 1, "minor": 0, "id": id, "replyTo": null, "kind": "command",
|
||||
"op": "publish", "body": body, "attachments": []})
|
||||
}
|
||||
|
||||
async fn malformed_frames_close_the_connection(via: Via) {
|
||||
let e = env(via).await;
|
||||
let admin = e.client("admin").await;
|
||||
admin.declare_topic("t.x", Retained::None).await.unwrap();
|
||||
let publish = |id: &str| envelope(id, json!({"topic": "t.x", "payload": {}}));
|
||||
let text = |v: Value| serde_json::to_string(&v).unwrap();
|
||||
let att = |name: &str| {
|
||||
json!({"name": name, "ref": {"storeId": "s", "artifactId": "a-1", "generation": "1",
|
||||
"byteLength": "1", "contentType": "x", "digest": null}, "ownerId": "own-1"})
|
||||
};
|
||||
let mut too_many = publish("msg-2");
|
||||
too_many["attachments"] = Value::Array((0..33).map(|i| att(&format!("a{i}"))).collect());
|
||||
let mut duplicate_names = publish("msg-2");
|
||||
duplicate_names["attachments"] = json!([att("a"), att("a")]);
|
||||
let mut unknown_field = publish("msg-2");
|
||||
unknown_field["priority"] = json!(1);
|
||||
let mut from_router = publish("msg-2");
|
||||
from_router["kind"] = json!("reply");
|
||||
let mut reply_to = publish("msg-2");
|
||||
reply_to["replyTo"] = json!("msg-1");
|
||||
let mut major = publish("msg-2");
|
||||
major["major"] = json!(2);
|
||||
let mut not_flybus = publish("msg-2");
|
||||
not_flybus["protocol"] = json!("flybusx");
|
||||
|
||||
let cases: Vec<(&str, Vec<u8>)> = vec![
|
||||
("nested duplicate key", br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-2","replyTo":null,"kind":"command","op":"publish","body":{"topic":"t.x","payload":{"a":{"b":1,"b":2}}},"attachments":[]}"#.to_vec()),
|
||||
("top-level duplicate key", br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-2","id":"msg-3","replyTo":null,"kind":"command","op":"publish","body":{"topic":"t.x","payload":{}},"attachments":[]}"#.to_vec()),
|
||||
("NaN", br#"{"protocol":"flybus","major":1,"minor":0,"id":"msg-2","replyTo":null,"kind":"command","op":"publish","body":{"topic":"t.x","payload":{"v":NaN}},"attachments":[]}"#.to_vec()),
|
||||
("invalid UTF-8", [&text(publish("msg-2")).into_bytes()[..60], b"\xff\xfe", &text(publish("msg-2")).into_bytes()[62..]].concat()),
|
||||
("trailing bytes", [text(publish("msg-2")).into_bytes(), b" {}".to_vec()].concat()),
|
||||
("not an object", b"[1,2,3]".to_vec()),
|
||||
("non-canonical id", text(publish("msg-02")).into_bytes()),
|
||||
("too many attachments", text(too_many).into_bytes()),
|
||||
("duplicate attachment names", text(duplicate_names).into_bytes()),
|
||||
("unknown envelope field", text(unknown_field).into_bytes()),
|
||||
("reply kind from a client", text(from_router).into_bytes()),
|
||||
("non-null replyTo", text(reply_to).into_bytes()),
|
||||
("wrong major after hello", text(major).into_bytes()),
|
||||
("wrong protocol", text(not_flybus).into_bytes()),
|
||||
];
|
||||
for (i, (what, bytes)) in cases.into_iter().enumerate() {
|
||||
let mut raw = e.raw_hello(&format!("bad-{i}")).await;
|
||||
raw.send_bytes(&bytes).await;
|
||||
let notice = raw
|
||||
.closing()
|
||||
.await
|
||||
.unwrap_or_else(|| panic!("{what}: no closing notice"));
|
||||
assert_eq!(notice["code"], "INVALID_ENVELOPE", "{what}");
|
||||
e.settle(what, |s| s.connections == 1).await;
|
||||
}
|
||||
|
||||
// Ids must increase.
|
||||
let mut raw = e.raw_hello("replay").await;
|
||||
raw.send_bytes(&serde_json::to_vec(&publish("msg-5")).unwrap())
|
||||
.await;
|
||||
raw.send_bytes(&serde_json::to_vec(&publish("msg-5")).unwrap())
|
||||
.await;
|
||||
expect_closed(&mut raw, "INVALID_ENVELOPE").await;
|
||||
|
||||
// Framing: a zero length, and a length over the limit with no body behind it.
|
||||
let mut raw = e.raw_hello("zero").await;
|
||||
raw.send_prefix(0).await;
|
||||
expect_closed(&mut raw, "INVALID_ENVELOPE").await;
|
||||
let mut raw = e.raw_hello("huge").await;
|
||||
raw.send_prefix(u32::MAX).await;
|
||||
expect_closed(&mut raw, "INVALID_ENVELOPE").await;
|
||||
// A frame cut short by the end of the stream just ends the connection.
|
||||
let mut raw = e.raw_hello("short").await;
|
||||
raw.send_prefix(100).await;
|
||||
drop(raw);
|
||||
e.settle("only the admin remains", |s| s.connections == 1)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn body_errors_keep_the_connection(via: Via) {
|
||||
let e = env(via).await;
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
let cases = [
|
||||
("no.such.op", json!({}), json!([])),
|
||||
(
|
||||
"topic.declare",
|
||||
json!({"name": "t.a", "retained": "none", "extra": 1}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"topic.declare",
|
||||
json!({"name": "t..a", "retained": "none"}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"topic.declare",
|
||||
json!({"name": "t.a", "retained": "sometimes"}),
|
||||
json!([]),
|
||||
),
|
||||
("topic.declare", json!({"name": "t.a"}), json!([])),
|
||||
(
|
||||
"subscribe",
|
||||
json!({"topic": "t.a", "mode": "bounded", "maxQueued": 0, "maxInFlight": 1, "replayLatest": false}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"subscribe",
|
||||
json!({"topic": "t.a", "mode": "bounded", "maxQueued": 1.0, "maxInFlight": 1, "replayLatest": false}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"subscribe",
|
||||
json!({"topic": "t.a", "mode": "bounded", "maxQueued": 65536, "maxInFlight": 1, "replayLatest": false}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"artifact.allocate",
|
||||
json!({"byteLength": 5, "contentType": "x"}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"artifact.allocate",
|
||||
json!({"byteLength": "18446744073709551616", "contentType": "x"}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"rpc.call",
|
||||
json!({"callId": "call-1", "target": "a.b", "expectedIncarnation": null, "method": "", "payload": {}}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"rpc.call",
|
||||
json!({"callId": "call-1", "target": "a.b", "expectedIncarnation": null, "method": "M", "payload": []}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"delivery.consumed",
|
||||
json!({"deliveryIds": "dlv-1"}),
|
||||
json!([]),
|
||||
),
|
||||
(
|
||||
"topic.declare",
|
||||
json!({"name": "t.a", "retained": "none"}),
|
||||
json!([{"name": "x", "ref": {"storeId": "s", "artifactId": "a-1", "generation": "1", "byteLength": "1", "contentType": "x", "digest": null}, "ownerId": "own-1"}]),
|
||||
),
|
||||
];
|
||||
for (op, body, atts) in cases {
|
||||
let r = raw.call_with(op, body.clone(), atts).await;
|
||||
assert_eq!(
|
||||
r,
|
||||
Err(("INVALID_ENVELOPE".into(), "not-dispatched".into())),
|
||||
"{op} {body}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
code(
|
||||
&raw.call("topic.declare", json!({"name": "t.a", "retained": "none"}))
|
||||
.await
|
||||
),
|
||||
"OK"
|
||||
);
|
||||
assert_eq!(e.stats().connections, 1);
|
||||
}
|
||||
|
||||
async fn envelope_size_limits(via: Via) {
|
||||
let e = env(via).await;
|
||||
// Exactly 65536 bytes is a frame the router reads (and answers); one more is not.
|
||||
let sized = |id: &str, total: usize| {
|
||||
let base = serde_json::to_vec(
|
||||
&json!({"protocol": "flybus", "major": 1, "minor": 0, "id": id, "replyTo": null,
|
||||
"kind": "command", "op": "no.such.op", "body": {"pad": ""}, "attachments": []}),
|
||||
)
|
||||
.unwrap();
|
||||
let pad = "p".repeat(total - base.len());
|
||||
let bytes = serde_json::to_vec(
|
||||
&json!({"protocol": "flybus", "major": 1, "minor": 0, "id": id, "replyTo": null,
|
||||
"kind": "command", "op": "no.such.op", "body": {"pad": pad}, "attachments": []}),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(bytes.len(), total);
|
||||
bytes
|
||||
};
|
||||
let mut raw = e.raw_hello("raw").await;
|
||||
raw.send_bytes(&sized("msg-2", MAX_ENVELOPE_BYTES)).await;
|
||||
assert_eq!(code(&raw.reply("msg-2").await), "INVALID_ENVELOPE");
|
||||
raw.send_bytes(&sized("msg-3", MAX_ENVELOPE_BYTES + 1))
|
||||
.await;
|
||||
expect_closed(&mut raw, "INVALID_ENVELOPE").await;
|
||||
|
||||
// The SDK refuses to send an oversized envelope and the connection lives on.
|
||||
let c = e.client("c").await;
|
||||
c.declare_topic("t.big", Retained::None).await.unwrap();
|
||||
let huge = c
|
||||
.publish(
|
||||
"t.big",
|
||||
obj(json!({"blob": "x".repeat(MAX_ENVELOPE_BYTES)})),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(huge.code, ErrorCode::InvalidEnvelope);
|
||||
// An envelope that fits but whose delivery (with its added ids) would not is refused at
|
||||
// admission rather than truncated later.
|
||||
let edge = c
|
||||
.publish("t.big", obj(json!({"blob": "x".repeat(65_300)})), &[])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(edge.code, ErrorCode::InvalidEnvelope);
|
||||
assert!(edge.message.contains("delivery"), "{edge}");
|
||||
assert_eq!(
|
||||
c.publish("t.big", obj(json!({"blob": "x".repeat(60_000)})), &[])
|
||||
.await
|
||||
.unwrap()
|
||||
.topic_sequence,
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
/// A client that sends commands and never reads replies is closed once its control lane is
|
||||
/// full, with a notice, instead of the router buffering without bound.
|
||||
async fn control_lane_exhaustion_closes(via: Via) {
|
||||
let limits = Limits {
|
||||
max_control_frames: 4,
|
||||
..Limits::default()
|
||||
};
|
||||
let e = env_with(via, limits, Policy::open()).await;
|
||||
let mut raw = e.raw_hello("flood").await;
|
||||
let mut writer = raw.take_writer();
|
||||
let flood = tokio::spawn(async move {
|
||||
for i in 2..20_000u64 {
|
||||
let env = json!({"protocol": "flybus", "major": 1, "minor": 0, "id": format!("msg-{i}"), "replyTo": null,
|
||||
"kind": "command", "op": "no.such.op", "body": {}, "attachments": []});
|
||||
if !writer.send(&serde_json::to_vec(&env).unwrap()).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
let mut replies = 0;
|
||||
let mut last = None;
|
||||
while let Some(env) = raw.recv().await {
|
||||
match env.kind {
|
||||
Kind::Reply => replies += 1,
|
||||
_ => last = Some(env),
|
||||
}
|
||||
}
|
||||
let last = last.expect("a closing notice");
|
||||
assert_eq!(
|
||||
(last.op.as_str(), last.body["code"].as_str()),
|
||||
("connection.closing", Some("QUOTA_EXCEEDED"))
|
||||
);
|
||||
assert!(replies < 19_998, "the router stopped answering");
|
||||
flood.abort();
|
||||
e.settle("flooder released", |s| s.connections == 0).await;
|
||||
}
|
||||
|
||||
both_transports!(
|
||||
hello_negotiation,
|
||||
hello_refusals,
|
||||
malformed_frames_close_the_connection,
|
||||
body_errors_keep_the_connection,
|
||||
envelope_size_limits,
|
||||
control_lane_exhaustion_closes,
|
||||
);
|
||||
Loading…
Add table
Reference in a new issue