diff --git a/Cargo.lock b/Cargo.lock index b65a238..2988e67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -515,6 +515,21 @@ dependencies = [ "toml 0.9.12+spec-1.1.0", ] +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.64" @@ -710,6 +725,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -736,7 +765,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.2", + "unicode-width 0.2.0", "windows-sys 0.59.0", ] @@ -748,7 +777,7 @@ checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" dependencies = [ "encode_unicode", "libc", - "unicode-width 0.2.2", + "unicode-width 0.2.0", "windows-sys 0.61.2", ] @@ -770,6 +799,35 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -838,6 +896,31 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crucible-client-types" version = "0.1.0" @@ -976,8 +1059,18 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88490bf1b990d87eaaa7ac8aa887f629a08e7359765b4911faf63c3763347d23" +dependencies = [ + "darling_core 0.24.0", + "darling_macro 0.24.0", ] [[package]] @@ -993,17 +1086,41 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "darling_core" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084e274f91c482280130e1e34e0b8d6e66776a060d7b6de7b84289ca778868c4" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.118", ] +[[package]] +name = "darling_macro" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5792fa0d41cd2325ce0ffa64f0a340eaebd4971a3a0c5e1ffd2cc488a355e" +dependencies = [ + "darling_core 0.24.0", + "quote", + "syn 3.0.3", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -1120,6 +1237,15 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dof" version = "0.4.0" @@ -1403,6 +1529,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -1714,6 +1846,17 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1722,7 +1865,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2061,7 +2204,7 @@ dependencies = [ "allocator-api2", "daft", "equivalent", - "foldhash", + "foldhash 0.2.0", "hashbrown 0.17.1", "ref-cast", "schemars 0.8.22", @@ -2133,7 +2276,7 @@ checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" dependencies = [ "console 0.16.3", "portable-atomic", - "unicode-width 0.2.2", + "unicode-width 0.2.0", "unit-prefix", "web-time", ] @@ -2156,6 +2299,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instability" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf84e73fa6f27f299dec58e13223cf70db80da872eb921d4f6138342a0eabc8" +dependencies = [ + "darling 0.24.0", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "instant" version = "0.1.13" @@ -2213,6 +2369,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2345,7 +2510,7 @@ dependencies = [ "knus-derive", "miette 7.6.0", "thiserror 2.0.18", - "unicode-width 0.2.2", + "unicode-width 0.2.0", ] [[package]] @@ -2469,6 +2634,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -2484,6 +2655,15 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2608,6 +2788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -3493,6 +3674,22 @@ dependencies = [ "unarray", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3687,6 +3884,27 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum 0.26.3", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + [[package]] name = "rats-corim" version = "0.1.0" @@ -3829,6 +4047,8 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", + "cookie", + "cookie_store", "futures-core", "futures-util", "http", @@ -4423,7 +4643,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.118", @@ -4491,6 +4711,27 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -5024,7 +5265,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" dependencies = [ - "unicode-width 0.2.2", + "unicode-width 0.2.0", ] [[package]] @@ -5326,6 +5567,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -5721,6 +5963,17 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + [[package]] name = "unicode-width" version = "0.1.14" @@ -5729,9 +5982,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unit-prefix" @@ -5877,8 +6130,11 @@ version = "0.1.0" dependencies = [ "anyhow", "attest-mock", + "base64 0.22.1", "camino", + "chrono", "clap", + "crossterm", "flate2", "futures", "indoc", @@ -5887,12 +6143,15 @@ dependencies = [ "oxnet", "propolis-client", "rack-init-config", + "ratatui", "reqwest 0.13.4", + "serde", "serde_json", "slog", "sprockets-tls-test-utils", "tar", "tokio", + "tokio-util", "toml 0.9.12+spec-1.1.0", "voxel-config", "wicketd-commission-client", diff --git a/Cargo.toml b/Cargo.toml index a33d9e3..543e644 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,11 @@ clap = { version = "4.6.1", features = ["derive", "env"] } expectorate = "1.2.0" # Our crates use only #[tokio::main] + tokio::time; libfalcon pulls its own # tokio features (they're additive), so we don't need "full". -tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "time", "sync", "process", "io-util"] } +tokio-util = { version = "0.7", features = ["rt"] } +ratatui = { version = "0.29", features = ["unstable-rendered-line-info"] } +crossterm = "0.28" +base64 = "0.22" libfalcon = { git = "https://github.com/oxidecomputer/falcon", branch = "main" } # Must be the rev libfalcon pins, so instance-spec component types unify. propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "36f20be9bb4c3b362029237f5feb6377c982395f" } @@ -29,6 +33,7 @@ serde = "1" camino = { version = "1.2.2", features = ["serde1"] } indoc = "2.0.7" toml_edit = "0.25" +reqwest = { version = "0.13", default-features = false, features = ["cookies", "json", "rustls"] } # sprockets/trust quorum related deps attest-mock = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d" } diff --git a/README.md b/README.md index 35f7d19..6f9cb91 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,9 @@ generation. - **`voxel-image/`**: image build machinery (`voxel image create`) and the install scripts that bake a control-plane image from an omicron commit. -See [`docs/parameters.md`](docs/parameters.md) for the `voxel.toml` reference, there are a LOT of tuning knobs. +See [`docs/parameters.md`](docs/parameters.md) for the `voxel.toml` reference, +there are a LOT of tuning knobs. Terminal-interface operators should also see +the [`voxel tui` guide](docs/voxel-tui.md). ## Building diff --git a/docs/voxel-tui.md b/docs/voxel-tui.md new file mode 100644 index 0000000..6655be4 --- /dev/null +++ b/docs/voxel-tui.md @@ -0,0 +1,98 @@ +# Voxel TUI operator guide + +`voxel tui` is the private, persistent control and observability interface for +a Voxel deployment. The TUI supervises the public `voxel launch`, `voxel +route`, and `voxel destroy` commands as opaque child processes. Its displayed +phase transitions are advisory hints parsed from output, not lifecycle APIs or +proof of deployment state. It tracks deployment state through live probes +implemented by collector and telemetry interfaces. + +## Usage + +Build Voxel and prepare images as described in the [project +quickstart](../README.md#quickstart), then start the interface: + +```console +$ cargo build -p voxel --bin voxel +$ pfexec ./target/debug/voxel tui +``` + +Logs are written to `/voxel-tui.log` and displayed in a +bounded on-screen history. The TUI can launch and destroy Voxel deployments, +and its session can attach to or detach from new or existing deployments. Use +`voxel tui resume` to resume the most recent detached session, or add `--choose` +to select one. + +## Views and actions + +The TUI has two alternate views, Deployment and Monitoring. Each view has a +number of foldable sections stacked within it. `Up` and `Down` move within +nested content and hand focus to the adjacent section at an effective edge; +section traversal wraps. `Tab` and `Shift-Tab` always move section focus, and +`Space` folds or expands the focused section. + +| Key | Action | +|---|---| +| `1` / `2` | Switch between views | +| `Tab` / `Shift-Tab` | Next / previous top-level section | +| `Space` | Fold / expand the focused section | +| `Left` / `Right` | Previous / next rack in Rack Summary | +| `Up` / `Down` | Navigate nested content, handing off at an edge | +| `PageUp` / `PageDown` | Page nested content without changing section focus | +| `Enter` | Open the selected topology resource detail | +| `?` / `F1` | Open or close Help | +| `Esc` | Dismiss the topmost confirmation, Help, detail, or selection | +| `f` | Cycle the level filter while Deployment Logs is focused | +| `l` / `r` | Request launch / route | +| `c` / `x` | Cancel and leave resources / cancel and destroy | +| `d` | Detach and leave deployment resources in place | +| `y` | Copy the full fallback command from the detach dialog | +| `n` | Reject a confirmation | +| `q` | Quit, destroying resources first unless observed stopped | + +### Deployment + +The Deployment view tracks the launch and teardown of a Voxel deployment, and +contains **Overall Progress**, **Phases**, **Status**, **Current Phase**, and +**Logs** in separate sections. Live probes are used to track observed deployment +and route state. + +Logs are displayed oldest to newest, with new entries edging older ones out of +the bounded view. Press `f` to filter them by level: All, Info, Warning, or +Error. The complete history remains in the durable log file. + +### Monitoring + +The Monitoring view exposes the state of a Voxel deployment, and contains +**Rack Summary**, **Topology**, and **Top Zones by Traffic** in separate +sections. +Rack Summary reports RSS readiness, aggregate RX/TX, health counts, and history. +Topology presents fleet routers, the shared switch-fabric bus, rack-local switch +zones, and sleds in the order of router → switch zone → sled. + +The resource inspector sits beside the topology on wide layouts and below it on +compact layouts. Health summarizes sled-agent, maintenance services, zones, and +NTP probes for sleds, and traffic collection for routers and switch zones. The +inspector also shows the last successful probe, latest collection error, +RX/TX/total and packet rates, a 60-second sparkline, and zones. Missing samples +say collecting or unavailable rather than reporting zero. For the purposes of +a basic labelled implementation, traffic is considered normal at or below 100 +KB/s, elevated above 100 KB/s through 5 MB/s, and high above 5 MB/s. **Top Zones +by Traffic** is a separate rack-wide ranking. + +### Display + +Wide terminals place topology and inspector side by side. Compact windows will +stack them and page resources rather than hiding labels, and prioritize focussed +sections while keeping folded sections minimized. Below `48x16`, the TUI +displays its minimum-size requirement instead of clipping an unsafe layout. + +## Testing + +```sh +cargo test -p voxel tui::app::tests -- --nocapture +cargo test -p voxel tui::terminal::tests -- --nocapture +cargo test -p voxel tui::ui::confirm_dialog::tests -- --nocapture +cargo test -p voxel tui::ui::widgets::tests -- --nocapture +cargo test -p voxel tui::effects::tests -- --nocapture +``` diff --git a/voxel/Cargo.toml b/voxel/Cargo.toml index af02929..790da80 100644 --- a/voxel/Cargo.toml +++ b/voxel/Cargo.toml @@ -12,6 +12,10 @@ path = "src/main.rs" [dependencies] tokio.workspace = true +tokio-util.workspace = true +ratatui.workspace = true +crossterm.workspace = true +base64.workspace = true libfalcon.workspace = true propolis-client.workspace = true anyhow.workspace = true @@ -31,11 +35,13 @@ wicketd-commission-client.workspace = true # `launch --wicket-setup` (drive setup through wicketd). toml = "0.9" serde_json = "1" +serde = { workspace = true, features = ["derive"] } # Unpack TUF control-plane composites (GNU tar format; illumos tar rejects it). tar = "0.4" flate2 = "1" -reqwest = { version = "0.13", default-features = false, features = ["rustls"] } libc = "0.2" +reqwest.workspace = true +chrono = { version = "0.4", features = ["serde"] } # Sprockets/trust-quorum test keys - generated at launch (replaces a4x2's # pre-generated per-sled sprockets material that prep.sh used to rsync in). diff --git a/voxel/src/main.rs b/voxel/src/main.rs index a50b143..db9b97b 100644 --- a/voxel/src/main.rs +++ b/voxel/src/main.rs @@ -43,6 +43,7 @@ mod sp_cmd; mod sp_host; mod topo; mod tufrepo; +mod tui; mod util; mod wicket_setup; @@ -79,6 +80,11 @@ struct Cli { #[derive(Subcommand)] enum Cmd { + /// Open the terminal interface. + Tui { + #[command(subcommand)] + cmd: Option, + }, /// Launch the rack and stream RSS bring-up progress. Launch { /// Don't watch RSS bring-up after launch. @@ -217,6 +223,16 @@ enum Cmd { }, } +#[derive(Subcommand)] +enum TuiCmd { + /// Resume a detached TUI session. + Resume { + /// Choose from all available sessions. + #[arg(long)] + choose: bool, + }, +} + #[derive(Subcommand)] enum ConfigCmd { /// Print the effective configuration. @@ -771,9 +787,55 @@ fn anchor_workdir( Ok(()) } +fn resolve_tui_context( + cli: &Cli, + config_path: &Utf8Path, + config: VoxelConfig, + invocation_dir: &Utf8Path, +) -> anyhow::Result { + let make_absolute = |path: Utf8PathBuf| { + if path.is_absolute() { path } else { invocation_dir.join(path) } + }; + let workdir = cli + .workdir + .clone() + .or_else(|| config.falcon.workdir.clone().map(Utf8PathBuf::from)) + .or_else(|| config_path.parent().map(Utf8Path::to_path_buf)) + .map(make_absolute) + .context("cannot resolve TUI workdir")?; + let dataset = std::env::var("FALCON_DATASET") + .unwrap_or_else(|_| "rpool/falcon".to_string()); + let build_root = std::env::var("BUILD_ROOT") + .map(Utf8PathBuf::from) + .unwrap_or_else(|_| { + Utf8PathBuf::from( + std::env::var("HOME").unwrap_or_else(|_| "/root".into()), + ) + .join("voxel-builds") + }); + + Ok(tui::TuiContext::new( + config_path.to_path_buf(), + workdir, + cli.name.clone(), + dataset, + make_absolute(build_root), + config, + std::env::current_exe().context("resolve current voxel executable")?, + std::env::vars_os().collect(), + )) +} + #[tokio::main] async fn main() -> Result<(), Error> { let cli = Cli::parse(); + if let Cmd::Tui { cmd: Some(TuiCmd::Resume { choose }) } = &cli.cmd { + return tui::resume(*choose); + } + let invocation_dir = Utf8PathBuf::try_from( + std::env::current_dir().context("resolve invocation directory")?, + ) + .context("invocation directory is not valid UTF-8")?; // Anchor to the project root before anything touches cargo-bay/.falcon. let config_path = discover_config(cli.config.as_deref()); // A missing config means defaults; an existing one that fails to load or @@ -786,6 +848,19 @@ async fn main() -> Result<(), Error> { resolve_falcon_env(&cli, cfg.as_ref()); anchor_workdir(&cli, cfg.as_ref(), &config_path)?; match &cli.cmd { + Cmd::Tui { cmd } => { + debug_assert!( + cmd.is_none(), + "resume dispatched before configuration" + ); + let context = resolve_tui_context( + &cli, + &config_path, + load_config(&config_path)?, + &invocation_dir, + )?; + tui::run(context).await + } Cmd::Launch { no_progress, no_route, emu, sp_firmware } => { // One flag: emulated SPs, the RoT bridge on top of them, and // wicketd-driven setup are the same configuration in practice, and @@ -1067,3 +1142,19 @@ mod tests { assert!(find("deadbeef").ends_with("omicron-deadbeef")); } } + +#[cfg(test)] +mod main_tests { + use super::Cli; + use clap::Parser; + + #[test] + fn parses_tui_commands_and_preserves_launch_parsing() { + assert!(Cli::try_parse_from(["voxel", "tui"]).is_ok()); + assert!(Cli::try_parse_from(["voxel", "tui", "resume"]).is_ok()); + assert!( + Cli::try_parse_from(["voxel", "tui", "resume", "--choose"]).is_ok() + ); + assert!(Cli::try_parse_from(["voxel", "launch", "--emu"]).is_ok()); + } +} diff --git a/voxel/src/tui/app.rs b/voxel/src/tui/app.rs new file mode 100644 index 0000000..8e18c46 --- /dev/null +++ b/voxel/src/tui/app.rs @@ -0,0 +1,2000 @@ +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::time::Instant; + +use crate::tui::event::{ + Action, AppEvent, CancelChoice, Confirmation, ConfirmationOptionAction, + DeploymentPane, Direction, Effect, MonitoringPane, OperationRequestId, + View, +}; +use crate::tui::operation::{ + CommandOutcome, LogLevel, OperationEvent, OperationKind, OperationPhase, + OperationWarning, +}; +use crate::tui::reconcile::{ + ObservedDeploymentState, ReconciliationResult, RouteEvidence, + RssObservation, +}; +use crate::tui::telemetry::{ + HealthDiagnostic, LatestSample, NodeAddresses, OximeterExceptions, RackId, + ResourceDescriptor, ResourceId, ResourceScope, TelemetryModel, + TrafficSample, TrafficSource, ZfsHeadroom, ZoneCpu, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TerminalSize { + pub width: u16, + pub height: u16, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PostOperationExit { + Detach, + Destroy, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SessionExit { + Detach, + Quit, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SessionState { + pub view: View, + pub deployment_pane: DeploymentPane, + pub monitoring_pane: MonitoringPane, + pub collapsed_deployment: BTreeSet, + pub collapsed_monitoring: BTreeSet, + pub phase_scroll: usize, + pub subtask_scroll: usize, + pub monitor_scroll: usize, + pub top_zones_scroll: usize, + pub help_scroll: usize, + pub help_open: bool, + pub external_monitoring_open: bool, + pub selected_rack: Option, + pub selected_resource: Option, + pub terminal: TerminalSize, + pub detail_open: bool, + pub confirmation: Option, + pub confirmation_selection: usize, + pub post_operation_exit: Option, + pub exit: Option, + pub quitting: bool, +} + +impl SessionState { + pub(crate) fn deployment_expanded(&self, pane: DeploymentPane) -> bool { + !self.collapsed_deployment.contains(&pane) + } + + pub(crate) fn monitoring_expanded(&self, pane: MonitoringPane) -> bool { + !self.collapsed_monitoring.contains(&pane) + } +} + +#[derive(Clone, Debug)] +pub struct DeploymentState { + pub topology: Vec, + pub observed: ObservedDeploymentState, + pub rss: BTreeMap>, + pub routes: RouteEvidence, + pub last_reconciliation: Option, + pub last_reconciliation_at: Option, + pub reconciliation_failure: Option, + pub reconciliation_in_progress: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Progress { + pub completed: usize, + pub total: usize, + pub message: String, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LogEntry { + pub source: LogSource, + pub level: LogLevel, + pub message: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LogSource { + Application, + Operation, +} + +impl LogEntry { + pub fn application(level: LogLevel, message: impl Into) -> Self { + Self { source: LogSource::Application, level, message: message.into() } + } + + pub fn operation(level: LogLevel, message: impl Into) -> Self { + Self { source: LogSource::Operation, level, message: message.into() } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LogFilter { + All, + Info, + Warning, + Error, +} +impl LogFilter { + fn next(self) -> Self { + match self { + Self::All => Self::Info, + Self::Info => Self::Warning, + Self::Warning => Self::Error, + Self::Error => Self::All, + } + } + pub fn accepts(self, level: LogLevel) -> bool { + self == Self::All + || matches!( + (self, level), + (Self::Info, LogLevel::Info) + | (Self::Warning, LogLevel::Warning) + | (Self::Error, LogLevel::Error) + ) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BoundedLogs { + pub entries: VecDeque, + pub capacity: usize, + pub scroll: usize, +} +impl BoundedLogs { + pub fn new(capacity: usize) -> Self { + Self { entries: VecDeque::new(), capacity, scroll: 0 } + } + pub fn push(&mut self, entry: LogEntry) { + if self.capacity == 0 { + self.entries.clear(); + self.scroll = 0; + return; + } + let evicted = self.entries.len() == self.capacity; + if evicted { + self.entries.pop_front(); + } else if self.scroll > 0 { + self.scroll = self.scroll.saturating_add(1); + } + self.entries.push_back(entry); + self.scroll = self.scroll.min(self.entries.len().saturating_sub(1)); + } + pub fn push_filtered(&mut self, entry: LogEntry, filter: LogFilter) { + let previous_scroll = self.scroll; + let accepted_append = filter.accepts(entry.level); + let accepted_eviction = self.entries.len() == self.capacity + && self + .entries + .front() + .is_some_and(|oldest| filter.accepts(oldest.level)); + self.push(entry); + if previous_scroll > 0 { + self.scroll = previous_scroll + .saturating_add(usize::from(accepted_append)) + .saturating_sub(usize::from(accepted_eviction)); + } + } + pub fn clamp(&mut self, visible: usize) { + self.scroll = + self.scroll.min(self.entries.len().saturating_sub(visible)); + } + pub fn scroll(&mut self, delta: isize, visible: usize) { + self.scroll = self.scroll.saturating_add_signed(delta); + self.clamp(visible); + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActiveOperation { + pub request_id: OperationRequestId, + pub kind: OperationKind, + pub phase: Option, + pub completed_phases: Vec, + pub progress: Option, + pub warnings: Vec, + pub cancelling: bool, + pub cancel_choice: Option, + pub force_stop_requested: bool, +} +impl ActiveOperation { + pub fn new(request_id: OperationRequestId, kind: OperationKind) -> Self { + Self { + request_id, + kind, + phase: None, + completed_phases: vec![], + progress: None, + warnings: vec![], + cancelling: false, + cancel_choice: None, + force_stop_requested: false, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PendingOperation { + pub request_id: OperationRequestId, + pub kind: OperationKind, +} +impl PendingOperation { + pub fn new(request_id: OperationRequestId, kind: OperationKind) -> Self { + Self { request_id, kind } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OperationState { + pub pending: Option, + pub active: Option, + pub retained: Option, + pub retained_warnings: Vec, + pub outcome: Option, + pub start_failure: Option, +} + +impl OperationState { + pub(crate) fn displayed(&self) -> Option<&ActiveOperation> { + self.active.as_ref().or(self.retained.as_ref()) + } +} + +#[derive(Clone, Debug)] +pub struct ObservabilityState { + pub telemetry: TelemetryModel, + pub health: BTreeMap>, + pub addresses: BTreeMap>, + pub traffic_failures: BTreeMap>, + direct_traffic: BTreeMap>, + pub zone_cpu: BTreeMap>>, + pub zfs_headroom: BTreeMap>>, + pub oximeter_exceptions: BTreeMap>, + pub latest_traffic_generation: Option, +} + +#[derive(Clone, Debug)] +pub struct App { + pub session: SessionState, + pub deployment: DeploymentState, + pub operation: OperationState, + pub observability: ObservabilityState, + pub logs: BoundedLogs, + pub logs_filter: LogFilter, + pub durable_log_path: String, + pub reattach_command: Option, + pub external_monitoring_endpoints: BTreeMap, + pub clipboard_copied: bool, + pub now: Option, + next_operation_request_id: Option, +} + +impl App { + pub fn new( + topology: Vec, + log_capacity: usize, + history_capacity: usize, + ) -> Self { + let topology = Self::normalize_topology(topology); + let telemetry = TelemetryModel::new(topology.clone(), history_capacity); + let health = topology + .iter() + .map(|d| (d.id.clone(), LatestSample::default())) + .collect(); + let traffic_failures = topology + .iter() + .map(|d| (d.id.clone(), LatestSample::default())) + .collect(); + let direct_traffic = topology + .iter() + .map(|d| (d.id.clone(), LatestSample::default())) + .collect(); + let addresses = topology + .iter() + .map(|d| (d.id.clone(), LatestSample::default())) + .collect(); + let rss = topology + .iter() + .filter_map(|d| d.rack) + .map(|rack| (rack, LatestSample::default())) + .collect(); + let racks = topology + .iter() + .filter_map(|descriptor| descriptor.rack) + .collect::>(); + let mut app = Self { + session: SessionState { + view: View::Deployment, + deployment_pane: DeploymentPane::Phases, + monitoring_pane: MonitoringPane::Topology, + collapsed_deployment: BTreeSet::new(), + collapsed_monitoring: BTreeSet::new(), + phase_scroll: 0, + subtask_scroll: 0, + monitor_scroll: 0, + top_zones_scroll: 0, + help_scroll: 0, + help_open: false, + external_monitoring_open: false, + selected_rack: None, + selected_resource: None, + terminal: TerminalSize { width: 0, height: 0 }, + detail_open: false, + confirmation: None, + confirmation_selection: 0, + post_operation_exit: None, + exit: None, + quitting: false, + }, + deployment: DeploymentState { + topology, + observed: ObservedDeploymentState::Unknown, + rss, + routes: RouteEvidence::Unknown, + last_reconciliation: None, + last_reconciliation_at: None, + reconciliation_failure: None, + reconciliation_in_progress: false, + }, + operation: OperationState { + pending: None, + active: None, + retained: None, + retained_warnings: vec![], + outcome: None, + start_failure: None, + }, + observability: ObservabilityState { + telemetry, + health, + addresses, + traffic_failures, + direct_traffic, + zone_cpu: racks + .iter() + .map(|rack| (*rack, LatestSample::default())) + .collect(), + zfs_headroom: racks + .iter() + .map(|rack| (*rack, LatestSample::default())) + .collect(), + oximeter_exceptions: racks + .iter() + .map(|rack| (*rack, LatestSample::default())) + .collect(), + latest_traffic_generation: None, + }, + logs: BoundedLogs::new(log_capacity), + logs_filter: LogFilter::All, + durable_log_path: "voxel-tui.log".into(), + reattach_command: None, + external_monitoring_endpoints: BTreeMap::new(), + clipboard_copied: false, + now: None, + next_operation_request_id: Some(OperationRequestId::FIRST), + }; + app.repair_selection(); + app + } + + pub fn update(&mut self, event: AppEvent) -> Vec { + let mut effects = vec![]; + match event { + AppEvent::Action(action) => return self.action(action), + AppEvent::Tick { now } => self.now = Some(now), + AppEvent::Resize { width, height } => { + self.session.terminal = TerminalSize { width, height } + } + AppEvent::ReconciliationStarted { at } => { + self.deployment.reconciliation_in_progress = true; + self.deployment.last_reconciliation_at = Some(at); + self.deployment.reconciliation_failure = None; + } + AppEvent::Reconciled { at, result, routes } => { + self.deployment.reconciliation_in_progress = false; + self.deployment.observed = result.state; + self.deployment.last_reconciliation = Some(result); + self.deployment.last_reconciliation_at = Some(at); + self.deployment.reconciliation_failure = None; + self.deployment.routes = routes; + } + AppEvent::ReconciliationFailed { at, message } => { + self.deployment.reconciliation_in_progress = false; + self.deployment.last_reconciliation_at = Some(at); + self.deployment.reconciliation_failure = Some(message); + } + AppEvent::Operation { request_id, event } => { + effects = self.operation_event(request_id, event) + } + AppEvent::OperationStartFailed { request_id, kind, message } => { + if self.operation.pending.as_ref().is_some_and(|pending| { + pending.request_id == request_id && pending.kind == kind + }) { + self.operation.pending = None; + self.operation.start_failure = Some(message); + self.session.post_operation_exit = None; + } + } + AppEvent::Traffic { id, at, sample } => { + if self.observability.telemetry.resources.contains_key(&id) + && Self::accept_attempt( + &self.observability.direct_traffic[&id], + at, + ) + { + let oximeter_is_active = self + .observability + .telemetry + .resources + .get(&id) + .is_some_and(|resource| { + resource.current_at.is_some() + && resource.current_sample.source + == TrafficSource::Oximeter + }); + self.observability + .direct_traffic + .entry(id.clone()) + .or_default() + .record_success(at, sample.clone()); + if !oximeter_is_active && self.accept_traffic_generation(at) + { + self.observability + .telemetry + .set_current_sample(&id, at, sample); + self.observability.telemetry.rebuild_aggregates(at); + self.observability + .traffic_failures + .entry(id) + .or_default() + .record_success(at, ()); + } + } + } + AppEvent::OximeterTraffic { id, at, samples } => { + if self.observability.telemetry.resources.contains_key(&id) + && !samples.is_empty() + && Self::accept_attempt( + &self.observability.traffic_failures[&id], + at, + ) + && self.accept_traffic_generation(at) + { + self.observability + .telemetry + .set_oximeter_samples(&id, at, samples); + self.observability.telemetry.rebuild_aggregates(at); + self.observability + .traffic_failures + .entry(id) + .or_default() + .record_success(at, ()); + } + } + AppEvent::OximeterTrafficFailed { rack, at, message } => { + if self.accept_traffic_generation(at) { + let ids = self + .observability + .telemetry + .resources + .keys() + .filter(|id| id.scope == ResourceScope::Rack(rack)) + .cloned() + .collect::>(); + for id in ids { + let fallback = self + .observability + .direct_traffic + .get(&id) + .and_then(|sample| sample.good.as_ref()) + .map(|good| good.value.clone()); + if let Some(sample) = fallback { + self.observability + .telemetry + .set_current_sample(&id, at, sample); + self.observability + .traffic_failures + .entry(id) + .or_default() + .record_success(at, ()); + } else { + self.observability + .traffic_failures + .entry(id) + .or_default() + .record_error(at, message.clone()); + } + } + self.observability.telemetry.rebuild_aggregates(at); + } + } + AppEvent::TrafficFailed { id, at, message } => { + if self.observability.direct_traffic.contains_key(&id) + && Self::accept_attempt( + &self.observability.direct_traffic[&id], + at, + ) + { + self.observability + .direct_traffic + .entry(id.clone()) + .or_default() + .record_error(at, message.clone()); + let oximeter_is_active = self + .observability + .telemetry + .resources + .get(&id) + .is_some_and(|resource| { + resource.current_at.is_some() + && resource.current_sample.source + == TrafficSource::Oximeter + }); + if !oximeter_is_active && self.accept_traffic_generation(at) + { + self.observability + .traffic_failures + .entry(id) + .or_default() + .record_error(at, message); + } + } + } + AppEvent::Health { id, at, diagnostic } => { + if self + .observability + .health + .get(&id) + .is_some_and(|sample| Self::accept_attempt(sample, at)) + { + let sample = + self.observability.health.get_mut(&id).unwrap(); + sample.record_success(at, diagnostic); + } + } + AppEvent::HealthFailed { id, at, message } => { + if self + .observability + .health + .get(&id) + .is_some_and(|sample| Self::accept_attempt(sample, at)) + { + let sample = + self.observability.health.get_mut(&id).unwrap(); + sample.record_error(at, message); + } + } + AppEvent::ZoneCpu { rack, at, zones } => { + if let Some(sample) = self.observability.zone_cpu.get_mut(&rack) + && Self::accept_attempt(sample, at) + { + sample.record_success(at, zones); + } + } + AppEvent::ZoneCpuFailed { rack, at, message } => { + if let Some(sample) = self.observability.zone_cpu.get_mut(&rack) + && Self::accept_attempt(sample, at) + { + sample.record_error(at, message); + } + } + AppEvent::ZfsHeadroom { rack, at, pools } => { + if let Some(sample) = + self.observability.zfs_headroom.get_mut(&rack) + && Self::accept_attempt(sample, at) + { + sample.record_success(at, pools); + } + } + AppEvent::ZfsHeadroomFailed { rack, at, message } => { + if let Some(sample) = + self.observability.zfs_headroom.get_mut(&rack) + && Self::accept_attempt(sample, at) + { + sample.record_error(at, message); + } + } + AppEvent::OximeterExceptions { rack, at, exceptions } => { + if let Some(sample) = + self.observability.oximeter_exceptions.get_mut(&rack) + && Self::accept_attempt(sample, at) + { + sample.record_success(at, exceptions); + } + } + AppEvent::OximeterExceptionsFailed { rack, at, message } => { + if let Some(sample) = + self.observability.oximeter_exceptions.get_mut(&rack) + && Self::accept_attempt(sample, at) + { + sample.record_error(at, message); + } + } + AppEvent::Addresses { id, at, addresses } => { + if self + .observability + .addresses + .get(&id) + .is_some_and(|sample| Self::accept_attempt(sample, at)) + { + self.observability + .addresses + .get_mut(&id) + .unwrap() + .record_success(at, addresses); + } + } + AppEvent::AddressesFailed { id, at, message } => { + if self + .observability + .addresses + .get(&id) + .is_some_and(|sample| Self::accept_attempt(sample, at)) + { + self.observability + .addresses + .get_mut(&id) + .unwrap() + .record_error(at, message); + } + } + AppEvent::Rss { rack, at, observation } => { + if self + .deployment + .rss + .get(&rack) + .is_some_and(|sample| Self::accept_attempt(sample, at)) + { + let sample = self.deployment.rss.get_mut(&rack).unwrap(); + sample.record_success(at, observation); + } + } + AppEvent::RssFailed { rack, at, message } => { + if self + .deployment + .rss + .get(&rack) + .is_some_and(|sample| Self::accept_attempt(sample, at)) + { + let sample = self.deployment.rss.get_mut(&rack).unwrap(); + sample.record_error(at, message); + } + } + AppEvent::DurableLogFailed { message } => self.logs.push_filtered( + LogEntry::application(LogLevel::Error, message), + self.logs_filter, + ), + } + self.clamp_log_scroll(); + effects + } + + fn action(&mut self, action: Action) -> Vec { + if let Some(confirmation) = self.session.confirmation.clone() { + let options = confirmation.options(self.can_cancel()); + return match (confirmation.clone(), action) { + (_, Action::Close) => { + self.session.confirmation = None; + vec![] + } + (_, Action::Scroll { delta, .. }) => { + self.session.confirmation_selection = self + .session + .confirmation_selection + .saturating_add_signed(delta) + .min(options.len().saturating_sub(1)); + vec![] + } + (_, Action::Activate) => { + self.activate_confirmation_option(confirmation) + } + (_, Action::Reject) => { + self.session.confirmation = None; + vec![] + } + (Confirmation::Detach, Action::CopyReattachCommand) => { + if self.can_detach() + && let Some(command) = self.reattach_command.clone() + { + self.clipboard_copied = true; + vec![Effect::CopyToClipboard(command)] + } else { + vec![] + } + } + // `y` confirms ordinary dialogs; detach reserves it for the + // reattach-command clipboard fallback above. + (_, Action::CopyReattachCommand) => { + self.session.confirmation = None; + self.confirm(confirmation) + } + _ => vec![], + }; + } + if self.session.post_operation_exit.is_none() && !self.session.quitting + { + match action { + Action::RequestDetach if self.can_detach() => { + self.open_confirmation(Confirmation::Detach); + return vec![]; + } + Action::RequestQuit => { + let confirmation = if self.resources_may_exist() { + Confirmation::QuitAndDestroy + } else { + Confirmation::Quit + }; + self.open_confirmation(confirmation); + return vec![]; + } + _ => {} + } + } + if self.session.help_open { + return match action { + Action::ToggleHelp | Action::Close => { + self.session.help_open = false; + vec![] + } + Action::Scroll { delta, page } => { + let amount = if page { + crate::tui::ui::help::page_capacity(self) + } else { + 1 + } as isize; + self.session.help_scroll = self + .session + .help_scroll + .saturating_add_signed(delta.saturating_mul(amount)); + vec![] + } + _ => vec![], + }; + } + if self.session.external_monitoring_open { + return match action { + Action::Close => { + self.session.external_monitoring_open = false; + vec![] + } + Action::CopyExternalMonitoringSelected => { + vec![Effect::CopyToClipboard( + crate::tui::ui::external_monitoring::selected_yaml( + self, + ), + )] + } + Action::CopyExternalMonitoringAll => { + vec![Effect::CopyToClipboard( + crate::tui::ui::external_monitoring::all_yaml(self), + )] + } + Action::CopyExternalMonitoringGuide => { + vec![Effect::CopyToClipboard( + crate::tui::ui::external_monitoring::GUIDE_URL.into(), + )] + } + _ => vec![], + }; + } + if self.session.detail_open { + return match action { + Action::Activate | Action::Close => { + self.session.detail_open = false; + vec![] + } + Action::ToggleHelp => { + self.session.help_open = true; + self.session.help_scroll = 0; + vec![] + } + Action::Scroll { delta, page: false } => { + if delta != 0 { + self.move_resource(if delta > 0 { + Direction::Next + } else { + Direction::Previous + }); + } + vec![] + } + _ => vec![], + }; + } + match action { + Action::SwitchView(view) => { + self.session.view = view; + if view == View::Deployment { + self.session.detail_open = false; + } + self.repair_selection(); + } + Action::NextRack + if self.session.view == View::Monitor + && self.session.monitoring_pane + == MonitoringPane::RackSummary + && self + .session + .monitoring_expanded(MonitoringPane::RackSummary) => + { + self.move_rack(Direction::Next) + } + Action::PreviousRack + if self.session.view == View::Monitor + && self.session.monitoring_pane + == MonitoringPane::RackSummary + && self + .session + .monitoring_expanded(MonitoringPane::RackSummary) => + { + self.move_rack(Direction::Previous) + } + Action::NextItem => self.move_item(Direction::Next), + Action::PreviousItem => self.move_item(Direction::Previous), + Action::ToggleSection => match self.session.view { + View::Deployment => { + let pane = self.session.deployment_pane; + if !self.session.collapsed_deployment.remove(&pane) { + self.session.collapsed_deployment.insert(pane); + } + } + View::Monitor => { + let pane = self.session.monitoring_pane; + if !self.session.collapsed_monitoring.remove(&pane) { + self.session.collapsed_monitoring.insert(pane); + } + } + }, + Action::Activate + if self.session.view == View::Monitor + && self.session.monitoring_pane + == MonitoringPane::Topology + && self + .session + .monitoring_expanded(MonitoringPane::Topology) + && self.session.selected_resource.is_some() + && !self.session.help_open + && self.session.confirmation.is_none() => + { + self.session.detail_open = !self.session.detail_open + } + Action::ToggleHelp if self.session.confirmation.is_none() => { + self.session.help_open = !self.session.help_open; + if self.session.help_open { + self.session.help_scroll = 0; + } + } + Action::RequestCancelAndDestroy + if self.session.view == View::Monitor + && self.session.confirmation.is_none() => + { + self.session.external_monitoring_open = true; + } + Action::Close => { + if self.session.confirmation.take().is_some() { + return vec![]; + } + if self.session.help_open { + self.session.help_open = false + } else if self.session.detail_open { + self.session.detail_open = false + } else { + self.session.selected_resource = None + } + } + Action::Reject => self.session.confirmation = None, + Action::RequestLaunch + if self.session.view == View::Deployment + && self.can_start(OperationKind::Launch) => + { + self.open_confirmation(Confirmation::Launch) + } + Action::RequestRoute + if self.session.view == View::Deployment + && self.can_start(OperationKind::Route) => + { + self.open_confirmation(Confirmation::Route) + } + Action::RequestCancelAndLeave + if self.session.view == View::Deployment + && self.can_cancel() => + { + self.open_confirmation(Confirmation::CancelAndLeave) + } + Action::RequestCancelAndLeave + if self.session.view == View::Deployment + && self.can_force_stop() => + { + self.open_confirmation(Confirmation::ForceStop) + } + Action::RequestCancelAndDestroy + if self.session.view == View::Deployment + && (self.can_cancel() + || self.can_upgrade_cancel_to_destroy()) => + { + self.open_confirmation(Confirmation::CancelAndDestroy) + } + Action::CycleLogFilter + if self.session.view == View::Deployment + && self.session.deployment_pane == DeploymentPane::Logs + && self + .session + .deployment_expanded(DeploymentPane::Logs) + && !self.session.help_open + && !self.session.detail_open + && self.session.confirmation.is_none() => + { + self.logs_filter = self.logs_filter.next(); + self.logs.scroll = 0; + } + Action::Scroll { delta, page: false } + if !self.try_move_focused_content(delta) && delta != 0 => + { + self.move_item(if delta > 0 { + Direction::Next + } else { + Direction::Previous + }); + } + Action::Scroll { delta, page: true } => match self.session.view { + View::Deployment + if self + .session + .deployment_expanded(self.session.deployment_pane) => + { + match self.session.deployment_pane { + DeploymentPane::OverallProgress + | DeploymentPane::Status => {} + DeploymentPane::Phases => { + let amount = crate::tui::ui::deployment::phase_content_height(self) + .max(1) as isize; + self.session.phase_scroll = + self.session.phase_scroll.saturating_add_signed( + delta.saturating_mul(amount), + ) + } + DeploymentPane::CurrentPhase => { + let amount = crate::tui::ui::deployment::subtask_content_height(self) + .max(1) as isize; + self.session.subtask_scroll = self + .session + .subtask_scroll + .saturating_add_signed( + delta.saturating_mul(amount), + ) + } + DeploymentPane::Logs => { + let visible = + crate::tui::ui::deployment::log_content_height( + self, + ); + let amount = visible.max(1) as isize; + self.logs.scroll( + delta.saturating_mul(amount).saturating_neg(), + visible, + ); + } + } + } + View::Deployment => {} + View::Monitor + if self.session.monitoring_pane + == MonitoringPane::Topology + && self + .session + .monitoring_expanded(MonitoringPane::Topology) => + { + let amount = crate::tui::ui::monitor::page_capacity(self) + .max(1) as isize; + let resources = self.resources(); + let current = + self.session.selected_resource.as_ref().and_then( + |selected| { + resources.iter().position(|id| id == selected) + }, + ); + let index = current.map_or_else( + || { + if delta < 0 { + resources.len().saturating_sub(1) + } else { + 0 + } + }, + |current| { + current + .saturating_add_signed( + delta.saturating_mul(amount), + ) + .min(resources.len().saturating_sub(1)) + }, + ); + self.session.selected_resource = + resources.get(index).cloned(); + self.session.monitor_scroll = index; + } + View::Monitor + if self.session.monitoring_pane + == MonitoringPane::TopZones + && self + .session + .monitoring_expanded(MonitoringPane::TopZones) => + { + let amount = + crate::tui::ui::monitor::top_zones_page_capacity(self) + as isize; + if amount > 0 { + let maximum = crate::tui::ui::monitor::top_zones_len( + self, + self.session.selected_rack, + ) + .saturating_sub(amount as usize); + let current = + self.session.top_zones_scroll.min(maximum); + self.session.top_zones_scroll = current + .saturating_add_signed(delta.saturating_mul(amount)) + .min(maximum); + } + } + View::Monitor => {} + }, + _ => {} + } + vec![] + } + pub(crate) fn can_start(&self, kind: OperationKind) -> bool { + !self.session.quitting + && self.operation.active.is_none() + && self.operation.pending.is_none() + && match kind { + OperationKind::Launch => { + self.deployment.observed == ObservedDeploymentState::Stopped + } + OperationKind::Route => matches!( + self.deployment.observed, + ObservedDeploymentState::Running + | ObservedDeploymentState::Degraded + ), + OperationKind::Destroy => { + self.deployment.observed != ObservedDeploymentState::Stopped + } + } + } + pub(crate) fn resources_may_exist(&self) -> bool { + self.deployment.observed != ObservedDeploymentState::Stopped + || self.operation.pending.is_some() + || self.operation.active.is_some() + } + pub(crate) fn can_cancel(&self) -> bool { + self.operation.active.as_ref().is_some_and(|active| !active.cancelling) + } + fn can_upgrade_cancel_to_destroy(&self) -> bool { + self.operation.active.as_ref().is_some_and(|active| { + active.cancelling + && active.cancel_choice == Some(CancelChoice::Leave) + }) + } + fn can_force_stop(&self) -> bool { + self.operation + .active + .as_ref() + .is_some_and(|active| !active.force_stop_requested) + } + fn open_confirmation(&mut self, confirmation: Confirmation) { + self.clipboard_copied = false; + self.session.confirmation_selection = + confirmation.options(self.can_cancel()).len().saturating_sub(1); + self.session.confirmation = Some(confirmation); + } + fn activate_confirmation_option( + &mut self, + confirmation: Confirmation, + ) -> Vec { + let action = confirmation + .options(self.can_cancel()) + .get(self.session.confirmation_selection) + .map(|option| option.action) + .unwrap_or(ConfirmationOptionAction::Reject); + match action { + ConfirmationOptionAction::Confirm => { + self.session.confirmation = None; + self.confirm(confirmation) + } + ConfirmationOptionAction::Reject => { + self.session.confirmation = None; + vec![] + } + } + } + fn confirm(&mut self, c: Confirmation) -> Vec { + match c { + Confirmation::Launch if self.can_start(OperationKind::Launch) => { + self.start(OperationKind::Launch) + } + Confirmation::Route if self.can_start(OperationKind::Route) => { + self.start(OperationKind::Route) + } + Confirmation::Detach + if !self.session.quitting && self.can_detach() => + { + self.detach() + } + Confirmation::Quit + if !self.session.quitting && !self.resources_may_exist() => + { + self.session.exit = Some(SessionExit::Quit); + self.session.quitting = true; + vec![Effect::Quit] + } + Confirmation::QuitAndDestroy + if !self.session.quitting && self.resources_may_exist() => + { + self.destroy_and_quit() + } + Confirmation::CancelAndLeave if self.can_cancel() => { + vec![self.cancel(CancelChoice::Leave)] + } + Confirmation::CancelAndDestroy + if self.can_cancel() + || self.can_upgrade_cancel_to_destroy() => + { + vec![self.cancel(CancelChoice::Destroy)] + } + Confirmation::ForceStop if self.can_force_stop() => { + let active = self.operation.active.as_mut().unwrap(); + active.force_stop_requested = true; + let request_id = active.request_id; + vec![Effect::ForceStop { request_id }] + } + _ => vec![], + } + } + fn detach(&mut self) -> Vec { + if !self.can_detach() { + return vec![]; + } + if let Some(active) = self.operation.active.as_ref() { + self.session.post_operation_exit = Some(PostOperationExit::Detach); + if active.cancelling { + return vec![]; + } + return vec![self.cancel(CancelChoice::Leave)]; + } + if self.operation.pending.is_some() { + self.session.post_operation_exit = Some(PostOperationExit::Detach); + return vec![]; + } + self.session.exit = Some(SessionExit::Detach); + self.session.quitting = true; + vec![Effect::Quit] + } + fn can_detach(&self) -> bool { + self.reattach_command.is_some() + && self.session.post_operation_exit.is_none() + && !self.operation.active.as_ref().is_some_and(|active| { + active.kind == OperationKind::Destroy + || active.cancel_choice == Some(CancelChoice::Destroy) + }) + && !self + .operation + .pending + .as_ref() + .is_some_and(|pending| pending.kind == OperationKind::Destroy) + } + fn destroy_and_quit(&mut self) -> Vec { + self.session.post_operation_exit = Some(PostOperationExit::Destroy); + if let Some(active) = self.operation.active.as_ref() { + if active.kind == OperationKind::Destroy + || active.cancel_choice == Some(CancelChoice::Destroy) + { + return vec![]; + } + return vec![self.cancel(CancelChoice::Destroy)]; + } + if self.operation.pending.is_some() { + return vec![]; + } + let effects = self.start(OperationKind::Destroy); + if effects.is_empty() { + self.session.post_operation_exit = None; + } + effects + } + fn racks(&self) -> Vec { + self.deployment + .topology + .iter() + .filter_map(|d| d.rack) + .collect::>() + .into_iter() + .collect() + } + fn resources(&self) -> Vec { + let descriptors = self + .deployment + .topology + .iter() + .filter(|d| { + self.session.view != View::Monitor + || d.rack == self.session.selected_rack + || d.kind == crate::tui::telemetry::ResourceKind::Router + }) + .cloned() + .collect::>(); + if self.session.view == View::Monitor { + crate::tui::ui::topology::semantic_order(&descriptors) + } else { + descriptors.into_iter().map(|descriptor| descriptor.id).collect() + } + } + fn moved( + items: &[T], + current: Option<&T>, + direction: Direction, + ) -> Option { + if items.is_empty() { + return None; + } + let Some(index) = + current.and_then(|c| items.iter().position(|v| v == c)) + else { + return Some(match direction { + Direction::Next => items[0].clone(), + Direction::Previous => items[items.len() - 1].clone(), + }); + }; + Some( + items[match direction { + Direction::Next => (index + 1) % items.len(), + Direction::Previous => (index + items.len() - 1) % items.len(), + }] + .clone(), + ) + } + fn move_rack(&mut self, direction: Direction) { + self.session.selected_rack = Self::moved( + &self.racks(), + self.session.selected_rack.as_ref(), + direction, + ); + self.session.monitor_scroll = 0; + self.session.top_zones_scroll = 0; + self.repair_resource(); + } + fn move_resource(&mut self, direction: Direction) { + self.session.selected_resource = Self::moved( + &self.resources(), + self.session.selected_resource.as_ref(), + direction, + ); + } + fn try_move_focused_content(&mut self, delta: isize) -> bool { + if delta == 0 { + return false; + } + match self.session.view { + View::Deployment + if self + .session + .deployment_expanded(self.session.deployment_pane) => + { + match self.session.deployment_pane { + DeploymentPane::Phases => { + let Some(operation) = self.operation.active.as_ref() + else { + return false; + }; + let phases = crate::tui::ui::deployment::phase_order( + operation.kind, + ); + if phases.is_empty() { + return false; + } + let visible = + crate::tui::ui::deployment::phase_content_height( + self, + ); + let active = operation.phase.and_then(|phase| { + phases.iter().position(|item| *item == phase) + }); + let current = + crate::tui::ui::deployment::phase_window_start( + self.session.phase_scroll, + visible, + phases.len(), + active, + ); + let candidate = + crate::tui::ui::deployment::phase_window_start( + current.saturating_add_signed(delta), + visible, + phases.len(), + active, + ); + self.session.phase_scroll = candidate; + candidate != current + } + DeploymentPane::CurrentPhase => false, + DeploymentPane::Logs => { + let visible = + crate::tui::ui::deployment::log_content_height( + self, + ); + let len = crate::tui::ui::logs::filtered_len(self); + if len == 0 { + return false; + } + let current = crate::tui::ui::logs::effective_scroll( + self.logs.scroll, + len, + visible, + ); + let candidate = crate::tui::ui::logs::effective_scroll( + current + .saturating_add_signed(delta.saturating_neg()), + len, + visible, + ); + self.logs.scroll = candidate; + candidate != current + } + DeploymentPane::OverallProgress + | DeploymentPane::Status => false, + } + } + View::Monitor + if self.session.monitoring_pane == MonitoringPane::Topology + && self + .session + .monitoring_expanded(MonitoringPane::Topology) => + { + let resources = self.resources(); + if resources.is_empty() { + return false; + } + let current = self.session.selected_resource.as_ref().and_then( + |selected| resources.iter().position(|id| id == selected), + ); + let candidate = current.map_or_else( + || if delta < 0 { resources.len() - 1 } else { 0 }, + |index| { + index + .saturating_add_signed(delta) + .min(resources.len().saturating_sub(1)) + }, + ); + self.session.selected_resource = + resources.get(candidate).cloned(); + self.session.monitor_scroll = candidate; + current != Some(candidate) + } + View::Monitor + if self.session.monitoring_pane == MonitoringPane::TopZones + && self + .session + .monitoring_expanded(MonitoringPane::TopZones) => + { + let capacity = + crate::tui::ui::monitor::top_zones_page_capacity(self); + if capacity == 0 { + return false; + } + let maximum = crate::tui::ui::monitor::top_zones_len( + self, + self.session.selected_rack, + ) + .saturating_sub(capacity); + let current = self.session.top_zones_scroll.min(maximum); + let candidate = + current.saturating_add_signed(delta).min(maximum); + self.session.top_zones_scroll = candidate; + candidate != current + } + _ => false, + } + } + fn move_item(&mut self, direction: Direction) { + match self.session.view { + View::Deployment => { + self.session.deployment_pane = Self::moved( + &DeploymentPane::ORDER, + Some(&self.session.deployment_pane), + direction, + ) + .unwrap(); + } + View::Monitor => { + self.session.monitoring_pane = Self::moved( + &MonitoringPane::ORDER, + Some(&self.session.monitoring_pane), + direction, + ) + .unwrap(); + } + } + } + fn repair_selection(&mut self) { + let racks = self.racks(); + if self + .session + .selected_rack + .as_ref() + .is_none_or(|r| !racks.contains(r)) + { + self.session.selected_rack = racks.first().copied(); + } + self.repair_resource(); + } + fn repair_resource(&mut self) { + let resources = self.resources(); + if self + .session + .selected_resource + .as_ref() + .is_none_or(|r| !resources.contains(r)) + { + self.session.selected_resource = resources.first().cloned(); + } + if resources.is_empty() { + self.session.selected_resource = None; + self.session.detail_open = false; + } + } + fn filtered_log_count(&self) -> usize { + self.logs + .entries + .iter() + .filter(|entry| self.logs_filter.accepts(entry.level)) + .count() + } + + fn clamp_log_scroll(&mut self) { + let visible = crate::tui::ui::deployment::log_content_height(self); + let maximum = self.filtered_log_count().saturating_sub(visible); + self.logs.scroll = self.logs.scroll.min(maximum); + } + fn operation_event( + &mut self, + request_id: OperationRequestId, + event: OperationEvent, + ) -> Vec { + let mut effects = vec![]; + // Stateful events are correlated to pending/active requests below. Logs remain an + // intentionally inclusive display stream, including delayed entries from old requests. + match event { + OperationEvent::Started { kind } => { + if self.operation.active.is_none() + && self.operation.pending.as_ref().is_some_and(|pending| { + pending.request_id == request_id && pending.kind == kind + }) + { + self.operation.pending = None; + self.operation.active = + Some(ActiveOperation::new(request_id, kind)); + self.operation.outcome = None; + let deferred = match self.session.post_operation_exit { + Some(PostOperationExit::Detach) => { + Some(CancelChoice::Leave) + } + Some(PostOperationExit::Destroy) + if kind != OperationKind::Destroy => + { + Some(CancelChoice::Destroy) + } + _ => None, + }; + if let Some(choice) = deferred { + effects.push(self.cancel(choice)); + } + } + } + OperationEvent::PhaseStarted { phase } => { + if let Some(active) = self.matching_active_mut(request_id) { + let phases = + crate::tui::ui::deployment::phase_order(active.kind); + if let Some(index) = + phases.iter().position(|candidate| *candidate == phase) + { + active.completed_phases = phases[..index].to_vec(); + active.phase = Some(phase); + } + } + } + OperationEvent::Log { level, message } => self.logs.push_filtered( + LogEntry::operation(level, message), + self.logs_filter, + ), + OperationEvent::Warning(warning) => { + if let Some(active) = self.matching_active_mut(request_id) + && !active.warnings.contains(&warning) + { + active.warnings.push(warning); + } + } + OperationEvent::Finished(outcome) => { + if self + .operation + .active + .as_ref() + .is_some_and(|active| active.request_id == request_id) + { + let mut active = self.operation.active.take().unwrap(); + let active_kind = active.kind; + let cancel_choice = active.cancel_choice; + self.operation.retained_warnings = active.warnings.clone(); + if command_succeeded(&outcome) { + active.completed_phases = + crate::tui::ui::deployment::phase_order( + active.kind, + ); + active.phase = None; + } + self.operation.retained = Some(active); + let should_start_destroy = active_kind + != OperationKind::Destroy + && (self.session.post_operation_exit + == Some(PostOperationExit::Destroy) + || (self.session.post_operation_exit.is_none() + && cancel_choice + == Some(CancelChoice::Destroy))); + let should_quit = match self.session.post_operation_exit { + Some(PostOperationExit::Detach) => true, + Some(PostOperationExit::Destroy) + if active_kind == OperationKind::Destroy => + { + command_succeeded(&outcome) + } + Some(PostOperationExit::Destroy) => false, + None => false, + }; + let exit = match self.session.post_operation_exit { + Some(PostOperationExit::Detach) => SessionExit::Detach, + Some(PostOperationExit::Destroy) => SessionExit::Quit, + None => SessionExit::Quit, + }; + self.operation.outcome = Some(outcome); + if should_start_destroy { + let destroy_effects = + self.start(OperationKind::Destroy); + if destroy_effects.is_empty() { + self.session.post_operation_exit = None; + } + effects.extend(destroy_effects); + } else { + self.session.post_operation_exit = None; + if should_quit { + self.session.exit = Some(exit); + self.session.quitting = true; + effects.push(Effect::Quit); + } + } + } + } + } + effects + } + + fn start(&mut self, kind: OperationKind) -> Vec { + let Some(request_id) = self.next_operation_request_id else { + self.operation.start_failure = Some( + "operation request ID space exhausted; restart the application before retrying" + .into(), + ); + return vec![]; + }; + self.next_operation_request_id = request_id.next(); + self.operation.retained = None; + self.operation.pending = Some(PendingOperation::new(request_id, kind)); + self.operation.start_failure = None; + vec![Effect::Start { request_id, kind }] + } + + fn cancel(&mut self, choice: CancelChoice) -> Effect { + let active = self.operation.active.as_mut().unwrap(); + active.cancelling = true; + if active.cancel_choice != Some(CancelChoice::Destroy) { + active.cancel_choice = Some(choice); + } + Effect::Cancel { + request_id: active.request_id, + choice: active.cancel_choice.expect("cancellation intent is set"), + } + } + + fn matching_active_mut( + &mut self, + request_id: OperationRequestId, + ) -> Option<&mut ActiveOperation> { + self.operation + .active + .as_mut() + .filter(|active| active.request_id == request_id) + } + + fn accept_attempt(sample: &LatestSample, at: Instant) -> bool { + sample.last_attempt.is_none_or(|latest| at >= latest) + && sample.good.as_ref().is_none_or(|good| at >= good.captured_at) + } + + fn accept_traffic_generation(&mut self, at: Instant) -> bool { + if self + .observability + .latest_traffic_generation + .is_some_and(|latest| at < latest) + { + return false; + } + self.observability.latest_traffic_generation = Some(at); + true + } + + fn normalize_topology( + mut topology: Vec, + ) -> Vec { + topology.sort_by(|a, b| { + a.id.cmp(&b.id) + .then_with(|| a.rack.cmp(&b.rack)) + .then_with(|| a.kind.cmp(&b.kind)) + .then_with(|| a.name.cmp(&b.name)) + .then_with(|| a.host.cmp(&b.host)) + }); + topology.dedup_by(|a, b| a.id == b.id); + topology + } +} + +fn command_succeeded(outcome: &CommandOutcome) -> bool { + matches!(outcome, CommandOutcome::Exited { status, .. } if status.success()) +} + +#[cfg(test)] +mod factual_outcome_tests { + use super::*; + use crate::tui::operation::LaunchPhase; + use crate::tui::telemetry::{ + BidirectionalRate, RackId, ResourceDescriptor, ResourceId, + ResourceKind, TrafficSample, TrafficSource, + }; + use std::os::unix::process::ExitStatusExt; + use std::time::Duration; + + fn active_app() -> App { + let mut app = App::new(vec![], 8, 8); + app.operation.active = Some(ActiveOperation::new( + OperationRequestId::FIRST, + OperationKind::Launch, + )); + app.next_operation_request_id = OperationRequestId::FIRST.next(); + app + } + + fn failed_outcome() -> CommandOutcome { + CommandOutcome::SpawnFailed { + message: "cancelled child settled".into(), + } + } + + fn successful_outcome() -> CommandOutcome { + CommandOutcome::Exited { + status: std::process::ExitStatus::from_raw(0), + stderr_summary: vec![], + } + } + + #[test] + fn switching_to_monitoring_selects_first_resource() { + let descriptor = ResourceDescriptor { + id: ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "g0".into(), + host: None, + }; + let mut app = App::new(vec![descriptor.clone()], 8, 8); + + app.update(AppEvent::Action(Action::SwitchView(View::Monitor))); + + assert_eq!(app.session.selected_resource, Some(descriptor.id)); + } + + #[test] + fn monitoring_x_opens_external_monitoring() { + let mut app = App::new(vec![], 8, 8); + app.session.view = View::Monitor; + + app.update(AppEvent::Action(Action::RequestCancelAndDestroy)); + + assert!(app.session.external_monitoring_open); + assert!(app.session.confirmation.is_none()); + } + + #[test] + fn oximeter_supersedes_direct_until_its_next_failure() { + let id = ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"); + let descriptor = ResourceDescriptor { + id: id.clone(), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "g0".into(), + host: None, + }; + let mut app = App::new(vec![descriptor], 8, 8); + let started = Instant::now(); + let direct = TrafficSample { + total: BidirectionalRate { + rx_bytes_sec: 10.0, + ..Default::default() + }, + ..Default::default() + }; + app.update(AppEvent::Traffic { + id: id.clone(), + at: started, + sample: direct, + }); + app.update(AppEvent::Traffic { + id: id.clone(), + at: started + Duration::from_secs(2), + sample: TrafficSample { + total: BidirectionalRate { + rx_bytes_sec: 30.0, + ..Default::default() + }, + ..Default::default() + }, + }); + app.update(AppEvent::OximeterTraffic { + id: id.clone(), + // The request completed after the newer direct probe, while the + // Oximeter sample itself retains its source timestamp. + at: started + Duration::from_secs(3), + samples: vec![( + started + Duration::from_secs(1), + TrafficSample { + total: BidirectionalRate { + rx_bytes_sec: 20.0, + ..Default::default() + }, + source: TrafficSource::Oximeter, + ..Default::default() + }, + )], + }); + + let state = &app.observability.telemetry.resources[&id]; + assert_eq!(state.current_sample.source, TrafficSource::Oximeter); + assert_eq!(state.current_rate.rx_bytes_sec, 20.0); + + app.update(AppEvent::OximeterTrafficFailed { + rack: RackId(0), + at: started + Duration::from_secs(4), + message: "Nexus timed out".into(), + }); + + let state = &app.observability.telemetry.resources[&id]; + assert_eq!(state.current_sample.source, TrafficSource::DirectProbe); + assert_eq!(state.current_rate.rx_bytes_sec, 30.0); + assert!(app.observability.traffic_failures[&id].latest_error.is_none()); + } + + #[test] + fn later_phase_completes_skipped_predecessors() { + let mut app = active_app(); + + app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::PhaseStarted { + phase: OperationPhase::Launch(LaunchPhase::Initialize), + }, + }); + + let operation = app.operation.active.as_ref().unwrap(); + assert_eq!( + operation.completed_phases, + vec![ + OperationPhase::Launch(LaunchPhase::Preflight), + OperationPhase::Launch(LaunchPhase::Stage), + OperationPhase::Launch(LaunchPhase::Boot), + ] + ); + assert_eq!( + operation.phase, + Some(OperationPhase::Launch(LaunchPhase::Initialize)) + ); + } + + #[test] + fn successful_operation_retains_complete_phase_history() { + let mut app = active_app(); + app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::PhaseStarted { + phase: OperationPhase::Launch(LaunchPhase::Initialize), + }, + }); + + app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::Finished(successful_outcome()), + }); + + assert!(app.operation.active.is_none()); + let operation = app.operation.displayed().unwrap(); + assert_eq!( + operation.completed_phases, + crate::tui::ui::deployment::phase_order(OperationKind::Launch) + ); + assert_eq!(operation.phase, None); + } + + #[test] + fn failed_operation_retains_only_proven_phase_history() { + let mut app = active_app(); + app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::PhaseStarted { + phase: OperationPhase::Launch(LaunchPhase::Initialize), + }, + }); + + app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::Finished(failed_outcome()), + }); + + let operation = app.operation.displayed().unwrap(); + assert_eq!(operation.completed_phases.len(), 3); + assert_eq!( + operation.phase, + Some(OperationPhase::Launch(LaunchPhase::Initialize)) + ); + } + + #[test] + fn new_operation_clears_retained_phase_history() { + let mut app = active_app(); + app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::Finished(successful_outcome()), + }); + assert!(app.operation.displayed().is_some()); + + app.start(OperationKind::Route); + + assert!(app.operation.displayed().is_none()); + } + + #[test] + fn confirmation_shortcut_confirms_launch() { + let mut app = App::new(vec![], 8, 8); + app.deployment.observed = ObservedDeploymentState::Stopped; + app.open_confirmation(Confirmation::Launch); + + assert_eq!( + app.update(AppEvent::Action(Action::CopyReattachCommand)), + vec![Effect::Start { + request_id: OperationRequestId::FIRST, + kind: OperationKind::Launch, + }] + ); + assert!(app.session.confirmation.is_none()); + } + + #[test] + fn force_stop_is_separate_and_defaults_to_back() { + let options = Confirmation::ForceStop.options(true); + assert!(options[0].label.contains("Force stop")); + assert_eq!(options[1].action, ConfirmationOptionAction::Reject); + assert!( + crate::tui::ui::confirm_dialog::prompt_text( + Confirmation::ForceStop + ) + .contains("Force stop") + ); + } + + #[test] + fn direct_cancel_destroy_starts_destroy_only_after_finished_and_stays_open() + { + let mut app = active_app(); + + assert_eq!( + app.update(AppEvent::Action(Action::RequestCancelAndDestroy)), + vec![] + ); + app.update(AppEvent::Action(Action::Scroll { delta: -1, page: false })); + assert_eq!( + app.update(AppEvent::Action(Action::Activate)), + vec![Effect::Cancel { + request_id: OperationRequestId::FIRST, + choice: CancelChoice::Destroy, + }] + ); + assert_eq!( + app.update(AppEvent::ReconciliationStarted { at: Instant::now() }), + vec![] + ); + assert!(app.operation.pending.is_none()); + + let effects = app.update(AppEvent::Operation { + request_id: OperationRequestId::FIRST, + event: OperationEvent::Finished(failed_outcome()), + }); + let destroy_request = OperationRequestId::FIRST.next().unwrap(); + assert_eq!( + effects, + vec![Effect::Start { + request_id: destroy_request, + kind: OperationKind::Destroy, + }] + ); + assert!(!app.session.quitting); + assert_eq!(app.session.exit, None); + + app.update(AppEvent::Operation { + request_id: destroy_request, + event: OperationEvent::Started { kind: OperationKind::Destroy }, + }); + assert_eq!( + app.update(AppEvent::Operation { + request_id: destroy_request, + event: OperationEvent::Finished(failed_outcome()), + }), + vec![] + ); + assert!(!app.session.quitting); + assert_eq!(app.session.exit, None); + } + + #[test] + fn cancellation_intent_only_escalates() { + let mut app = active_app(); + app.cancel(CancelChoice::Leave); + assert!(app.can_upgrade_cancel_to_destroy()); + assert_eq!( + app.update(AppEvent::Action(Action::RequestCancelAndDestroy)), + vec![] + ); + app.update(AppEvent::Action(Action::Scroll { delta: -1, page: false })); + assert_eq!( + app.update(AppEvent::Action(Action::Activate)), + vec![Effect::Cancel { + request_id: OperationRequestId::FIRST, + choice: CancelChoice::Destroy, + }] + ); + assert!(!app.can_upgrade_cancel_to_destroy()); + assert_eq!( + app.cancel(CancelChoice::Leave), + Effect::Cancel { + request_id: OperationRequestId::FIRST, + choice: CancelChoice::Destroy, + } + ); + + assert_eq!( + app.operation.active.unwrap().cancel_choice, + Some(CancelChoice::Destroy) + ); + } + + #[test] + fn force_stop_can_be_requested_once_while_safe_waiting() { + let mut app = active_app(); + app.cancel(CancelChoice::Leave); + + app.open_confirmation(Confirmation::ForceStop); + app.update(AppEvent::Action(Action::Scroll { delta: -1, page: false })); + assert_eq!( + app.update(AppEvent::Action(Action::Activate)), + vec![Effect::ForceStop { request_id: OperationRequestId::FIRST }] + ); + app.open_confirmation(Confirmation::ForceStop); + } +} diff --git a/voxel/src/tui/collector.rs b/voxel/src/tui/collector.rs new file mode 100644 index 0000000..c086208 --- /dev/null +++ b/voxel/src/tui/collector.rs @@ -0,0 +1,2611 @@ +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + use tokio::net::TcpListener; + use tokio::sync::Notify; + use voxel_config::VoxelConfig; + + fn response(generation: u64, command: &str) -> String { + if command == KSTAT { + format!( + "0:link:net0:rbytes64\t{}\n0:link:net0:obytes64\t{}\n0:link:net0:ipackets64\t{}\n0:link:net0:opackets64\t{}\n", + 100 + generation, + 200 + generation, + 10 + generation, + 20 + generation + ) + } else if command == DLADM { + "net0:oxz_nexus_deadbeef0\n".into() + } else if command == LINUX_COUNTERS { + format!( + "eth0 {} {} {} {}\n", + 100 + generation, + 200 + generation, + 10 + generation, + 20 + generation + ) + } else if command == IPADM { + "net0/v4:192.0.2.10/24\nnet0/v6:fd00::10/64\n".into() + } else if command == LINUX_ADDRESSES { + "1: lo inet 127.0.0.1/8 scope host lo\n2: eth0 inet6 fd00::20/64 scope global\n".into() + } else if command == SLED_AGENT_STATE { + "online\n".into() + } else if command == MAINTENANCE_SERVICES { + "svc:/oxide/broken:default\n".into() + } else if command == "zoneadm list -cp | cut -d: -f2" { + "global\noxz_ntp_deadbeef\n".into() + } else if command + == "zlogin oxz_ntp_deadbeef chronyc -n tracking 2>/dev/null" + { + "Stratum : 3\nLeap status : Normal\n".into() + } else if command.starts_with("curl -s --max-time 5 ") { + r#"{"status":"initializing","step":"sled_init"}"#.into() + } else { + String::new() + } + } + + struct FakeExecutor { + generation: AtomicU64, + calls: std::sync::Mutex>, + } + + impl NodeExecutor for FakeExecutor { + fn execute<'a>( + &'a self, + target: &'a NodeTarget, + command: &'a str, + ) -> BoxFuture<'a, anyhow::Result> { + self.calls + .lock() + .unwrap() + .push((target.clone(), command.to_owned())); + let n = self.generation.load(Ordering::SeqCst); + Box::pin(async move { Ok(response(n, command)) }) + } + } + + struct Gate { + open: AtomicBool, + notify: Notify, + } + + impl Gate { + fn closed() -> Self { + Self { open: AtomicBool::new(false), notify: Notify::new() } + } + + async fn wait(&self) { + while !self.open.load(Ordering::Acquire) { + let notified = self.notify.notified(); + if self.open.load(Ordering::Acquire) { + break; + } + notified.await; + } + } + + fn release(&self) { + self.open.store(true, Ordering::Release); + self.notify.notify_waiters(); + } + } + + struct ActiveCall<'a> { + executor: &'a ControlledExecutor, + traffic: bool, + } + + impl Drop for ActiveCall<'_> { + fn drop(&mut self) { + self.executor.active.fetch_sub(1, Ordering::SeqCst); + if self.traffic { + self.executor.traffic_active.fetch_sub(1, Ordering::SeqCst); + } + } + } + + struct ControlledExecutor { + calls: std::sync::Mutex>, + active: AtomicUsize, + max_active: AtomicUsize, + traffic_active: AtomicUsize, + overlap: AtomicBool, + block_traffic: bool, + block_router: bool, + gate: Gate, + } + + impl ControlledExecutor { + fn new(block_traffic: bool, block_router: bool) -> Self { + Self { + calls: Default::default(), + active: AtomicUsize::new(0), + max_active: AtomicUsize::new(0), + traffic_active: AtomicUsize::new(0), + overlap: AtomicBool::new(false), + block_traffic, + block_router, + gate: Gate::closed(), + } + } + + fn has_command(&self, command: &str) -> bool { + self.calls.lock().unwrap().iter().any(|(_, c)| c == command) + } + } + + impl NodeExecutor for ControlledExecutor { + fn execute<'a>( + &'a self, + target: &'a NodeTarget, + command: &'a str, + ) -> BoxFuture<'a, anyhow::Result> { + self.calls + .lock() + .unwrap() + .push((target.clone(), command.to_owned())); + Box::pin(async move { + let traffic = command == KSTAT + || command == DLADM + || command == LINUX_COUNTERS; + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.max_active.fetch_max(active, Ordering::SeqCst); + if traffic { + self.traffic_active.fetch_add(1, Ordering::SeqCst); + } else if self.traffic_active.load(Ordering::SeqCst) > 0 { + self.overlap.store(true, Ordering::SeqCst); + } + let _active = ActiveCall { executor: self, traffic }; + let should_block = (self.block_traffic && traffic) + || (self.block_router + && matches!(target, NodeTarget::Router { .. })); + if should_block { + self.gate.wait().await; + } + Ok(response(0, command)) + }) + } + } + + async fn wait_until(mut condition: impl FnMut() -> bool) { + tokio::time::timeout(Duration::from_secs(1), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .expect("condition was not reached"); + } + + fn collector_with_targets( + executor: Arc, + resources: Vec, + concurrency: usize, + ) -> Arc> { + Arc::new(Collector { + executor, + targets: CollectorTargets { resources, rss: vec![] }, + baselines: Default::default(), + nexus: Default::default(), + sled_serials: Default::default(), + oximeter_attempts: Default::default(), + oximeter_diagnostic_attempts: Default::default(), + concurrency, + }) + } + + fn resource_target(name: &str, kind: ResourceKind) -> ResourceTarget { + let id = match kind { + ResourceKind::Router => ResourceId::fleet(kind, name), + _ => ResourceId::rack(RackId(0), kind, name), + }; + ResourceTarget { + id, + kind, + target: match kind { + ResourceKind::Sled => NodeTarget::Sled { name: name.into() }, + ResourceKind::SwitchZone => NodeTarget::SwitchZone { + host: name.into(), + zone: "oxz_switch".into(), + }, + ResourceKind::Router => { + NodeTarget::Router { name: name.into() } + } + }, + } + } + + #[test] + fn discovers_typed_targets_and_one_rss_target_per_rack() { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 2\nsleds = 3\nrouters = [\"ce\", \"cr1\"]\n", + ) + .unwrap(); + let targets = CollectorTargets::from_config(&config); + assert_eq!(targets.resources.len(), 12); + assert_eq!(targets.rss.len(), 2); + assert_eq!(targets.rss[0].rack, crate::tui::telemetry::RackId(0)); + assert_eq!( + targets.rss[0].target, + NodeTarget::Sled { name: "g0".into() } + ); + assert_eq!( + targets.rss[1].target, + NodeTarget::Sled { name: "g3".into() } + ); + assert!(targets.resources.iter().any(|target| matches!( + &target.target, + NodeTarget::SwitchZone { host, zone } if host == "g5" && zone == "oxz_switch" + ))); + assert!(targets.resources.iter().any(|target| matches!( + &target.target, + NodeTarget::Router { name } if name == "ce" + ))); + } + + #[test] + fn isolated_node_addresses_are_available_without_serial_discovery() { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 3\nrouters = [\"ce\"]\n\ + [external]\nmode = \"isolated\"\n", + ) + .unwrap(); + + let cache = + IpCache::from_config_and_addresses(&config, BTreeMap::new()); + + assert_eq!( + cache.resolved.get("g0").map(String::as_str), + Some("172.30.199.10") + ); + assert_eq!( + cache.resolved.get("ce").map(String::as_str), + Some("172.30.199.13") + ); + assert!(cache.in_flight.is_empty()); + + assert!( + IpCache::from_config_and_addresses( + &VoxelConfig::default(), + BTreeMap::new(), + ) + .resolved + .is_empty() + ); + let resumed = IpCache::from_config_and_addresses( + &VoxelConfig::default(), + [("g0".to_string(), "192.168.1.184".to_string())] + .into_iter() + .collect(), + ); + assert_eq!( + resumed.resolved.get("g0").map(String::as_str), + Some("192.168.1.184") + ); + assert!(resumed.address_for_probe("g0", true).is_ok()); + assert!( + IpCache::default() + .address_for_probe("g0", true) + .unwrap_err() + .to_string() + .contains("unavailable after resuming") + ); + assert_eq!( + IpCache::default().address_for_probe("g0", false).unwrap(), + None + ); + } + + #[test] + fn oximeter_attempts_are_throttled_from_completion_per_rack() { + let start = Instant::now(); + let mut attempts = BTreeMap::new(); + + assert!(oximeter_due(&attempts, RackId(0), start)); + attempts.insert(RackId(0), start); + assert!(!oximeter_due( + &attempts, + RackId(0), + start + OXIMETER_INTERVAL - Duration::from_nanos(1) + )); + assert!(oximeter_due(&attempts, RackId(0), start + OXIMETER_INTERVAL)); + assert!(oximeter_due( + &attempts, + RackId(1), + start + Duration::from_secs(1) + )); + } + + #[test] + fn scheduler_configuration_rejects_zero_values() { + assert!( + SchedulerConfig::new( + std::time::Duration::ZERO, + std::time::Duration::from_secs(1), + 1, + ) + .is_err() + ); + assert!( + SchedulerConfig::new( + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + 0, + ) + .is_err() + ); + } + + #[tokio::test] + async fn first_sample_is_zero_with_detail_and_all_resource_kinds_emit() { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 1\nrouters = [\"ce\"]\n", + ) + .unwrap(); + let fake = Arc::new(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + }); + let collector = Collector::new(fake.clone(), &config, 2).unwrap(); + let (sender, mut receiver) = mpsc::channel(8); + assert!( + collector.collect_traffic(&sender, &CancellationToken::new()).await + ); + let mut events = Vec::new(); + while let Ok(event) = receiver.try_recv() { + events.push(event); + } + assert_eq!(events.len(), 3); + let mut kinds = std::collections::BTreeSet::new(); + let mut timestamp = None; + for event in events { + let AppEvent::Traffic { id, at, sample } = event else { + panic!("unexpected failure event") + }; + assert_eq!(sample.total.total_bytes_sec(), 0.0); + assert!(sample.links.is_empty()); + assert_eq!(timestamp.get_or_insert(at), &at); + kinds.insert(id.kind); + } + assert_eq!(kinds.len(), 3); + assert!(fake.calls.lock().unwrap().iter().any( + |(target, _)| matches!(target, NodeTarget::SwitchZone { host, .. } if host == "g0") + )); + let calls = fake.calls.lock().unwrap(); + assert!(calls.iter().any(|(target, command)| + matches!(target, NodeTarget::Sled { name } if name == "g0") + && command == "kstat -p -c net link:::rbytes64 link:::obytes64 link:::ipackets64 link:::opackets64")); + assert!(calls.iter().any( + |(target, command)| matches!(target, NodeTarget::Sled { name } if name == "g0") + && command == "dladm show-linkprop -c -p zone -o link,value" + )); + assert!(calls.iter().any(|(target, command)| + matches!(target, NodeTarget::SwitchZone { host, zone } if host == "g0" && zone == "oxz_switch") + && command == "kstat -p -c net link:::rbytes64 link:::obytes64 link:::ipackets64 link:::opackets64")); + assert!(calls.iter().any( + |(target, command)| matches!(target, NodeTarget::Router { name } if name == "ce") + && command == LINUX_COUNTERS + )); + } + + #[tokio::test] + async fn second_generation_uses_elapsed_time_and_retains_link_and_zone_detail() + { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 1\nrouters = []\n", + ) + .unwrap(); + let fake = Arc::new(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + }); + let collector = Collector::new(fake.clone(), &config, 2).unwrap(); + let (sender, mut receiver) = mpsc::channel(16); + assert!( + collector.collect_traffic(&sender, &CancellationToken::new()).await + ); + while receiver.try_recv().is_ok() {} + for state in collector.baselines.lock().await.values_mut() { + state.baseline.as_mut().unwrap().captured_at -= + Duration::from_secs(2); + } + fake.generation.store(100, Ordering::SeqCst); + assert!( + collector.collect_traffic(&sender, &CancellationToken::new()).await + ); + let sample = std::iter::from_fn(|| receiver.try_recv().ok()) + .find_map(|event| match event { + AppEvent::Traffic { id, sample, .. } + if id.kind == ResourceKind::Sled => + { + Some(sample) + } + _ => None, + }) + .unwrap(); + assert!((90.0..110.0).contains(&sample.total.total_bytes_sec())); + assert!(sample.links.contains_key("net0")); + assert_eq!(sample.zones[0].name, "oxz_nexus_deadbeef0"); + assert!(sample.zones[0].rate.total_bytes_sec() > 0.0); + } + + #[tokio::test] + async fn addresses_sled_health_and_multi_rack_rss_publish_typed_results_in_order() + { + assert_eq!( + SLED_AGENT_STATE, + "svcs -H -o state svc:/oxide/sled-agent:default" + ); + assert_eq!( + MAINTENANCE_SERVICES, + "svcs -H -o state,fmri -a | awk '$1 == \"maintenance\" { print $2 }'" + ); + let config = VoxelConfig::from_toml( + "[topology]\nracks = 2\nsleds = 2\nrouters = [\"ce\"]\n", + ) + .unwrap(); + let fake = Arc::new(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + }); + let collector = Collector::new(fake.clone(), &config, 2).unwrap(); + let (sender, mut receiver) = mpsc::channel(32); + assert!( + collector.collect_health(&sender, &CancellationToken::new()).await + ); + let events: Vec<_> = + std::iter::from_fn(|| receiver.try_recv().ok()).collect(); + for kind in + [ResourceKind::Sled, ResourceKind::SwitchZone, ResourceKind::Router] + { + assert!( + events + .iter() + .any(|e| matches!(e, AppEvent::Addresses { id, .. } if id.kind == kind)) + ); + } + assert_eq!(events.iter().filter(|e| matches!(e, AppEvent::Health { diagnostic, .. } if diagnostic.sled_agent == Some(crate::tui::telemetry::ServiceState::Online) && diagnostic.ntp.synchronized == Some(true))).count(), 4); + assert_eq!( + events + .iter() + .filter_map(|event| match event { + AppEvent::Health { diagnostic, .. } => Some(diagnostic), + _ => None, + }) + .filter(|diagnostic| { + diagnostic.sled_agent + == Some(crate::tui::telemetry::ServiceState::Online) + && diagnostic.failed_services == ["oxide/broken"] + && diagnostic.ntp.synchronized == Some(true) + }) + .count(), + 4 + ); + assert_eq!( + events.iter().filter(|e| matches!(e, AppEvent::Rss { .. })).count(), + 2 + ); + let calls = fake.calls.lock().unwrap(); + let zone = calls + .iter() + .position(|(_, c)| c == "zoneadm list -cp | cut -d: -f2") + .unwrap(); + let ntp = calls + .iter() + .position(|(_, c)| { + c == "zlogin oxz_ntp_deadbeef chronyc -n tracking 2>/dev/null" + }) + .unwrap(); + assert!(zone < ntp); + assert!(calls.iter().any(|(_, command)| command == SLED_AGENT_STATE)); + assert!( + calls.iter().any(|(_, command)| command == MAINTENANCE_SERVICES) + ); + assert!(!calls.iter().any(|(_, command)| command == "svcs -xHo fmri")); + } + + struct EmptyRouter(FakeExecutor); + impl NodeExecutor for EmptyRouter { + fn execute<'a>( + &'a self, + target: &'a NodeTarget, + command: &'a str, + ) -> BoxFuture<'a, anyhow::Result> { + if matches!(target, NodeTarget::Router { .. }) + && command == LINUX_COUNTERS + { + Box::pin(async { Ok(String::new()) }) + } else { + self.0.execute(target, command) + } + } + } + + #[tokio::test] + async fn empty_router_counters_fail_only_router_at_peer_generation_timestamp() + { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 1\nrouters = [\"ce\"]\n", + ) + .unwrap(); + let fake = Arc::new(EmptyRouter(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + })); + let collector = Collector::new(fake, &config, 3).unwrap(); + let (sender, mut receiver) = mpsc::channel(8); + assert!( + collector.collect_traffic(&sender, &CancellationToken::new()).await + ); + let events: Vec<_> = + std::iter::from_fn(|| receiver.try_recv().ok()).collect(); + let times: std::collections::BTreeSet<_> = events + .iter() + .map(|e| match e { + AppEvent::Traffic { at, .. } + | AppEvent::TrafficFailed { at, .. } => *at, + _ => unreachable!(), + }) + .collect(); + assert_eq!(times.len(), 1); + assert_eq!(events.iter().filter(|e| matches!(e, AppEvent::TrafficFailed { id, .. } if id.kind == ResourceKind::Router)).count(), 1); + assert_eq!( + events + .iter() + .filter(|e| matches!(e, AppEvent::Traffic { .. })) + .count(), + 2 + ); + } + + #[tokio::test] + async fn cancellation_unblocks_bounded_sender_without_post_cancel_event() { + let (sender, mut receiver) = mpsc::channel(1); + sender.send(AppEvent::Tick { now: Instant::now() }).await.unwrap(); + let cancel = CancellationToken::new(); + let task = tokio::spawn({ + let sender = sender.clone(); + let cancel = cancel.clone(); + async move { + Collector::::send( + &sender, + &cancel, + AppEvent::Tick { now: Instant::now() }, + ) + .await + } + }); + cancel.cancel(); + assert!( + !tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + ); + assert!(matches!(receiver.recv().await, Some(AppEvent::Tick { .. }))); + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn receiver_closure_terminates_collection_cleanly() { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 1\nrouters = []\n", + ) + .unwrap(); + let fake = Arc::new(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + }); + let collector = Collector::new(fake, &config, 1).unwrap(); + let (sender, receiver) = mpsc::channel(1); + drop(receiver); + assert!( + !tokio::time::timeout( + Duration::from_secs(1), + collector.collect_traffic(&sender, &CancellationToken::new()) + ) + .await + .unwrap() + ); + } + + #[tokio::test] + async fn blocked_ssh_work_is_dropped_on_cancellation_without_an_event() { + let fake = Arc::new(ControlledExecutor::new(true, false)); + let collector = collector_with_targets( + fake.clone(), + vec![resource_target("g0", ResourceKind::Sled)], + 1, + ); + let (sender, mut receiver) = mpsc::channel(1); + let cancel = CancellationToken::new(); + let task = tokio::spawn({ + let collector = collector.clone(); + let cancel = cancel.clone(); + async move { collector.collect_traffic(&sender, &cancel).await } + }); + wait_until(|| fake.active.load(Ordering::SeqCst) == 1).await; + + cancel.cancel(); + + assert!( + !tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + ); + assert_eq!(fake.active.load(Ordering::SeqCst), 0); + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn cancelled_router_call_finishes_its_safe_boundary_before_returning() + { + let fake = Arc::new(ControlledExecutor::new(false, true)); + let collector = collector_with_targets( + fake.clone(), + vec![resource_target("ce", ResourceKind::Router)], + 1, + ); + let (sender, mut receiver) = mpsc::channel(1); + let cancel = CancellationToken::new(); + let task = tokio::spawn({ + let collector = collector.clone(); + let cancel = cancel.clone(); + async move { collector.collect_traffic(&sender, &cancel).await } + }); + wait_until(|| fake.active.load(Ordering::SeqCst) == 1).await; + + cancel.cancel(); + tokio::task::yield_now().await; + assert!( + !task.is_finished(), + "an in-flight Falcon serial call must reach its timeout/result boundary" + ); + fake.gate.release(); + + assert!( + !tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + ); + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn full_sender_cancellation_does_not_drop_an_in_flight_router_call() { + let fake = Arc::new(ControlledExecutor::new(false, true)); + let collector = collector_with_targets( + fake.clone(), + vec![ + resource_target("g0", ResourceKind::Sled), + resource_target("ce", ResourceKind::Router), + ], + 2, + ); + let (sender, mut receiver) = mpsc::channel(1); + sender.send(AppEvent::Tick { now: Instant::now() }).await.unwrap(); + let cancel = CancellationToken::new(); + let task = tokio::spawn({ + let collector = collector.clone(); + let cancel = cancel.clone(); + async move { collector.collect_traffic(&sender, &cancel).await } + }); + wait_until(|| { + fake.calls.lock().unwrap().iter().any(|(target, command)| { + matches!(target, NodeTarget::Router { .. }) + && command == LINUX_COUNTERS + }) + }) + .await; + + cancel.cancel(); + tokio::task::yield_now().await; + assert!(!task.is_finished()); + fake.gate.release(); + + assert!( + !tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + ); + assert!(matches!(receiver.recv().await, Some(AppEvent::Tick { .. }))); + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn scheduler_uses_its_concurrency_limit_and_never_overlaps_cadences() + { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 2\nsleds = 2\nrouters = [\"ce\"]\n", + ) + .unwrap(); + let fake = Arc::new(ControlledExecutor::new(true, false)); + // Deliberately differ from SchedulerConfig to prove the scheduler's + // configured limit, rather than the direct-collection default, wins. + let collector = + Arc::new(Collector::new(fake.clone(), &config, 8).unwrap()); + let schedule = SchedulerConfig::new( + Duration::from_secs(3_600), + Duration::from_secs(3_600), + 2, + ) + .unwrap(); + let (sender, _receiver) = mpsc::channel(64); + let cancel = CancellationToken::new(); + let task = tokio::spawn(collector.run_gated( + schedule, + sender, + cancel.clone(), + Arc::new(tokio::sync::Semaphore::new(1)), + )); + wait_until(|| fake.max_active.load(Ordering::SeqCst) >= 2).await; + + assert_eq!(fake.max_active.load(Ordering::SeqCst), 2); + assert!(!fake.has_command(IPADM)); + fake.gate.release(); + wait_until(|| fake.has_command(IPADM)).await; + + assert!( + !fake.overlap.load(Ordering::SeqCst), + "health cadence began before the traffic run completed" + ); + assert!(fake.max_active.load(Ordering::SeqCst) <= 2); + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn stalled_nexus_does_not_block_direct_traffic_cadence() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = reqwest::Url::parse(&format!( + "http://{}/", + listener.local_addr().unwrap() + )) + .unwrap(); + let server = tokio::spawn(async move { + while let Ok((connection, _)) = listener.accept().await { + tokio::spawn(async move { + let _connection = connection; + std::future::pending::<()>().await; + }); + } + }); + let fake = Arc::new(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + }); + let collector = Arc::new(Collector { + executor: fake, + targets: CollectorTargets { + resources: vec![resource_target("g0", ResourceKind::Sled)], + rss: vec![], + }, + baselines: Default::default(), + nexus: BTreeMap::from([( + RackId(0), + Arc::new( + NexusClient::new( + vec![endpoint], + RecoveryLogin { + silo: "recovery".into(), + username: "recovery".into(), + password: "oxide".into(), + }, + Duration::from_secs(5), + ) + .unwrap(), + ), + )]), + sled_serials: Default::default(), + oximeter_attempts: Default::default(), + oximeter_diagnostic_attempts: Default::default(), + concurrency: 1, + }); + let schedule = SchedulerConfig::new( + Duration::from_millis(20), + Duration::from_secs(3_600), + 1, + ) + .unwrap(); + let (sender, mut receiver) = mpsc::channel(64); + let cancel = CancellationToken::new(); + let task = tokio::spawn(collector.run_gated( + schedule, + sender, + cancel.clone(), + Arc::new(tokio::sync::Semaphore::new(1)), + )); + + let direct_samples = + tokio::time::timeout(Duration::from_millis(250), async { + let mut count = 0; + while count < 3 { + if matches!( + receiver.recv().await, + Some(AppEvent::Traffic { .. }) + ) { + count += 1; + } + } + count + }) + .await + .expect("direct probes were blocked by the stalled Nexus request"); + + assert_eq!(direct_samples, 3); + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + server.abort(); + } + + struct FailingSecondRackRss(FakeExecutor); + + impl NodeExecutor for FailingSecondRackRss { + fn execute<'a>( + &'a self, + target: &'a NodeTarget, + command: &'a str, + ) -> BoxFuture<'a, anyhow::Result> { + if matches!(target, NodeTarget::Sled { name } if name == "g2") + && command.starts_with("curl -s --max-time 5 ") + { + Box::pin(async { Err(anyhow!("rack 2 unreachable")) }) + } else { + self.0.execute(target, command) + } + } + } + + #[tokio::test] + async fn one_rss_failure_does_not_suppress_another_racks_observation() { + let config = VoxelConfig::from_toml( + "[topology]\nracks = 2\nsleds = 2\nrouters = []\n", + ) + .unwrap(); + let fake = Arc::new(FailingSecondRackRss(FakeExecutor { + generation: AtomicU64::new(0), + calls: Default::default(), + })); + let collector = Collector::new(fake, &config, 2).unwrap(); + let (sender, mut receiver) = mpsc::channel(32); + + assert!( + collector.collect_health(&sender, &CancellationToken::new()).await + ); + let events: Vec<_> = + std::iter::from_fn(|| receiver.try_recv().ok()).collect(); + assert!(events.iter().any(|event| matches!( + event, + AppEvent::Rss { rack: RackId(0), .. } + ))); + assert!(events.iter().any(|event| matches!( + event, + AppEvent::RssFailed { + rack: RackId(1), + message, + .. + } if message == "rack 2 unreachable" + ))); + } +} +use std::collections::{BTreeMap, BTreeSet}; +use std::process::Stdio; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, anyhow, ensure}; +use chrono::{DateTime, Utc}; +use futures::future::BoxFuture; +use futures::{StreamExt, stream}; +use tokio::sync::{Mutex, mpsc, oneshot, watch}; +use tokio_util::sync::CancellationToken; +use voxel_config::VoxelConfig; + +use crate::tui::event::AppEvent; +use crate::tui::nexus::{ + MetricType, NexusClient, OxqlQueryResult, RecoveryLogin, +}; +use crate::tui::reconcile::RssObservation; +use crate::tui::telemetry::{ + BidirectionalErrors, BidirectionalRate, CounterSnapshot, HealthDiagnostic, + OximeterExceptions, RackId, ResourceId, ResourceKind, ResourceTelemetry, + TrafficSample, TrafficSource, ZfsHeadroom, ZoneCpu, ZoneTraffic, + parse_chrony_tracking, parse_dladm_zone_vnics, parse_failed_services, + parse_ipadm_addresses, parse_kstat_link_counters, parse_linux_ip_addresses, + parse_linux_link_counters, parse_service_state, parse_zone_diagnostics, + resource_descriptors, +}; + +const KSTAT: &str = "kstat -p -c net link:::rbytes64 link:::obytes64 link:::ipackets64 link:::opackets64"; +const DLADM: &str = "dladm show-linkprop -c -p zone -o link,value"; +const IPADM: &str = "ipadm show-addr -p -o addrobj,addr"; +const LINUX_COUNTERS: &str = "for n in /sys/class/net/*; do i=${n##*/}; echo $i $(cat $n/statistics/rx_bytes) $(cat $n/statistics/tx_bytes) $(cat $n/statistics/rx_packets) $(cat $n/statistics/tx_packets); done"; +const LINUX_ADDRESSES: &str = "ip -o addr show"; +const SLED_AGENT_STATE: &str = "svcs -H -o state svc:/oxide/sled-agent:default"; +const MAINTENANCE_SERVICES: &str = + "svcs -H -o state,fmri -a | awk '$1 == \"maintenance\" { print $2 }'"; +const OXIMETER_TRAFFIC_METRICS: [&str; 6] = [ + "sled_data_link:bytes_received", + "sled_data_link:bytes_sent", + "sled_data_link:packets_received", + "sled_data_link:packets_sent", + "sled_data_link:errors_received", + "sled_data_link:errors_sent", +]; +const OXIMETER_INTERVAL: Duration = Duration::from_secs(30); +const ZONE_CPU_ALIGNMENT: Duration = Duration::from_secs(10); +const ZONE_CPU_QUERY: &str = "get zone:cpu_nsec + | filter timestamp > @now() - 2m + | align mean_within(10s) + | group_by [sled_serial, zone_name, zone_type, state], sum + | last 1"; +const ZFS_QUERY: &str = r#"{ +get zfs_pool:bytes_allocated; +get zfs_pool:bytes_total +} | last 1"#; +const OXIMETER_EXCEPTIONS_QUERY: &str = r#"{ +get oximeter_collector:failed_collections; +get oximeter_collector:database_samples_dropped +} | filter timestamp > @now() - 5m"#; + +fn oximeter_traffic_queries( + rack: RackId, + sled_serials: &BTreeMap, +) -> Vec { + sled_serials + .iter() + .filter(|(_, (sled_rack, _, _))| *sled_rack == rack) + .flat_map(|(serial, _)| { + let serial = serde_json::to_string(serial) + .expect("a string always serializes as JSON"); + OXIMETER_TRAFFIC_METRICS.into_iter().map(move |metric| { + format!( + "get {metric} | filter sled_serial == {serial} | last 1" + ) + }) + }) + .collect() +} + +fn oximeter_due( + attempts: &BTreeMap, + rack: RackId, + now: Instant, +) -> bool { + !attempts.get(&rack).is_some_and(|previous| { + now.duration_since(*previous) < OXIMETER_INTERVAL + }) +} + +#[derive(Default)] +struct PartialLinkRate { + rate: BidirectionalRate, + errors: BidirectionalErrors, + present: u8, +} + +#[derive(Default)] +struct PartialTrafficSample { + timestamp: Option>, + links: BTreeMap<(String, String), PartialLinkRate>, +} + +fn oximeter_sample_instant(timestamp: DateTime, now: Instant) -> Instant { + Utc::now() + .signed_duration_since(timestamp) + .to_std() + .ok() + .and_then(|age| now.checked_sub(age)) + .unwrap_or(now) +} + +fn oximeter_resource_ids( + rack: RackId, + serial: &str, + zone: &str, + sled_serials: &BTreeMap, +) -> anyhow::Result> { + let (sled_rack, sled, scrimlet) = sled_serials + .get(serial) + .with_context(|| format!("unknown Oximeter sled serial {serial}"))?; + ensure!(*sled_rack == rack, "Oximeter sled belongs to another rack"); + let mut ids = vec![ResourceId::rack(rack, ResourceKind::Sled, sled)]; + if zone.starts_with("oxz_switch") && *scrimlet { + ids.push(ResourceId::rack(rack, ResourceKind::SwitchZone, sled)); + } + Ok(ids) +} + +fn parse_oximeter_traffic( + result: OxqlQueryResult, + rack: RackId, + sled_serials: &BTreeMap, + now: Instant, +) -> anyhow::Result>> { + const TABLES: [&str; 6] = [ + "sled_data_link:bytes_received", + "sled_data_link:bytes_sent", + "sled_data_link:packets_received", + "sled_data_link:packets_sent", + "sled_data_link:errors_received", + "sled_data_link:errors_sent", + ]; + ensure!( + TABLES.iter().all(|required| result + .tables + .iter() + .any(|table| table.name == *required)), + "Oximeter traffic response omitted a required table" + ); + let mut partial = BTreeMap::::new(); + for table in result.tables { + let field = + TABLES.iter().position(|name| *name == table.name).ok_or_else( + || anyhow!("unexpected Oximeter traffic table {}", table.name), + )?; + for series in table.timeseries { + let serial = series + .field("sled_serial") + .context("Oximeter link series omitted sled_serial")?; + let zone = series + .field("zone_name") + .context("Oximeter link series omitted zone_name")?; + let link = series + .field("link_name") + .context("Oximeter link series omitted link_name")?; + let ids = oximeter_resource_ids(rack, serial, zone, sled_serials)?; + ensure!( + series.points.values.first().is_some_and(|column| { + column.metric_type == MetricType::Delta + }), + "Oximeter link counter was not converted to deltas" + ); + let points = series.integer_points()?; + ensure!( + points.len() == 1, + "Oximeter traffic query did not return the latest sample" + ); + for (start, timestamp, value) in points { + let start = + start.context("Oximeter link delta omitted start time")?; + let seconds = timestamp + .signed_duration_since(start) + .to_std() + .context("Oximeter link delta has an invalid interval")? + .as_secs_f64(); + ensure!(seconds > 0.0, "Oximeter link delta interval is empty"); + let value = value.context("Oximeter link delta is null")?; + ensure!(value >= 0, "Oximeter link delta is negative"); + for id in &ids { + let sample = partial.entry(id.clone()).or_default(); + sample.timestamp = + Some(sample.timestamp.map_or(timestamp, |current| { + current.max(timestamp) + })); + let rate = sample + .links + .entry((zone.to_owned(), link.to_owned())) + .or_default(); + let value = value as f64 / seconds; + match field { + 0 => rate.rate.rx_bytes_sec = value, + 1 => rate.rate.tx_bytes_sec = value, + 2 => rate.rate.rx_packets_sec = value, + 3 => rate.rate.tx_packets_sec = value, + 4 => rate.errors.rx_sec = value, + 5 => rate.errors.tx_sec = value, + _ => unreachable!(), + } + rate.present |= 1 << field; + } + } + } + } + let mut samples = + BTreeMap::>::new(); + for (id, partial) in partial { + let timestamp = partial + .timestamp + .context("Oximeter traffic sample omitted its timestamp")?; + let mut sample = TrafficSample { + source: TrafficSource::Oximeter, + ..Default::default() + }; + let mut zones = + BTreeMap::::new(); + for ((zone, link), rate) in partial.links { + ensure!( + rate.present == 0b11_1111, + "Oximeter link sample is incomplete" + ); + sample.total += rate.rate; + sample.errors.rx_sec += rate.errors.rx_sec; + sample.errors.tx_sec += rate.errors.tx_sec; + sample.links.insert(format!("{zone}/{link}"), rate.rate); + if zone != "global" { + let entry = zones.entry(zone).or_default(); + entry.0 += rate.rate; + entry.1.rx_sec += rate.errors.rx_sec; + entry.1.tx_sec += rate.errors.tx_sec; + } + } + sample.zones = zones + .into_iter() + .map(|(name, (rate, errors))| ZoneTraffic { + short_name: name + .strip_prefix("oxz_") + .unwrap_or(&name) + .split('_') + .next() + .unwrap_or(&name) + .to_owned(), + name, + rate, + errors, + }) + .collect(); + ensure!(!sample.links.is_empty(), "Oximeter traffic sample is empty"); + samples + .entry(id) + .or_default() + .push((oximeter_sample_instant(timestamp, now), sample)); + } + ensure!( + !samples.is_empty(), + "Oximeter traffic response contained no samples" + ); + Ok(samples) +} + +fn parse_zone_cpu( + result: OxqlQueryResult, + rack: RackId, + sled_serials: &BTreeMap, +) -> anyhow::Result> { + let table = result + .tables + .into_iter() + .find(|table| table.name == "zone:cpu_nsec") + .context("Oximeter response omitted zone:cpu_nsec")?; + let mut zones = BTreeMap::<(ResourceId, String, String), ZoneCpu>::new(); + for series in table.timeseries { + let serial = series + .field("sled_serial") + .context("zone CPU series omitted sled_serial")?; + ensure!( + sled_serials.get(serial).is_some_and(|(r, _, _)| *r == rack), + "zone CPU series belongs to another rack" + ); + let name = series + .field("zone_name") + .context("zone CPU series omitted zone_name")?; + let kind = series + .field("zone_type") + .context("zone CPU series omitted zone_type")?; + let state = + series.field("state").context("zone CPU series omitted state")?; + let ids = oximeter_resource_ids(rack, serial, name, sled_serials)?; + let metric_type = series + .points + .values + .first() + .map(|value| value.metric_type) + .context("zone CPU series omitted values")?; + ensure!( + matches!(metric_type, MetricType::Delta | MetricType::Gauge), + "zone CPU metric has an unsupported type" + ); + let mut latest = None; + for (start, timestamp, value) in series.numeric_points()? { + let seconds = match metric_type { + MetricType::Delta => timestamp + .signed_duration_since( + start.context("zone CPU delta omitted start time")?, + ) + .to_std() + .context("zone CPU delta has an invalid interval")? + .as_secs_f64(), + MetricType::Gauge => ZONE_CPU_ALIGNMENT.as_secs_f64(), + MetricType::Cumulative => unreachable!(), + }; + ensure!(seconds > 0.0, "zone CPU delta interval is empty"); + let percent = value + .map(|value| { + ensure!( + value.is_finite() && value >= 0.0, + "zone CPU value is invalid" + ); + Ok(value / 1_000_000_000.0 / seconds * 100.0) + }) + .transpose()?; + if latest.as_ref().is_none_or(|(latest_timestamp, _)| { + timestamp > *latest_timestamp + }) { + latest = Some((timestamp, percent)); + } + } + let percent = latest + .context("zone CPU series has no samples")? + .1 + .context("latest zone CPU delta is null")?; + for id in ids { + let zone = zones + .entry((id.clone(), name.into(), kind.into())) + .or_insert_with(|| ZoneCpu { + id, + name: name.into(), + kind: kind.into(), + user_percent: 0.0, + system_percent: 0.0, + wait_percent: 0.0, + }); + match state { + "user" => zone.user_percent += percent, + "sys" => zone.system_percent += percent, + "waitrq" => zone.wait_percent += percent, + _ => return Err(anyhow!("unknown zone CPU state {state}")), + } + } + } + Ok(zones.into_values().collect()) +} + +fn parse_zfs_headroom( + result: OxqlQueryResult, + rack: RackId, + sled_serials: &BTreeMap, +) -> anyhow::Result> { + let mut pools = + BTreeMap::<(ResourceId, String), (Option, Option)>::new(); + for table in result.tables { + let field = match table.name.as_str() { + "zfs_pool:bytes_allocated" => 0, + "zfs_pool:bytes_total" => 1, + _ => continue, + }; + for series in table.timeseries { + ensure!( + series + .points + .values + .first() + .is_some_and(|v| v.metric_type == MetricType::Gauge), + "ZFS headroom metric is not a gauge" + ); + let serial = series + .field("sled_serial") + .context("ZFS series omitted sled_serial")?; + let (sled_rack, sled, _) = sled_serials + .get(serial) + .context("ZFS series has unknown sled_serial")?; + ensure!(*sled_rack == rack, "ZFS series belongs to another rack"); + let pool = series + .field("pool_name") + .context("ZFS series omitted pool_name")?; + let value = series + .integer_points()? + .into_iter() + .filter_map(|(_, _, value)| value) + .next_back() + .context("ZFS series has no value")?; + ensure!(value >= 0, "ZFS byte gauge is negative"); + let pair = pools + .entry(( + ResourceId::rack(rack, ResourceKind::Sled, sled), + pool.into(), + )) + .or_default(); + if field == 0 { + pair.0 = Some(value as u64) + } else { + pair.1 = Some(value as u64) + } + } + } + pools + .into_iter() + .map(|((id, pool), (allocated, total))| { + let allocated_bytes = + allocated.context("ZFS pool omitted allocated bytes")?; + let total_bytes = total.context("ZFS pool omitted total bytes")?; + ensure!( + allocated_bytes <= total_bytes, + "ZFS pool allocation exceeds total" + ); + Ok(ZfsHeadroom { id, pool, allocated_bytes, total_bytes }) + }) + .collect() +} + +fn parse_oximeter_exceptions( + result: OxqlQueryResult, +) -> anyhow::Result { + let mut exceptions = OximeterExceptions::default(); + for table in result.tables { + let target = match table.name.as_str() { + "oximeter_collector:failed_collections" => { + &mut exceptions.failed_collections + } + "oximeter_collector:database_samples_dropped" => { + &mut exceptions.dropped_samples + } + _ => continue, + }; + for series in table.timeseries { + ensure!( + series + .points + .values + .first() + .is_some_and(|v| v.metric_type == MetricType::Delta), + "Oximeter exception counter was not converted to deltas" + ); + for (_, _, value) in series.integer_points()? { + if let Some(value) = value { + ensure!(value >= 0, "Oximeter exception delta is negative"); + *target = target.saturating_add(value as u64); + } + } + } + } + Ok(exceptions) +} + +#[cfg(test)] +mod oximeter_tests { + use super::*; + use serde_json::json; + + fn series(value: Option) -> serde_json::Value { + json!({ + "fields": { + "sled_serial": {"type": "string", "value": "2FAKE000"}, + "zone_name": {"type": "string", "value": "oxz_nexus_deadbeef0"}, + "link_name": {"type": "string", "value": "net0"} + }, + "points": { + "start_times": ["2026-08-17T00:00:00Z"], + "timestamps": ["2026-08-17T00:00:10Z"], + "values": [{ + "values": {"type": "integer", "values": [value]}, + "metric_type": "delta" + }] + } + }) + } + + fn traffic_result(null_table: Option<&str>) -> OxqlQueryResult { + let tables = [ + "sled_data_link:bytes_received", + "sled_data_link:bytes_sent", + "sled_data_link:packets_received", + "sled_data_link:packets_sent", + "sled_data_link:errors_received", + "sled_data_link:errors_sent", + ] + .into_iter() + .enumerate() + .map(|(index, name)| { + json!({ + "name": name, + "timeseries": [series((null_table != Some(name)).then_some((index + 1) as i64))] + }) + }) + .collect::>(); + serde_json::from_value(json!({"tables": tables})).unwrap() + } + + #[test] + fn traffic_queries_bound_each_response_to_one_sled() { + let serials = BTreeMap::from([ + ("2FAKE000".into(), (RackId(0), "g0".into(), true)), + ("2FAKE\"001".into(), (RackId(0), "g1".into(), false)), + ("2FAKE002".into(), (RackId(1), "g2".into(), false)), + ]); + + let queries = oximeter_traffic_queries(RackId(0), &serials); + + assert_eq!(queries.len(), 12); + assert!(queries.iter().any(|query| { + query + == "get sled_data_link:bytes_received | filter sled_serial == \"2FAKE000\" | last 1" + })); + assert!(queries.iter().any(|query| { + query + == "get sled_data_link:errors_sent | filter sled_serial == \"2FAKE\\\"001\" | last 1" + })); + assert!(queries.iter().all(|query| !query.contains("2FAKE002"))); + } + + #[test] + fn traffic_parser_maps_deltas_and_rejects_partial_samples() { + let serials = BTreeMap::from([( + "2FAKE000".into(), + (RackId(0), "g0".into(), true), + )]); + let parsed = parse_oximeter_traffic( + traffic_result(None), + RackId(0), + &serials, + Instant::now(), + ) + .unwrap(); + let sample = &parsed + [&ResourceId::rack(RackId(0), ResourceKind::Sled, "g0")][0] + .1; + assert_eq!(sample.source, TrafficSource::Oximeter); + assert_eq!(sample.total.rx_bytes_sec, 0.1); + assert_eq!(sample.errors.tx_sec, 0.6); + assert_eq!(sample.zones[0].name, "oxz_nexus_deadbeef0"); + + assert!( + parse_oximeter_traffic( + traffic_result(Some("sled_data_link:bytes_sent")), + RackId(0), + &serials, + Instant::now(), + ) + .is_err() + ); + } + + #[test] + fn traffic_parser_combines_latest_metrics_with_different_timestamps() { + let serials = BTreeMap::from([( + "2FAKE000".into(), + (RackId(0), "g0".into(), true), + )]); + let mut result = traffic_result(None); + for (index, table) in result.tables.iter_mut().enumerate() { + table.timeseries[0].points.timestamps[0] = + "2026-08-17T00:00:10Z".parse::>().unwrap() + + chrono::Duration::seconds(index as i64); + } + + let parsed = + parse_oximeter_traffic(result, RackId(0), &serials, Instant::now()) + .unwrap(); + + assert_eq!( + parsed[&ResourceId::rack(RackId(0), ResourceKind::Sled, "g0")] + .len(), + 1 + ); + } + + #[test] + fn diagnostic_parsers_aggregate_cpu_pair_zfs_and_count_exceptions() { + let fields = |extra: serde_json::Value| { + let mut fields = serde_json::Map::from_iter([( + "sled_serial".into(), + json!({"type": "string", "value": "2FAKE000"}), + )]); + fields.extend(extra.as_object().unwrap().clone()); + fields + }; + let points = + |metric_type: &str, start_times: serde_json::Value, value| { + json!({ + "start_times": start_times, + "timestamps": ["2026-08-17T00:00:10Z"], + "values": [{ + "values": {"type": "integer", "values": [value]}, + "metric_type": metric_type + }] + }) + }; + let serials = BTreeMap::from([( + "2FAKE000".into(), + (RackId(0), "g0".into(), true), + )]); + let cpu_tables = ["user", "sys", "waitrq"] + .into_iter() + .map(|state| { + json!({ + "fields": fields(json!({ + "zone_name": {"type": "string", "value": "oxz_nexus_deadbeef0"}, + "zone_type": {"type": "string", "value": "nexus"}, + "state": {"type": "string", "value": state} + })), + "points": { + "start_times": [ + "2026-08-17T00:00:00Z", + "2026-08-17T00:00:10Z" + ], + "timestamps": [ + "2026-08-17T00:00:10Z", + "2026-08-17T00:00:20Z" + ], + "values": [{ + "values": { + "type": "integer", + "values": [ + 1_000_000_000_i64, + 2_000_000_000_i64 + ] + }, + "metric_type": "delta" + }] + } + }) + }) + .collect::>(); + let cpu: OxqlQueryResult = serde_json::from_value(json!({ + "tables": [{"name": "zone:cpu_nsec", "timeseries": cpu_tables}] + })) + .unwrap(); + let cpu = parse_zone_cpu(cpu, RackId(0), &serials).unwrap(); + assert_eq!(cpu[0].user_percent, 20.0); + assert_eq!(cpu[0].system_percent, 20.0); + assert_eq!(cpu[0].wait_percent, 20.0); + + let zfs_tables = [ + ("zfs_pool:bytes_allocated", 40_i64), + ("zfs_pool:bytes_total", 100_i64), + ] + .into_iter() + .map(|(name, value)| { + json!({ + "name": name, + "timeseries": [{ + "fields": fields(json!({ + "pool_name": {"type": "string", "value": "oxp_test"} + })), + "points": points("gauge", serde_json::Value::Null, value) + }] + }) + }) + .collect::>(); + let zfs = parse_zfs_headroom( + serde_json::from_value(json!({"tables": zfs_tables})).unwrap(), + RackId(0), + &serials, + ) + .unwrap(); + assert_eq!(zfs[0].available_bytes(), 60); + + let exception_tables = [ + ("oximeter_collector:failed_collections", 2_i64), + ("oximeter_collector:database_samples_dropped", 3_i64), + ] + .into_iter() + .map(|(name, value)| json!({ + "name": name, + "timeseries": [{ + "fields": { + "collector_id": { + "type": "uuid", + "value": "00000000-0000-0000-0000-000000000001" + }, + "collector_ip": {"type": "ip_addr", "value": "192.0.2.1"}, + "collector_port": {"type": "u16", "value": 12223} + }, + "points": points("delta", json!(["2026-08-17T00:00:00Z"]), value) + }] + })) + .collect::>(); + let exceptions = parse_oximeter_exceptions( + serde_json::from_value(json!({"tables": exception_tables})) + .unwrap(), + ) + .unwrap(); + assert_eq!(exceptions.failed_collections, 2); + assert_eq!(exceptions.dropped_samples, 3); + } + + #[test] + fn zone_cpu_parser_accepts_aligned_grouped_values() { + let serials = BTreeMap::from([( + "2FAKE000".into(), + (RackId(0), "g0".into(), true), + )]); + let timeseries = ["user", "sys", "waitrq"] + .into_iter() + .map(|state| { + json!({ + "fields": { + "sled_serial": {"type": "string", "value": "2FAKE000"}, + "zone_name": {"type": "string", "value": "oxz_nexus_deadbeef0"}, + "zone_type": {"type": "string", "value": "nexus"}, + "state": {"type": "string", "value": state} + }, + "points": { + "timestamps": ["2026-08-17T00:00:10Z"], + "values": [{ + "values": {"type": "double", "values": [2_000_000_000.0]}, + "metric_type": "gauge" + }] + } + }) + }) + .collect::>(); + let result = serde_json::from_value(json!({ + "tables": [{"name": "zone:cpu_nsec", "timeseries": timeseries}] + })) + .unwrap(); + + let cpu = parse_zone_cpu(result, RackId(0), &serials).unwrap(); + + assert_eq!(cpu[0].user_percent, 20.0); + assert_eq!(cpu[0].system_percent, 20.0); + assert_eq!(cpu[0].wait_percent, 20.0); + } +} + +fn json_string_field(input: &str, key: &str) -> String { + let pattern = format!("\"{key}\":\""); + input + .find(&pattern) + .and_then(|start| input[start + pattern.len()..].split('"').next()) + .unwrap_or_default() + .to_string() +} + +fn parse_rss_observation(body: &str) -> RssObservation { + if body.trim().is_empty() { + return RssObservation::Unavailable; + } + match json_string_field(body, "status").as_str() { + "initializing" => { + let step = body + .find("\"step\"") + .map(|start| json_string_field(&body[start..], "status")) + .filter(|step| !step.is_empty()); + RssObservation::Initializing { step } + } + "initialized" => match json_string_field(body, "id") { + id if id.is_empty() => RssObservation::StaleInitialized, + id => RssObservation::Initialized { id }, + }, + "initialization_failed" => RssObservation::Failed { + message: json_string_field(body, "message"), + }, + _ => RssObservation::UnknownResponse, + } +} + +async fn ssh_capture_bounded( + ip: &str, + remote: &str, + timeout: Duration, +) -> anyhow::Result { + let askpass = crate::util::temp_dir().join("voxel-tui-empty-askpass.sh"); + if !askpass.exists() { + std::fs::write(&askpass, "#!/bin/sh\necho\n")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + &askpass, + std::fs::Permissions::from_mode(0o700), + )?; + } + } + let mut command = tokio::process::Command::new("ssh"); + command + .kill_on_drop(true) + .env("SSH_ASKPASS", &askpass) + .env("SSH_ASKPASS_REQUIRE", "force") + .stdin(Stdio::null()) + .args(crate::net::EPHEMERAL_HOST_OPTS) + .args([ + "-o", + "PreferredAuthentications=password", + "-o", + "PubkeyAuthentication=no", + "-o", + "NumberOfPasswordPrompts=1", + "-o", + "ConnectTimeout=8", + "-o", + "ServerAliveInterval=5", + "-o", + "ServerAliveCountMax=2", + ]) + .arg(format!("root@{ip}")) + .arg(remote); + let output = tokio::time::timeout(timeout, command.output()) + .await + .map_err(|_| anyhow!("SSH command on {ip} timed out"))??; + if !output.status.success() { + return Err(anyhow!( + "SSH command on {ip} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + String::from_utf8(output.stdout).map_err(Into::into) +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum NodeTarget { + Sled { name: String }, + SwitchZone { host: String, zone: String }, + Router { name: String }, +} + +pub trait NodeExecutor: Send + Sync + 'static { + fn execute<'a>( + &'a self, + target: &'a NodeTarget, + command: &'a str, + ) -> BoxFuture<'a, anyhow::Result>; +} + +/// Production execution boundary: Falcon is used only for identity and bounded, +/// one-time host-LAN discovery. Recurring illumos probes use bounded SSH. +#[derive(Default)] +struct IpCache { + resolved: BTreeMap, + in_flight: + BTreeMap>>>, +} + +impl IpCache { + fn from_config_and_addresses( + config: &VoxelConfig, + mut resolved: BTreeMap, + ) -> Self { + if config.external.isolated() { + resolved.extend(config.static_external_ips()); + } + Self { resolved, in_flight: BTreeMap::new() } + } + + fn address_for_probe( + &self, + host: &str, + resumed: bool, + ) -> anyhow::Result> { + match self.resolved.get(host) { + Some(ip) => Ok(Some(ip.clone())), + None if resumed => Err(anyhow!( + "telemetry for {host} is unavailable after resuming without a saved address" + )), + None => Ok(None), + } + } +} + +pub(crate) struct FalconExecutor { + topo: Arc, + ips: Arc>, + router_commands: Arc>>, + serial_tasks: Arc>>>, + resumed: bool, + timeout: Duration, +} + +impl FalconExecutor { + pub(crate) fn new( + topo: crate::topo::Topo, + config: &VoxelConfig, + addresses: BTreeMap, + resumed: bool, + timeout: Duration, + ) -> Self { + Self { + topo: Arc::new(topo), + ips: Arc::new(Mutex::new(IpCache::from_config_and_addresses( + config, addresses, + ))), + router_commands: Default::default(), + serial_tasks: Default::default(), + resumed, + timeout, + } + } + + fn track_serial_task(&self, task: tokio::task::JoinHandle<()>) { + // Registration must not yield after spawning: otherwise cancellation + // could drop the JoinHandle and leave untracked Falcon serial work. + let mut tasks = self.serial_tasks.lock().unwrap(); + tasks.retain(|task| !task.is_finished()); + tasks.push(task); + } + + /// Wait for Falcon serial workers that reached their caller's timeout. + /// Service supervisors call this before admitting mutating work so a timed + /// out read-only probe cannot outlive telemetry or reconciliation ownership. + pub(crate) async fn drain_serial_tasks(&self) { + loop { + let tasks = { + let mut tracked = self.serial_tasks.lock().unwrap(); + if tracked.is_empty() { + break; + } + std::mem::take(&mut *tracked) + }; + for task in tasks { + let _ = task.await; + } + } + } + + pub(crate) async fn resolved_addresses(&self) -> BTreeMap { + self.ips.lock().await.resolved.clone() + } + + async fn sled_ip(&self, host: &str) -> anyhow::Result { + let mut cache = self.ips.lock().await; + if let Some(ip) = cache.address_for_probe(host, self.resumed)? { + return Ok(ip); + } + let node = self + .topo + .node_ref(host) + .ok_or_else(|| anyhow!("unknown Falcon sled {host}"))?; + let mut result = if let Some(in_flight) = cache.in_flight.get(host) { + in_flight.clone() + } else { + let (sender, receiver) = watch::channel(None); + cache.in_flight.insert(host.into(), receiver.clone()); + let topo = self.topo.clone(); + let ips = self.ips.clone(); + let host = host.to_owned(); + let task = tokio::spawn(async move { + let result = + crate::net::node_external_ip(&topo.runner, node, false) + .await + .map_err(|error| error.to_string()); + let mut cache = ips.lock().await; + cache.in_flight.remove(&host); + if let Ok(ip) = &result { + cache.resolved.insert(host, ip.clone()); + } + let _ = sender.send(Some(result)); + }); + self.track_serial_task(task); + receiver + }; + drop(cache); + + let wait = async { + loop { + if let Some(result) = result.borrow().clone() { + return result.map_err(|message| anyhow!(message)); + } + result + .changed() + .await + .map_err(|_| anyhow!("host-LAN discovery task stopped"))?; + } + }; + tokio::time::timeout(self.timeout, wait) + .await + .map_err(|_| anyhow!("host-LAN discovery for {host} timed out"))? + } + + async fn router_exec( + &self, + name: &str, + command: &str, + ) -> anyhow::Result { + if self.resumed { + return Err(anyhow!( + "router telemetry is unavailable after resuming a TUI session" + )); + } + let node = self + .topo + .node_ref(name) + .ok_or_else(|| anyhow!("unknown Falcon router {name}"))?; + { + let mut commands = self.router_commands.lock().await; + if !commands.insert(name.into()) { + return Err(anyhow!( + "a previous router command on {name} is still running" + )); + } + } + let (sender, receiver) = oneshot::channel(); + let topo = self.topo.clone(); + let commands = self.router_commands.clone(); + let name = name.to_owned(); + let task_name = name.clone(); + let command = command.to_owned(); + let task = tokio::spawn(async move { + let result = topo + .runner + .exec(node, &command) + .await + .map_err(|error| error.to_string()); + commands.lock().await.remove(&task_name); + let _ = sender.send(result); + }); + self.track_serial_task(task); + tokio::time::timeout(self.timeout, receiver) + .await + .map_err(|_| anyhow!("router command on {name} timed out"))? + .map_err(|_| anyhow!("router command task on {name} stopped"))? + .map_err(|error| anyhow!("router command on {name}: {error}")) + } +} + +impl NodeExecutor for FalconExecutor { + fn execute<'a>( + &'a self, + target: &'a NodeTarget, + command: &'a str, + ) -> BoxFuture<'a, anyhow::Result> { + Box::pin(async move { + match target { + NodeTarget::Sled { name } => { + let ip = self.sled_ip(name).await?; + ssh_capture_bounded(&ip, command, self.timeout).await + } + NodeTarget::SwitchZone { host, zone: _ } => { + let ip = self.sled_ip(host).await?; + ssh_capture_bounded( + &ip, + &crate::net::zlogin(command), + self.timeout, + ) + .await + } + NodeTarget::Router { name } => { + self.router_exec(name, command).await + } + } + }) + } +} + +#[derive(Clone, Debug)] +pub struct ResourceTarget { + pub id: ResourceId, + pub kind: ResourceKind, + pub target: NodeTarget, +} + +#[derive(Clone, Debug)] +pub struct RssTarget { + pub rack: RackId, + pub target: NodeTarget, + pub bootstrap_addr: String, +} + +#[derive(Clone, Debug)] +pub struct CollectorTargets { + pub resources: Vec, + pub rss: Vec, +} + +impl CollectorTargets { + pub fn from_config(config: &VoxelConfig) -> Self { + let resources = resource_descriptors(config) + .into_iter() + .map(|descriptor| ResourceTarget { + id: descriptor.id, + kind: descriptor.kind, + target: match descriptor.kind { + ResourceKind::Sled => { + NodeTarget::Sled { name: descriptor.name } + } + ResourceKind::SwitchZone => NodeTarget::SwitchZone { + host: descriptor + .host + .expect("switch descriptor has host"), + zone: "oxz_switch".into(), + }, + ResourceKind::Router => { + NodeTarget::Router { name: descriptor.name } + } + }, + }) + .collect(); + let mut seen = std::collections::BTreeSet::new(); + let rss = config + .sleds() + .into_iter() + .filter(|sled| sled.rss && seen.insert(sled.rack)) + .map(|sled| RssTarget { + rack: RackId(sled.rack), + target: NodeTarget::Sled { name: sled.name.clone() }, + bootstrap_addr: sled.bootstrap_addr(), + }) + .collect(); + Self { resources, rss } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct SchedulerConfig { + pub traffic_interval: Duration, + pub health_interval: Duration, + pub concurrency: usize, +} + +impl SchedulerConfig { + pub fn new( + traffic_interval: Duration, + health_interval: Duration, + concurrency: usize, + ) -> anyhow::Result { + if traffic_interval.is_zero() + || health_interval.is_zero() + || concurrency == 0 + { + return Err(anyhow!( + "collector intervals and concurrency must be nonzero" + )); + } + Ok(Self { traffic_interval, health_interval, concurrency }) + } +} + +pub struct Collector { + executor: Arc, + targets: CollectorTargets, + baselines: Arc>>, + nexus: BTreeMap>, + sled_serials: BTreeMap, + oximeter_attempts: Mutex>, + oximeter_diagnostic_attempts: Mutex>, + #[cfg(test)] + concurrency: usize, +} + +impl Collector { + pub fn new( + executor: Arc, + config: &VoxelConfig, + concurrency: usize, + ) -> anyhow::Result { + if concurrency == 0 { + return Err(anyhow!("collector concurrency must be nonzero")); + } + let login = RecoveryLogin::from_config(config); + let nexus = if cfg!(test) { + BTreeMap::new() + } else if let Some(login) = login { + (0..config.topology.racks()) + .map(|rack| { + Ok(( + RackId(rack), + Arc::new(NexusClient::new( + crate::tui::nexus::rack_endpoints(config, rack)?, + login.clone(), + Duration::from_secs(60), + )?), + )) + }) + .collect::>()? + } else { + BTreeMap::new() + }; + let sled_serials = config + .sleds() + .into_iter() + .map(|sled| { + ( + sled.serial_number, + (RackId(sled.rack), sled.name, sled.scrimlet), + ) + }) + .collect(); + Ok(Self { + executor, + targets: CollectorTargets::from_config(config), + baselines: Default::default(), + nexus, + sled_serials, + oximeter_attempts: Default::default(), + oximeter_diagnostic_attempts: Default::default(), + #[cfg(test)] + concurrency, + }) + } + + async fn call( + &self, + cancel: &CancellationToken, + target: &NodeTarget, + command: &str, + ) -> anyhow::Result { + if cancel.is_cancelled() { + return Err(anyhow!("cancelled")); + } + if matches!(target, NodeTarget::Router { .. }) { + // Falcon serial exec is bounded but not cancellation-safe on drop. + // Finish this safe-boundary call, then observe cancellation before + // publishing or beginning another command. + let result = self.executor.execute(target, command).await; + if cancel.is_cancelled() { + Err(anyhow!("cancelled")) + } else { + result + } + } else { + tokio::select! { biased; + _ = cancel.cancelled() => Err(anyhow!("cancelled")), + result = self.executor.execute(target, command) => result, + } + } + } + + async fn send( + sender: &mpsc::Sender, + cancel: &CancellationToken, + event: AppEvent, + ) -> bool { + tokio::select! { biased; + _ = cancel.cancelled() => false, + result = sender.send(event) => result.is_ok(), + } + } + + #[cfg(test)] + async fn collect_traffic( + &self, + sender: &mpsc::Sender, + cancel: &CancellationToken, + ) -> bool { + self.collect_traffic_with_limit(sender, cancel, self.concurrency).await + } + + async fn collect_traffic_with_limit( + &self, + sender: &mpsc::Sender, + cancel: &CancellationToken, + concurrency: usize, + ) -> bool { + let at = Instant::now(); + let jobs = self.targets.resources.clone().into_iter().map( + |resource| async move { + let result = self.traffic_one(&resource, at, cancel).await; + (resource.id, result) + }, + ); + // Finish all command futures before awaiting bounded event sends. This + // prevents send cancellation/closure from dropping an in-flight Falcon + // serial future held by the unordered job stream. + let results = stream::iter(jobs) + .buffer_unordered(concurrency) + .collect::>() + .await; + for event in results.into_iter().map(|(id, result)| match result { + Ok(sample) => AppEvent::Traffic { id, at, sample }, + Err(error) => { + AppEvent::TrafficFailed { id, at, message: error.to_string() } + } + }) { + if !Self::send(sender, cancel, event).await { + return false; + } + } + true + } + + async fn collect_oximeter( + &self, + sender: &mpsc::Sender, + cancel: &CancellationToken, + ) -> bool { + let at = Instant::now(); + for (rack, client) in &self.nexus { + let due = { + let attempts = self.oximeter_attempts.lock().await; + oximeter_due(&attempts, *rack, at) + }; + if !due { + continue; + } + let result = self.oximeter_traffic(*rack, client, at, cancel).await; + let completed_at = Instant::now(); + self.oximeter_attempts.lock().await.insert(*rack, completed_at); + match result { + Ok(samples) => { + for (id, samples) in samples { + if !Self::send( + sender, + cancel, + AppEvent::OximeterTraffic { + id, + at: completed_at, + samples, + }, + ) + .await + { + return false; + } + } + } + Err(error) => { + if !Self::send( + sender, + cancel, + AppEvent::OximeterTrafficFailed { + rack: *rack, + at: completed_at, + message: format!("Oximeter traffic: {error}"), + }, + ) + .await + { + return false; + } + } + } + + let diagnostics_due = { + let attempts = self.oximeter_diagnostic_attempts.lock().await; + oximeter_due(&attempts, *rack, at) + }; + if diagnostics_due { + for event in self + .oximeter_diagnostic_events(*rack, client, at, cancel) + .await + { + if !Self::send(sender, cancel, event).await { + return false; + } + } + self.oximeter_diagnostic_attempts + .lock() + .await + .insert(*rack, Instant::now()); + } + } + true + } + + async fn oximeter_traffic( + &self, + rack: RackId, + client: &NexusClient, + now: Instant, + cancel: &CancellationToken, + ) -> anyhow::Result>> + { + let queries = oximeter_traffic_queries(rack, &self.sled_serials); + let mut results = Vec::with_capacity(queries.len()); + // Keeping each response below one sled's data avoids stalled Nexus + // streams while retaining link and zone detail. + for query in queries { + results.push(client.query(&query, cancel).await?); + } + let result = OxqlQueryResult { + tables: results + .into_iter() + .flat_map(|result| result.tables) + .collect(), + }; + parse_oximeter_traffic(result, rack, &self.sled_serials, now) + } + + async fn traffic_one( + &self, + resource: &ResourceTarget, + at: Instant, + cancel: &CancellationToken, + ) -> anyhow::Result { + let counters = match resource.kind { + ResourceKind::Router => parse_linux_link_counters( + &self.call(cancel, &resource.target, LINUX_COUNTERS).await?, + ), + _ => parse_kstat_link_counters( + &self.call(cancel, &resource.target, KSTAT).await?, + ), + }; + if counters.is_empty() { + return Err(anyhow!("counter output contained no complete links")); + } + let zones = if resource.kind == ResourceKind::Sled { + parse_dladm_zone_vnics( + &self.call(cancel, &resource.target, DLADM).await?, + ) + } else { + vec![] + }; + let mut baselines = self.baselines.lock().await; + let state = baselines.entry(resource.id.clone()).or_default(); + state.zones = zones; + state.update(CounterSnapshot::new(at, counters)); + Ok(TrafficSample { + total: state.total_rate(), + links: state.link_rates.clone(), + zones: state.top_zones(usize::MAX), + ..Default::default() + }) + } + + #[cfg(test)] + async fn collect_health( + &self, + sender: &mpsc::Sender, + cancel: &CancellationToken, + ) -> bool { + self.collect_health_with_limit(sender, cancel, self.concurrency).await + } + + async fn collect_health_with_limit( + &self, + sender: &mpsc::Sender, + cancel: &CancellationToken, + concurrency: usize, + ) -> bool { + let at = Instant::now(); + let resource_jobs = self.targets.resources.clone().into_iter().map( + |resource| async move { + self.health_events(&resource, at, cancel).await + }, + ); + let mut events = stream::iter(resource_jobs) + .buffer_unordered(concurrency) + .collect::>() + .await + .into_iter() + .flatten() + .collect::>(); + let rss_jobs = + self.targets.rss.clone().into_iter().map(|rss| async move { + self.rss_event(&rss, at, cancel).await + }); + events.extend( + stream::iter(rss_jobs) + .buffer_unordered(concurrency) + .collect::>() + .await, + ); + for event in events { + if !Self::send(sender, cancel, event).await { + return false; + } + } + true + } + + async fn oximeter_diagnostic_events( + &self, + rack: RackId, + client: &NexusClient, + at: Instant, + cancel: &CancellationToken, + ) -> Vec { + // A slow high-cardinality query must not prevent independent rack + // diagnostics from reaching the UI during the same collection cycle. + let (cpu, zfs, exceptions) = tokio::join!( + client.query(ZONE_CPU_QUERY, cancel), + client.query(ZFS_QUERY, cancel), + client.query(OXIMETER_EXCEPTIONS_QUERY, cancel), + ); + let cpu = cpu.and_then(|result| { + parse_zone_cpu(result, rack, &self.sled_serials) + }); + let zfs = zfs.and_then(|result| { + parse_zfs_headroom(result, rack, &self.sled_serials) + }); + let exceptions = exceptions.and_then(parse_oximeter_exceptions); + vec![ + match cpu { + Ok(zones) => AppEvent::ZoneCpu { rack, at, zones }, + Err(error) => AppEvent::ZoneCpuFailed { + rack, + at, + message: error.to_string(), + }, + }, + match zfs { + Ok(pools) => AppEvent::ZfsHeadroom { rack, at, pools }, + Err(error) => AppEvent::ZfsHeadroomFailed { + rack, + at, + message: error.to_string(), + }, + }, + match exceptions { + Ok(exceptions) => { + AppEvent::OximeterExceptions { rack, at, exceptions } + } + Err(error) => AppEvent::OximeterExceptionsFailed { + rack, + at, + message: error.to_string(), + }, + }, + ] + } + + async fn health_events( + &self, + resource: &ResourceTarget, + at: Instant, + cancel: &CancellationToken, + ) -> Vec { + let addresses = match resource.kind { + ResourceKind::Router => self + .call(cancel, &resource.target, LINUX_ADDRESSES) + .await + .map(|s| parse_linux_ip_addresses(&s)), + _ => self + .call(cancel, &resource.target, IPADM) + .await + .map(|s| parse_ipadm_addresses(&s)), + }; + let mut events = vec![match addresses { + Ok(addresses) => { + AppEvent::Addresses { id: resource.id.clone(), at, addresses } + } + Err(error) => AppEvent::AddressesFailed { + id: resource.id.clone(), + at, + message: error.to_string(), + }, + }]; + if resource.kind == ResourceKind::Sled { + let diagnostic = self.sled_health(resource, cancel).await; + events.push(match diagnostic { + Ok(diagnostic) => { + AppEvent::Health { id: resource.id.clone(), at, diagnostic } + } + Err(error) => AppEvent::HealthFailed { + id: resource.id.clone(), + at, + message: error.to_string(), + }, + }); + } + events + } + + async fn rss_event( + &self, + rss: &RssTarget, + at: Instant, + cancel: &CancellationToken, + ) -> AppEvent { + let command = format!( + "curl -s --max-time 5 http://[{}]:8080/rack-initialize 2>/dev/null", + rss.bootstrap_addr + ); + match self.call(cancel, &rss.target, &command).await { + Ok(body) if !body.trim().is_empty() => AppEvent::Rss { + rack: rss.rack, + at, + observation: parse_rss_observation(&body), + }, + Ok(_) => AppEvent::RssFailed { + rack: rss.rack, + at, + message: "empty RSS response".into(), + }, + Err(error) => AppEvent::RssFailed { + rack: rss.rack, + at, + message: error.to_string(), + }, + } + } + + async fn sled_health( + &self, + resource: &ResourceTarget, + cancel: &CancellationToken, + ) -> anyhow::Result { + let service = + self.call(cancel, &resource.target, SLED_AGENT_STATE).await?; + let sled_agent = parse_service_state(&service) + .ok_or_else(|| anyhow!("invalid sled-agent service state"))?; + let failed_services = parse_failed_services( + &self.call(cancel, &resource.target, MAINTENANCE_SERVICES).await?, + ); + let zones = parse_zone_diagnostics( + &self + .call( + cancel, + &resource.target, + "zoneadm list -cp | cut -d: -f2", + ) + .await?, + ); + let ntp_zone = zones + .zones + .iter() + .find(|zone| zone.starts_with("oxz_ntp_")) + .ok_or_else(|| anyhow!("NTP zone not found"))?; + let ntp = parse_chrony_tracking( + &self + .call( + cancel, + &resource.target, + &format!( + "zlogin {ntp_zone} chronyc -n tracking 2>/dev/null" + ), + ) + .await?, + ); + if ntp.synchronized.is_none() { + return Err(anyhow!("invalid NTP tracking state")); + } + Ok(HealthDiagnostic { + sled_agent: Some(sled_agent), + failed_services, + ntp, + zones, + notes: vec![], + }) + } + + pub(crate) async fn run_gated( + self: Arc, + config: SchedulerConfig, + sender: mpsc::Sender, + cancel: CancellationToken, + gate: Arc, + ) { + let probes = async { + let mut traffic = tokio::time::interval(config.traffic_interval); + let mut health = tokio::time::interval(config.health_interval); + traffic.set_missed_tick_behavior( + tokio::time::MissedTickBehavior::Skip, + ); + health.set_missed_tick_behavior( + tokio::time::MissedTickBehavior::Skip, + ); + loop { + let health_cadence = tokio::select! { + biased; + _ = cancel.cancelled() => break, + _ = traffic.tick() => false, + _ = health.tick() => true, + }; + let permit = tokio::select! { + _ = cancel.cancelled() => break, + permit = gate.clone().acquire_owned() => match permit { + Ok(permit) => permit, + Err(_) => break, + }, + }; + let keep_running = if health_cadence { + self.collect_health_with_limit( + &sender, + &cancel, + config.concurrency, + ) + .await + } else { + self.collect_traffic_with_limit( + &sender, + &cancel, + config.concurrency, + ) + .await + }; + drop(permit); + if !keep_running { + break; + } + } + }; + let oximeter = async { + let mut interval = tokio::time::interval(config.traffic_interval); + interval.set_missed_tick_behavior( + tokio::time::MissedTickBehavior::Skip, + ); + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = interval.tick() => {}, + } + if sender.is_closed() + || !self.collect_oximeter(&sender, &cancel).await + { + break; + } + } + }; + tokio::join!(probes, oximeter); + } +} diff --git a/voxel/src/tui/context.rs b/voxel/src/tui/context.rs new file mode 100644 index 0000000..22a4544 --- /dev/null +++ b/voxel/src/tui/context.rs @@ -0,0 +1,167 @@ +use camino::Utf8PathBuf; +use std::ffi::OsString; +use std::path::PathBuf; +use voxel_config::VoxelConfig; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PublicCommand { + Launch, + Route, + Destroy, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CommandSpec { + pub(crate) program: PathBuf, + pub(crate) args: Vec, + pub(crate) current_dir: PathBuf, + pub(crate) env: Vec<(OsString, OsString)>, +} + +#[derive(Clone, Debug)] +pub(crate) struct TuiContext { + pub(crate) config_path: Utf8PathBuf, + pub(crate) workdir: Utf8PathBuf, + pub(crate) name: String, + pub(crate) dataset: String, + pub(crate) build_root: Utf8PathBuf, + pub(crate) config: VoxelConfig, + executable: PathBuf, + effective_env: Vec<(OsString, OsString)>, +} + +impl TuiContext { + pub(crate) fn executable(&self) -> &std::path::Path { + &self.executable + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + config_path: Utf8PathBuf, + workdir: Utf8PathBuf, + name: String, + dataset: String, + build_root: Utf8PathBuf, + config: VoxelConfig, + executable: PathBuf, + effective_env: Vec<(OsString, OsString)>, + ) -> Self { + Self { + config_path, + workdir, + name, + dataset, + build_root, + config, + executable, + effective_env, + } + } + + pub(crate) fn command_spec(&self, command: PublicCommand) -> CommandSpec { + let subcommand = match command { + PublicCommand::Launch => "launch", + PublicCommand::Route => "route", + PublicCommand::Destroy => "destroy", + }; + let args = [ + OsString::from("--config"), + self.config_path.as_os_str().to_owned(), + OsString::from("--workdir"), + self.workdir.as_os_str().to_owned(), + OsString::from("--name"), + OsString::from(&self.name), + OsString::from("--dataset"), + OsString::from(&self.dataset), + OsString::from("--build-root"), + self.build_root.as_os_str().to_owned(), + OsString::from(subcommand), + ] + .into_iter() + .collect(); + + CommandSpec { + program: self.executable.clone(), + args, + current_dir: self.workdir.clone().into_std_path_buf(), + env: self.effective_env.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{PublicCommand, TuiContext}; + use camino::Utf8PathBuf; + use std::ffi::OsString; + use std::path::PathBuf; + use voxel_config::VoxelConfig; + + fn context() -> TuiContext { + TuiContext { + config_path: Utf8PathBuf::from("/cfg/voxel.toml"), + workdir: Utf8PathBuf::from("/work/voxel"), + name: "demo".to_string(), + dataset: "rpool/falcon".to_string(), + build_root: Utf8PathBuf::from("/build/voxel"), + config: VoxelConfig::default(), + executable: PathBuf::from("/bin/voxel"), + effective_env: vec![ + ( + OsString::from("FALCON_DATASET"), + OsString::from("rpool/falcon"), + ), + (OsString::from("BUILD_ROOT"), OsString::from("/build/voxel")), + ( + OsString::from("VOXEL_OMICRON_SRC"), + OsString::from("/build/voxel/omicron-main"), + ), + ], + } + } + + fn expected_args(command: &str) -> Vec { + [ + "--config", + "/cfg/voxel.toml", + "--workdir", + "/work/voxel", + "--name", + "demo", + "--dataset", + "rpool/falcon", + "--build-root", + "/build/voxel", + command, + ] + .into_iter() + .map(OsString::from) + .collect() + } + + #[test] + fn constructs_exact_public_command_specs() { + let context = context(); + + for (command, subcommand) in [ + (PublicCommand::Launch, "launch"), + (PublicCommand::Route, "route"), + (PublicCommand::Destroy, "destroy"), + ] { + let spec = context.command_spec(command); + assert_eq!(spec.program, PathBuf::from("/bin/voxel")); + assert_eq!(spec.args, expected_args(subcommand)); + assert_eq!(spec.current_dir, PathBuf::from("/work/voxel")); + assert_eq!(spec.env, context.effective_env); + } + } + + #[test] + fn launch_does_not_enable_optional_flags() { + let args = context().command_spec(PublicCommand::Launch).args; + + for flag in ["--no-progress", "--no-route", "--emu", "--sp-firmware"] { + assert!(!args.contains(&OsString::from(flag))); + } + } +} diff --git a/voxel/src/tui/effects.rs b/voxel/src/tui/effects.rs new file mode 100644 index 0000000..93b13da --- /dev/null +++ b/voxel/src/tui/effects.rs @@ -0,0 +1,416 @@ +use std::{sync::Arc, time::Duration}; + +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; +use tokio_util::sync::CancellationToken; + +use super::{ + context::{PublicCommand, TuiContext}, + event::{AppEvent, Effect, OperationRequestId}, + logging::DurableLog, + operation::{ + CommandOutcome, DestroyPhase, LaunchPhase, LogLevel, OperationEvent, + OperationKind, OperationPhase, OperationWarning, OutputStream, + RoutePhase, + }, + phase::{PhaseClassifier, PhaseHint}, + process::{ + ChildResult, ChildSupervisor, DrainFailure, ForceStopResult, OutputLine, + }, + reconcile::{self, LifecycleIntent}, +}; + +const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(25); +const OUTPUT_CHANNEL_CAPACITY: usize = 256; +const PARTIAL_STATE_WARNING: &str = "ALARM: force stopping only the direct Voxel child; descendants and Falcon state may be partial"; + +struct ActiveChild { + request_id: OperationRequestId, + kind: OperationKind, + child: ChildSupervisor, + output: mpsc::Receiver, + output_done: bool, + classifier: PhaseClassifier, + forced: Option, + last_poll_error_log: Option, + _admission: OwnedSemaphorePermit, +} + +/// Owns the sole lifecycle child. Runtime shutdown must keep this future alive +/// until it returns; dropping it uses `kill_on_drop` only as a failure fallback. +pub(crate) struct Effects { + context: Arc, + durable: DurableLog, + events: mpsc::Sender, + executor: Arc, + gate: Arc, + shutdown: CancellationToken, +} + +impl Effects { + pub(crate) fn new( + context: Arc, + durable: DurableLog, + events: mpsc::Sender, + executor: Arc, + gate: Arc, + shutdown: CancellationToken, + ) -> Self { + Self { context, durable, events, executor, gate, shutdown } + } + + pub(crate) async fn run( + mut self, + mut effects: mpsc::UnboundedReceiver, + ) { + let mut active: Option = None; + let mut effects_open = true; + let mut poll = tokio::time::interval(CHILD_POLL_INTERVAL); + loop { + tokio::select! { + biased; + line = async { active.as_mut().unwrap().output.recv().await }, if active.as_ref().is_some_and(|child| !child.output_done) => { + if let Some(line) = line { + self.output(active.as_mut().unwrap(), line).await; + } else { + active.as_mut().unwrap().output_done = true; + } + } + effect = effects.recv(), if effects_open => match effect { + Some(effect) => self.effect(effect, &mut active).await, + None if active.is_none() => break, + None => effects_open = false, + }, + _ = poll.tick(), if active.is_some() => { + let settled = match active.as_mut().unwrap().child.is_settled() { + Ok(settled) => settled, + Err(error) => { + let child = active.as_mut().unwrap(); + let now = std::time::Instant::now(); + if child.last_poll_error_log.is_none_or(|last| now.duration_since(last) >= Duration::from_secs(5)) { + child.last_poll_error_log = Some(now); + self.operation(child.request_id, OperationEvent::Log { + level: LogLevel::Error, + message: format!("child_poll_error request_id={:?} error={:?}", child.request_id, error.message), + }).await; + } + false + }, + }; + if settled { + self.settle(active.take().unwrap()).await; + if !effects_open { + break; + } + } + } + _ = self.shutdown.cancelled(), if active.is_none() => break, + } + } + } + + async fn effect( + &mut self, + effect: Effect, + active: &mut Option, + ) { + match effect { + Effect::Start { request_id, kind } => { + if active.is_some() { + self.start_failed( + request_id, + kind, + "another lifecycle child is active", + ) + .await; + return; + } + let admission = match self.gate.clone().acquire_owned().await { + Ok(admission) => admission, + Err(_) => return, + }; + self.executor.drain_serial_tasks().await; + let command = match kind { + OperationKind::Launch => PublicCommand::Launch, + OperationKind::Route => PublicCommand::Route, + OperationKind::Destroy => PublicCommand::Destroy, + }; + let (tx, output) = mpsc::channel(OUTPUT_CHANNEL_CAPACITY); + match ChildSupervisor::spawn( + self.context.command_spec(command), + self.durable.clone(), + tx, + ) { + Ok(child) => { + *active = Some(ActiveChild { + request_id, + kind, + child, + output, + output_done: false, + classifier: PhaseClassifier::default(), + forced: None, + last_poll_error_log: None, + _admission: admission, + }); + self.operation( + request_id, + OperationEvent::Started { kind }, + ) + .await; + } + Err(error) => { + self.start_failed(request_id, kind, &error.to_string()) + .await + } + } + } + Effect::Cancel { request_id, .. } => { + // Safe cancellation is intent only. In particular, do not signal + // an opaque child that may own a Falcon serial operation. + if active + .as_ref() + .is_some_and(|child| child.request_id == request_id) + { + self.operation(request_id, OperationEvent::Log { level: LogLevel::Warning, message: "Cancellation pending; waiting for the Voxel command to settle safely".into() }).await; + } + } + Effect::ForceStop { request_id } => { + let Some(child) = active + .as_mut() + .filter(|child| child.request_id == request_id) + else { + return; + }; + if child.forced.is_some() { + return; + } + self.operation( + request_id, + OperationEvent::Warning(OperationWarning { + message: PARTIAL_STATE_WARNING.into(), + resource: None, + }), + ) + .await; + let result = child.child.force_stop().await; + if let ForceStopResult::Failed(message) = &result { + self.operation( + request_id, + OperationEvent::Log { + level: LogLevel::Error, + message: format!( + "direct child force-stop attempt failed: {message}" + ), + }, + ) + .await; + } + child.forced = Some(result); + } + Effect::CopyToClipboard(_) | Effect::Quit => {} + } + } + + async fn output(&self, active: &mut ActiveChild, line: OutputLine) { + self.operation( + active.request_id, + OperationEvent::Log { + level: if line.stream == OutputStream::Stderr { + LogLevel::Error + } else { + LogLevel::Info + }, + message: line.text.clone(), + }, + ) + .await; + if let Some(PhaseHint::Started(phase)) = + active.classifier.classify(active.kind, line.stream, &line.text) + { + self.operation( + active.request_id, + OperationEvent::PhaseStarted { phase }, + ) + .await; + } + } + + async fn settle(&self, active: ActiveChild) { + let request_id = active.request_id; + let kind = active.kind; + let forced = active.forced.clone(); + let result = active.child.wait().await; + let outcome = match (result, forced) { + (Ok(result), forced) => { + for (stream, error) in &result.output_errors { + match error { + DrainFailure::Output(message) => { + self.operation( + request_id, + OperationEvent::Log { + level: LogLevel::Error, + message: format!( + "{stream:?} drain failed: {message}" + ), + }, + ) + .await; + } + DrainFailure::DurableLog(message) => { + let _ = self + .events + .send(AppEvent::DurableLogFailed { + message: message.clone(), + }) + .await; + } + } + } + match forced { + Some(kill) => force_outcome(result, kill), + None => child_outcome(result), + } + } + (Err(error), Some(kill)) => CommandOutcome::ForceStopped { + status: None, + kill_error: match kill { + ForceStopResult::Failed(message) => Some(message), + _ => Some(error.message), + }, + }, + (Err(error), None) => CommandOutcome::SpawnFailed { + message: format!("child wait failed: {}", error.message), + }, + }; + self.operation( + request_id, + OperationEvent::PhaseStarted { phase: reconciliation_phase(kind) }, + ) + .await; + self.reconcile(kind).await; + self.operation(request_id, OperationEvent::Finished(outcome)).await; + } + + async fn reconcile(&self, kind: OperationKind) { + let at = std::time::Instant::now(); + let _ = self.events.send(AppEvent::ReconciliationStarted { at }).await; + let intent = match kind { + OperationKind::Launch => LifecycleIntent::Launch, + OperationKind::Destroy => LifecycleIntent::Destroy, + _ => LifecycleIntent::Idle, + }; + self.executor.drain_serial_tasks().await; + match reconcile::collect( + &self.context, + &self.executor, + intent, + &self.shutdown, + ) + .await + { + Ok(evidence) => { + let routes = evidence.routes; + let result = reconcile::reduce(&evidence); + let _ = self + .events + .send(AppEvent::Reconciled { + at: std::time::Instant::now(), + result, + routes, + }) + .await; + } + Err(error) => { + let _ = self + .events + .send(AppEvent::ReconciliationFailed { + at: std::time::Instant::now(), + message: error.to_string(), + }) + .await; + } + } + } + + async fn operation( + &self, + request_id: OperationRequestId, + event: OperationEvent, + ) { + let _ = + self.events.send(AppEvent::Operation { request_id, event }).await; + } + + async fn start_failed( + &self, + request_id: OperationRequestId, + kind: OperationKind, + message: &str, + ) { + let _ = self + .events + .send(AppEvent::OperationStartFailed { + request_id, + kind, + message: message.into(), + }) + .await; + } +} + +fn child_outcome(result: ChildResult) -> CommandOutcome { + CommandOutcome::Exited { + status: result.status, + stderr_summary: result.stderr_summary, + } +} + +fn reconciliation_phase(kind: OperationKind) -> OperationPhase { + match kind { + OperationKind::Launch => OperationPhase::Launch(LaunchPhase::Reconcile), + OperationKind::Destroy => { + OperationPhase::Destroy(DestroyPhase::Reconcile) + } + OperationKind::Route => OperationPhase::Route(RoutePhase::Reconcile), + } +} + +fn force_outcome(result: ChildResult, kill: ForceStopResult) -> CommandOutcome { + if matches!( + kill, + ForceStopResult::AlreadyExited | ForceStopResult::Failed(_) + ) { + return child_outcome(result); + } + CommandOutcome::ForceStopped { + status: Some(result.status), + kill_error: match kill { + ForceStopResult::Failed(message) => Some(message), + _ => None, + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::operation::{ + DestroyPhase, LaunchPhase, OperationPhase, RoutePhase, + }; + + #[test] + fn every_operation_has_a_tui_owned_reconciliation_phase() { + assert_eq!( + reconciliation_phase(OperationKind::Launch), + OperationPhase::Launch(LaunchPhase::Reconcile) + ); + assert_eq!( + reconciliation_phase(OperationKind::Destroy), + OperationPhase::Destroy(DestroyPhase::Reconcile) + ); + assert_eq!( + reconciliation_phase(OperationKind::Route), + OperationPhase::Route(RoutePhase::Reconcile) + ); + } +} diff --git a/voxel/src/tui/event.rs b/voxel/src/tui/event.rs new file mode 100644 index 0000000..430527f --- /dev/null +++ b/voxel/src/tui/event.rs @@ -0,0 +1,305 @@ +use std::time::Instant; + +use crate::tui::operation::{OperationEvent, OperationKind}; +use crate::tui::reconcile::{ + ReconciliationResult, RouteEvidence, RssObservation, +}; +use crate::tui::telemetry::{ + HealthDiagnostic, NodeAddresses, OximeterExceptions, RackId, ResourceId, + TrafficSample, ZfsHeadroom, ZoneCpu, +}; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum View { + Deployment, + Monitor, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum DeploymentPane { + OverallProgress, + Phases, + Status, + CurrentPhase, + Logs, +} + +impl DeploymentPane { + pub(crate) const ORDER: [Self; 5] = [ + Self::OverallProgress, + Self::Phases, + Self::Status, + Self::CurrentPhase, + Self::Logs, + ]; +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum MonitoringPane { + RackSummary, + Topology, + TopZones, +} + +impl MonitoringPane { + pub(crate) const ORDER: [Self; 3] = + [Self::RackSummary, Self::Topology, Self::TopZones]; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Direction { + Next, + Previous, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Confirmation { + Launch, + Route, + Detach, + Quit, + QuitAndDestroy, + CancelAndLeave, + CancelAndDestroy, + ForceStop, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConfirmationOptionAction { + Confirm, + Reject, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ConfirmationOption { + pub label: &'static str, + pub action: ConfirmationOptionAction, +} + +use ConfirmationOptionAction::*; +const LAUNCH_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { label: "Launch deployment", action: Confirm }, + ConfirmationOption { label: "Cancel", action: Reject }, +]; +const ROUTE_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { label: "Apply routes", action: Confirm }, + ConfirmationOption { label: "Cancel", action: Reject }, +]; +const DETACH_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { label: "Detach and leave resources", action: Confirm }, + ConfirmationOption { label: "Back", action: Reject }, +]; +const QUIT_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { label: "Quit Voxel TUI", action: Confirm }, + ConfirmationOption { label: "Back", action: Reject }, +]; +const QUIT_AND_DESTROY_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { + label: "Destroy deployment and quit", + action: Confirm, + }, + ConfirmationOption { label: "Back", action: Reject }, +]; +const LEAVE_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { label: "Cancel and leave resources", action: Confirm }, + ConfirmationOption { label: "Back", action: Reject }, +]; +const DESTROY_CANCEL_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { + label: "Cancel and destroy resources", + action: Confirm, + }, + ConfirmationOption { label: "Back", action: Reject }, +]; +const FORCE_STOP_OPTIONS: &[ConfirmationOption] = &[ + ConfirmationOption { label: "Force stop direct child", action: Confirm }, + ConfirmationOption { label: "Back", action: Reject }, +]; +impl Confirmation { + pub fn options(&self, _can_cancel: bool) -> &'static [ConfirmationOption] { + match self { + Self::Launch => LAUNCH_OPTIONS, + Self::Route => ROUTE_OPTIONS, + Self::Detach => DETACH_OPTIONS, + Self::Quit => QUIT_OPTIONS, + Self::QuitAndDestroy => QUIT_AND_DESTROY_OPTIONS, + Self::CancelAndLeave => LEAVE_OPTIONS, + Self::CancelAndDestroy => DESTROY_CANCEL_OPTIONS, + Self::ForceStop => FORCE_STOP_OPTIONS, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CancelChoice { + Leave, + Destroy, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct OperationRequestId(u64); + +impl OperationRequestId { + pub(crate) const FIRST: Self = Self(1); + + pub(crate) fn next(self) -> Option { + self.0.checked_add(1).map(Self) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Effect { + Start { request_id: OperationRequestId, kind: OperationKind }, + Cancel { request_id: OperationRequestId, choice: CancelChoice }, + ForceStop { request_id: OperationRequestId }, + CopyToClipboard(String), + Quit, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Action { + SwitchView(View), + NextRack, + PreviousRack, + NextItem, + PreviousItem, + Activate, + ToggleSection, + ToggleHelp, + CopyExternalMonitoringSelected, + CopyExternalMonitoringAll, + CopyExternalMonitoringGuide, + Close, + RequestLaunch, + RequestRoute, + RequestDetach, + RequestQuit, + RequestCancelAndLeave, + RequestCancelAndDestroy, + CopyReattachCommand, + Reject, + CycleLogFilter, + Scroll { delta: isize, page: bool }, +} + +#[derive(Clone, Debug)] +pub enum AppEvent { + Action(Action), + Tick { + now: Instant, + }, + Resize { + width: u16, + height: u16, + }, + ReconciliationStarted { + at: Instant, + }, + Reconciled { + at: Instant, + result: ReconciliationResult, + routes: RouteEvidence, + }, + ReconciliationFailed { + at: Instant, + message: String, + }, + Operation { + request_id: OperationRequestId, + event: OperationEvent, + }, + OperationStartFailed { + request_id: OperationRequestId, + kind: OperationKind, + message: String, + }, + Traffic { + id: ResourceId, + at: Instant, + sample: TrafficSample, + }, + OximeterTraffic { + id: ResourceId, + at: Instant, + samples: Vec<(Instant, TrafficSample)>, + }, + OximeterTrafficFailed { + rack: RackId, + at: Instant, + message: String, + }, + TrafficFailed { + id: ResourceId, + at: Instant, + message: String, + }, + Health { + id: ResourceId, + at: Instant, + diagnostic: HealthDiagnostic, + }, + HealthFailed { + id: ResourceId, + at: Instant, + message: String, + }, + ZoneCpu { + rack: RackId, + at: Instant, + zones: Vec, + }, + ZoneCpuFailed { + rack: RackId, + at: Instant, + message: String, + }, + ZfsHeadroom { + rack: RackId, + at: Instant, + pools: Vec, + }, + ZfsHeadroomFailed { + rack: RackId, + at: Instant, + message: String, + }, + OximeterExceptions { + rack: RackId, + at: Instant, + exceptions: OximeterExceptions, + }, + OximeterExceptionsFailed { + rack: RackId, + at: Instant, + message: String, + }, + Addresses { + id: ResourceId, + at: Instant, + addresses: NodeAddresses, + }, + AddressesFailed { + id: ResourceId, + at: Instant, + message: String, + }, + Rss { + rack: RackId, + at: Instant, + observation: RssObservation, + }, + RssFailed { + rack: RackId, + at: Instant, + message: String, + }, + DurableLogFailed { + message: String, + }, +} + +impl From for AppEvent { + fn from(value: Action) -> Self { + Self::Action(value) + } +} diff --git a/voxel/src/tui/logging.rs b/voxel/src/tui/logging.rs new file mode 100644 index 0000000..adbc9fa --- /dev/null +++ b/voxel/src/tui/logging.rs @@ -0,0 +1,70 @@ +use std::{ + fs::{File, OpenOptions}, + io::{self, Write}, + path::Path, + sync::{Arc, Mutex}, +}; + +use slog::Drain; + +#[derive(Clone)] +pub(crate) struct DurableLog(Arc>); + +impl DurableLog { + pub(crate) fn open(path: &Path) -> io::Result { + let file = OpenOptions::new().create(true).append(true).open(path)?; + Ok(Self(Arc::new(Mutex::new(file)))) + } + + pub(crate) fn write_line(&self, line: &str) -> io::Result<()> { + let mut file = + self.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + writeln!(file, "{line}")?; + file.flush() + } + + pub(crate) fn falcon_logger(&self) -> slog::Logger { + slog::Logger::root(DurableDrain(self.clone()).fuse(), slog::o!()) + } +} + +struct DurableDrain(DurableLog); + +impl slog::Drain for DurableDrain { + type Ok = (); + type Err = io::Error; + + fn log( + &self, + record: &slog::Record<'_>, + _values: &slog::OwnedKVList, + ) -> Result { + self.0.write_line(&format!("{} {}", record.level(), record.msg())) + } +} + +#[cfg(test)] +mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + + use slog::warn; + + use super::*; + + #[test] + fn falcon_logger_writes_to_durable_log() { + let nonce = + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let path = std::env::temp_dir().join(format!( + "voxel-tui-falcon-log-{}-{nonce}", + std::process::id() + )); + let durable = DurableLog::open(&path).unwrap(); + + warn!(durable.falcon_logger(), "serial retry"; "node" => "g0"); + + let contents = std::fs::read_to_string(&path).unwrap(); + assert!(contents.contains("WARN serial retry"), "{contents:?}"); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/voxel/src/tui/mod.rs b/voxel/src/tui/mod.rs new file mode 100644 index 0000000..ff5aecb --- /dev/null +++ b/voxel/src/tui/mod.rs @@ -0,0 +1,380 @@ +mod app; +mod collector; +mod context; +mod effects; +mod event; +mod logging; +mod nexus; +mod operation; +mod phase; +mod process; +mod reconcile; +mod session; +mod telemetry; +mod terminal; +mod ui; + +use std::{fs::OpenOptions, sync::Arc, time::Duration}; + +use anyhow::Context; +pub(crate) use app::App; +pub(crate) use context::TuiContext; +#[cfg(test)] +pub(crate) use event::Action; +pub(crate) use event::{AppEvent, Effect}; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +pub(crate) fn resume(choose: bool) -> anyhow::Result<()> { + session::resume(choose) +} + +fn prepare_probe_mounts( + workdir: &camino::Utf8Path, + config: &voxel_config::VoxelConfig, +) -> anyhow::Result<()> { + let cargo_bay = workdir.join("cargo-bay"); + for node in config + .sleds() + .into_iter() + .map(|sled| sled.name) + .chain(config.topology.routers.iter().cloned()) + { + std::fs::create_dir_all(cargo_bay.join(&node)) + .with_context(|| format!("create TUI probe mount for {node}"))?; + } + Ok(()) +} + +async fn reserve_initial_reconciliation( + admission: &Arc, +) -> anyhow::Result { + admission + .clone() + .acquire_owned() + .await + .context("reserve initial TUI reconciliation") +} + +pub(crate) async fn run(context: TuiContext) -> anyhow::Result<()> { + let resumed_claim = session::claim_from_environment()?; + + // Everything that must survive terminal setup failure is created first. + std::fs::create_dir_all(&context.workdir)?; + let durable = logging::DurableLog::open( + context.workdir.join("voxel-tui.log").as_std_path(), + ) + .context("open durable TUI log")?; + let (mut detached, reattach) = session::prepare_direct(&context)?; + let topology = telemetry::resource_descriptors(&context.config); + prepare_probe_mounts(&context.workdir, &context.config)?; + let mut topo = crate::topo::build_topo(&context.config, &context.name)?; + topo.runner.log = durable.falcon_logger(); + let executor = Arc::new(collector::FalconExecutor::new( + topo, + &context.config, + resumed_claim + .as_ref() + .map(|claim| claim.addresses().clone()) + .unwrap_or_default(), + resumed_claim.is_some(), + Duration::from_secs(10), + )); + let collector = Arc::new(collector::Collector::new( + executor.clone(), + &context.config, + 8, + )?); + let schedule = collector::SchedulerConfig::new( + Duration::from_secs(2), + Duration::from_secs(10), + 8, + )?; + let context = Arc::new(context); + let (events_tx, mut events_rx) = terminal::event_channel(); + let (effects_tx, effects_rx) = mpsc::unbounded_channel(); + let shutdown = CancellationToken::new(); + let admission = Arc::new(tokio::sync::Semaphore::new(1)); + // Reserve admission before telemetry can contend for it. If collection + // wins startup, a timed-out Falcon serial probe can leave a tracked task + // that initial reconciliation then waits on indefinitely while holding + // this gate, preventing telemetry from ever publishing another sample. + let initial_permit = reserve_initial_reconciliation(&admission).await?; + + let terminal_writer = OpenOptions::new() + .read(true) + .write(true) + .open("/dev/tty") + .context("open controlling terminal")?; + let clipboard_writer = terminal_writer.try_clone()?; + let mut terminal = terminal::TerminalSession::enter(terminal_writer) + .context("enter terminal UI")?; + let mut app = App::new(topology, 500, 120); + app.reattach_command = Some(reattach); + app.external_monitoring_endpoints = (0..context.config.topology.racks()) + .filter_map(|rack| { + nexus::rack_endpoints(&context.config, rack) + .ok()? + .into_iter() + .find(|endpoint| endpoint.scheme() == "https") + .map(|endpoint| (telemetry::RackId(rack), endpoint.to_string())) + }) + .collect(); + + let mut input = terminal::spawn_input(events_tx.clone(), shutdown.clone()); + let mut ticks = terminal::spawn_ticks( + events_tx.clone(), + shutdown.clone(), + Duration::from_millis(250), + )?; + let mut collection = tokio::spawn(collector.run_gated( + schedule, + events_tx.clone(), + shutdown.clone(), + admission.clone(), + )); + let executor_drain = executor.clone(); + let services = effects::Effects::new( + context.clone(), + durable, + events_tx.clone(), + executor, + admission.clone(), + shutdown.clone(), + ); + let mut effect_task = tokio::spawn(services.run(effects_rx)); + let initial_context = context.clone(); + let initial_events = events_tx.clone(); + let initial_shutdown = shutdown.clone(); + let initial_executor = executor_drain.clone(); + let mut initial = tokio::spawn(async move { + initial_executor.drain_serial_tasks().await; + let at = std::time::Instant::now(); + let _ = + initial_events.send(AppEvent::ReconciliationStarted { at }).await; + match reconcile::collect( + &initial_context, + &initial_executor, + reconcile::LifecycleIntent::Idle, + &initial_shutdown, + ) + .await + { + Ok(evidence) => { + let routes = evidence.routes; + let _ = initial_events + .send(AppEvent::Reconciled { + at: std::time::Instant::now(), + result: reconcile::reduce(&evidence), + routes, + }) + .await; + } + Err(error) => { + let _ = initial_events + .send(AppEvent::ReconciliationFailed { + at: std::time::Instant::now(), + message: error.to_string(), + }) + .await; + } + } + drop(initial_permit); + initial_shutdown.cancelled().await; + }); + drop(events_tx); + + let mut renderer = ui::renderer::Renderer::new(Duration::from_millis(33)); + let mut clipboard = std::io::BufWriter::new(clipboard_writer); + let mut loop_future = Box::pin(ui::renderer::run_loop( + terminal.terminal_mut(), + &mut clipboard, + &mut renderer, + &mut app, + &mut events_rx, + &effects_tx, + shutdown.clone(), + )); + let mut input_result = None; + let mut ticks_result = None; + let mut collection_result = None; + let mut effects_result = None; + let mut initial_result = None; + let mut primary = None; + let mut loop_result = tokio::select! { + result = &mut loop_future => Some(result), + result = &mut input => { input_result = Some(result); primary = Some("terminal input stopped unexpectedly".to_string()); None }, + result = &mut ticks => { ticks_result = Some(result); primary = Some("terminal ticks stopped unexpectedly".to_string()); None }, + result = &mut collection => { collection_result = Some(result); primary = Some("TUI collector stopped unexpectedly".to_string()); None }, + result = &mut effect_task => { effects_result = Some(result); primary = Some("TUI effects task stopped unexpectedly".to_string()); None }, + result = &mut initial => { initial_result = Some(result); primary = Some("initial reconciliation stopped unexpectedly".to_string()); None }, + }; + shutdown.cancel(); + if loop_result.is_none() { + loop_result = Some(loop_future.as_mut().await); + } + drop(loop_future); + drop(effects_tx); + + match loop_result.as_ref().expect("renderer result captured") { + Err(error) => { + primary.get_or_insert_with(|| { + format!("terminal renderer failed: {error}") + }); + } + Ok(ui::renderer::LoopExit::ConfirmedExit) => {} + Ok(ui::renderer::LoopExit::RenderFailed { message, .. }) => { + primary.get_or_insert_with(|| { + format!("render terminal UI: {message}") + }); + } + Ok(ui::renderer::LoopExit::EffectChannelClosed { .. }) => { + primary.get_or_insert_with(|| "TUI effect channel closed".into()); + } + Ok(ui::renderer::LoopExit::EventChannelClosed) => { + primary.get_or_insert_with(|| "TUI event channel closed".into()); + } + Ok(ui::renderer::LoopExit::Cancelled { .. }) => { + primary.get_or_insert_with(|| { + "TUI renderer cancelled unexpectedly".into() + }); + } + } + + // Restore before Falcon or a lifecycle child can make shutdown wait without + // a bound. The receiver remains live below so every producer can settle. + if let Err(error) = terminal.restore() { + let cleanup = format!("restore terminal: {error}"); + primary = Some(primary.map_or(cleanup.clone(), |error| { + format!("{error}; cleanup error: {cleanup}") + })); + } + + while input_result.is_none() + || ticks_result.is_none() + || collection_result.is_none() + || effects_result.is_none() + || initial_result.is_none() + { + tokio::select! { + result = &mut input, if input_result.is_none() => input_result = Some(result), + result = &mut ticks, if ticks_result.is_none() => ticks_result = Some(result), + result = &mut collection, if collection_result.is_none() => collection_result = Some(result), + result = &mut effect_task, if effects_result.is_none() => effects_result = Some(result), + result = &mut initial, if initial_result.is_none() => initial_result = Some(result), + event = events_rx.recv() => if let Some(event) = event { let _ = app.update(event); }, + } + } + while let Ok(event) = events_rx.try_recv() { + let _ = app.update(event); + } + executor_drain.drain_serial_tasks().await; + let mut cleanup = Vec::new(); + match &input_result { + Some(Ok(Err(error))) => { + cleanup.push(format!("terminal input failed: {error}")) + } + Some(Err(error)) => { + cleanup.push(format!("terminal input task failed: {error}")) + } + _ => {} + } + if let Some(Err(error)) = ticks_result { + cleanup.push(format!("terminal ticks task failed: {error}")); + } + if let Some(Err(error)) = collection_result { + cleanup.push(format!("TUI collector task failed: {error}")); + } + if let Some(Err(error)) = effects_result { + cleanup.push(format!("TUI effects task failed: {error}")); + } + if let Some(Err(error)) = initial_result { + cleanup.push(format!("initial reconciliation task failed: {error}")); + } + if !cleanup.is_empty() { + let cleanup = cleanup.join("; "); + primary = Some(primary.map_or(cleanup.clone(), |error| { + format!("{error}; cleanup error: {cleanup}") + })); + } + if let Some(error) = primary { + return Err(anyhow::anyhow!(error)); + } + match app.session.exit { + Some(app::SessionExit::Detach) => { + if resumed_claim.is_none() { + detached + .set_addresses(executor_drain.resolved_addresses().await); + session::detach(&detached)?; + } + Ok(()) + } + Some(app::SessionExit::Quit) => { + if let Some(claim) = resumed_claim { + claim.finish()?; + } + Ok(()) + } + None => Err(anyhow::anyhow!("TUI stopped without a confirmed exit")), + } +} + +#[cfg(test)] +mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + + use camino::Utf8PathBuf; + use tokio::sync::TryAcquireError; + use voxel_config::VoxelConfig; + + #[tokio::test] + async fn initial_reconciliation_reserves_admission_before_collection() { + let admission = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let initial = + super::reserve_initial_reconciliation(&admission).await.unwrap(); + + assert!(matches!( + admission.clone().try_acquire_owned(), + Err(TryAcquireError::NoPermits) + )); + drop(initial); + assert!(admission.try_acquire_owned().is_ok()); + } + + #[test] + fn probe_mount_preparation_creates_only_empty_node_directories() { + let nonce = + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let workdir = Utf8PathBuf::from_path_buf(std::env::temp_dir()) + .unwrap() + .join(format!( + "voxel-tui-probe-mounts-{}-{nonce}", + std::process::id() + )); + std::fs::create_dir_all(&workdir).unwrap(); + let config = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 2\nrouters = [\"ce\", \"cr1\"]\n", + ) + .unwrap(); + + super::prepare_probe_mounts(&workdir, &config).unwrap(); + + let mut entries = std::fs::read_dir(workdir.join("cargo-bay")) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect::>(); + entries.sort(); + assert_eq!(entries, ["ce", "cr1", "g0", "g1"]); + for node in entries { + assert!( + std::fs::read_dir(workdir.join("cargo-bay").join(node)) + .unwrap() + .next() + .is_none() + ); + } + assert!(!workdir.join(".falcon").exists()); + std::fs::remove_dir_all(workdir).unwrap(); + } +} diff --git a/voxel/src/tui/nexus.rs b/voxel/src/tui/nexus.rs new file mode 100644 index 0000000..255607c --- /dev/null +++ b/voxel/src/tui/nexus.rs @@ -0,0 +1,817 @@ +use std::{net::Ipv4Addr, time::Duration}; + +use anyhow::{Context, anyhow, ensure}; +use chrono::{DateTime, Utc}; +use futures::{StreamExt, stream::FuturesUnordered}; +use reqwest::{RequestBuilder, Response, StatusCode, Url}; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; +use voxel_config::{Network, RecoverySiloCfg, VoxelConfig}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RecoveryLogin { + pub(crate) silo: String, + pub(crate) username: String, + pub(crate) password: String, +} + +impl RecoveryLogin { + pub(crate) fn from_config(config: &VoxelConfig) -> Option { + (config.recovery_silo == RecoverySiloCfg::default()).then(|| Self { + silo: config.recovery_silo.silo_name.clone(), + username: config.recovery_silo.user_name.clone(), + password: "oxide".into(), + }) + } +} + +fn api_candidates(network: &Network) -> Vec { + let Ok(first) = network.service_pool_first.parse::() else { + return Vec::new(); + }; + let last = network.service_pool_last.parse::().unwrap_or(first); + let lo = u32::from(first); + let hi = u32::from(last).max(lo).min(lo.saturating_add(31)); + let dns: Vec<&str> = + network.external_dns_ips.iter().map(String::as_str).collect(); + let (rest, dns_members): (Vec, Vec) = (lo..=hi) + .map(Ipv4Addr::from) + .partition(|addr| !dns.contains(&addr.to_string().as_str())); + rest.into_iter().chain(dns_members).collect() +} + +pub(crate) fn rack_endpoints( + config: &VoxelConfig, + rack: usize, +) -> anyhow::Result> { + ensure!(rack < config.topology.racks(), "unknown rack {}", rack + 1); + let network = config.network.for_rack(rack); + let candidates = api_candidates(&network); + ["http", "https"] + .into_iter() + .flat_map(|scheme| { + candidates.iter().map(move |address| (scheme, address)) + }) + .map(|(scheme, address)| { + Url::parse(&format!("{scheme}://{address}")) + .context("build Nexus endpoint candidate") + }) + .collect() +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OxqlQueryResult { + pub(crate) tables: Vec, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OxqlTable { + pub(crate) name: String, + pub(crate) timeseries: Vec, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OxqlTimeseries { + pub(crate) fields: std::collections::BTreeMap, + pub(crate) points: OxqlPoints, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct FieldValue { + #[serde(rename = "type")] + kind: String, + value: serde_json::Value, +} + +impl FieldValue { + pub(crate) fn as_str(&self) -> Option<&str> { + match self.kind.as_str() { + "string" | "uuid" => self.value.as_str(), + _ => None, + } + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OxqlPoints { + #[serde(default)] + pub(crate) start_times: Option>>, + pub(crate) timestamps: Vec>, + pub(crate) values: Vec, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OxqlValues { + pub(crate) values: ValueArray, + pub(crate) metric_type: MetricType, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MetricType { + Gauge, + Cumulative, + Delta, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type", content = "values", rename_all = "snake_case")] +pub(crate) enum ValueArray { + Integer(Vec>), + Double(Vec>), +} + +pub(crate) type IntegerPoint = + (Option>, DateTime, Option); + +impl OxqlTimeseries { + pub(crate) fn field(&self, name: &str) -> Option<&str> { + self.fields.get(name).and_then(FieldValue::as_str) + } + + pub(crate) fn integer_points(&self) -> anyhow::Result> { + ensure!( + self.points.values.len() == 1, + "expected one OxQL value column" + ); + let ValueArray::Integer(values) = &self.points.values[0].values else { + return Err(anyhow!("expected integer OxQL values")); + }; + ensure!( + values.len() == self.points.timestamps.len(), + "OxQL value and timestamp lengths differ" + ); + if let Some(starts) = &self.points.start_times { + ensure!( + starts.len() == values.len(), + "OxQL interval lengths differ" + ); + } + Ok(values + .iter() + .enumerate() + .map(|(index, value)| { + ( + self.points + .start_times + .as_ref() + .map(|starts| starts[index]), + self.points.timestamps[index], + *value, + ) + }) + .collect()) + } + + #[allow(clippy::type_complexity)] + pub(crate) fn numeric_points( + &self, + ) -> anyhow::Result>, DateTime, Option)>> + { + ensure!( + self.points.values.len() == 1, + "expected one OxQL value column" + ); + let values = match &self.points.values[0].values { + ValueArray::Integer(values) => values + .iter() + .map(|value| value.map(|value| value as f64)) + .collect(), + ValueArray::Double(values) => values.clone(), + }; + ensure!( + values.len() == self.points.timestamps.len(), + "OxQL value and timestamp lengths differ" + ); + if let Some(starts) = &self.points.start_times { + ensure!( + starts.len() == values.len(), + "OxQL interval lengths differ" + ); + } + Ok(values + .into_iter() + .enumerate() + .map(|(index, value)| { + ( + self.points + .start_times + .as_ref() + .map(|starts| starts[index]), + self.points.timestamps[index], + value, + ) + }) + .collect()) + } +} + +#[derive(Serialize)] +struct LoginRequest<'a> { + username: &'a str, + password: &'a str, +} + +#[derive(Serialize)] +struct QueryRequest<'a> { + query: &'a str, + include_summaries: bool, +} + +pub(crate) struct NexusClient { + http: reqwest::Client, + endpoints: Vec, + endpoint: Mutex>, + authenticated: Mutex>, + operation: Mutex<()>, + login: RecoveryLogin, + timeout: Duration, +} + +impl NexusClient { + pub(crate) fn new( + endpoints: Vec, + login: RecoveryLogin, + timeout: Duration, + ) -> anyhow::Result { + ensure!(!endpoints.is_empty(), "Nexus endpoint list is empty"); + ensure!(!timeout.is_zero(), "Nexus request timeout must be nonzero"); + let http = reqwest::Client::builder() + .cookie_store(true) + // Voxel's wicket setup intentionally installs a self-signed + // development certificate. This client is private to virtual racks. + .danger_accept_invalid_certs(true) + .redirect(reqwest::redirect::Policy::none()) + .build() + .context("build Voxel Nexus client")?; + Ok(Self { + http, + endpoints, + endpoint: Mutex::new(None), + authenticated: Mutex::new(None), + operation: Mutex::new(()), + login, + timeout, + }) + } + + async fn send( + &self, + request: RequestBuilder, + cancel: &CancellationToken, + ) -> anyhow::Result { + tokio::select! { + biased; + _ = cancel.cancelled() => Err(anyhow!("cancelled")), + result = tokio::time::timeout(self.timeout, request.send()) => { + result + .map_err(|_| anyhow!("Nexus request timed out after {:?}", self.timeout))? + .context("send Nexus request") + } + } + } + + async fn endpoint( + &self, + cancel: &CancellationToken, + excluded: &[Url], + ) -> anyhow::Result { + if let Some(endpoint) = self.endpoint.lock().await.clone() + && !excluded.contains(&endpoint) + { + return Ok(endpoint); + } + let mut probes = FuturesUnordered::new(); + for endpoint in self + .endpoints + .iter() + .filter(|endpoint| !excluded.contains(endpoint)) + { + let endpoint = endpoint.clone(); + probes.push(async move { + let ping = + endpoint.join("v1/ping").context("build Nexus ping URL")?; + Ok::<_, anyhow::Error>(( + endpoint, + self.send(self.http.get(ping), cancel).await, + )) + }); + } + let mut errors = Vec::new(); + while let Some(probe) = probes.next().await { + let (endpoint, response) = probe?; + match response { + Ok(response) if response.status().is_success() => { + *self.endpoint.lock().await = Some(endpoint.clone()); + return Ok(endpoint); + } + Ok(response) => errors.push(format!( + "{} returned {}", + endpoint, + response.status() + )), + Err(error) if cancel.is_cancelled() => return Err(error), + Err(error) => errors.push(format!("{endpoint}: {error:#}")), + } + } + Err(anyhow!("Nexus is unavailable: {}", errors.join("; "))) + } + + async fn login( + &self, + endpoint: &Url, + cancel: &CancellationToken, + ) -> anyhow::Result<()> { + let url = endpoint + .join(&format!("v1/login/{}/local", self.login.silo)) + .context("build Nexus login URL")?; + let response = self + .send( + self.http.post(url).json(&LoginRequest { + username: &self.login.username, + password: &self.login.password, + }), + cancel, + ) + .await?; + if !response.status().is_success() { + return Err(anyhow!("Nexus login returned {}", response.status())); + } + *self.authenticated.lock().await = Some(endpoint.clone()); + Ok(()) + } + + async fn query_once( + &self, + endpoint: &Url, + query: &str, + cancel: &CancellationToken, + ) -> anyhow::Result { + let url = endpoint + .join("v1/system/timeseries/query") + .context("build Nexus timeseries URL")?; + self.send( + self.http + .post(url) + .json(&QueryRequest { query, include_summaries: false }), + cancel, + ) + .await + } + + async fn query_from( + &self, + endpoint: &Url, + query: &str, + cancel: &CancellationToken, + ) -> anyhow::Result { + let mut response = self.query_once(endpoint, query, cancel).await?; + if response.status() == StatusCode::UNAUTHORIZED { + let _operation = tokio::select! { + biased; + _ = cancel.cancelled() => return Err(anyhow!("cancelled")), + operation = self.operation.lock() => operation, + }; + *self.authenticated.lock().await = None; + tokio::select! { + biased; + _ = cancel.cancelled() => return Err(anyhow!("cancelled")), + _ = tokio::time::sleep(Duration::from_millis(100)) => {} + } + self.login(endpoint, cancel).await?; + response = self.query_once(endpoint, query, cancel).await?; + } + if !response.status().is_success() { + return Err(anyhow!( + "Nexus timeseries query returned {}", + response.status() + )); + } + tokio::select! { + biased; + _ = cancel.cancelled() => Err(anyhow!("cancelled")), + result = tokio::time::timeout(self.timeout, response.json()) => { + result + .map_err(|_| anyhow!("Nexus response timed out after {:?}", self.timeout))? + .context("decode Nexus timeseries response") + } + } + } + + pub(crate) async fn query( + &self, + query: &str, + cancel: &CancellationToken, + ) -> anyhow::Result { + let mut excluded = Vec::new(); + let mut first_error = None; + for _ in 0..self.endpoints.len() { + let endpoint = { + let _operation = tokio::select! { + biased; + _ = cancel.cancelled() => return Err(anyhow!("cancelled")), + operation = self.operation.lock() => operation, + }; + let endpoint = self.endpoint(cancel, &excluded).await?; + if self.authenticated.lock().await.as_ref() != Some(&endpoint) { + self.login(&endpoint, cancel).await?; + } + endpoint + }; + match self.query_from(&endpoint, query, cancel).await { + Ok(result) => return Ok(result), + Err(error) if cancel.is_cancelled() => return Err(error), + Err(error) => { + first_error.get_or_insert(error); + let _operation = self.operation.lock().await; + if self.endpoint.lock().await.as_ref() == Some(&endpoint) { + *self.endpoint.lock().await = None; + } + if self.authenticated.lock().await.as_ref() + == Some(&endpoint) + { + *self.authenticated.lock().await = None; + } + excluded.push(endpoint); + } + } + } + Err(first_error.expect("query attempt failed without an error")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tokio_util::sync::CancellationToken; + use voxel_config::VoxelConfig; + + struct TestResponse { + status: &'static str, + headers: &'static str, + body: &'static str, + delay: Duration, + } + + async fn test_server( + responses: Vec, + ) -> (Url, Arc>>, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = requests.clone(); + let task = tokio::spawn(async move { + let mut responses = VecDeque::from(responses); + while let Some(response) = responses.pop_front() { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let mut buffer = [0; 4096]; + loop { + let count = stream.read(&mut buffer).await.unwrap(); + if count == 0 { + break; + } + bytes.extend_from_slice(&buffer[..count]); + let Some(headers_end) = bytes + .windows(4) + .position(|window| window == b"\r\n\r\n") + else { + continue; + }; + let headers = + String::from_utf8_lossy(&bytes[..headers_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .and_then(|value| value.trim().parse().ok()) + }) + .unwrap_or(0); + if bytes.len() >= headers_end + 4 + content_length { + break; + } + } + recorded + .lock() + .unwrap() + .push(String::from_utf8(bytes).unwrap()); + let reply = format!( + "HTTP/1.1 {}\r\nContent-Length: {}\r\nConnection: close\r\n{}\r\n", + response.status, + response.body.len(), + response.headers, + ); + stream.write_all(reply.as_bytes()).await.unwrap(); + tokio::time::sleep(response.delay).await; + stream.write_all(response.body.as_bytes()).await.unwrap(); + } + }); + (Url::parse(&format!("http://{address}/")).unwrap(), requests, task) + } + + #[test] + fn rack_endpoints_follow_service_pool_probe_order() { + let config = VoxelConfig::default(); + + let endpoints = rack_endpoints(&config, 0).unwrap(); + + assert_eq!(endpoints.len(), 20); + assert_eq!(endpoints[0].as_str(), "http://198.51.100.22/"); + assert_eq!(endpoints[8].as_str(), "http://198.51.100.20/"); + assert_eq!(endpoints[10].as_str(), "https://198.51.100.22/"); + } + + #[test] + fn automatic_login_requires_the_default_recovery_identity() { + let mut config = VoxelConfig::default(); + assert_eq!( + RecoveryLogin::from_config(&config), + Some(RecoveryLogin { + silo: "recovery".into(), + username: "recovery".into(), + password: "oxide".into(), + }) + ); + + config.recovery_silo.user_name = "operator".into(); + assert_eq!(RecoveryLogin::from_config(&config), None); + } + + #[test] + fn oxql_fields_decode_every_api_field_type() { + let result: OxqlQueryResult = serde_json::from_value(serde_json::json!({ + "tables": [{ + "name": "test:metric", + "timeseries": [{ + "fields": { + "string": {"type": "string", "value": "value"}, + "i8": {"type": "i8", "value": -1}, + "u8": {"type": "u8", "value": 1}, + "i16": {"type": "i16", "value": -2}, + "u16": {"type": "u16", "value": 2}, + "i32": {"type": "i32", "value": -3}, + "u32": {"type": "u32", "value": 3}, + "i64": {"type": "i64", "value": -4}, + "u64": {"type": "u64", "value": 4}, + "ip_addr": {"type": "ip_addr", "value": "192.0.2.1"}, + "uuid": { + "type": "uuid", + "value": "00000000-0000-0000-0000-000000000001" + }, + "bool": {"type": "bool", "value": true} + }, + "points": { + "timestamps": [], + "values": [] + } + }] + }] + })) + .unwrap(); + let fields = &result.tables[0].timeseries[0].fields; + + assert_eq!(fields["string"].as_str(), Some("value")); + assert_eq!( + fields["uuid"].as_str(), + Some("00000000-0000-0000-0000-000000000001") + ); + assert_eq!(fields["ip_addr"].as_str(), None); + } + + #[tokio::test] + async fn query_logs_in_and_reauthenticates_once_after_unauthorized() { + let (endpoint, requests, server) = test_server(vec![ + TestResponse { + status: "200 OK", + headers: "", + body: "pong", + delay: Duration::ZERO, + }, + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=first; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "401 Unauthorized", + headers: "", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=second; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "200 OK", + headers: "Content-Type: application/json\r\n", + body: r#"{"tables":[],"query_summaries":null}"#, + delay: Duration::ZERO, + }, + ]) + .await; + let client = NexusClient::new( + vec![endpoint], + RecoveryLogin { + silo: "recovery".into(), + username: "recovery".into(), + password: "oxide".into(), + }, + Duration::from_secs(2), + ) + .unwrap(); + + let result = client + .query("get sled_data_link:bytes_sent", &CancellationToken::new()) + .await + .unwrap(); + + assert!(result.tables.is_empty()); + server.await.unwrap(); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 5); + assert!(requests[0].starts_with("GET /v1/ping ")); + assert!(requests[1].starts_with("POST /v1/login/recovery/local ")); + assert!( + requests[1] + .contains(r#"{"username":"recovery","password":"oxide"}"#) + ); + assert!(requests[2].contains("cookie: session=first")); + assert!(requests[3].starts_with("POST /v1/login/recovery/local ")); + assert!(requests[4].contains("cookie: session=second")); + assert!(requests[4].contains(r#""include_summaries":false"#)); + } + + #[tokio::test] + async fn query_retries_another_endpoint_when_a_response_stalls() { + let (stalled, stalled_requests, stalled_server) = test_server(vec![ + TestResponse { + status: "200 OK", + headers: "", + body: "pong", + delay: Duration::ZERO, + }, + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=stalled; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "200 OK", + headers: "Content-Type: application/json\r\n", + body: r#"{"tables":[],"query_summaries":null}"#, + delay: Duration::from_millis(200), + }, + ]) + .await; + let (healthy, healthy_requests, healthy_server) = test_server(vec![ + TestResponse { + status: "200 OK", + headers: "", + body: "pong", + delay: Duration::from_millis(20), + }, + TestResponse { + status: "200 OK", + headers: "", + body: "pong", + delay: Duration::ZERO, + }, + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=healthy; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "200 OK", + headers: "Content-Type: application/json\r\n", + body: r#"{"tables":[],"query_summaries":null}"#, + delay: Duration::ZERO, + }, + ]) + .await; + let client = NexusClient::new( + vec![stalled, healthy], + RecoveryLogin { + silo: "recovery".into(), + username: "recovery".into(), + password: "oxide".into(), + }, + Duration::from_millis(50), + ) + .unwrap(); + + let result = client + .query("get sled_data_link:bytes_sent", &CancellationToken::new()) + .await + .unwrap(); + + assert!(result.tables.is_empty()); + stalled_server.await.unwrap(); + healthy_server.await.unwrap(); + assert_eq!(stalled_requests.lock().unwrap().len(), 3); + assert_eq!(healthy_requests.lock().unwrap().len(), 4); + } + + #[tokio::test] + async fn query_tries_every_endpoint_before_failing() { + let stalled_query = TestResponse { + status: "200 OK", + headers: "Content-Type: application/json\r\n", + body: r#"{"tables":[],"query_summaries":null}"#, + delay: Duration::from_millis(200), + }; + let (first, _, first_server) = test_server(vec![ + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=first; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + stalled_query, + ]) + .await; + let (second, _, second_server) = test_server(vec![ + TestResponse { + status: "200 OK", + headers: "", + body: "pong", + delay: Duration::ZERO, + }, + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=second; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "200 OK", + headers: "Content-Type: application/json\r\n", + body: r#"{"tables":[],"query_summaries":null}"#, + delay: Duration::from_millis(200), + }, + ]) + .await; + let (third, third_requests, third_server) = test_server(vec![ + TestResponse { + status: "503 Service Unavailable", + headers: "", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "200 OK", + headers: "", + body: "pong", + delay: Duration::ZERO, + }, + TestResponse { + status: "204 No Content", + headers: "Set-Cookie: session=third; Path=/; HttpOnly\r\n", + body: "", + delay: Duration::ZERO, + }, + TestResponse { + status: "200 OK", + headers: "Content-Type: application/json\r\n", + body: r#"{"tables":[],"query_summaries":null}"#, + delay: Duration::ZERO, + }, + ]) + .await; + let client = NexusClient::new( + vec![first.clone(), second, third], + RecoveryLogin { + silo: "recovery".into(), + username: "recovery".into(), + password: "oxide".into(), + }, + Duration::from_millis(50), + ) + .unwrap(); + *client.endpoint.lock().await = Some(first); + + let result = client + .query("get sled_data_link:bytes_sent", &CancellationToken::new()) + .await + .unwrap(); + + assert!(result.tables.is_empty()); + first_server.await.unwrap(); + second_server.await.unwrap(); + third_server.await.unwrap(); + assert_eq!(third_requests.lock().unwrap().len(), 4); + } +} diff --git a/voxel/src/tui/operation.rs b/voxel/src/tui/operation.rs new file mode 100644 index 0000000..d986467 --- /dev/null +++ b/voxel/src/tui/operation.rs @@ -0,0 +1,121 @@ +use std::process::ExitStatus; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum OperationKind { + Launch, + Destroy, + Route, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum LaunchPhase { + Preflight, + Stage, + Boot, + Initialize, + RackSetup, + Route, + Reconcile, +} + +impl LaunchPhase { + pub(crate) const ORDER: [Self; 7] = [ + Self::Preflight, + Self::Stage, + Self::Boot, + Self::Initialize, + Self::RackSetup, + Self::Route, + Self::Reconcile, + ]; + + pub(crate) fn ordinal(self) -> usize { + Self::ORDER.iter().position(|phase| *phase == self).unwrap() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DestroyPhase { + OrphanCleanup, + FalconTeardown, + StorageCleanup, + Reconcile, +} + +impl DestroyPhase { + pub(crate) const ORDER: [Self; 4] = [ + Self::OrphanCleanup, + Self::FalconTeardown, + Self::StorageCleanup, + Self::Reconcile, + ]; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RoutePhase { + Validate, + Apply, + Reconcile, +} + +impl RoutePhase { + pub(crate) const ORDER: [Self; 3] = + [Self::Validate, Self::Apply, Self::Reconcile]; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum OperationPhase { + Launch(LaunchPhase), + Destroy(DestroyPhase), + Route(RoutePhase), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum LogLevel { + Info, + Warning, + Error, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct OperationWarning { + pub(crate) message: String, + pub(crate) resource: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum OutputStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum OperationEvent { + Started { kind: OperationKind }, + PhaseStarted { phase: OperationPhase }, + Log { level: LogLevel, message: String }, + Warning(OperationWarning), + Finished(CommandOutcome), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CommandOutcome { + Exited { status: ExitStatus, stderr_summary: Vec }, + SpawnFailed { message: String }, + ForceStopped { status: Option, kill_error: Option }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn phase_orders_preserve_static_display_sequence() { + assert_eq!(LaunchPhase::ORDER[0], LaunchPhase::Preflight); + assert_eq!(DestroyPhase::ORDER[0], DestroyPhase::OrphanCleanup); + assert_eq!( + RoutePhase::ORDER, + [RoutePhase::Validate, RoutePhase::Apply, RoutePhase::Reconcile] + ); + } +} diff --git a/voxel/src/tui/phase.rs b/voxel/src/tui/phase.rs new file mode 100644 index 0000000..747992a --- /dev/null +++ b/voxel/src/tui/phase.rs @@ -0,0 +1,211 @@ +use super::operation::{ + DestroyPhase, LaunchPhase, OperationKind, OperationPhase, OutputStream, + RoutePhase, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PhaseHint { + Started(OperationPhase), +} + +#[derive(Default)] +pub(crate) struct PhaseClassifier { + launch: Option, + destroy: Option, + route: Option, +} + +impl PhaseClassifier { + pub(crate) fn classify( + &mut self, + kind: OperationKind, + _stream: OutputStream, + line: &str, + ) -> Option { + let lower = line.to_ascii_lowercase(); + let (phase, ordinal, last) = match kind { + OperationKind::Launch => { + let phase = launch_phase(&lower)?; + ( + OperationPhase::Launch(phase), + phase.ordinal(), + &mut self.launch, + ) + } + OperationKind::Destroy => { + let phase = destroy_phase(&lower)?; + let ordinal = DestroyPhase::ORDER + .iter() + .position(|item| *item == phase) + .unwrap(); + (OperationPhase::Destroy(phase), ordinal, &mut self.destroy) + } + OperationKind::Route => { + let phase = route_phase(&lower)?; + let ordinal = RoutePhase::ORDER + .iter() + .position(|item| *item == phase) + .unwrap(); + (OperationPhase::Route(phase), ordinal, &mut self.route) + } + }; + if last.is_some_and(|previous| ordinal <= previous) { + return None; + } + *last = Some(ordinal); + Some(PhaseHint::Started(phase)) + } +} + +fn launch_phase(line: &str) -> Option { + if line.contains("external route set:") || line.contains(" external route:") + { + Some(LaunchPhase::Route) + } else if line.contains(": watching rss progress on the rss node ...") { + Some(LaunchPhase::RackSetup) + } else if line.contains(": launch start") { + Some(LaunchPhase::Initialize) + } else if line.contains("creating nodes") { + Some(LaunchPhase::Boot) + } else if line.contains("creating links") { + Some(LaunchPhase::Stage) + } else if line.contains("starting preflight for deployment") { + Some(LaunchPhase::Preflight) + } else { + None + } +} + +fn destroy_phase(line: &str) -> Option { + if line.contains("orphaned propolis") { + Some(DestroyPhase::OrphanCleanup) + } else { + None + } +} + +fn route_phase(line: &str) -> Option { + if line.contains("external route") { Some(RoutePhase::Apply) } else { None } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::operation::{ + LaunchPhase, OperationKind, OperationPhase, OutputStream, + }; + + #[test] + fn known_current_main_launch_output_advances_monotonically() { + let mut classifier = PhaseClassifier::default(); + let fixtures = [ + ( + "starting preflight for deployment vtui_demo2", + LaunchPhase::Preflight, + ), + ("creating links", LaunchPhase::Stage), + ("creating nodes", LaunchPhase::Boot), + ("g0: launch start", LaunchPhase::Initialize), + ( + "rack-init: watching RSS progress on the RSS node ...", + LaunchPhase::RackSetup, + ), + ( + "external route set: 198.51.100.0/24 -> 172.30.199.14 (ce)", + LaunchPhase::Route, + ), + ]; + for (line, phase) in fixtures { + assert_eq!( + classifier.classify( + OperationKind::Launch, + OutputStream::Stdout, + line + ), + Some(PhaseHint::Started(OperationPhase::Launch(phase))) + ); + } + assert_eq!( + classifier.classify( + OperationKind::Launch, + OutputStream::Stdout, + "gimlet-0: launch start" + ), + None + ); + assert_eq!( + classifier.classify( + OperationKind::Launch, + OutputStream::Stdout, + "unrecognized output" + ), + None + ); + assert_eq!( + classifier.classify( + OperationKind::Launch, + OutputStream::Stdout, + "reconcile complete" + ), + None + ); + } + + #[test] + fn invented_launch_messages_remain_unclassified() { + for line in + ["launching topology", "staging node disks", "booting nodes"] + { + assert_eq!( + PhaseClassifier::default().classify( + OperationKind::Launch, + OutputStream::Stdout, + line, + ), + None + ); + } + } + + #[test] + fn route_and_destroy_fixtures_are_optional_and_monotonic() { + let mut classifier = PhaseClassifier::default(); + assert_eq!( + classifier.classify( + OperationKind::Route, + OutputStream::Stdout, + "external route set", + ), + Some(PhaseHint::Started(OperationPhase::Route( + crate::tui::operation::RoutePhase::Apply + ))) + ); + assert_eq!( + classifier.classify( + OperationKind::Route, + OutputStream::Stdout, + "validating route", + ), + None + ); + let mut classifier = PhaseClassifier::default(); + assert_eq!( + classifier.classify( + OperationKind::Destroy, + OutputStream::Stdout, + "orphaned propolis processes", + ), + Some(PhaseHint::Started(OperationPhase::Destroy( + crate::tui::operation::DestroyPhase::OrphanCleanup + ))) + ); + assert_eq!( + classifier.classify( + OperationKind::Destroy, + OutputStream::Stdout, + "destroy complete", + ), + None + ); + } +} diff --git a/voxel/src/tui/process.rs b/voxel/src/tui/process.rs new file mode 100644 index 0000000..a3ee0f7 --- /dev/null +++ b/voxel/src/tui/process.rs @@ -0,0 +1,471 @@ +use std::{collections::VecDeque, io, process::ExitStatus}; + +use tokio::{ + io::{AsyncRead, AsyncReadExt, BufReader}, + process::{Child, Command}, + sync::mpsc, + task::JoinHandle, +}; + +use super::{ + context::CommandSpec, logging::DurableLog, operation::OutputStream, +}; + +pub(crate) const STDERR_SUMMARY_LINES: usize = 128; +pub(crate) const MAX_OUTPUT_CHUNK_BYTES: usize = 16 * 1024; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct OutputLine { + pub(crate) stream: OutputStream, + pub(crate) text: String, +} + +#[derive(Debug)] +pub(crate) struct ChildResult { + pub(crate) status: ExitStatus, + pub(crate) stderr_summary: Vec, + pub(crate) output_errors: Vec<(OutputStream, DrainFailure)>, +} + +#[derive(Debug, Eq, PartialEq)] +pub(crate) struct ChildWaitFailure { + pub(crate) message: String, +} + +impl From for ChildWaitFailure { + fn from(error: io::Error) -> Self { + Self { message: error.to_string() } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum DrainFailure { + Output(String), + DurableLog(String), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ForceStopResult { + Killed, + AlreadyExited, + Failed(String), +} + +struct DrainResult { + stderr: VecDeque, + errors: Vec, +} + +pub(crate) struct ChildSupervisor { + child: Child, + stdout: JoinHandle, + stderr: JoinHandle, +} + +impl ChildSupervisor { + pub(crate) fn spawn( + spec: CommandSpec, + durable: DurableLog, + ui: mpsc::Sender, + ) -> io::Result { + let mut command = Command::new(spec.program); + command + .args(spec.args) + .current_dir(spec.current_dir) + .envs(spec.env) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let mut child = command.spawn()?; + let stdout = child.stdout.take().expect("configured stdout pipe"); + let stderr = child.stderr.take().expect("configured stderr pipe"); + Ok(Self { + child, + stdout: tokio::spawn(drain( + stdout, + OutputStream::Stdout, + durable.clone(), + ui.clone(), + )), + stderr: tokio::spawn(drain( + stderr, + OutputStream::Stderr, + durable, + ui, + )), + }) + } + + pub(crate) async fn force_stop(&mut self) -> ForceStopResult { + match self.child.try_wait() { + Ok(Some(_)) => ForceStopResult::AlreadyExited, + Ok(None) => match self.child.kill().await { + Ok(()) => ForceStopResult::Killed, + Err(error) => match self.child.try_wait() { + Ok(Some(_)) => ForceStopResult::AlreadyExited, + _ => ForceStopResult::Failed(error.to_string()), + }, + }, + Err(error) => ForceStopResult::Failed(error.to_string()), + } + } + + pub(crate) fn is_settled(&mut self) -> Result { + self.child + .try_wait() + .map(|status| status.is_some()) + .map_err(ChildWaitFailure::from) + } + + pub(crate) async fn wait( + mut self, + ) -> Result { + let status = self.child.wait().await.map_err(ChildWaitFailure::from)?; + let stdout = join_drain(self.stdout, OutputStream::Stdout).await; + let stderr = join_drain(self.stderr, OutputStream::Stderr).await; + let mut output_errors = Vec::new(); + output_errors.extend( + stdout + .errors + .into_iter() + .map(|error| (OutputStream::Stdout, error)), + ); + output_errors.extend( + stderr + .errors + .into_iter() + .map(|error| (OutputStream::Stderr, error)), + ); + Ok(ChildResult { + status, + stderr_summary: stderr.stderr.into(), + output_errors, + }) + } +} + +async fn join_drain( + task: JoinHandle, + stream: OutputStream, +) -> DrainResult { + task.await.unwrap_or_else(|error| DrainResult { + stderr: VecDeque::new(), + errors: vec![DrainFailure::Output(format!( + "{stream:?} drain task failed: {error}" + ))], + }) +} + +async fn drain( + pipe: impl AsyncRead + Unpin, + stream: OutputStream, + durable: DurableLog, + ui: mpsc::Sender, +) -> DrainResult { + let mut reader = BufReader::new(pipe); + let mut bytes = [0; MAX_OUTPUT_CHUNK_BYTES]; + let mut pending = Vec::with_capacity(MAX_OUTPUT_CHUNK_BYTES); + let mut stderr = VecDeque::with_capacity(STDERR_SUMMARY_LINES); + let mut errors = Vec::new(); + loop { + match reader.read(&mut bytes).await { + Ok(0) => break, + Ok(read) => { + for byte in &bytes[..read] { + if *byte == b'\n' { + if pending.last() == Some(&b'\r') { + pending.pop(); + } + emit_line( + &pending, + stream, + &durable, + &ui, + &mut stderr, + &mut errors, + ); + pending.clear(); + } else { + if pending.len() == MAX_OUTPUT_CHUNK_BYTES { + emit_line( + &pending, + stream, + &durable, + &ui, + &mut stderr, + &mut errors, + ); + pending.clear(); + } + pending.push(*byte); + } + } + } + Err(error) => { + errors.push(DrainFailure::Output(error.to_string())); + return DrainResult { stderr, errors }; + } + } + } + if !pending.is_empty() { + emit_line(&pending, stream, &durable, &ui, &mut stderr, &mut errors); + } + DrainResult { stderr, errors } +} + +fn emit_line( + bytes: &[u8], + stream: OutputStream, + durable: &DurableLog, + ui: &mpsc::Sender, + stderr: &mut VecDeque, + errors: &mut Vec, +) { + let text = String::from_utf8_lossy(bytes).into_owned(); + if let Err(error) = durable.write_line(&text) + && !errors + .iter() + .any(|failure| matches!(failure, DrainFailure::DurableLog(_))) + { + errors.push(DrainFailure::DurableLog(error.to_string())); + } + if stream == OutputStream::Stderr { + if stderr.len() == STDERR_SUMMARY_LINES { + stderr.pop_front(); + } + stderr.push_back(text.clone()); + } + let _ = ui.try_send(OutputLine { stream, text }); +} + +#[cfg(test)] +mod tests { + use std::{ + ffi::OsString, + io::{self, Write}, + path::PathBuf, + time::{Duration, SystemTime}, + }; + + use tokio::{sync::mpsc, time::timeout}; + + use super::*; + use crate::tui::{context::CommandSpec, logging::DurableLog}; + + const TWO_MIB: usize = 2 * 1024 * 1024; + + #[test] + fn process_helper() { + let Ok(mode) = std::env::var("VOXEL_TUI_PROCESS_HELPER") else { + return; + }; + match mode.as_str() { + "both-full" => { + let out = vec![b'o'; TWO_MIB]; + let err = vec![b'e'; TWO_MIB]; + let stdout = std::thread::spawn(move || { + io::stdout().write_all(&out).unwrap() + }); + let stderr = std::thread::spawn(move || { + io::stderr().write_all(&err).unwrap() + }); + stdout.join().unwrap(); + stderr.join().unwrap(); + } + "exit-7" => { + for n in 0..300 { + eprintln!("stderr-{n:03}"); + } + std::process::exit(7); + } + "wait" => { + println!("started"); + io::stdout().flush().unwrap(); + std::thread::sleep(Duration::from_millis(600)); + println!("settled"); + } + "unterminated" => { + io::stdout().write_all(b"last line").unwrap(); + io::stdout().flush().unwrap(); + } + "split-line" => { + let mut stdout = io::stdout(); + stdout.write_all(b"rack external ").unwrap(); + stdout.flush().unwrap(); + std::thread::sleep(Duration::from_millis(100)); + stdout.write_all(b"route: timed out\n").unwrap(); + stdout.flush().unwrap(); + } + "invalid-utf8" => { + io::stdout().write_all(b"bad \xff text\n").unwrap() + } + other => panic!("unknown helper mode {other}"), + } + std::process::exit(0); + } + + fn paths(mode: &str) -> (CommandSpec, PathBuf) { + let nonce = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let log = std::env::temp_dir().join(format!( + "voxel-tui-{mode}-{}-{nonce}.log", + std::process::id() + )); + let test_name = "tui::process::tests::process_helper"; + let spec = CommandSpec { + program: std::env::current_exe().unwrap(), + args: ["--exact", test_name, "--nocapture"] + .into_iter() + .map(OsString::from) + .collect(), + current_dir: std::env::current_dir().unwrap(), + env: vec![( + OsString::from("VOXEL_TUI_PROCESS_HELPER"), + OsString::from(mode), + )], + }; + (spec, log) + } + + async fn spawn( + mode: &str, + capacity: usize, + ) -> (ChildSupervisor, mpsc::Receiver, PathBuf) { + let (spec, path) = paths(mode); + let (tx, rx) = mpsc::channel(capacity); + let child = + ChildSupervisor::spawn(spec, DurableLog::open(&path).unwrap(), tx) + .unwrap(); + (child, rx, path) + } + + #[tokio::test] + async fn drains_large_stdout_and_stderr_without_deadlock() { + let (child, _rx, path) = spawn("both-full", 1).await; + let result = timeout(Duration::from_secs(5), child.wait()) + .await + .unwrap() + .unwrap(); + assert!(result.status.success()); + let bytes = std::fs::read(&path).unwrap(); + assert!(bytes.iter().filter(|b| **b == b'o').count() >= TWO_MIB); + assert!(bytes.iter().filter(|b| **b == b'e').count() >= TWO_MIB); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn segments_newline_free_output_into_bounded_chunks() { + let (child, mut rx, path) = spawn("both-full", 16).await; + let result = timeout(Duration::from_secs(5), child.wait()) + .await + .unwrap() + .unwrap(); + assert!(result.status.success()); + while let Ok(line) = rx.try_recv() { + assert!(line.text.len() <= MAX_OUTPUT_CHUNK_BYTES); + } + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(bytes.iter().filter(|byte| **byte == b'o').count(), TWO_MIB); + assert!(bytes.iter().filter(|byte| **byte == b'e').count() >= TWO_MIB); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn child_wait_errors_are_structured() { + let failure = ChildWaitFailure::from(io::Error::other("wait broke")); + assert_eq!(failure.message, "wait broke"); + } + + #[test] + fn drain_failures_distinguish_pipe_and_durable_log() { + assert_ne!( + DrainFailure::Output("read broke".into()), + DrainFailure::DurableLog("disk full".into()) + ); + } + + #[tokio::test] + async fn captures_final_unterminated_line() { + let (child, mut rx, path) = spawn("unterminated", 8).await; + child.wait().await.unwrap(); + assert!( + std::fs::read_to_string(&path).unwrap().contains("last line\n") + ); + let mut found = false; + while let Ok(line) = rx.try_recv() { + found |= line.text == "last line"; + } + assert!(found); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn assembles_a_line_written_in_multiple_flushed_pieces() { + let (child, mut rx, path) = spawn("split-line", 8).await; + child.wait().await.unwrap(); + let lines: Vec<_> = std::iter::from_fn(|| rx.try_recv().ok()) + .filter(|line| line.stream == OutputStream::Stdout) + .map(|line| line.text) + .collect(); + assert!( + lines.iter().any(|line| line == "rack external route: timed out") + ); + assert!(!lines.iter().any(|line| line == "rack external ")); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn lossily_preserves_invalid_utf8() { + let (child, _rx, path) = spawn("invalid-utf8", 8).await; + child.wait().await.unwrap(); + assert!(std::fs::read_to_string(&path).unwrap().contains("bad � text")); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn nonzero_exit_keeps_bounded_stderr_summary() { + let (child, _rx, path) = spawn("exit-7", 1).await; + let result = child.wait().await.unwrap(); + assert_eq!(result.status.code(), Some(7)); + assert_eq!(result.stderr_summary.len(), STDERR_SUMMARY_LINES); + assert_eq!(result.stderr_summary.last().unwrap(), "stderr-299"); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn waiting_does_not_terminate_child() { + let (child, _rx, path) = spawn("wait", 8).await; + let result = child.wait().await.unwrap(); + assert!(result.status.success()); + assert!(std::fs::read_to_string(&path).unwrap().contains("settled")); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn force_stop_settles_direct_child() { + let (mut child, _rx, path) = spawn("wait", 8).await; + tokio::time::sleep(Duration::from_millis(100)).await; + let kill = child.force_stop().await; + let result = child.wait().await.unwrap(); + assert!(matches!( + kill, + ForceStopResult::Killed | ForceStopResult::AlreadyExited + )); + assert!(!result.status.success()); + std::fs::remove_file(path).unwrap(); + } + + #[tokio::test] + async fn closed_ui_receiver_does_not_block_drain() { + let (child, rx, path) = spawn("both-full", 1).await; + drop(rx); + timeout(Duration::from_secs(5), child.wait()).await.unwrap().unwrap(); + assert!( + std::fs::metadata(&path).unwrap().len() >= (TWO_MIB * 2) as u64 + ); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/voxel/src/tui/reconcile.rs b/voxel/src/tui/reconcile.rs new file mode 100644 index 0000000..eff1bb7 --- /dev/null +++ b/voxel/src/tui/reconcile.rs @@ -0,0 +1,674 @@ +use std::collections::{BTreeMap, HashSet}; +use std::io; +use std::process::Stdio; +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +use super::collector::{FalconExecutor, NodeExecutor, NodeTarget}; +use super::context::TuiContext; + +const PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ObservedDeploymentState { + Unknown, + Stopped, + Starting, + Running, + Degraded, + Stopping, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum LifecycleIntent { + Idle, + Launch, + Destroy, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NodeEvidence { + Missing, + Booting, + Running, + Failed, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct NodeObservation { + pub(crate) id: String, + pub(crate) state: NodeEvidence, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RouteEvidence { + Applied, + NotRequired, + Unavailable, + Failed, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum RssObservation { + Initializing { step: Option }, + Initialized { id: String }, + StaleInitialized, + Failed { message: String }, + Unavailable, + UnknownResponse, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TopologyEvidence { + pub(crate) node_ids: Vec, + pub(crate) rack_ids: Vec, + pub(crate) available: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RackRssEvidence { + pub(crate) rack: usize, + pub(crate) observation: RssObservation, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationEvidence { + pub(crate) topology: TopologyEvidence, + pub(crate) nodes: Vec, + pub(crate) racks: Vec, + pub(crate) required_rss_rack_ids: Vec, + pub(crate) routes: RouteEvidence, + pub(crate) intent: LifecycleIntent, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct ReconciliationSummary { + pub(crate) expected_nodes: usize, + pub(crate) running_nodes: usize, + pub(crate) live_nodes: usize, + pub(crate) configured_racks: usize, + pub(crate) initialized_racks: usize, + pub(crate) warnings: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationResult { + pub(crate) state: ObservedDeploymentState, + pub(crate) summary: ReconciliationSummary, +} + +pub(crate) fn reduce( + evidence: &ReconciliationEvidence, +) -> ReconciliationResult { + let node_ids: HashSet<_> = evidence.topology.node_ids.iter().collect(); + let rack_ids: HashSet<_> = evidence.topology.rack_ids.iter().collect(); + let seen_nodes: HashSet<_> = + evidence.nodes.iter().map(|node| &node.id).collect(); + let seen_racks: HashSet<_> = + evidence.racks.iter().map(|rack| &rack.rack).collect(); + let required_rss_ids: HashSet<_> = + evidence.required_rss_rack_ids.iter().collect(); + let malformed_nodes = node_ids.len() != evidence.topology.node_ids.len() + || seen_nodes.len() != evidence.nodes.len() + || seen_nodes != node_ids; + let malformed_racks = rack_ids.len() != evidence.topology.rack_ids.len() + || seen_racks.len() != evidence.racks.len() + || seen_racks != rack_ids; + let malformed_required_racks = required_rss_ids.len() + != evidence.required_rss_rack_ids.len() + || !required_rss_ids.is_subset(&rack_ids); + let running = evidence + .nodes + .iter() + .filter(|node| node.state == NodeEvidence::Running) + .count(); + let live = evidence + .nodes + .iter() + .filter(|node| { + !matches!(node.state, NodeEvidence::Missing | NodeEvidence::Unknown) + }) + .count(); + let initialized = evidence + .racks + .iter() + .filter(|rack| { + required_rss_ids.contains(&rack.rack) + && matches!( + rack.observation, + RssObservation::Initialized { .. } + ) + }) + .count(); + let fatal = + evidence.nodes.iter().any(|node| node.state == NodeEvidence::Failed) + || evidence.racks.iter().any(|rack| { + required_rss_ids.contains(&rack.rack) + && matches!(rack.observation, RssObservation::Failed { .. }) + }) + || evidence.routes == RouteEvidence::Failed; + let invalid_topology = !evidence.topology.available + || malformed_nodes + || malformed_racks + || malformed_required_racks; + let node_uncertain = + evidence.nodes.iter().any(|node| node.state == NodeEvidence::Unknown); + let route_satisfied = matches!( + evidence.routes, + RouteEvidence::Applied | RouteEvidence::NotRequired + ); + let rss_uncertain = evidence.racks.iter().any(|rack| { + required_rss_ids.contains(&rack.rack) + && matches!( + rack.observation, + RssObservation::Unavailable | RssObservation::UnknownResponse + ) + }); + let mut warnings = Vec::new(); + for id in node_ids.difference(&seen_nodes) { + warnings.push(format!("node {id} observation is missing")); + } + if seen_nodes.len() != evidence.nodes.len() { + warnings.push("duplicate node observations".into()); + } + for id in rack_ids.difference(&seen_racks) { + warnings.push(format!("rack {id} observation is missing")); + } + if seen_racks.len() != evidence.racks.len() { + warnings.push("duplicate rack observations".into()); + } + if required_rss_ids.len() != evidence.required_rss_rack_ids.len() { + warnings.push("duplicate required RSS rack IDs".into()); + } + for id in required_rss_ids.difference(&rack_ids) { + warnings.push(format!("required RSS rack {id} is not in topology")); + } + for node in &evidence.nodes { + if node.state != NodeEvidence::Running { + warnings.push(format!("node {} is {:?}", node.id, node.state)); + } + } + for rack in &evidence.racks { + if required_rss_ids.contains(&rack.rack) + && !matches!(rack.observation, RssObservation::Initialized { .. }) + { + warnings.push(format!( + "rack {} RSS is {:?}", + rack.rack, rack.observation + )); + } + } + if !route_satisfied { + warnings.push("required routes are not confirmed applied".into()); + } + + let state = if malformed_required_racks { + ObservedDeploymentState::Unknown + } else if fatal { + ObservedDeploymentState::Degraded + } else if invalid_topology { + ObservedDeploymentState::Unknown + } else if live == 0 && !node_uncertain { + ObservedDeploymentState::Stopped + } else if evidence.intent == LifecycleIntent::Destroy && live > 0 { + ObservedDeploymentState::Stopping + } else if evidence.intent == LifecycleIntent::Launch + && evidence.nodes.iter().any(|node| { + matches!(node.state, NodeEvidence::Booting | NodeEvidence::Running) + }) + { + ObservedDeploymentState::Starting + } else if node_uncertain + || (evidence.intent == LifecycleIntent::Idle + && (rss_uncertain + || matches!( + evidence.routes, + RouteEvidence::Unavailable | RouteEvidence::Unknown + ))) + { + ObservedDeploymentState::Unknown + } else if running == evidence.topology.node_ids.len() + && initialized == required_rss_ids.len() + && route_satisfied + { + ObservedDeploymentState::Running + } else { + ObservedDeploymentState::Degraded + }; + + ReconciliationResult { + state, + summary: ReconciliationSummary { + expected_nodes: evidence.topology.node_ids.len(), + running_nodes: running, + live_nodes: live, + configured_racks: evidence.topology.rack_ids.len(), + initialized_racks: initialized, + warnings, + }, + } +} + +async fn bounded_output( + mut command: tokio::process::Command, + label: &str, + shutdown: &CancellationToken, +) -> anyhow::Result { + command.kill_on_drop(true).stdin(Stdio::null()); + tokio::select! { + biased; + _ = shutdown.cancelled() => anyhow::bail!("{label} cancelled"), + result = tokio::time::timeout(PROBE_TIMEOUT, command.output()) => { + result + .map_err(|_| anyhow::anyhow!("{label} timed out after {PROBE_TIMEOUT:?}"))? + .map_err(|error| anyhow::anyhow!("{label}: {error}")) + } + } +} + +async fn deployment_storage_exists( + context: &TuiContext, + shutdown: &CancellationToken, +) -> anyhow::Result> { + let dataset = format!("{}/topo/{}", context.dataset, context.name); + let mut command = tokio::process::Command::new("zfs"); + command.args(["list", "-H", "-o", "name", &dataset]); + match bounded_output(command, "deployment storage probe", shutdown).await { + Ok(output) if output.status.success() => Ok(Some(true)), + Ok(output) + if String::from_utf8_lossy(&output.stderr) + .to_ascii_lowercase() + .contains("does not exist") => + { + Ok(Some(false)) + } + Ok(_) => Ok(None), + Err(error) if shutdown.is_cancelled() => Err(error), + Err(_) => Ok(None), + } +} + +fn json_string_field(input: &str, key: &str) -> String { + let pattern = format!("\"{key}\":\""); + input + .find(&pattern) + .and_then(|start| input[start + pattern.len()..].split('"').next()) + .unwrap_or_default() + .to_string() +} + +fn parse_rss_observation(body: &str) -> RssObservation { + if body.trim().is_empty() { + return RssObservation::Unavailable; + } + match json_string_field(body, "status").as_str() { + "initializing" => { + let step = body + .find("\"step\"") + .map(|start| json_string_field(&body[start..], "status")) + .filter(|step| !step.is_empty()); + RssObservation::Initializing { step } + } + "initialized" => { + let id = json_string_field(body, "id"); + if id.is_empty() { + RssObservation::StaleInitialized + } else { + RssObservation::Initialized { id } + } + } + "initialization_failed" => RssObservation::Failed { + message: json_string_field(body, "message"), + }, + _ => RssObservation::UnknownResponse, + } +} + +fn route_evidence_from_table( + context: &TuiContext, + table: &str, +) -> RouteEvidence { + let applied = (0..context.config.topology.racks()).all(|rack| { + let prefix = context.config.network.for_rack(rack).infra_prefix; + let destination = prefix.split('/').next().unwrap_or(&prefix); + table + .lines() + .any(|line| line.split_whitespace().next() == Some(destination)) + }); + if applied { RouteEvidence::Applied } else { RouteEvidence::Unknown } +} + +fn classify_pid_probe(result: io::Result<()>) -> NodeEvidence { + match result { + Ok(()) => NodeEvidence::Running, + Err(error) if error.raw_os_error() == Some(libc::EPERM) => { + NodeEvidence::Running + } + Err(error) if error.raw_os_error() == Some(libc::ESRCH) => { + NodeEvidence::Missing + } + Err(_) => NodeEvidence::Unknown, + } +} + +fn probe_pid(raw: &str) -> NodeEvidence { + let Ok(pid) = raw.trim().parse::() else { + return NodeEvidence::Unknown; + }; + if pid <= 0 { + return NodeEvidence::Unknown; + } + + // SAFETY: signal 0 does not deliver a signal; a positive pid limits the + // probe to the one process named by Falcon's pid file. + let result = if unsafe { libc::kill(pid, 0) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + }; + classify_pid_probe(result) +} + +pub(crate) async fn collect( + context: &TuiContext, + executor: &FalconExecutor, + intent: LifecycleIntent, + shutdown: &CancellationToken, +) -> anyhow::Result { + let sleds = context.config.sleds(); + let nodes_and_targets: Vec<_> = sleds + .iter() + .map(|sled| { + (sled.name.clone(), NodeTarget::Sled { name: sled.name.clone() }) + }) + .chain(context.config.topology.routers.iter().map(|name| { + (name.clone(), NodeTarget::Router { name: name.clone() }) + })) + .collect(); + let node_ids = + nodes_and_targets.iter().map(|(id, _)| id.clone()).collect::>(); + let rack_ids: Vec<_> = (0..context.config.topology.racks()).collect(); + let mut pid_states = Vec::with_capacity(node_ids.len()); + let mut any_live = false; + let mut all_pid_files_absent = !node_ids.is_empty(); + for id in &node_ids { + let path = context.workdir.join(".falcon").join(format!("{id}.pid")); + let state = match tokio::fs::read_to_string(path).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + NodeEvidence::Missing + } + Err(_) => { + all_pid_files_absent = false; + NodeEvidence::Unknown + } + Ok(pid) => { + all_pid_files_absent = false; + let state = probe_pid(&pid); + if state == NodeEvidence::Running { + any_live = true; + } + state + } + }; + pid_states.push(state); + } + + let mut nodes = Vec::with_capacity(node_ids.len()); + for ((id, target), mut state) in nodes_and_targets + .iter() + .zip(pid_states) + .map(|(node, state)| ((node.0.clone(), &node.1), state)) + { + if state == NodeEvidence::Running { + state = match executor.execute(target, "true").await { + Ok(_) => NodeEvidence::Running, + Err(_) if intent == LifecycleIntent::Launch => { + NodeEvidence::Booting + } + Err(_) => NodeEvidence::Unknown, + }; + if shutdown.is_cancelled() { + anyhow::bail!("node observation cancelled"); + } + } + nodes.push(NodeObservation { id, state }); + } + if all_pid_files_absent + && deployment_storage_exists(context, shutdown).await? != Some(false) + && !nodes.is_empty() + { + nodes[0].state = NodeEvidence::Unknown; + } + + let mut rack_observations: BTreeMap<_, _> = rack_ids + .iter() + .map(|rack| (*rack, RssObservation::Unavailable)) + .collect(); + let mut seen_racks = HashSet::new(); + let rss_sleds = sleds + .iter() + .filter(|sled| sled.rss && seen_racks.insert(sled.rack)) + .collect::>(); + let mut observed_rss_rack_ids = + rss_sleds.iter().map(|sled| sled.rack).collect::>(); + if any_live { + for sled in rss_sleds { + let command = format!( + "curl -s --max-time 5 http://[{}]:8080/rack-initialize 2>/dev/null", + sled.bootstrap_addr() + ); + let target = NodeTarget::Sled { name: sled.name.clone() }; + let observation = match executor.execute(&target, &command).await { + Ok(body) => parse_rss_observation(&body), + Err(_) => RssObservation::Unavailable, + }; + if shutdown.is_cancelled() { + anyhow::bail!("RSS observation cancelled"); + } + rack_observations.insert(sled.rack, observation); + } + } + observed_rss_rack_ids.sort_unstable(); + let required_rss_rack_ids = + observed_rss_rack_ids.into_iter().take(1).collect(); + + let routes = + if !context.config.topology.routers.iter().any(|router| router == "ce") + { + RouteEvidence::NotRequired + } else { + let mut command = tokio::process::Command::new("netstat"); + command.args(["-rn", "-f", "inet"]); + match bounded_output(command, "route table probe", shutdown).await { + Ok(output) if output.status.success() => { + route_evidence_from_table( + context, + &String::from_utf8_lossy(&output.stdout), + ) + } + Ok(_) => RouteEvidence::Failed, + Err(error) if shutdown.is_cancelled() => return Err(error), + Err(error) if error.to_string().contains("No such file") => { + RouteEvidence::Unavailable + } + Err(_) => RouteEvidence::Unknown, + } + }; + + executor.drain_serial_tasks().await; + Ok(ReconciliationEvidence { + topology: TopologyEvidence { node_ids, rack_ids, available: true }, + nodes, + racks: rack_observations + .into_iter() + .map(|(rack, observation)| RackRssEvidence { rack, observation }) + .collect(), + required_rss_rack_ids, + routes, + intent, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(id: &str, state: NodeEvidence) -> NodeObservation { + NodeObservation { id: id.into(), state } + } + + fn rack(rack: usize, observation: RssObservation) -> RackRssEvidence { + RackRssEvidence { rack, observation } + } + + fn base() -> ReconciliationEvidence { + ReconciliationEvidence { + topology: TopologyEvidence { + node_ids: vec!["n1".into(), "n2".into()], + rack_ids: vec![0, 1], + available: true, + }, + nodes: vec![ + node("n1", NodeEvidence::Running), + node("n2", NodeEvidence::Running), + ], + racks: vec![ + rack(0, RssObservation::Initialized { id: "r0".into() }), + rack(1, RssObservation::Initialized { id: "r1".into() }), + ], + required_rss_rack_ids: vec![0, 1], + routes: RouteEvidence::Applied, + intent: LifecycleIntent::Idle, + } + } + + #[test] + fn successful_pid_probe_means_running() { + assert_eq!(classify_pid_probe(Ok(())), NodeEvidence::Running); + } + + #[test] + fn pid_probe_permission_denied_still_means_running() { + assert_eq!( + classify_pid_probe(Err(std::io::Error::from_raw_os_error( + libc::EPERM + ))), + NodeEvidence::Running + ); + } + + #[test] + fn pid_probe_missing_process_means_missing() { + assert_eq!( + classify_pid_probe(Err(std::io::Error::from_raw_os_error( + libc::ESRCH + ))), + NodeEvidence::Missing + ); + } + + #[test] + fn pid_probe_unexpected_error_is_unknown() { + assert_eq!( + classify_pid_probe(Err(std::io::Error::from_raw_os_error( + libc::EINVAL + ))), + NodeEvidence::Unknown + ); + } + + #[test] + fn reduces_complete_live_evidence_to_running() { + assert_eq!(reduce(&base()).state, ObservedDeploymentState::Running); + } + + #[test] + fn covers_stopped_starting_stopping_degraded_and_unknown() { + let mut evidence = base(); + evidence + .nodes + .iter_mut() + .for_each(|node| node.state = NodeEvidence::Missing); + assert_eq!(reduce(&evidence).state, ObservedDeploymentState::Stopped); + + evidence.nodes[0].state = NodeEvidence::Booting; + evidence.intent = LifecycleIntent::Launch; + assert_eq!(reduce(&evidence).state, ObservedDeploymentState::Starting); + + evidence.nodes[0].state = NodeEvidence::Running; + evidence.intent = LifecycleIntent::Destroy; + assert_eq!(reduce(&evidence).state, ObservedDeploymentState::Stopping); + + evidence.intent = LifecycleIntent::Idle; + evidence.nodes[0].state = NodeEvidence::Failed; + assert_eq!(reduce(&evidence).state, ObservedDeploymentState::Degraded); + + evidence.nodes[0].state = NodeEvidence::Running; + evidence.routes = RouteEvidence::Unknown; + assert_eq!(reduce(&evidence).state, ObservedDeploymentState::Unknown); + } + + #[test] + fn only_required_rss_racks_affect_running_state() { + for observation in [ + RssObservation::Unavailable, + RssObservation::StaleInitialized, + RssObservation::Failed { message: "held rack failed".into() }, + ] { + let mut evidence = base(); + evidence.required_rss_rack_ids = vec![0]; + evidence.racks[1].observation = observation; + let result = reduce(&evidence); + assert_eq!(result.state, ObservedDeploymentState::Running); + assert!( + !result + .summary + .warnings + .iter() + .any(|warning| warning.contains("rack 1 RSS")) + ); + } + } + + #[test] + fn malformed_required_rss_racks_are_unknown() { + for required in [vec![0, 99], vec![0, 0]] { + let mut evidence = base(); + evidence.required_rss_rack_ids = required; + evidence.nodes[0].state = NodeEvidence::Failed; + assert_eq!( + reduce(&evidence).state, + ObservedDeploymentState::Unknown + ); + } + } + + #[test] + fn live_evidence_overrides_successful_launch_outcome() { + let mut stopped_after_successful_launch = base(); + stopped_after_successful_launch + .nodes + .iter_mut() + .for_each(|node| node.state = NodeEvidence::Missing); + assert_eq!( + reduce(&stopped_after_successful_launch).state, + ObservedDeploymentState::Stopped + ); + } + + #[test] + fn live_evidence_overrides_failed_destroy_outcome() { + let mut stopped_after_failed_destroy = base(); + stopped_after_failed_destroy + .nodes + .iter_mut() + .for_each(|node| node.state = NodeEvidence::Missing); + assert_eq!( + reduce(&stopped_after_failed_destroy).state, + ObservedDeploymentState::Stopped + ); + } +} diff --git a/voxel/src/tui/session.rs b/voxel/src/tui/session.rs new file mode 100644 index 0000000..574f9ff --- /dev/null +++ b/voxel/src/tui/session.rs @@ -0,0 +1,438 @@ +use std::{ + collections::BTreeMap, + ffi::OsString, + fs::{self, OpenOptions}, + io::{self, Write}, + path::{Path, PathBuf}, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, anyhow}; +use serde::{Deserialize, Serialize}; + +#[cfg(unix)] +use std::os::fd::AsRawFd; + +use super::TuiContext; + +const VERSION: u32 = 1; +const CLAIM_ENV: &str = "VOXEL_TUI_SESSION_CLAIM"; +const ELEVATED_ENV: &str = "VOXEL_TUI_RESUME_ELEVATED"; +const SYSTEM_DIRECTORY: &str = "/var/run/voxel/tui-sessions"; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct DetachedSession { + version: u32, + detached_at_ns: u128, + executable: PathBuf, + config_path: PathBuf, + workdir: PathBuf, + name: String, + dataset: String, + build_root: PathBuf, + #[serde(default)] + addresses: BTreeMap, +} + +impl DetachedSession { + fn from_context(context: &TuiContext) -> anyhow::Result { + Ok(Self { + version: VERSION, + detached_at_ns: SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock precedes Unix epoch")? + .as_nanos(), + executable: context.executable().to_owned(), + config_path: context.config_path.clone().into_std_path_buf(), + workdir: context.workdir.clone().into_std_path_buf(), + name: context.name.clone(), + dataset: context.dataset.clone(), + build_root: context.build_root.clone().into_std_path_buf(), + addresses: BTreeMap::new(), + }) + } + + pub(crate) fn addresses(&self) -> &BTreeMap { + &self.addresses + } + + pub(crate) fn set_addresses( + &mut self, + addresses: BTreeMap, + ) { + self.addresses = addresses; + } + + fn arguments(&self) -> Vec { + [ + OsString::from("--config"), + self.config_path.as_os_str().to_owned(), + OsString::from("--workdir"), + self.workdir.as_os_str().to_owned(), + OsString::from("--name"), + OsString::from(&self.name), + OsString::from("--dataset"), + OsString::from(&self.dataset), + OsString::from("--build-root"), + self.build_root.as_os_str().to_owned(), + OsString::from("tui"), + ] + .into_iter() + .collect() + } + + pub(crate) fn reattach_command(&self) -> anyhow::Result { + let mut words = vec!["pfexec".to_owned(), quote(&self.executable)?]; + for argument in self.arguments() { + words.push(crate::util::shell_quote( + argument.to_str().ok_or_else(|| { + anyhow!("recorded TUI argument is not valid UTF-8") + })?, + )); + } + Ok(words.join(" ")) + } +} + +fn quote(path: &Path) -> anyhow::Result { + Ok(crate::util::shell_quote( + path.to_str() + .ok_or_else(|| anyhow!("recorded TUI path is not valid UTF-8"))?, + )) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct Stored { + path: PathBuf, + session: DetachedSession, +} + +struct Store { + directory: PathBuf, +} + +impl Store { + fn system() -> Self { + Self { directory: SYSTEM_DIRECTORY.into() } + } + + #[cfg(test)] + fn at(directory: PathBuf) -> Self { + Self { directory } + } + + fn write(&self, session: &DetachedSession) -> anyhow::Result { + ensure_private_dir(&self.directory)?; + let path = self.directory.join(format!( + "{:032}-{}.json", + session.detached_at_ns, + std::process::id() + )); + let temporary = path.with_extension("json.tmp"); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(&temporary) + .with_context(|| format!("create {}", temporary.display()))?; + file.write_all(&serde_json::to_vec_pretty(session)?) + .with_context(|| format!("write {}", temporary.display()))?; + file.sync_all()?; + fs::rename(&temporary, &path)?; + Ok(path) + } + + fn load(&self) -> anyhow::Result<(Vec, Vec)> { + let entries = match fs::read_dir(&self.directory) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Ok((vec![], vec![])); + } + Err(error) => return Err(error.into()), + }; + let mut records = vec![]; + let mut warnings = vec![]; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") + { + continue; + } + match read_private(&path) { + Ok(session) if session.version == VERSION => { + records.push(Stored { path, session }) + } + Ok(session) => warnings.push(format!( + "skip {}: unsupported session schema {}", + path.display(), + session.version + )), + Err(error) => { + warnings.push(format!("skip {}: {error:#}", path.display())) + } + } + } + records.sort_by(|a, b| { + b.session + .detached_at_ns + .cmp(&a.session.detached_at_ns) + .then_with(|| b.path.cmp(&a.path)) + }); + Ok((records, warnings)) + } +} + +fn ensure_private_dir(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent() + && path == Path::new(SYSTEM_DIRECTORY) + { + let mut parent_builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + parent_builder.mode(0o700); + } + match parent_builder.create(parent) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + } + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = fs::symlink_metadata(path)?; + if !metadata.is_dir() + || metadata.file_type().is_symlink() + || metadata.mode() & 0o077 != 0 + || metadata.uid() != unsafe { libc::geteuid() } + { + return Err(anyhow!("TUI session directory is not private")); + } + } + Ok(()) +} + +fn read_private(path: &Path) -> anyhow::Result { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = fs::symlink_metadata(path)?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.mode() & 0o077 != 0 + || metadata.nlink() != 1 + || metadata.uid() != unsafe { libc::geteuid() } + { + return Err(anyhow!("session record is not private")); + } + } + Ok(serde_json::from_slice(&fs::read(path)?)?) +} + +pub(crate) fn prepare_direct( + context: &TuiContext, +) -> anyhow::Result<(DetachedSession, String)> { + let session = DetachedSession::from_context(context)?; + let command = session.reattach_command()?; + Ok((session, command)) +} + +pub(crate) fn detach(session: &DetachedSession) -> anyhow::Result { + Store::system().write(session) +} + +pub(crate) struct SessionClaim { + path: PathBuf, + _file: fs::File, + session: DetachedSession, +} + +impl SessionClaim { + pub(crate) fn addresses(&self) -> &BTreeMap { + self.session.addresses() + } + + pub(crate) fn finish(self) -> anyhow::Result<()> { + fs::remove_file(&self.path).with_context(|| { + format!("remove resumed session {}", self.path.display()) + }) + } +} + +pub(crate) fn claim_from_environment() -> anyhow::Result> { + let Some(path) = std::env::var_os(CLAIM_ENV).map(PathBuf::from) else { + return Ok(None); + }; + if path.parent() != Some(Path::new(SYSTEM_DIRECTORY)) { + return Err(anyhow!("invalid detached TUI session claim path")); + } + let session = read_private(&path)?; + let file = + OpenOptions::new().read(true).write(true).open(&path).with_context( + || format!("open session claim {}", path.display()), + )?; + #[cfg(unix)] + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } + != 0 + { + return Err(io::Error::last_os_error()) + .context("detached TUI session is already being resumed"); + } + Ok(Some(SessionClaim { path, _file: file, session })) +} + +pub(crate) fn resume(choose: bool) -> anyhow::Result<()> { + elevate(choose)?; + let (records, warnings) = Store::system().load()?; + for warning in warnings { + eprintln!("warning: {warning}"); + } + if records.is_empty() { + return Err(anyhow!("no detached Voxel TUI session found")); + } + let Some(record) = select(&records, choose)? else { + return Ok(()); + }; + let status = Command::new(&record.session.executable) + .args(record.session.arguments()) + .env(CLAIM_ENV, &record.path) + .env_remove(ELEVATED_ENV) + .status() + .context("spawn detached TUI")?; + if !status.success() { + return Err(anyhow!("resumed TUI exited with status {status}")); + } + Ok(()) +} + +fn select(records: &[Stored], choose: bool) -> anyhow::Result> { + if records.is_empty() { + return Ok(None); + } + if !choose { + return Ok(records.first()); + } + for (index, record) in records.iter().enumerate() { + eprintln!( + " {}. {} {}", + index + 1, + record.session.name, + record.session.workdir.display() + ); + } + eprint!("Select session [1-{}, blank to cancel]: ", records.len()); + io::stderr().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if input.trim().is_empty() { + return Ok(None); + } + let selected: usize = + input.trim().parse().context("session selection must be a number")?; + records + .get(selected.saturating_sub(1)) + .map(Some) + .ok_or_else(|| anyhow!("session selection {selected} is out of range")) +} + +#[cfg(unix)] +fn elevate(choose: bool) -> anyhow::Result<()> { + if unsafe { libc::geteuid() } == 0 { + return Ok(()); + } + if std::env::var_os(ELEVATED_ENV).is_some() { + return Err(anyhow!("pfexec did not run Voxel with effective UID 0")); + } + use std::os::unix::process::CommandExt; + let mut command = Command::new("/usr/bin/pfexec"); + command + .arg(std::env::current_exe()?) + .args(["tui", "resume"]) + .env(ELEVATED_ENV, "1"); + if choose { + command.arg("--choose"); + } + Err(command.exec()).context("elevate detached TUI session resume") +} + +#[cfg(not(unix))] +fn elevate(_: bool) -> anyhow::Result<()> { + Err(anyhow!("resuming detached TUI sessions requires Unix")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn record(at: u128, name: &str) -> DetachedSession { + DetachedSession { + version: VERSION, + detached_at_ns: at, + executable: "/opt/voxel bin/voxel".into(), + config_path: "/cfg/v.toml".into(), + workdir: "/work".into(), + name: name.into(), + dataset: "pool/falcon".into(), + build_root: "/build".into(), + addresses: BTreeMap::new(), + } + } + fn directory() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "voxel-session-{}-{}", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() + )); + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + path + } + + #[test] + fn round_trip_and_newest_selection_preserve_exact_values() { + let directory = directory(); + let store = Store::at(directory.clone()); + store.write(&record(1, "old")).unwrap(); + let mut expected = record(2, "new"); + expected.set_addresses( + [("g0".to_string(), "192.168.1.10".to_string())] + .into_iter() + .collect(), + ); + store.write(&expected).unwrap(); + let (records, warnings) = store.load().unwrap(); + assert!(warnings.is_empty()); + let selected = select(&records, false).unwrap().unwrap(); + assert_eq!(selected.session, expected); + assert_eq!( + selected.session.addresses().get("g0").map(String::as_str), + Some("192.168.1.10") + ); + assert!(!expected.arguments().iter().any(|arg| arg == "--rss-gen")); + assert!( + expected + .reattach_command() + .unwrap() + .starts_with("pfexec '/opt/voxel bin/voxel'") + ); + fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/voxel/src/tui/telemetry.rs b/voxel/src/tui/telemetry.rs new file mode 100644 index 0000000..7ba9b95 --- /dev/null +++ b/voxel/src/tui/telemetry.rs @@ -0,0 +1,1556 @@ +//! Telemetry domain types and parsers. + +use std::collections::{BTreeMap, VecDeque}; +use std::ops::AddAssign; +use std::time::{Duration, Instant}; + +use voxel_config::VoxelConfig; + +const HISTORY_WINDOW: Duration = Duration::from_secs(60); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RackId(pub usize); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ResourceKind { + Sled, + SwitchZone, + Router, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ResourceScope { + Fleet, + Rack(RackId), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ResourceId { + pub scope: ResourceScope, + pub kind: ResourceKind, + pub name: String, +} + +impl ResourceId { + pub fn rack( + rack: RackId, + kind: ResourceKind, + name: impl Into, + ) -> Self { + Self { scope: ResourceScope::Rack(rack), kind, name: name.into() } + } + + pub fn fleet(kind: ResourceKind, name: impl Into) -> Self { + Self { scope: ResourceScope::Fleet, kind, name: name.into() } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResourceDescriptor { + pub id: ResourceId, + pub rack: Option, + pub kind: ResourceKind, + pub name: String, + /// Hosting sled for a switch zone. + pub host: Option, +} + +pub fn resource_descriptors(config: &VoxelConfig) -> Vec { + let sleds = config.sleds(); + let mut result = Vec::new(); + let mut switch_slots = BTreeMap::::new(); + for sled in sleds { + let rack = RackId(sled.rack); + result.push(ResourceDescriptor { + id: ResourceId::rack(rack, ResourceKind::Sled, &sled.name), + rack: Some(rack), + kind: ResourceKind::Sled, + name: sled.name.clone(), + host: None, + }); + if sled.scrimlet { + let slot = switch_slots.entry(sled.rack).or_default(); + let name = format!("switch{slot}"); + *slot += 1; + result.push(ResourceDescriptor { + // The rack-local slot is presentation/targeting data; the hosting + // scrimlet's configured name remains stable if slots are reordered. + id: ResourceId::rack( + rack, + ResourceKind::SwitchZone, + &sled.name, + ), + rack: Some(rack), + kind: ResourceKind::SwitchZone, + name, + host: Some(sled.name), + }); + } + } + for name in &config.topology.routers { + result.push(ResourceDescriptor { + id: ResourceId::fleet(ResourceKind::Router, name), + rack: None, + kind: ResourceKind::Router, + name: name.clone(), + host: None, + }); + } + result.sort_by(|a, b| a.id.cmp(&b.id)); + result +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct LinkCounters { + pub rx_bytes: u64, + pub tx_bytes: u64, + pub rx_packets: u64, + pub tx_packets: u64, +} + +impl LinkCounters { + #[cfg(test)] + fn with_packets(mut self, rx: u64, tx: u64) -> Self { + self.rx_packets = rx; + self.tx_packets = tx; + self + } + + fn rate_from( + self, + previous: Self, + elapsed: Duration, + ) -> Option { + let seconds = elapsed.as_secs_f64(); + if seconds == 0.0 + || self.rx_bytes < previous.rx_bytes + || self.tx_bytes < previous.tx_bytes + || self.rx_packets < previous.rx_packets + || self.tx_packets < previous.tx_packets + { + return None; + } + Some(BidirectionalRate { + rx_bytes_sec: (self.rx_bytes - previous.rx_bytes) as f64 / seconds, + tx_bytes_sec: (self.tx_bytes - previous.tx_bytes) as f64 / seconds, + rx_packets_sec: (self.rx_packets - previous.rx_packets) as f64 + / seconds, + tx_packets_sec: (self.tx_packets - previous.tx_packets) as f64 + / seconds, + }) + } +} + +#[derive(Debug, Clone)] +pub struct CounterSnapshot { + pub captured_at: Instant, + pub links: BTreeMap, +} + +impl CounterSnapshot { + pub fn new( + captured_at: Instant, + links: BTreeMap, + ) -> Self { + Self { captured_at, links } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct BidirectionalRate { + pub rx_bytes_sec: f64, + pub tx_bytes_sec: f64, + pub rx_packets_sec: f64, + pub tx_packets_sec: f64, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct BidirectionalErrors { + pub rx_sec: f64, + pub tx_sec: f64, +} + +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub enum TrafficSource { + Oximeter, + #[default] + DirectProbe, +} + +impl TrafficSource { + pub fn label(self) -> &'static str { + match self { + Self::Oximeter => "Oximeter", + Self::DirectProbe => "direct probe", + } + } +} + +impl BidirectionalRate { + pub fn total_bytes_sec(self) -> f64 { + self.rx_bytes_sec + self.tx_bytes_sec + } +} + +impl AddAssign for BidirectionalRate { + fn add_assign(&mut self, rhs: Self) { + self.rx_bytes_sec += rhs.rx_bytes_sec; + self.tx_bytes_sec += rhs.tx_bytes_sec; + self.rx_packets_sec += rhs.rx_packets_sec; + self.tx_packets_sec += rhs.tx_packets_sec; + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZoneInfo { + pub name: String, + pub short_name: String, + pub vnics: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ZoneTraffic { + pub name: String, + pub short_name: String, + pub rate: BidirectionalRate, + pub errors: BidirectionalErrors, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TrafficSample { + pub total: BidirectionalRate, + pub links: BTreeMap, + pub zones: Vec, + pub errors: BidirectionalErrors, + pub source: TrafficSource, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ZoneCpu { + pub id: ResourceId, + pub name: String, + pub kind: String, + pub user_percent: f64, + pub system_percent: f64, + pub wait_percent: f64, +} + +impl ZoneCpu { + pub fn total_percent(&self) -> f64 { + self.user_percent + self.system_percent + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZfsHeadroom { + pub id: ResourceId, + pub pool: String, + pub allocated_bytes: u64, + pub total_bytes: u64, +} + +impl ZfsHeadroom { + pub fn available_bytes(&self) -> u64 { + self.total_bytes.saturating_sub(self.allocated_bytes) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OximeterExceptions { + pub failed_collections: u64, + pub dropped_samples: u64, +} + +impl Default for TrafficSample { + fn default() -> Self { + Self { + total: Default::default(), + links: Default::default(), + zones: Default::default(), + errors: Default::default(), + source: TrafficSource::DirectProbe, + } + } +} + +impl From for TrafficSample { + fn from(total: BidirectionalRate) -> Self { + Self { total, ..Self::default() } + } +} + +#[derive(Debug, Clone, Default)] +pub struct ResourceTelemetry { + pub baseline: Option, + pub link_rates: BTreeMap, + pub zones: Vec, +} + +impl ResourceTelemetry { + pub fn update(&mut self, snapshot: CounterSnapshot) { + self.link_rates.clear(); + if let Some(previous) = &self.baseline + && let Some(elapsed) = snapshot + .captured_at + .checked_duration_since(previous.captured_at) + { + for (name, current) in &snapshot.links { + if let Some(rate) = previous + .links + .get(name) + .and_then(|old| current.rate_from(*old, elapsed)) + { + self.link_rates.insert(name.clone(), rate); + } + } + } + // Every update, including resets and non-forward timestamps, is the next baseline. + self.baseline = Some(snapshot); + } + + pub fn total_rate(&self) -> BidirectionalRate { + self.link_rates.values().copied().fold( + BidirectionalRate::default(), + |mut a, r| { + a += r; + a + }, + ) + } + + pub fn top_zones(&self, limit: usize) -> Vec { + let mut owners = BTreeMap::<&str, usize>::new(); + for zone in &self.zones { + let unique: std::collections::BTreeSet<_> = + zone.vnics.iter().map(String::as_str).collect(); + for vnic in unique { + *owners.entry(vnic).or_default() += 1; + } + } + let mut zones: Vec<_> = self + .zones + .iter() + .map(|zone| { + let rate = zone + .vnics + .iter() + .map(String::as_str) + .collect::>() + .into_iter() + .filter(|v| owners.get(v).copied() == Some(1)) + .filter_map(|v| self.link_rates.get(v)) + .copied() + .fold(BidirectionalRate::default(), |mut a, r| { + a += r; + a + }); + ZoneTraffic { + name: zone.name.clone(), + short_name: zone.short_name.clone(), + rate, + errors: Default::default(), + } + }) + .collect(); + zones.sort_by(|a, b| { + b.rate + .total_bytes_sec() + .total_cmp(&a.rate.total_bytes_sec()) + .then_with(|| a.name.cmp(&b.name)) + }); + zones.truncate(limit); + zones + } +} + +#[derive(Debug, Clone, Copy)] +pub struct HistoryPoint { + pub captured_at: Instant, + pub rate: BidirectionalRate, +} + +#[derive(Debug, Clone)] +pub struct TrafficHistory { + points: VecDeque, + max_count: usize, +} + +impl TrafficHistory { + pub fn new(max_count: usize) -> Self { + Self { points: VecDeque::new(), max_count } + } + #[cfg(test)] + pub fn len(&self) -> usize { + self.points.len() + } + pub fn points(&self) -> &VecDeque { + &self.points + } + pub fn push(&mut self, point: HistoryPoint) { + if let Some(last) = self.points.back() { + if point.captured_at < last.captured_at { + return; + } + if point.captured_at == last.captured_at { + self.points.pop_back(); + } + } + let cutoff = point.captured_at.checked_sub(HISTORY_WINDOW); + while self + .points + .front() + .is_some_and(|p| cutoff.is_some_and(|c| p.captured_at < c)) + { + self.points.pop_front(); + } + self.points.push_back(point); + while self.points.len() > self.max_count { + self.points.pop_front(); + } + } +} + +#[derive(Debug, Clone)] +pub struct ResourceState { + pub descriptor: ResourceDescriptor, + pub current_rate: BidirectionalRate, + pub current_sample: TrafficSample, + /// Sample generation that produced `current_rate`. + pub current_at: Option, + pub history: TrafficHistory, +} + +#[derive(Debug, Clone)] +pub struct TelemetryModel { + pub resources: BTreeMap, + pub fleet_rate: BidirectionalRate, + pub rack_rates: BTreeMap, + pub fleet_history: TrafficHistory, + pub rack_histories: BTreeMap, +} + +impl TelemetryModel { + pub fn new( + descriptors: Vec, + history_max: usize, + ) -> Self { + let mut rack_rates = BTreeMap::new(); + let mut rack_histories = BTreeMap::new(); + let resources = descriptors + .into_iter() + .map(|descriptor| { + if let Some(rack) = descriptor.rack { + rack_rates.entry(rack).or_default(); + rack_histories + .entry(rack) + .or_insert_with(|| TrafficHistory::new(history_max)); + } + ( + descriptor.id.clone(), + ResourceState { + descriptor, + current_rate: Default::default(), + current_sample: Default::default(), + current_at: None, + history: TrafficHistory::new(history_max), + }, + ) + }) + .collect(); + Self { + resources, + fleet_rate: Default::default(), + rack_rates, + fleet_history: TrafficHistory::new(history_max), + rack_histories, + } + } + + pub fn set_current_sample( + &mut self, + id: &ResourceId, + now: Instant, + sample: TrafficSample, + ) { + if let Some(resource) = self.resources.get_mut(id) { + resource.current_rate = sample.total; + resource.current_sample = sample; + resource.current_at = Some(now); + resource.history.push(HistoryPoint { + captured_at: now, + rate: resource.current_rate, + }); + } + } + + pub fn set_oximeter_samples( + &mut self, + id: &ResourceId, + generation: Instant, + samples: impl IntoIterator, + ) { + let Some(resource) = self.resources.get_mut(id) else { + return; + }; + for (captured_at, sample) in samples { + resource.current_rate = sample.total; + resource.current_sample = sample; + resource.history.push(HistoryPoint { + captured_at, + rate: resource.current_rate, + }); + } + if resource.current_sample.source == TrafficSource::Oximeter { + resource.current_at = Some(generation); + } + } + + #[cfg(test)] + pub fn set_current_rate( + &mut self, + id: &ResourceId, + now: Instant, + rate: BidirectionalRate, + ) { + self.set_current_sample(id, now, rate.into()); + } + + pub fn rebuild_aggregates(&mut self, now: Instant) { + self.fleet_rate = Default::default(); + for rate in self.rack_rates.values_mut() { + *rate = Default::default(); + } + for resource in self + .resources + .values() + .filter(|resource| resource.current_at == Some(now)) + { + self.fleet_rate += resource.current_rate; + if let Some(rack) = resource.descriptor.rack { + *self.rack_rates.entry(rack).or_default() += + resource.current_rate; + } + } + self.fleet_history + .push(HistoryPoint { captured_at: now, rate: self.fleet_rate }); + for (rack, rate) in &self.rack_rates { + if let Some(history) = self.rack_histories.get_mut(rack) { + history.push(HistoryPoint { captured_at: now, rate: *rate }); + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrafficSeverity { + Normal, + Elevated, + High, +} + +impl TrafficSeverity { + pub fn for_bytes_per_sec(value: f64) -> Self { + if value > 5_000_000.0 { + Self::High + } else if value > 100_000.0 { + Self::Elevated + } else { + Self::Normal + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NodeAddresses { + pub ipv4: Vec, + pub ipv6: Vec, +} + +pub fn parse_kstat_link_counters( + output: &str, +) -> BTreeMap { + #[derive(Default)] + struct Fields { + rx_bytes: Option, + tx_bytes: Option, + rx_packets: Option, + tx_packets: Option, + } + + let mut fields = BTreeMap::::new(); + for line in output.lines() { + let Some((key, raw)) = line.split_once('\t') else { + continue; + }; + let parts: Vec<_> = key.split(':').collect(); + if parts.len() != 4 { + continue; + } + let Ok(value) = raw.trim().parse::() else { + continue; + }; + let field = match parts[3] { + "rbytes64" => 0, + "obytes64" => 1, + "ipackets64" => 2, + "opackets64" => 3, + _ => continue, + }; + let entry = fields.entry(parts[2].to_owned()).or_default(); + match field { + 0 => entry.rx_bytes = Some(value), + 1 => entry.tx_bytes = Some(value), + 2 => entry.rx_packets = Some(value), + _ => entry.tx_packets = Some(value), + } + } + fields + .into_iter() + .filter_map(|(name, f)| { + Some(( + name, + LinkCounters { + rx_bytes: f.rx_bytes?, + tx_bytes: f.tx_bytes?, + rx_packets: f.rx_packets?, + tx_packets: f.tx_packets?, + }, + )) + }) + .collect() +} + +fn short_zone_name(name: &str) -> String { + if name == "global" { + return name.into(); + } + let stripped = name.strip_prefix("oxz_").unwrap_or(name); + if let Some((prefix, suffix)) = stripped.rsplit_once('_') + && suffix.len() > 8 + && suffix.chars().take(8).all(|c| c.is_ascii_hexdigit()) + { + return prefix.into(); + } + stripped.into() +} + +pub fn parse_dladm_zone_vnics(output: &str) -> Vec { + let mut mapped = + BTreeMap::>::new(); + let mut owners = + BTreeMap::>::new(); + for line in output.lines() { + if line.matches(':').count() != 1 || line.contains('\t') { + continue; + } + let Some((link, zone)) = line.split_once(':') else { + continue; + }; + let (link, zone) = (link.trim(), zone.trim()); + if link.is_empty() || !(zone == "global" || zone.starts_with("oxz_")) { + continue; + } + mapped.entry(zone.into()).or_default().insert(link.into()); + owners.entry(link.into()).or_default().insert(zone.into()); + } + mapped + .into_iter() + .filter_map(|(name, vnics)| { + let vnics = vnics + .into_iter() + .filter(|vnic| { + owners.get(vnic).is_some_and(|zones| zones.len() == 1) + }) + .collect::>(); + (!vnics.is_empty()).then(|| ZoneInfo { + short_name: short_zone_name(&name), + name, + vnics, + }) + }) + .collect() +} + +pub fn parse_linux_link_counters( + output: &str, +) -> BTreeMap { + let mut result = BTreeMap::new(); + for line in output.lines() { + let p: Vec<_> = line.split_whitespace().collect(); + if p.len() < 5 { + continue; + } + let Ok(values) = p[1..5] + .iter() + .map(|v| v.parse::()) + .collect::, _>>() + else { + continue; + }; + result.insert( + p[0].into(), + LinkCounters { + rx_bytes: values[0], + tx_bytes: values[1], + rx_packets: values[2], + tx_packets: values[3], + }, + ); + } + result +} + +pub fn parse_ipadm_addresses(output: &str) -> NodeAddresses { + let mut result = NodeAddresses::default(); + for line in output.lines() { + let Some((object, raw)) = line.split_once(':') else { + continue; + }; + if object.starts_with("lo0") || object.ends_with("/ll") { + continue; + } + let address = raw.replace("\\:", ":"); + let target = if address.contains('.') { + if object.starts_with("vioif") && !result.ipv4.contains(&address) { + result.ipv4.insert(0, address); + continue; + } + &mut result.ipv4 + } else if address.contains(':') && !address.starts_with("fe80") { + if (object.contains("sled") || object.contains("underlay")) + && !result.ipv6.contains(&address) + { + result.ipv6.insert(0, address); + continue; + } + &mut result.ipv6 + } else { + continue; + }; + if !target.contains(&address) { + target.push(address); + } + } + result +} + +pub fn parse_linux_ip_addresses(output: &str) -> NodeAddresses { + let mut result = NodeAddresses::default(); + for line in output.lines() { + if line.contains(": lo ") + || line.contains("docker") + || line.contains("veth") + { + continue; + } + let (marker, target, ipv6) = if line.contains(" inet6 ") { + (" inet6 ", &mut result.ipv6, true) + } else if line.contains(" inet ") { + (" inet ", &mut result.ipv4, false) + } else { + continue; + }; + let Some(raw) = line + .split_once(marker) + .and_then(|(_, rest)| rest.split_whitespace().next()) + else { + continue; + }; + let valid_family = raw + .split('/') + .next() + .and_then(|address| address.parse::().ok()) + .is_some_and(|address| address.is_ipv6() == ipv6); + if !valid_family { + continue; + } + if raw.starts_with("127.") + || raw.starts_with("fe80") + || raw.starts_with("::1") + { + continue; + } + let address = raw.to_owned(); + if !target.contains(&address) { + target.push(address); + } + } + result +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServiceState { + Online, + Offline, + Disabled, + Maintenance, + Degraded, +} + +pub fn parse_service_state(output: &str) -> Option { + match output.trim() { + "online" => Some(ServiceState::Online), + "offline" => Some(ServiceState::Offline), + "disabled" => Some(ServiceState::Disabled), + "maintenance" => Some(ServiceState::Maintenance), + "degraded" => Some(ServiceState::Degraded), + _ => None, + } +} + +pub fn parse_failed_services(output: &str) -> Vec { + let mut services: Vec<_> = output + .lines() + .filter_map(|line| line.trim().strip_prefix("svc:/")) + .filter_map(|service| { + service.rsplit_once(':').map(|(name, _)| name.to_owned()) + }) + .collect(); + services.sort(); + services.dedup(); + services +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ZoneDiagnostic { + pub zones: Vec, +} + +pub fn parse_zone_diagnostics(output: &str) -> ZoneDiagnostic { + let mut zones: Vec<_> = output + .lines() + .map(str::trim) + .filter(|s| !s.is_empty() && !s.chars().any(char::is_whitespace)) + .map(str::to_owned) + .collect(); + zones.sort(); + zones.dedup(); + ZoneDiagnostic { zones } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NtpDiagnostic { + pub stratum: Option, + pub synchronized: Option, +} + +pub fn parse_chrony_tracking(output: &str) -> NtpDiagnostic { + let mut result = NtpDiagnostic::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + match key.trim().to_ascii_lowercase().as_str() { + "stratum" => result.stratum = value.trim().parse().ok(), + "leap status" => { + let value = value.trim().to_ascii_lowercase(); + result.synchronized = Some( + !(value.contains("not synchron") + || value.contains("unsynchron")), + ); + } + _ => {} + } + } + if result.synchronized.is_none() { + result.synchronized = result.stratum.map(|s| s > 0 && s < 16); + } + result +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealthState { + Checking, + Healthy, + Degraded, + Failed, + Unknown, + Stale, + Unavailable, + Stopped, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HealthDiagnostic { + pub sled_agent: Option, + pub failed_services: Vec, + pub ntp: NtpDiagnostic, + pub zones: ZoneDiagnostic, + pub notes: Vec, +} + +impl HealthDiagnostic { + pub fn state(&self) -> HealthState { + if matches!( + self.sled_agent, + Some(ServiceState::Offline | ServiceState::Maintenance) + ) { + return HealthState::Failed; + } + if !self.failed_services.is_empty() { + return HealthState::Degraded; + } + if self.sled_agent.is_none() && self.ntp.synchronized.is_none() { + return HealthState::Unknown; + } + if self.sled_agent != Some(ServiceState::Online) + || self.ntp.synchronized != Some(true) + { + HealthState::Degraded + } else { + HealthState::Healthy + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Freshness { + Fresh, + Stale, + Unavailable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealthContext { + Checking, + Active, + Stopped, +} + +pub fn derive_health_state( + context: HealthContext, + diagnostic: Option<&HealthDiagnostic>, + freshness: Freshness, +) -> HealthState { + match context { + HealthContext::Checking => HealthState::Checking, + HealthContext::Stopped => HealthState::Stopped, + HealthContext::Active => match freshness { + Freshness::Stale => HealthState::Stale, + Freshness::Unavailable => HealthState::Unavailable, + Freshness::Fresh => diagnostic + .map(HealthDiagnostic::state) + .unwrap_or(HealthState::Unknown), + }, + } +} + +#[derive(Debug, Clone)] +pub struct TimedValue { + pub captured_at: Instant, + pub value: T, +} +#[derive(Debug, Clone)] +pub struct CollectionError { + pub attempted_at: Instant, + pub message: String, +} + +#[derive(Debug, Clone)] +pub struct LatestSample { + pub good: Option>, + pub last_attempt: Option, + pub latest_error: Option, +} + +impl Default for LatestSample { + fn default() -> Self { + Self { good: None, last_attempt: None, latest_error: None } + } +} + +impl LatestSample { + pub fn record_success(&mut self, captured_at: Instant, value: T) { + self.last_attempt = Some(captured_at); + self.good = Some(TimedValue { captured_at, value }); + self.latest_error = None; + } + pub fn record_error( + &mut self, + attempted_at: Instant, + message: impl Into, + ) { + self.last_attempt = Some(attempted_at); + self.latest_error = + Some(CollectionError { attempted_at, message: message.into() }); + } + /// Boundary semantics: age equal to `stale_after` remains fresh and age equal + /// to `unavailable_after` remains stale. Threshold order errors degrade safely. + pub fn freshness( + &self, + now: Instant, + stale_after: Duration, + unavailable_after: Duration, + ) -> Freshness { + let Some(good) = &self.good else { + return Freshness::Unavailable; + }; + let age = + now.checked_duration_since(good.captured_at).unwrap_or_default(); + if age <= stale_after { + Freshness::Fresh + } else if age <= unavailable_after.max(stale_after) { + Freshness::Stale + } else { + Freshness::Unavailable + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use std::time::{Duration, Instant}; + use voxel_config::VoxelConfig; + + fn counters(rx: u64, tx: u64) -> LinkCounters { + LinkCounters { + rx_bytes: rx, + tx_bytes: tx, + rx_packets: rx / 10, + tx_packets: tx / 10, + } + } + + #[test] + fn discovers_stable_multi_rack_resources() { + let cfg = VoxelConfig::from_toml( + "[topology]\nracks = 2\nsleds = 3\nrouters = [\"ce\", \"cr1\"]\n", + ) + .unwrap(); + let resources = resource_descriptors(&cfg); + assert_eq!( + resources.iter().filter(|r| r.kind == ResourceKind::Sled).count(), + 6 + ); + let switches: Vec<_> = resources + .iter() + .filter(|r| r.kind == ResourceKind::SwitchZone) + .collect(); + assert_eq!( + switches + .iter() + .map(|r| (r.rack, r.name.as_str(), r.host.as_deref())) + .collect::>(), + vec![ + (Some(RackId(0)), "switch0", Some("g0")), + (Some(RackId(0)), "switch1", Some("g2")), + (Some(RackId(1)), "switch0", Some("g3")), + (Some(RackId(1)), "switch1", Some("g5")), + ] + ); + assert!( + resources + .iter() + .filter(|r| r.kind == ResourceKind::Router) + .all(|r| r.rack.is_none()) + ); + assert_eq!( + resources + .iter() + .map(|r| &r.id) + .collect::>() + .len(), + resources.len() + ); + + let reordered = VoxelConfig::from_toml( + "[topology]\nracks = 1\nsleds = 3\nscrimlets = [\"g2\", \"g0\"]\nrouters = []\n", + ) + .unwrap(); + let switches: BTreeMap<_, _> = resource_descriptors(&reordered) + .into_iter() + .filter(|r| r.kind == ResourceKind::SwitchZone) + .map(|r| (r.host.clone().unwrap(), (r.id, r.name))) + .collect(); + assert_eq!(switches["g2"].0.name, "g2"); + assert!(switches["g2"].1.starts_with("switch")); + assert_eq!(switches["g0"].0.name, "g0"); + assert!(switches["g0"].1.starts_with("switch")); + } + + #[test] + fn parses_production_counter_and_zone_fixtures_resiliently() { + let kstat = "0:link:vioif0:rbytes64\t1000\n0:link:vioif0:obytes64\t2000\n0:link:vioif0:ipackets64\t50\n0:link:vioif0:opackets64\t60\n0:link:bad:rbytes64\tnot-a-number\nbroken\n"; + assert_eq!( + parse_kstat_link_counters(kstat)["vioif0"], + counters(1000, 2000).with_packets(50, 60) + ); + assert!(!parse_kstat_link_counters(kstat).contains_key("bad")); + + let zones = parse_dladm_zone_vnics( + "vnic2:oxz_ntp_aabbccddeeff\nvnic2:oxz_ntp_aabbccddeeff\nvnic1:oxz_nexus_0123456789\nvnic0:global\nshared:oxz_one_deadbeef0\nshared:oxz_two_deadbeef0\n:global\nbad:zone:line\n", + ); + assert_eq!( + zones[0], + ZoneInfo { + name: "global".into(), + short_name: "global".into(), + vnics: vec!["vnic0".into()] + } + ); + assert_eq!(zones[1].short_name, "nexus"); + assert_eq!(zones[2].short_name, "ntp"); + assert!( + zones.iter().all(|zone| !zone.vnics.iter().any(|v| v == "shared")) + ); + assert_eq!(zones[2].vnics, vec!["vnic2"]); + + let linux = parse_linux_link_counters( + "eth1 10 20 1 2 ignored extra\neth0 30 40 3 4\nbad 1 nope 3 4 5\nshort 1\n", + ); + assert_eq!( + linux.keys().cloned().collect::>(), + vec!["eth0", "eth1"] + ); + } + + #[test] + fn parses_addresses_and_health_diagnostics() { + let illumos = parse_ipadm_addresses( + "net0/v4:10.0.0.1/24\nvioif3/v4:192.168.68.122/22\nnet0/v6:2001\\:db8\\:\\:2/64\nunderlay0/sled6:fd00\\:1122\\:3344\\:101\\:\\:1/64\nvioif1/ll:fe80\\:\\:1/10\nmalformed\n", + ); + assert_eq!(illumos.ipv4, vec!["192.168.68.122/22", "10.0.0.1/24"]); + assert_eq!( + illumos.ipv6, + vec!["fd00:1122:3344:101::1/64", "2001:db8::2/64"] + ); + let linux = parse_linux_ip_addresses( + "1: lo inet 127.0.0.1/8 scope host lo\n5: enp0s11 inet 192.168.68.108/22 scope global enp0s11\n5: enp0s11 inet6 2001:db8::1/64 scope global\n6: docker0 inet 172.17.0.1/16 scope global docker0\ntruncated inet \n7: enp0s12 inet not-an-address scope global\nmalformed\n", + ); + assert_eq!(linux.ipv4, vec!["192.168.68.108/22"]); + assert_eq!(linux.ipv6, vec!["2001:db8::1/64"]); + + assert_eq!( + parse_zone_diagnostics( + "global\noxz_switch\noxz_switch\noxz_ntp_deadbeef0\nbad zone\n" + ) + .zones, + vec!["global", "oxz_ntp_deadbeef0", "oxz_switch"] + ); + assert_eq!( + parse_service_state(" online\n"), + Some(ServiceState::Online) + ); + assert_eq!(parse_service_state("garbage"), None); + assert_eq!( + parse_failed_services( + "svc:/oxide/sled-agent:default\nnoise\nsvc:/network/dns/client:default\n" + ), + vec!["network/dns/client", "oxide/sled-agent"] + ); + let ntp = parse_chrony_tracking( + "Reference ID : A29FC87B\nStratum : 4\nLeap status : Normal\n", + ); + assert_eq!( + ntp, + NtpDiagnostic { stratum: Some(4), synchronized: Some(true) } + ); + assert_eq!( + parse_chrony_tracking( + "Stratum : 0\nLeap status : Not synchronised" + ), + NtpDiagnostic { stratum: Some(0), synchronized: Some(false) } + ); + assert_eq!( + parse_chrony_tracking( + "Stratum : nope\nmalformed\nLeap status : Unsynchronised" + ), + NtpDiagnostic { stratum: None, synchronized: Some(false) } + ); + } + + #[test] + fn rates_use_elapsed_time_reset_baseline_and_recover() { + let start = Instant::now(); + let mut state = ResourceTelemetry::default(); + state.update(CounterSnapshot::new( + start, + BTreeMap::from([("net0".into(), counters(100, 200))]), + )); + state.update(CounterSnapshot::new( + start + Duration::from_secs(4), + BTreeMap::from([("net0".into(), counters(500, 1000))]), + )); + assert_eq!(state.link_rates["net0"].rx_bytes_sec, 100.0); + state.update(CounterSnapshot::new( + start + Duration::from_secs(6), + BTreeMap::from([("net0".into(), counters(10, 20))]), + )); + assert!(!state.link_rates.contains_key("net0")); + state.update(CounterSnapshot::new( + start + Duration::from_secs(8), + BTreeMap::from([("net0".into(), counters(110, 220))]), + )); + assert_eq!(state.link_rates["net0"].rx_bytes_sec, 50.0); + state.update(CounterSnapshot::new( + start + Duration::from_secs(8), + BTreeMap::from([("net0".into(), counters(210, 420))]), + )); + assert!(state.link_rates.is_empty(), "zero elapsed has no rates"); + state.update(CounterSnapshot::new( + start + Duration::from_secs(10), + BTreeMap::from([("net0".into(), counters(310, 620))]), + )); + assert_eq!( + state.link_rates["net0"].rx_bytes_sec, 50.0, + "zero elapsed still advances baseline" + ); + } + + #[test] + fn traffic_sample_preserves_total_links_and_zone_attribution() { + let sample = TrafficSample { + total: BidirectionalRate { + rx_bytes_sec: 7.0, + tx_bytes_sec: 3.0, + ..Default::default() + }, + links: BTreeMap::from([( + "net0".into(), + BidirectionalRate::default(), + )]), + zones: vec![ZoneTraffic { + name: "oxz_nexus_deadbeef0".into(), + short_name: "nexus".into(), + rate: BidirectionalRate::default(), + errors: Default::default(), + }], + ..Default::default() + }; + assert_eq!(sample.total.total_bytes_sec(), 10.0); + assert!(sample.links.contains_key("net0")); + assert_eq!(sample.zones[0].short_name, "nexus"); + } + + #[test] + fn history_is_trailing_sixty_seconds_and_count_bounded() { + let start = Instant::now(); + let mut history = TrafficHistory::new(3); + for (secs, total) in [(0, 1.0), (30, 2.0), (59, 3.0), (61, 4.0)] { + history.push(HistoryPoint { + captured_at: start + Duration::from_secs(secs), + rate: BidirectionalRate { + rx_bytes_sec: total, + ..Default::default() + }, + }); + } + assert_eq!( + history + .points() + .iter() + .map(|p| p.rate.rx_bytes_sec) + .collect::>(), + vec![2.0, 3.0, 4.0] + ); + history.push(HistoryPoint { + captured_at: start + Duration::from_secs(62), + rate: Default::default(), + }); + assert_eq!(history.len(), 3); + + let mut ordered = TrafficHistory::new(8); + for secs in [0, 60] { + ordered.push(HistoryPoint { + captured_at: start + Duration::from_secs(secs), + rate: BidirectionalRate { + rx_bytes_sec: secs as f64, + ..Default::default() + }, + }); + } + assert_eq!(ordered.len(), 2, "exact sixty-second boundary is retained"); + ordered.push(HistoryPoint { + captured_at: start + Duration::from_secs(60), + rate: BidirectionalRate { + rx_bytes_sec: 61.0, + ..Default::default() + }, + }); + ordered.push(HistoryPoint { + captured_at: start + Duration::from_secs(30), + rate: BidirectionalRate { + rx_bytes_sec: 30.0, + ..Default::default() + }, + }); + assert_eq!(ordered.len(), 2); + assert_eq!(ordered.points().back().unwrap().rate.rx_bytes_sec, 61.0); + assert!(ordered.points().iter().map(|p| p.captured_at).is_sorted()); + } + + #[test] + fn aggregates_each_scoped_resource_once_and_ranks_zones() { + let cfg = VoxelConfig::from_toml( + "[topology]\nracks = 2\nsleds = 3\nrouters = []\n", + ) + .unwrap(); + let mut model = TelemetryModel::new(resource_descriptors(&cfg), 64); + let now = Instant::now(); + let sled0 = ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"); + let switch0 = + ResourceId::rack(RackId(0), ResourceKind::SwitchZone, "g0"); + model.set_current_rate( + &sled0, + now, + BidirectionalRate { + rx_bytes_sec: 10.0, + tx_bytes_sec: 20.0, + ..Default::default() + }, + ); + model.set_current_rate( + &switch0, + now, + BidirectionalRate { + rx_bytes_sec: 1.0, + tx_bytes_sec: 2.0, + ..Default::default() + }, + ); + model.rebuild_aggregates(now); + assert_eq!(model.fleet_rate.total_bytes_sec(), 33.0); + assert_eq!(model.rack_rates[&RackId(0)].total_bytes_sec(), 33.0); + assert_eq!(model.rack_rates[&RackId(1)].total_bytes_sec(), 0.0); + assert_eq!(model.fleet_history.len(), 1); + model.rebuild_aggregates(now); + assert_eq!( + model.fleet_history.len(), + 1, + "same generation replaces history" + ); + let next = now + Duration::from_secs(1); + model.set_current_rate( + &sled0, + next, + BidirectionalRate { rx_bytes_sec: 4.0, ..Default::default() }, + ); + model.rebuild_aggregates(next); + assert_eq!( + model.fleet_rate.total_bytes_sec(), + 4.0, + "stale switch excluded" + ); + + let state = ResourceTelemetry { + link_rates: BTreeMap::from([ + ( + "a".into(), + BidirectionalRate { + rx_bytes_sec: 3.0, + ..Default::default() + }, + ), + ( + "b".into(), + BidirectionalRate { + rx_bytes_sec: 8.0, + ..Default::default() + }, + ), + ]), + zones: vec![ + ZoneInfo { + name: "zone-z".into(), + short_name: "z".into(), + vnics: vec!["a".into()], + }, + ZoneInfo { + name: "zone-a".into(), + short_name: "a".into(), + vnics: vec!["b".into(), "b".into(), "a".into()], + }, + ], + ..Default::default() + }; + assert_eq!( + state + .top_zones(2) + .iter() + .map(|z| z.name.as_str()) + .collect::>(), + vec!["zone-a", "zone-z"] + ); + assert_eq!(state.top_zones(2)[0].rate.total_bytes_sec(), 8.0); + assert_eq!(state.top_zones(2)[1].rate.total_bytes_sec(), 0.0); + } + + #[test] + fn severity_boundaries_are_strict() { + assert_eq!( + TrafficSeverity::for_bytes_per_sec(100_000.0), + TrafficSeverity::Normal + ); + assert_eq!( + TrafficSeverity::for_bytes_per_sec(100_000.1), + TrafficSeverity::Elevated + ); + assert_eq!( + TrafficSeverity::for_bytes_per_sec(5_000_000.0), + TrafficSeverity::Elevated + ); + assert_eq!( + TrafficSeverity::for_bytes_per_sec(5_000_000.1), + TrafficSeverity::High + ); + } + + #[test] + fn latest_good_survives_errors_with_deterministic_freshness() { + let start = Instant::now(); + let mut sample = LatestSample::default(); + assert_eq!( + sample.freshness( + start, + Duration::from_secs(10), + Duration::from_secs(30) + ), + Freshness::Unavailable + ); + sample.record_success(start, 7u32); + sample.record_error(start + Duration::from_secs(5), "timeout"); + assert_eq!(sample.good.as_ref().map(|g| g.value), Some(7)); + assert_eq!(sample.latest_error.as_ref().unwrap().message, "timeout"); + assert_eq!( + sample.freshness( + start + Duration::from_secs(10), + Duration::from_secs(10), + Duration::from_secs(30) + ), + Freshness::Fresh + ); + assert_eq!( + sample.freshness( + start + Duration::from_secs(11), + Duration::from_secs(10), + Duration::from_secs(30) + ), + Freshness::Stale + ); + assert_eq!( + sample.freshness( + start + Duration::from_secs(30), + Duration::from_secs(10), + Duration::from_secs(30) + ), + Freshness::Stale + ); + assert_eq!( + sample.freshness( + start + Duration::from_secs(31), + Duration::from_secs(10), + Duration::from_secs(30) + ), + Freshness::Unavailable + ); + sample.record_success(start + Duration::from_secs(32), 8); + assert_eq!(sample.good.as_ref().map(|g| g.value), Some(8)); + assert!( + sample.latest_error.is_none(), + "success clears the active error" + ); + } + + #[test] + fn health_does_not_use_zone_count_as_truth() { + let diagnostic = HealthDiagnostic { + sled_agent: Some(ServiceState::Online), + failed_services: vec![], + ntp: NtpDiagnostic { stratum: Some(4), synchronized: Some(true) }, + zones: ZoneDiagnostic { zones: vec!["global".into()] }, + notes: vec![], + }; + assert_eq!(diagnostic.state(), HealthState::Healthy); + let mut failed = diagnostic.clone(); + failed.failed_services.push("oxide/foo".into()); + assert_eq!(failed.state(), HealthState::Degraded); + + assert_eq!( + derive_health_state( + HealthContext::Checking, + None, + Freshness::Unavailable + ), + HealthState::Checking + ); + assert_eq!( + derive_health_state( + HealthContext::Checking, + Some(&failed), + Freshness::Stale + ), + HealthState::Checking + ); + assert_eq!( + derive_health_state( + HealthContext::Active, + Some(&diagnostic), + Freshness::Fresh + ), + HealthState::Healthy + ); + assert_eq!( + derive_health_state( + HealthContext::Active, + Some(&failed), + Freshness::Fresh + ), + HealthState::Degraded + ); + assert_eq!( + derive_health_state( + HealthContext::Active, + Some(&diagnostic), + Freshness::Stale + ), + HealthState::Stale + ); + assert_eq!( + derive_health_state( + HealthContext::Active, + Some(&diagnostic), + Freshness::Unavailable + ), + HealthState::Unavailable + ); + assert_eq!( + derive_health_state(HealthContext::Active, None, Freshness::Fresh), + HealthState::Unknown + ); + assert_eq!( + derive_health_state( + HealthContext::Stopped, + Some(&failed), + Freshness::Fresh + ), + HealthState::Stopped + ); + assert_eq!( + derive_health_state( + HealthContext::Stopped, + Some(&failed), + Freshness::Stale + ), + HealthState::Stopped + ); + + let mut hard_failure = diagnostic; + hard_failure.sled_agent = Some(ServiceState::Maintenance); + assert_eq!( + derive_health_state( + HealthContext::Active, + Some(&hard_failure), + Freshness::Fresh + ), + HealthState::Failed + ); + + let only_failed_services = HealthDiagnostic { + failed_services: vec!["oxide/foo".into()], + ..Default::default() + }; + assert_eq!(only_failed_services.state(), HealthState::Degraded); + } +} diff --git a/voxel/src/tui/terminal.rs b/voxel/src/tui/terminal.rs new file mode 100644 index 0000000..72832b3 --- /dev/null +++ b/voxel/src/tui/terminal.rs @@ -0,0 +1,587 @@ +use std::{io, thread, time::Duration}; + +use base64::Engine; + +use crossterm::{ + cursor::{Hide, Show}, + event::{self, Event, KeyCode, KeyEvent, KeyEventKind}, + execute, + terminal::{ + EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, + enable_raw_mode, + }, +}; +use ratatui::{Terminal, backend::CrosstermBackend}; +use tokio::{sync::mpsc, task::JoinHandle}; +use tokio_util::sync::CancellationToken; + +use crate::tui::event::{Action, AppEvent, View}; + +pub type TuiTerminal = Terminal>>; + +pub const DEFAULT_EVENT_CAPACITY: usize = 64; +const INPUT_POLL: Duration = Duration::from_millis(50); +const SEND_RETRY: Duration = Duration::from_millis(5); + +pub fn event_channel() -> (mpsc::Sender, mpsc::Receiver) { + event_channel_with_capacity(DEFAULT_EVENT_CAPACITY) +} + +pub fn event_channel_with_capacity( + capacity: usize, +) -> (mpsc::Sender, mpsc::Receiver) { + assert!(capacity > 0, "event channel capacity must be nonzero"); + mpsc::channel(capacity) +} + +pub fn key_action(key: KeyEvent) -> Option { + if !matches!(key.kind, KeyEventKind::Press | KeyEventKind::Repeat) { + return None; + } + Some(match key.code { + KeyCode::Char('1') => Action::SwitchView(View::Deployment), + KeyCode::Char('2') => Action::SwitchView(View::Monitor), + KeyCode::Tab => Action::NextItem, + KeyCode::BackTab => Action::PreviousItem, + KeyCode::Left => Action::PreviousRack, + KeyCode::Right => Action::NextRack, + KeyCode::Enter => Action::Activate, + KeyCode::Char(' ') => Action::ToggleSection, + KeyCode::Char('?') | KeyCode::F(1) => Action::ToggleHelp, + KeyCode::Char('s') => Action::CopyExternalMonitoringSelected, + KeyCode::Char('a') => Action::CopyExternalMonitoringAll, + KeyCode::Char('u') => Action::CopyExternalMonitoringGuide, + KeyCode::Up => Action::Scroll { delta: -1, page: false }, + KeyCode::Down => Action::Scroll { delta: 1, page: false }, + KeyCode::PageUp => Action::Scroll { delta: -1, page: true }, + KeyCode::PageDown => Action::Scroll { delta: 1, page: true }, + KeyCode::Char('f') => Action::CycleLogFilter, + KeyCode::Esc => Action::Close, + KeyCode::Char('y') => Action::CopyReattachCommand, + KeyCode::Char('n') => Action::Reject, + KeyCode::Char('l') => Action::RequestLaunch, + KeyCode::Char('r') => Action::RequestRoute, + KeyCode::Char('d') => Action::RequestDetach, + KeyCode::Char('q') => Action::RequestQuit, + KeyCode::Char('c') => Action::RequestCancelAndLeave, + KeyCode::Char('x') => Action::RequestCancelAndDestroy, + _ => return None, + }) +} + +pub fn osc52(text: &str) -> String { + let encoded = base64::engine::general_purpose::STANDARD.encode(text); + format!("\u{1b}]52;c;{encoded}\u{7}") +} + +pub fn write_osc52(mut writer: impl io::Write, text: &str) -> io::Result<()> { + writer.write_all(osc52(text).as_bytes())?; + writer.flush() +} + +fn handoff( + tx: &mpsc::Sender, + cancel: &CancellationToken, + mut event: AppEvent, +) -> bool { + loop { + if cancel.is_cancelled() { + return false; + } + match tx.try_send(event) { + Ok(()) => return true, + Err(mpsc::error::TrySendError::Closed(_)) => return false, + Err(mpsc::error::TrySendError::Full(returned)) => { + event = returned; + thread::sleep(SEND_RETRY); + } + } + } +} + +trait EventSource { + fn next(&mut self, wait: Duration) -> io::Result>; +} +struct CrosstermEvents; +impl EventSource for CrosstermEvents { + fn next(&mut self, wait: Duration) -> io::Result> { + if event::poll(wait)? { event::read().map(Some) } else { Ok(None) } + } +} +fn input_loop( + mut source: impl EventSource, + tx: mpsc::Sender, + cancel: CancellationToken, +) -> io::Result<()> { + while !cancel.is_cancelled() { + if tx.is_closed() { + break; + } + let message = match source.next(INPUT_POLL)? { + Some(Event::Key(key)) => key_action(key).map(AppEvent::Action), + Some(Event::Resize(width, height)) => { + Some(AppEvent::Resize { width, height }) + } + _ => None, + }; + if message.is_some_and(|message| !handoff(&tx, &cancel, message)) { + break; + } + } + Ok(()) +} +pub fn spawn_input( + tx: mpsc::Sender, + cancel: CancellationToken, +) -> JoinHandle> { + tokio::task::spawn_blocking(move || input_loop(CrosstermEvents, tx, cancel)) +} + +pub fn spawn_ticks( + tx: mpsc::Sender, + cancel: CancellationToken, + period: Duration, +) -> io::Result> { + if period.is_zero() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "tick period must be nonzero", + )); + } + Ok(tokio::spawn(async move { + let mut interval = tokio::time::interval(period); + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = interval.tick() => { + let send = tx.send(AppEvent::Tick { now: std::time::Instant::now() }); + tokio::select! { _ = cancel.cancelled() => break, result = send => if result.is_err() { break } } + } + } + } + })) +} + +trait LifecycleOps { + fn raw(&mut self) -> io::Result<()>; + fn enter(&mut self) -> io::Result<()>; + fn hide(&mut self) -> io::Result<()>; + fn initialize(&mut self) -> io::Result<()>; + fn show(&mut self) -> io::Result<()>; + fn leave(&mut self) -> io::Result<()>; + fn disable(&mut self) -> io::Result<()>; +} +#[derive(Default)] +struct Lifecycle { + raw: bool, + alternate: bool, + hidden: bool, +} +impl Lifecycle { + fn setup(ops: &mut impl LifecycleOps) -> io::Result { + let mut state = Self::default(); + ops.raw()?; + state.raw = true; + if let Err(e) = ops.enter() { + let _ = state.restore(ops); + return Err(e); + } + state.alternate = true; + if let Err(e) = ops.hide() { + let _ = state.restore(ops); + return Err(e); + } + state.hidden = true; + if let Err(e) = ops.initialize() { + let _ = state.restore(ops); + return Err(e); + } + Ok(state) + } + fn restore(&mut self, ops: &mut impl LifecycleOps) -> io::Result<()> { + let mut first = None; + if self.hidden { + match ops.show() { + Ok(()) => self.hidden = false, + Err(e) => first = Some(e), + } + } + if self.alternate { + match ops.leave() { + Ok(()) => self.alternate = false, + Err(e) => { + if first.is_none() { + first = Some(e); + } + } + } + } + if self.raw { + match ops.disable() { + Ok(()) => self.raw = false, + Err(e) => { + if first.is_none() { + first = Some(e); + } + } + } + } + first.map_or(Ok(()), Err) + } +} +struct RealOps { + fallback: io::BufWriter, + backend_writer: Option>, + terminal: Option, +} +impl LifecycleOps for RealOps { + fn raw(&mut self) -> io::Result<()> { + enable_raw_mode() + } + fn enter(&mut self) -> io::Result<()> { + execute!(self.fallback, EnterAlternateScreen) + } + fn hide(&mut self) -> io::Result<()> { + execute!(self.fallback, Hide) + } + fn initialize(&mut self) -> io::Result<()> { + let writer = self.backend_writer.take().expect("unused backend writer"); + let mut t = Terminal::new(CrosstermBackend::new(writer))?; + t.clear()?; + self.terminal = Some(t); + Ok(()) + } + fn show(&mut self) -> io::Result<()> { + if let Some(t) = &mut self.terminal { + execute!(t.backend_mut(), Show) + } else { + execute!(self.fallback, Show) + } + } + fn leave(&mut self) -> io::Result<()> { + if let Some(t) = &mut self.terminal { + execute!(t.backend_mut(), LeaveAlternateScreen) + } else { + execute!(self.fallback, LeaveAlternateScreen) + } + } + fn disable(&mut self) -> io::Result<()> { + disable_raw_mode() + } +} +pub struct TerminalSession { + terminal: Option, + fallback: Option>, + lifecycle: Lifecycle, +} +impl TerminalSession { + pub fn enter(writer: std::fs::File) -> io::Result { + let fallback = io::BufWriter::new(writer.try_clone()?); + let mut ops = RealOps { + fallback, + backend_writer: Some(io::BufWriter::new(writer)), + terminal: None, + }; + let lifecycle = Lifecycle::setup(&mut ops)?; + Ok(Self { + terminal: ops.terminal, + fallback: Some(ops.fallback), + lifecycle, + }) + } + pub fn terminal_mut(&mut self) -> &mut TuiTerminal { + self.terminal.as_mut().expect("active terminal") + } + pub fn restore(&mut self) -> io::Result<()> { + let mut ops = RealOps { + fallback: self.fallback.take().expect("terminal fallback writer"), + backend_writer: None, + terminal: self.terminal.take(), + }; + let result = self.lifecycle.restore(&mut ops); + self.terminal = ops.terminal; + self.fallback = Some(ops.fallback); + result + } +} +impl Drop for TerminalSession { + fn drop(&mut self) { + let _ = self.restore(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crossterm::event::KeyModifiers; + use std::{ + collections::VecDeque, + sync::{Arc, Mutex, mpsc as std_mpsc}, + }; + #[test] + fn maps_all_documented_keys() { + let k = |c| KeyEvent::new(c, KeyModifiers::NONE); + let cases = [ + (KeyCode::Char('1'), Action::SwitchView(View::Deployment)), + (KeyCode::Char('2'), Action::SwitchView(View::Monitor)), + (KeyCode::Tab, Action::NextItem), + (KeyCode::BackTab, Action::PreviousItem), + (KeyCode::Left, Action::PreviousRack), + (KeyCode::Right, Action::NextRack), + (KeyCode::Enter, Action::Activate), + (KeyCode::Char(' '), Action::ToggleSection), + (KeyCode::Char('?'), Action::ToggleHelp), + (KeyCode::F(1), Action::ToggleHelp), + (KeyCode::Char('s'), Action::CopyExternalMonitoringSelected), + (KeyCode::Char('a'), Action::CopyExternalMonitoringAll), + (KeyCode::Char('u'), Action::CopyExternalMonitoringGuide), + (KeyCode::Esc, Action::Close), + (KeyCode::Char('l'), Action::RequestLaunch), + (KeyCode::Char('r'), Action::RequestRoute), + (KeyCode::Char('d'), Action::RequestDetach), + (KeyCode::Char('q'), Action::RequestQuit), + (KeyCode::Char('c'), Action::RequestCancelAndLeave), + (KeyCode::Char('x'), Action::RequestCancelAndDestroy), + (KeyCode::Char('y'), Action::CopyReattachCommand), + (KeyCode::Char('n'), Action::Reject), + ]; + for (c, a) in cases { + assert_eq!(key_action(k(c)), Some(a)); + } + assert_eq!(key_action(k(KeyCode::Char('3'))), None); + assert_eq!(key_action(k(KeyCode::Char('4'))), None); + assert_eq!(key_action(k(KeyCode::Char('o'))), None); + assert_eq!( + key_action(KeyEvent { + kind: KeyEventKind::Release, + ..k(KeyCode::Char('q')) + }), + None + ); + assert_eq!(key_action(k(KeyCode::F(12))), None); + } + + #[test] + fn osc52_encodes_exact_text_without_a_newline() { + assert_eq!( + osc52("voxel tui resume"), + "\u{1b}]52;c;dm94ZWwgdHVpIHJlc3VtZQ==\u{7}" + ); + } + + #[test] + fn maps_log_navigation_and_filter_keys() { + let key = |code| KeyEvent::new(code, KeyModifiers::NONE); + assert_eq!( + key_action(key(KeyCode::Up)), + Some(Action::Scroll { delta: -1, page: false }) + ); + assert_eq!( + key_action(key(KeyCode::Down)), + Some(Action::Scroll { delta: 1, page: false }) + ); + assert_eq!( + key_action(key(KeyCode::PageUp)), + Some(Action::Scroll { delta: -1, page: true }) + ); + assert_eq!( + key_action(key(KeyCode::PageDown)), + Some(Action::Scroll { delta: 1, page: true }) + ); + assert_eq!( + key_action(key(KeyCode::Char('f'))), + Some(Action::CycleLogFilter) + ); + } + #[derive(Default)] + struct Fake { + calls: Vec<&'static str>, + fail: Vec<&'static str>, + } + impl Fake { + fn call(&mut self, n: &'static str) -> io::Result<()> { + self.calls.push(n); + if self.fail.contains(&n) { + Err(io::Error::other(n)) + } else { + Ok(()) + } + } + } + impl LifecycleOps for Fake { + fn raw(&mut self) -> io::Result<()> { + self.call("raw") + } + fn enter(&mut self) -> io::Result<()> { + self.call("enter") + } + fn hide(&mut self) -> io::Result<()> { + self.call("hide") + } + fn initialize(&mut self) -> io::Result<()> { + self.call("init") + } + fn show(&mut self) -> io::Result<()> { + self.call("show") + } + fn leave(&mut self) -> io::Result<()> { + self.call("leave") + } + fn disable(&mut self) -> io::Result<()> { + self.call("disable") + } + } + #[test] + fn setup_rolls_back_each_successful_stage() { + for (failure, want) in [ + ("raw", vec!["raw"]), + ("enter", vec!["raw", "enter", "disable"]), + ("hide", vec!["raw", "enter", "hide", "leave", "disable"]), + ( + "init", + vec![ + "raw", "enter", "hide", "init", "show", "leave", "disable", + ], + ), + ] { + let mut f = Fake { fail: vec![failure], ..Default::default() }; + assert!(Lifecycle::setup(&mut f).is_err()); + assert_eq!(f.calls, want); + } + } + #[test] + fn restore_returns_first_error_and_attempts_every_cleanup_step() { + let mut f = Fake::default(); + let mut s = Lifecycle::setup(&mut f).unwrap(); + f.calls.clear(); + f.fail = vec!["show", "leave", "disable"]; + assert_eq!(s.restore(&mut f).unwrap_err().to_string(), "show"); + assert_eq!(f.calls, vec!["show", "leave", "disable"]); + } + #[test] + fn restore_retries_only_cleanup_steps_that_failed() { + let mut f = Fake::default(); + let mut s = Lifecycle::setup(&mut f).unwrap(); + f.calls.clear(); + f.fail = vec!["show", "disable"]; + assert_eq!(s.restore(&mut f).unwrap_err().to_string(), "show"); + assert_eq!(f.calls, vec!["show", "leave", "disable"]); + + f.calls.clear(); + f.fail.clear(); + assert!(s.restore(&mut f).is_ok()); + assert_eq!(f.calls, vec!["show", "disable"]); + + f.calls.clear(); + assert!(s.restore(&mut f).is_ok()); + assert!(f.calls.is_empty()); + } + #[test] + fn restore_is_idempotent() { + let mut f = Fake::default(); + let mut s = Lifecycle::setup(&mut f).unwrap(); + f.calls.clear(); + assert!(s.restore(&mut f).is_ok()); + assert_eq!(f.calls, vec!["show", "leave", "disable"]); + f.calls.clear(); + assert!(s.restore(&mut f).is_ok()); + assert!(f.calls.is_empty()); + } + struct Source(Arc>>); + impl EventSource for Source { + fn next(&mut self, _: Duration) -> io::Result> { + Ok(self.0.lock().unwrap().pop_front()) + } + } + #[tokio::test] + async fn fake_source_events_arrive_in_source_order() { + let (tx, mut rx) = event_channel_with_capacity(3); + let events = Arc::new(Mutex::new(VecDeque::from([ + Event::Resize(2, 3), + Event::Key(KeyEvent::new(KeyCode::Char('q'), KeyModifiers::NONE)), + ]))); + let cancel = CancellationToken::new(); + let (done_tx, done_rx) = std_mpsc::channel(); + let h = std::thread::spawn(move || { + let result = input_loop(Source(events), tx, cancel); + let _ = done_tx.send(()); + result + }); + assert!(matches!( + tokio::time::timeout(Duration::from_millis(100), rx.recv()) + .await + .expect("first source event timed out"), + Some(AppEvent::Resize { width: 2, height: 3 }) + )); + assert!(matches!( + tokio::time::timeout(Duration::from_millis(100), rx.recv()) + .await + .expect("second source event timed out"), + Some(AppEvent::Action(Action::RequestQuit)) + )); + drop(rx); + done_rx + .recv_timeout(Duration::from_millis(100)) + .expect("input loop did not stop after receiver closed"); + assert!(h.join().unwrap().is_ok()); + } + #[test] + fn input_full_channel_cancellation_joins_promptly() { + let (tx, mut rx) = event_channel_with_capacity(1); + tx.try_send(AppEvent::Resize { width: 1, height: 1 }).unwrap(); + let cancel = CancellationToken::new(); + let c = cancel.clone(); + let events = Arc::new(Mutex::new(VecDeque::from([ + Event::Resize(2, 2), + Event::Resize(3, 3), + ]))); + let (done_tx, done_rx) = std_mpsc::channel(); + let h = std::thread::spawn(move || { + let result = input_loop(Source(events), tx, c); + let _ = done_tx.send(()); + result + }); + std::thread::sleep(Duration::from_millis(15)); + cancel.cancel(); + let started = std::time::Instant::now(); + done_rx + .recv_timeout(Duration::from_millis(100)) + .expect("input loop did not stop after cancellation"); + assert!(h.join().unwrap().is_ok()); + assert!(started.elapsed() < Duration::from_millis(100)); + assert!(matches!(rx.try_recv(), Ok(AppEvent::Resize { width: 1, .. }))); + } + #[test] + fn dropped_receiver_stops_input_loop_promptly() { + let (tx, rx) = event_channel_with_capacity(1); + drop(rx); + let events = + Arc::new(Mutex::new(VecDeque::from([Event::Resize(4, 5)]))); + let started = std::time::Instant::now(); + assert!( + input_loop(Source(events), tx, CancellationToken::new()).is_ok() + ); + assert!(started.elapsed() < Duration::from_millis(100)); + } + #[tokio::test] + async fn tick_with_full_channel_stops_on_cancel_and_zero_is_rejected() { + let (tx, _rx) = event_channel_with_capacity(1); + tx.try_send(AppEvent::Resize { width: 1, height: 1 }).unwrap(); + let cancel = CancellationToken::new(); + let h = + spawn_ticks(tx, cancel.clone(), Duration::from_millis(1)).unwrap(); + tokio::time::sleep(Duration::from_millis(5)).await; + cancel.cancel(); + tokio::time::timeout(Duration::from_millis(100), h) + .await + .unwrap() + .unwrap(); + let (tx, _) = event_channel(); + assert_eq!( + spawn_ticks(tx, CancellationToken::new(), Duration::ZERO) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + } +} diff --git a/voxel/src/tui/ui/colors.rs b/voxel/src/tui/ui/colors.rs new file mode 100644 index 0000000..b5f72b0 --- /dev/null +++ b/voxel/src/tui/ui/colors.rs @@ -0,0 +1,81 @@ +use ratatui::style::Color; +pub const OX_YELLOW: Color = Color::Rgb(0xF5, 0xCF, 0x65); +pub const OX_OFF_WHITE: Color = Color::Rgb(0xE0, 0xE0, 0xE0); +pub const OX_RED: Color = Color::Rgb(255, 145, 173); +pub const OX_GREEN_LIGHT: Color = Color::Rgb(0x48, 0xD5, 0x97); +#[cfg(test)] +pub const OX_GREEN_DARK: Color = Color::Rgb(0x11, 0x27, 0x25); +pub const OX_GREEN_DARKEST: Color = Color::Rgb(0x0B, 0x14, 0x18); +#[cfg(test)] +pub const OX_GRAY: Color = Color::Rgb(0x9C, 0x9F, 0xA0); +#[cfg(test)] +pub const OX_GRAY_DARK: Color = Color::Rgb(0x62, 0x66, 0x68); +#[cfg(test)] +pub const OX_WHITE: Color = Color::Rgb(0xE7, 0xE7, 0xE8); +#[cfg(test)] +pub const OX_PINK: Color = Color::Rgb(0xE6, 0x68, 0x86); +#[cfg(test)] +pub const OX_YELLOW_DIM: Color = Color::Rgb(0xAE, 0x96, 0x4E); +#[cfg(test)] +pub const TUI_BLACK: Color = Color::Rgb(0x1E, 0x1E, 0x22); +pub const TUI_YELLOW: Color = Color::Rgb(0xF1, 0xD7, 0x8F); +pub const TUI_GREEN: Color = Color::Rgb(0x8F, 0xEF, 0xBF); +#[cfg(test)] +pub const TUI_GREEN_DARK: Color = Color::Rgb(0x2E, 0x81, 0x60); +pub const TUI_GREY: Color = Color::Rgb(0x78, 0x78, 0x7A); +pub const TUI_PURPLE: Color = Color::Rgb(0xBE, 0x95, 0xEB); +#[cfg(test)] +pub const TUI_PURPLE_DIM: Color = Color::Rgb(0x6C, 0x55, 0x84); +pub const TUI_GREY_DARK: Color = Color::Rgb(66, 66, 69); + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn palette_values_are_exact() { + assert_eq!( + [ + OX_YELLOW, + OX_OFF_WHITE, + OX_RED, + OX_GREEN_LIGHT, + OX_GREEN_DARK, + OX_GREEN_DARKEST, + OX_GRAY, + OX_GRAY_DARK, + OX_WHITE, + OX_PINK, + OX_YELLOW_DIM, + TUI_BLACK, + TUI_YELLOW, + TUI_GREEN, + TUI_GREEN_DARK, + TUI_GREY, + TUI_PURPLE, + TUI_PURPLE_DIM, + TUI_GREY_DARK + ], + [ + Color::Rgb(0xF5, 0xCF, 0x65), + Color::Rgb(0xE0, 0xE0, 0xE0), + Color::Rgb(255, 145, 173), + Color::Rgb(0x48, 0xD5, 0x97), + Color::Rgb(0x11, 0x27, 0x25), + Color::Rgb(0x0B, 0x14, 0x18), + Color::Rgb(0x9C, 0x9F, 0xA0), + Color::Rgb(0x62, 0x66, 0x68), + Color::Rgb(0xE7, 0xE7, 0xE8), + Color::Rgb(0xE6, 0x68, 0x86), + Color::Rgb(0xAE, 0x96, 0x4E), + Color::Rgb(0x1E, 0x1E, 0x22), + Color::Rgb(0xF1, 0xD7, 0x8F), + Color::Rgb(0x8F, 0xEF, 0xBF), + Color::Rgb(0x2E, 0x81, 0x60), + Color::Rgb(0x78, 0x78, 0x7A), + Color::Rgb(0xBE, 0x95, 0xEB), + Color::Rgb(0x6C, 0x55, 0x84), + Color::Rgb(66, 66, 69) + ] + ); + } +} diff --git a/voxel/src/tui/ui/confirm_dialog.rs b/voxel/src/tui/ui/confirm_dialog.rs new file mode 100644 index 0000000..9a8fee1 --- /dev/null +++ b/voxel/src/tui/ui/confirm_dialog.rs @@ -0,0 +1,341 @@ +use super::{ + colors::{OX_GREEN_DARKEST, OX_OFF_WHITE, OX_YELLOW, TUI_PURPLE}, + widgets::{overlay_area, terminal_width}, +}; +use crate::tui::{App, event::Confirmation}; +use ratatui::{ + layout::{Constraint, Layout, Rect}, + style::{Modifier, Style}, + text::{Line, Text}, + widgets::{ + Block, Borders, Clear, HighlightSpacing, List, ListItem, ListState, + Paragraph, Wrap, + }, +}; + +fn prompt(confirmation: &Confirmation) -> &'static str { + match confirmation { + Confirmation::Launch => "Launch deployment?", + Confirmation::Route => "Apply routes?", + Confirmation::Detach => "Detach from Voxel TUI?", + Confirmation::Quit => "Quit Voxel TUI?", + Confirmation::QuitAndDestroy => { + "Quit TUI and destroy Voxel deployment?" + } + Confirmation::CancelAndLeave => "Cancel operation and leave resources?", + Confirmation::CancelAndDestroy => { + "Cancel operation and DESTROY resources?" + } + Confirmation::ForceStop => "Force stop the direct Voxel child?", + } +} + +#[cfg(test)] +pub(crate) fn prompt_text(confirmation: Confirmation) -> &'static str { + prompt(&confirmation) +} + +fn body(app: &App, confirmation: &Confirmation) -> Text<'static> { + let mut lines = vec![Line::from(prompt(confirmation))]; + if matches!(confirmation, Confirmation::Detach) { + if app.operation.active.is_some() || app.operation.pending.is_some() { + lines.push(Line::from( + "The active operation will be cancelled; resources will be left in place.", + )); + } + lines.push(Line::from("Resume later with:")); + lines.push(Line::from("voxel tui resume")); + if app.clipboard_copied { + lines.push(Line::from("Full fallback command copied.")); + } + } + if matches!( + confirmation, + Confirmation::CancelAndLeave | Confirmation::CancelAndDestroy + ) { + lines.push(Line::from( + "The Voxel command is still running; waiting for the Voxel command to settle without interrupting an opaque Falcon boundary.", + )); + } + if matches!(confirmation, Confirmation::ForceStop) { + lines.push(Line::from( + "Force stop can leave partial deployment state. Only the direct child is terminated; reconciliation still follows.", + )); + } + Text::from(lines) +} + +fn visible_shortcuts( + confirmation: &Confirmation, + available_width: usize, +) -> &'static str { + let wide = if matches!(confirmation, Confirmation::Detach) { + "↑/↓ move · Enter select · y copy fallback · n/Esc back" + } else { + "↑/↓ move · Enter select · n/Esc back" + }; + if terminal_width(wide) <= available_width { + wide + } else if matches!(confirmation, Confirmation::Detach) { + "↑/↓ Enter y copy n/Esc" + } else { + "↑/↓ Enter n/Esc" + } +} + +pub(crate) fn dialog_area(app: &App, overlay: Rect) -> Rect { + let confirmation = + app.session.confirmation.as_ref().expect("open confirmation"); + let options = confirmation.options(app.can_cancel()); + let dialog_body = body(app, confirmation); + let content_width = dialog_body + .lines + .iter() + .map(Line::width) + .chain(options.iter().map(|option| terminal_width(option.label))) + .chain(std::iter::once(terminal_width( + "↑/↓ move · Enter select · y copy fallback · n/Esc back", + ))) + .max() + .unwrap_or_default() as u16; + let side_margin = if overlay.width >= 4 { 4 } else { 0 }; + let width = content_width + .saturating_add(6) + .min(if matches!(confirmation, Confirmation::Detach) { 96 } else { 64 }) + .min(overlay.width.saturating_sub(side_margin).max(1)); + let inner_width = width.saturating_sub(2).max(1); + let body_height = Paragraph::new(dialog_body) + .wrap(Wrap { trim: false }) + .line_count(inner_width) as u16; + let height = body_height + .saturating_add(options.len() as u16) + .saturating_add(3) + .min(overlay.height); + Rect::new( + overlay.x + overlay.width.saturating_sub(width) / 2, + overlay.y + overlay.height.saturating_sub(height) / 2, + width, + height, + ) +} + +pub fn draw(frame: &mut ratatui::Frame<'_>, app: &App) { + let Some(confirmation) = app.session.confirmation.as_ref() else { + return; + }; + let options = confirmation.options(app.can_cancel()); + let dialog_body = body(app, confirmation); + let area = dialog_area(app, overlay_area(frame.area())); + frame.render_widget(Clear, area); + let block = Block::default() + .borders(Borders::ALL) + .title(" Confirmation ") + .border_style(Style::default().fg(OX_YELLOW)) + .style(Style::default().bg(OX_GREEN_DARKEST).fg(OX_OFF_WHITE)); + let inner = block.inner(area); + frame.render_widget(block, area); + let body_height = Paragraph::new(dialog_body.clone()) + .wrap(Wrap { trim: false }) + .line_count(inner.width) as u16; + let rows = Layout::vertical([ + Constraint::Length(body_height), + Constraint::Length(options.len() as u16), + Constraint::Length(1), + ]) + .split(inner); + frame.render_widget( + Paragraph::new(dialog_body).wrap(Wrap { trim: false }), + rows[0], + ); + let list = + List::new(options.iter().map(|option| ListItem::new(option.label))) + .highlight_symbol("▶ ") + .highlight_spacing(HighlightSpacing::Always) + .highlight_style( + Style::default().fg(TUI_PURPLE).add_modifier(Modifier::BOLD), + ); + let selected = + app.session.confirmation_selection.min(options.len().saturating_sub(1)); + let mut state = ListState::default().with_selected(Some(selected)); + frame.render_stateful_widget(list, rows[1], &mut state); + frame.render_widget( + Paragraph::new(visible_shortcuts(confirmation, rows[2].width as usize)), + rows[2], + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + tui::operation::OperationKind, + tui::{app::ActiveOperation, event::OperationRequestId}, + }; + use ratatui::{Terminal, backend::TestBackend, buffer::Buffer}; + + fn app_for(confirmation: Confirmation) -> App { + let mut app = App::new(vec![], 2, 2); + app.reattach_command = Some("pfexec '/opt/voxel bin/voxel' --config '/cfg/voxel.toml' --workdir '/work dir' --name 'rack' --dataset 'pool/falcon' --build-root '/build root' tui".into()); + app.session.confirmation_selection = + confirmation.options(app.can_cancel()).len() - 1; + app.session.confirmation = Some(confirmation); + app + } + fn render(app: &App, width: u16, height: u16) -> Buffer { + let mut terminal = + Terminal::new(TestBackend::new(width, height)).unwrap(); + terminal.draw(|frame| draw(frame, app)).unwrap(); + terminal.backend().buffer().clone() + } + fn rendered_dialog_text(app: &App, width: u16, height: u16) -> String { + let buffer = render(app, width, height); + let area = + dialog_area(app, overlay_area(Rect::new(0, 0, width, height))); + (area.y..area.bottom()) + .map(|y| { + (area.x..area.right()) + .map(|x| buffer[(x, y)].symbol()) + .collect::() + .trim() + .to_owned() + }) + .collect::>() + .join(" ") + } + #[test] + fn lifecycle_prompts_match_exactly() { + for (confirmation, expected) in [ + (Confirmation::Detach, "Detach from Voxel TUI?"), + (Confirmation::Quit, "Quit Voxel TUI?"), + ( + Confirmation::QuitAndDestroy, + "Quit TUI and destroy Voxel deployment?", + ), + ] { + let text = rendered_dialog_text(&app_for(confirmation), 100, 22); + assert!(text.contains(expected), "missing {expected:?}: {text}"); + } + } + #[test] + fn detach_renders_short_resume_selected_back_copy_and_warning() { + for (width, height) in [(100, 22), (80, 20)] { + let app = app_for(Confirmation::Detach); + let text = rendered_dialog_text(&app, width, height); + assert!( + text.contains("Resume later with:") + && text.contains("voxel tui resume"), + "{text}" + ); + assert!(!text.contains("--config"), "{text}"); + assert!( + text.contains("▶ Back") + && text.contains("y copy") + && text.contains("n/Esc back"), + "{text}" + ); + } + let mut copied = app_for(Confirmation::Detach); + copied.clipboard_copied = true; + assert!( + rendered_dialog_text(&copied, 80, 20) + .contains("Full fallback command copied.") + ); + let mut app = app_for(Confirmation::Detach); + app.reattach_command = Some("voxel --name rack tui".into()); + app.operation.active = Some(ActiveOperation::new( + OperationRequestId::FIRST, + OperationKind::Launch, + )); + let text = rendered_dialog_text(&app, 48, 16); + assert!( + text.contains("voxel tui resume") && text.contains("▶ Back"), + "{text}" + ); + assert!( + text.contains("The active operation will be cancelled;") + && text.contains("resources will be left in place."), + "{text}" + ); + } + #[test] + fn dialog_geometry_is_bounded_and_uses_lifecycle_caps() { + for (confirmation, affirmative, selected_reject) in [ + (Confirmation::Launch, "Launch deployment", "▶ Cancel"), + (Confirmation::Route, "Apply routes", "▶ Cancel"), + (Confirmation::Detach, "Detach and leave resources", "▶ Back"), + (Confirmation::Quit, "Quit Voxel TUI", "▶ Back"), + ( + Confirmation::QuitAndDestroy, + "Destroy deployment and quit", + "▶ Back", + ), + ( + Confirmation::CancelAndLeave, + "Cancel and leave resources", + "▶ Back", + ), + ( + Confirmation::CancelAndDestroy, + "Cancel and destroy resources", + "▶ Back", + ), + ] { + for (width, height) in [(160, 38), (100, 22), (80, 20), (48, 16)] { + let mut app = app_for(confirmation.clone()); + if width == 48 && matches!(confirmation, Confirmation::Detach) { + app.reattach_command = Some("voxel tui".into()); + } + let overlay = overlay_area(Rect::new(0, 0, width, height)); + let area = dialog_area(&app, overlay); + assert!( + area.x >= overlay.x + && area.y >= overlay.y + && area.right() <= overlay.right() + && area.bottom() <= overlay.bottom() + ); + if overlay.width >= 4 { + assert!(area.width <= overlay.width - 4); + } + assert!( + area.width + <= if matches!(confirmation, Confirmation::Detach) { + 96 + } else { + 64 + } + ); + let text = rendered_dialog_text(&app, width, height); + assert!( + text.contains(affirmative), + "missing affirmative {affirmative:?} for {confirmation:?} {width}x{height}: {text}" + ); + assert!( + text.contains(selected_reject), + "missing selected reject {selected_reject:?} for {confirmation:?} {width}x{height}: {text}" + ); + assert!( + text.contains("n/Esc"), + "{confirmation:?} {width}x{height}: {text}" + ); + assert_eq!( + text.contains("y copy"), + matches!(confirmation, Confirmation::Detach), + "{confirmation:?} {width}x{height}: {text}" + ); + } + } + } + #[test] + fn selected_back_row_keeps_existing_list_style() { + let app = app_for(Confirmation::Quit); + let buffer = render(&app, 80, 20); + let area = dialog_area(&app, overlay_area(Rect::new(0, 0, 80, 20))); + let back = (area.y..area.bottom()) + .flat_map(|y| (area.x..area.right()).map(move |x| (x, y))) + .find(|&(x, y)| buffer[(x, y)].symbol() == "▶") + .unwrap(); + assert_eq!(buffer[back].fg, TUI_PURPLE); + assert!(buffer[back].modifier.contains(Modifier::BOLD)); + assert_eq!(buffer[(area.x, area.y)].fg, OX_YELLOW); + } +} diff --git a/voxel/src/tui/ui/deployment.rs b/voxel/src/tui/ui/deployment.rs new file mode 100644 index 0000000..c95b98e --- /dev/null +++ b/voxel/src/tui/ui/deployment.rs @@ -0,0 +1,572 @@ +use super::{ + colors::*, + logs, + renderer::LayoutMode, + widgets::{ + fit_terminal_width, section_block, section_heights, section_rects, + }, +}; +use crate::{ + tui::operation::{ + CommandOutcome, DestroyPhase, LaunchPhase, OperationKind, + OperationPhase, RoutePhase, + }, + tui::{app::App, event::DeploymentPane}, +}; +use ratatui::{ + layout::Rect, + style::{Modifier, Style}, + text::{Line, Span}, + widgets::{Borders, Gauge, List, ListItem, Paragraph, Wrap}, +}; + +pub fn draw( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + mode: LayoutMode, +) { + let rows = deployment_rows(area, app, mode); + + draw_progress(frame, rows[0], app); + draw_phases(frame, rows[1], app); + draw_status(frame, rows[2], app); + draw_subtasks(frame, rows[3], app); + logs::draw(frame, rows[4], app); +} + +pub(crate) fn deployment_rows( + area: Rect, + app: &App, + mode: LayoutMode, +) -> Vec { + let has_subtasks = false; + let current_height = match mode { + LayoutMode::Wide if has_subtasks => 5, + LayoutMode::Wide => 3, + LayoutMode::Compact if has_subtasks => 4, + LayoutMode::Compact => 3, + LayoutMode::Minimum => 1, + }; + let preferred = match mode { + LayoutMode::Wide => { + [3, 5, status_height(app, area.width, mode), current_height, 7] + } + LayoutMode::Compact => { + [3, 4, status_height(app, area.width, mode), current_height, 7] + } + LayoutMode::Minimum => [1; 5], + }; + let expanded = + DeploymentPane::ORDER.map(|pane| app.session.deployment_expanded(pane)); + let focused = DeploymentPane::ORDER + .iter() + .position(|pane| *pane == app.session.deployment_pane) + .unwrap_or(1); + let heights = section_heights( + area.height, + &expanded, + focused, + &preferred, + &[2, 1, 0, 3, 4], + 1, + ); + section_rects(area, &heights) +} + +pub(crate) fn log_content_height(app: &App) -> usize { + if !app.session.deployment_expanded(DeploymentPane::Logs) { + return 0; + } + pane_content_height(app, 4) +} + +fn pane_content_height(app: &App, pane: usize) -> usize { + let (content, mode) = super::widgets::content_area(app); + if mode == LayoutMode::Minimum { + return 0; + } + deployment_rows(content, app, mode)[pane].height.saturating_sub(2) as usize +} + +pub(crate) fn phase_content_height(app: &App) -> usize { + pane_content_height(app, 1) +} + +pub(crate) fn subtask_content_height(app: &App) -> usize { + pane_content_height(app, 3) +} + +pub(crate) fn phase_order(kind: OperationKind) -> Vec { + match kind { + OperationKind::Launch => { + LaunchPhase::ORDER.into_iter().map(OperationPhase::Launch).collect() + } + OperationKind::Destroy => DestroyPhase::ORDER + .into_iter() + .map(OperationPhase::Destroy) + .collect(), + OperationKind::Route => { + RoutePhase::ORDER.into_iter().map(OperationPhase::Route).collect() + } + } +} + +pub(crate) fn phase_name(phase: OperationPhase) -> &'static str { + match phase { + OperationPhase::Launch(LaunchPhase::Preflight) => "Preflight", + OperationPhase::Launch(LaunchPhase::Stage) => "Stage", + OperationPhase::Launch(LaunchPhase::Boot) => "Boot", + OperationPhase::Launch(LaunchPhase::Initialize) => "Initialize", + OperationPhase::Launch(LaunchPhase::RackSetup) => "Rack setup", + OperationPhase::Launch(LaunchPhase::Route) => "Route", + OperationPhase::Launch(LaunchPhase::Reconcile) => "Reconcile", + OperationPhase::Destroy(DestroyPhase::OrphanCleanup) => { + "Orphan cleanup" + } + OperationPhase::Destroy(DestroyPhase::FalconTeardown) => { + "Falcon teardown" + } + OperationPhase::Destroy(DestroyPhase::StorageCleanup) => { + "Storage cleanup" + } + OperationPhase::Destroy(DestroyPhase::Reconcile) => "Reconcile", + OperationPhase::Route(RoutePhase::Validate) => "Validate", + OperationPhase::Route(RoutePhase::Apply) => "Apply", + OperationPhase::Route(RoutePhase::Reconcile) => "Reconcile", + } +} + +fn draw_progress(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let pane = DeploymentPane::OverallProgress; + let expanded = app.session.deployment_expanded(pane); + let focused = app.session.deployment_pane == pane; + let block = section_block(" Overall Progress ", expanded, focused); + if area.height <= 1 || !expanded { + frame.render_widget(block, area); + return; + } + let (completed, total, ratio, message) = progress_summary(app); + if area.height == 2 { + // The following section owns the separator below this compact region. + // Omitting this block's bottom border leaves one real summary row. + let compact_block = + block.borders(Borders::TOP | Borders::LEFT | Borders::RIGHT); + frame.render_widget( + Paragraph::new(Line::from(Span::styled( + format!("{completed}/{total} {message}"), + Style::default().fg(TUI_GREEN), + ))) + .block(compact_block), + area, + ); + } else { + frame.render_widget( + Gauge::default() + .block(block) + .gauge_style(Style::default().fg(TUI_GREEN)) + .ratio(ratio.clamp(0.0, 1.0)) + .label(format!("{completed}/{total} {message}")), + area, + ); + } +} + +fn progress_summary(app: &App) -> (usize, usize, f64, String) { + app.operation.displayed().map_or( + (0, 0, 0.0, "Idle".to_owned()), + |operation| { + let phases = phase_order(operation.kind); + let phase_completed = + operation.completed_phases.len().min(phases.len()); + let phase_ratio = if phases.is_empty() { + 0.0 + } else { + phase_completed as f64 / phases.len() as f64 + }; + operation + .progress + .as_ref() + .and_then(|progress| { + (progress.total > 0 && progress.completed <= progress.total) + .then(|| { + ( + progress.completed, + progress.total, + progress.completed as f64 + / progress.total as f64, + progress.message.clone(), + ) + }) + }) + .unwrap_or(( + phase_completed, + phases.len(), + phase_ratio, + if phase_completed == phases.len() { + "Complete".to_owned() + } else { + phase_name( + operation.phase.unwrap_or(phases[phase_completed]), + ) + .to_owned() + }, + )) + }, + ) +} + +fn draw_phases(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let pane = DeploymentPane::Phases; + let focused = app.session.deployment_pane == pane; + let expanded = app.session.deployment_expanded(pane); + let block = section_block(" Deployment Phases ", expanded, focused); + if area.height <= 1 || !expanded { + frame.render_widget(block, area); + return; + } + let Some(operation) = app.operation.displayed() else { + frame.render_widget( + List::new([ListItem::new("○ No active deployment operation")]) + .block(block), + area, + ); + return; + }; + let phases = phase_order(operation.kind); + let visible = area.height.saturating_sub(2) as usize; + let active_index = operation + .phase + .and_then(|active| phases.iter().position(|p| *p == active)); + let start = phase_window_start( + app.session.phase_scroll, + visible, + phases.len(), + active_index, + ); + let failed_phase = None; + let items = phases.into_iter().skip(start).take(visible).map(|phase| { + let (icon, state, style) = if failed_phase == Some(phase) { + ( + "✗", + "Failed", + Style::default().fg(OX_RED).add_modifier(Modifier::BOLD), + ) + } else if operation.completed_phases.contains(&phase) { + ("●", "Complete", Style::default().fg(TUI_GREEN)) + } else if operation.phase == Some(phase) { + ( + "◐", + "Active", + Style::default().fg(TUI_YELLOW).add_modifier(Modifier::BOLD), + ) + } else { + ("○", "Pending", Style::default().fg(TUI_GREY)) + }; + ListItem::new(Line::from(vec![ + Span::styled(format!("{icon} {:<18}", phase_name(phase)), style), + Span::styled(state, style), + ])) + }); + frame.render_widget(List::new(items).block(block), area); +} + +fn draw_status(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let pane = DeploymentPane::Status; + let focused = app.session.deployment_pane == pane; + let expanded = app.session.deployment_expanded(pane); + let block = section_block(" Status ", expanded, focused); + if area.height <= 1 || !expanded { + frame.render_widget(block, area); + return; + } + let lines = status_lines(app, area.width.saturating_sub(2) as usize) + .into_iter() + .map(Line::from) + .collect::>(); + frame.render_widget( + Paragraph::new(lines).wrap(Wrap { trim: true }).block(block), + area, + ); +} + +fn status_lines(app: &App, width: usize) -> Vec { + let mut state = format!("Observed: {}", observed_name(app)); + if let Some(active) = app.operation.active.as_ref() { + state.push_str(&format!( + " · {}{}{}", + operation_name(active.kind), + active + .phase + .map(|p| format!(" · {}", phase_name(p))) + .unwrap_or_default(), + if active.cancelling { " · Cancelling" } else { "" } + )); + if let Some(progress) = active.progress.as_ref() { + state.push_str(&format!(" · {}", progress.message)); + } + } else if let Some(pending) = app.operation.pending.as_ref() { + state.push_str(&format!(" · {} pending", operation_name(pending.kind))); + } + + let mut failures = Vec::new(); + if let Some(failure) = app.operation.start_failure.as_ref() { + failures.push(format!("Start: {failure}")); + } + if let Some(failure) = app.deployment.reconciliation_failure.as_ref() { + failures.push(format!("Reconciliation: {failure}")); + } + let mut result = vec![fit_status_line(state, width)]; + + let mut terminal = + app.operation.outcome.as_ref().map(|outcome| match outcome { + CommandOutcome::Exited { status, .. } if status.success() => { + "Outcome: Command succeeded".into() + } + CommandOutcome::Exited { status, .. } => { + format!("Outcome: Command exited {status}") + } + CommandOutcome::SpawnFailed { message } => { + format!("Outcome: Failed · {message}") + } + CommandOutcome::ForceStopped { kill_error, .. } => { + kill_error.as_ref().map_or_else( + || "Outcome: Force stopped".into(), + |error| format!("Outcome: Force stop failed · {error}"), + ) + } + }); + if !failures.is_empty() { + let failure_summary = + format!("Failures: {} · {}", failures.len(), failures.join(" · ")); + terminal = Some(match terminal { + Some(outcome) => format!("{outcome} · {failure_summary}"), + None => failure_summary, + }); + } + if let Some(terminal) = terminal { + result.push(fit_status_line(terminal, width)); + } + + if let Some(first) = app.operation.retained_warnings.first() { + let count = app.operation.retained_warnings.len(); + let remainder = count + .checked_sub(1) + .filter(|remaining| *remaining > 0) + .map(|remaining| format!(" · {remaining} more")) + .unwrap_or_default(); + result.push(fit_status_line( + format!("Warnings: {count} · {}{remainder}", first.message), + width, + )); + } + result +} + +fn fit_status_line(line: String, width: usize) -> String { + fit_terminal_width(&line, width) +} + +fn status_height(app: &App, width: u16, mode: LayoutMode) -> u16 { + let content = + status_lines(app, width.saturating_sub(2).max(1) as usize).len() as u16; + content.saturating_add(2).max(match mode { + LayoutMode::Wide => 5, + LayoutMode::Compact => 4, + LayoutMode::Minimum => 0, + }) +} + +fn draw_subtasks(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let pane = DeploymentPane::CurrentPhase; + let focused = app.session.deployment_pane == pane; + let expanded = app.session.deployment_expanded(pane); + let block = section_block(" Current Phase ", expanded, focused); + if area.height <= 1 || !expanded { + frame.render_widget(block, area); + return; + } + let rows: Vec<(Line<'_>, bool)> = Vec::new(); + let lines = if rows.is_empty() { + vec![Line::from("No subtasks yet")] + } else { + let visible = area.height.saturating_sub(2) as usize; + let important = rows + .iter() + .enumerate() + .filter_map(|(index, (_, important))| important.then_some(index)) + .collect::>(); + let start = subtask_window_start( + app.session.subtask_scroll, + visible, + rows.len(), + &important, + ); + rows.into_iter() + .skip(start) + .take(visible) + .map(|(line, _)| line) + .collect() + }; + frame.render_widget(Paragraph::new(lines).block(block), area); +} + +pub(crate) fn phase_window_start( + requested: usize, + visible: usize, + len: usize, + important: Option, +) -> usize { + let mut start = requested.min(len.saturating_sub(visible)); + if let Some(index) = important { + if index < start { + start = index; + } + if index >= start.saturating_add(visible) { + start = index.saturating_add(1).saturating_sub(visible); + } + } + start +} + +pub(crate) fn subtask_window_start( + requested: usize, + visible: usize, + len: usize, + important: &[usize], +) -> usize { + let maximum = len.saturating_sub(visible); + if visible == 0 || important.is_empty() { + return requested.min(maximum); + } + let first = important[0]; + let last = *important.last().unwrap_or(&first); + if last.saturating_sub(first) < visible { + return requested + .min(first) + .max(last.saturating_add(1).saturating_sub(visible)) + .min(maximum); + } + (0..=maximum) + .max_by_key(|start| { + let end = start.saturating_add(visible); + ( + important + .iter() + .filter(|index| **index >= *start && **index < end) + .count(), + std::cmp::Reverse(*start), + ) + }) + .unwrap_or(0) +} + +fn operation_name(kind: OperationKind) -> &'static str { + match kind { + OperationKind::Launch => "Launch", + OperationKind::Destroy => "Destroy", + OperationKind::Route => "Route", + } +} +fn observed_name(app: &App) -> &'static str { + match app.deployment.observed { + crate::tui::reconcile::ObservedDeploymentState::Stopped => "Stopped", + crate::tui::reconcile::ObservedDeploymentState::Starting => "Starting", + crate::tui::reconcile::ObservedDeploymentState::Running => "Running", + crate::tui::reconcile::ObservedDeploymentState::Degraded => "Degraded", + crate::tui::reconcile::ObservedDeploymentState::Stopping => "Stopping", + crate::tui::reconcile::ObservedDeploymentState::Unknown => "Unknown", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::{ + app::ActiveOperation, + event::OperationRequestId, + operation::{LaunchPhase, OperationKind, OperationPhase}, + }; + + #[test] + fn retained_success_renders_complete_progress() { + let mut app = App::new(vec![], 8, 8); + let mut operation = ActiveOperation::new( + OperationRequestId::FIRST, + OperationKind::Launch, + ); + operation.completed_phases = LaunchPhase::ORDER + .into_iter() + .map(OperationPhase::Launch) + .collect(); + app.operation.retained = Some(operation); + + assert_eq!(progress_summary(&app), (7, 7, 1.0, "Complete".into())); + } + + #[test] + fn retained_failure_renders_last_proven_phase() { + let mut app = App::new(vec![], 8, 8); + let mut operation = ActiveOperation::new( + OperationRequestId::FIRST, + OperationKind::Launch, + ); + operation.completed_phases = LaunchPhase::ORDER[..3] + .iter() + .copied() + .map(OperationPhase::Launch) + .collect(); + operation.phase = Some(OperationPhase::Launch(LaunchPhase::Initialize)); + app.operation.retained = Some(operation); + + assert_eq!( + progress_summary(&app), + (3, 7, 3.0 / 7.0, "Initialize".into()) + ); + } + + #[test] + fn absent_operation_renders_idle_progress() { + let app = App::new(vec![], 8, 8); + + assert_eq!(progress_summary(&app), (0, 0, 0.0, "Idle".into())); + } + + #[test] + fn status_lines_fit_terminal_columns() { + for (line, width, expected) in [ + ("warning", 0, ""), + ("warning", 1, "…"), + ("警告ab", 4, "警…"), + ("e\u{301}rror", 3, "e\u{301}r…"), + ] { + let fitted = fit_status_line(line.to_owned(), width); + assert_eq!(fitted, expected); + assert!(Line::from(fitted).width() <= width); + } + } + + #[test] + fn compact_deployment_keeps_all_five_section_headers() { + let mut app = App::new(vec![], 8, 8); + app.session.terminal = + crate::tui::app::TerminalSize { width: 48, height: 16 }; + let rows = + deployment_rows(Rect::new(0, 0, 48, 11), &app, LayoutMode::Compact); + assert_eq!(rows.len(), DeploymentPane::ORDER.len()); + assert!(rows.iter().all(|row| row.height >= 1)); + assert_eq!(rows.iter().map(|row| row.height).sum::(), 11); + } + + #[test] + fn important_window_contains_a_separated_range_when_it_fits() { + assert_eq!(subtask_window_start(99, 6, 12, &[3, 8]), 3); + } + + #[test] + fn important_window_uses_deterministic_dense_coverage_when_range_cannot_fit() + { + assert_eq!(subtask_window_start(99, 3, 12, &[1, 4, 5, 9]), 3); + assert_eq!(subtask_window_start(0, 0, 12, &[1, 4]), 0); + } +} diff --git a/voxel/src/tui/ui/external_monitoring.rs b/voxel/src/tui/ui/external_monitoring.rs new file mode 100644 index 0000000..f07db7f --- /dev/null +++ b/voxel/src/tui/ui/external_monitoring.rs @@ -0,0 +1,193 @@ +use crate::tui::{App, telemetry::RackId}; +use ratatui::{ + layout::Rect, + style::{Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Clear, Paragraph, Wrap}, +}; + +use super::{colors::*, widgets::overlay_area}; + +pub const GUIDE_URL: &str = + "https://docs.oxide.computer/guides/integrations/opentelemetry"; + +const PATTERNS: &[&str] = &[ + "sled_data_link:.*", + "zone:cpu_nsec", + "zfs_pool:.*", + "oximeter_collector:.*", + ".*http.*latency.*", +]; + +fn yaml_for<'a>(racks: impl IntoIterator) -> String { + let mut yaml = String::from( + "# Receiver-only fragment. Add these receiver IDs to\n\ + # service.pipelines.metrics.receivers and choose your own processors/exporters.\n\ + # Create one least-privilege fleet-viewer token per rack.\n\ + receivers:\n", + ); + for (rack, host) in racks { + yaml.push_str(&format!( + " oxide/rack{}:\n host: {:?}\n token: ${{env:OXIDE_TOKEN_RACK{}}}\n # Voxel virtual racks only; do not copy this TLS setting to production racks.\n insecure_skip_verify: true\n metric_patterns:\n", + rack.0, host, rack.0 + )); + for pattern in PATTERNS { + yaml.push_str(&format!(" - {pattern:?}\n")); + } + } + yaml +} + +pub(crate) fn selected_yaml(app: &App) -> String { + let selected = app + .session + .selected_rack + .or_else(|| app.external_monitoring_endpoints.keys().next().copied()); + yaml_for(selected.into_iter().filter_map(|rack| { + app.external_monitoring_endpoints + .get(&rack) + .map(|host| (rack, host.as_str())) + })) +} + +pub(crate) fn all_yaml(app: &App) -> String { + yaml_for( + app.external_monitoring_endpoints + .iter() + .map(|(rack, host)| (*rack, host.as_str())), + ) +} + +pub fn draw(frame: &mut ratatui::Frame<'_>, app: &App) { + let root = overlay_area(frame.area()); + let width = root.width.min(92); + let height = root.height.min(26); + let area = Rect::new( + root.x + root.width.saturating_sub(width) / 2, + root.y + root.height.saturating_sub(height) / 2, + width, + height, + ); + if area.width <= 2 || area.height <= 2 { + return; + } + frame.render_widget(Clear, area); + let selected = app.session.selected_rack.map_or_else( + || "first configured rack".into(), + |rack| format!("rack {}", rack.0), + ); + let body = vec![ + Line::styled( + "The Voxel TUI is a deployment assistant, not a monitoring platform", + Style::default().fg(TUI_YELLOW).add_modifier(Modifier::BOLD), + ), + Line::default(), + Line::from( + "Use the official Oxide receiver to ingest Oximeter metrics,", + ), + Line::from("then connect your own observability stack."), + Line::default(), + Line::from( + "1. Create a least-privilege fleet-viewer token for each virtual rack.", + ), + Line::from( + "2. Copy the receiver fragment and set OXIDE_TOKEN_RACKN in the Collector environment.", + ), + Line::from( + "3. Add oxide/rack to your list of receivers, and choose your exporters.", + ), + Line::default(), + Line::from(vec![ + Span::styled("s", Style::default().fg(TUI_YELLOW)), + Span::raw(format!(" copy {selected} YAML ")), + Span::styled("a", Style::default().fg(TUI_YELLOW)), + Span::raw(" copy all-rack YAML "), + Span::styled("u", Style::default().fg(TUI_YELLOW)), + Span::raw(" copy official guide URL"), + ]), + Line::from("Esc close this window"), + Line::default(), + Line::from("Documentation:"), + Line::from(format!("1. {GUIDE_URL}")), + Line::from( + "2. https://docs.oxide.computer/guides/metrics/oxql-tutorial", + ), + Line::from( + "3. https://docs.oxide.computer/guides/metrics/timeseries-schemas", + ), + Line::default(), + Line::styled( + "NOTE: insecure_skip_verify is included only because Voxel uses self-signed virtual-rack certificates.", + Style::default().fg(OX_RED), + ), + ]; + let block = Block::bordered() + .title(" External monitoring ") + .border_style(Style::default().fg(TUI_PURPLE)) + .style(Style::default().bg(OX_GREEN_DARKEST)); + frame.render_widget( + Paragraph::new(body).wrap(Wrap { trim: false }).block(block), + area, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::{ + Action, Effect, + event::View, + telemetry::{ResourceDescriptor, ResourceId, ResourceKind}, + }; + use std::collections::BTreeMap; + + #[test] + fn generated_yaml_is_deterministic_and_contains_no_session_secret() { + let yaml = yaml_for([ + (RackId(0), "https://198.51.100.22"), + (RackId(1), "https://198.51.100.86"), + ]); + assert!(yaml.contains("oxide/rack0:")); + assert!(yaml.contains("${env:OXIDE_TOKEN_RACK1}")); + assert!(yaml.contains("sled_data_link:.*")); + assert!(yaml.contains("Voxel virtual racks only")); + assert!(!yaml.contains("password")); + assert!(!yaml.contains("oxide\n")); + } + + #[test] + fn overlay_actions_copy_selected_all_and_guide_then_close() { + let descriptor = ResourceDescriptor { + id: ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "g0".into(), + host: None, + }; + let mut app = App::new(vec![descriptor], 4, 4); + app.session.view = View::Monitor; + app.external_monitoring_endpoints = BTreeMap::from([ + (RackId(0), "https://198.51.100.22/".into()), + (RackId(1), "https://198.51.100.86/".into()), + ]); + + app.update(Action::RequestCancelAndDestroy.into()); + assert!(app.session.external_monitoring_open); + app.update(Action::RequestCancelAndDestroy.into()); + assert!(app.session.external_monitoring_open); + assert!(matches!( + app.update(Action::CopyExternalMonitoringSelected.into()).as_slice(), + [Effect::CopyToClipboard(yaml)] if yaml.contains("oxide/rack0:") && !yaml.contains("oxide/rack1:") + )); + assert!(matches!( + app.update(Action::CopyExternalMonitoringAll.into()).as_slice(), + [Effect::CopyToClipboard(yaml)] if yaml.contains("oxide/rack0:") && yaml.contains("oxide/rack1:") + )); + assert_eq!( + app.update(Action::CopyExternalMonitoringGuide.into()), + vec![Effect::CopyToClipboard(GUIDE_URL.into())] + ); + app.update(Action::Close.into()); + assert!(!app.session.external_monitoring_open); + } +} diff --git a/voxel/src/tui/ui/help.rs b/voxel/src/tui/ui/help.rs new file mode 100644 index 0000000..1767ba2 --- /dev/null +++ b/voxel/src/tui/ui/help.rs @@ -0,0 +1,558 @@ +use super::{colors::*, widgets::overlay_area}; +use crate::tui::App; +use ratatui::{ + layout::Rect, + style::{Modifier, Style}, + text::{Line, Span, Text}, + widgets::{Block, Clear, Paragraph, Wrap}, +}; + +const GLOBAL: &[(&str, &str)] = &[ + ("1 / 2", "Deployment / Monitoring"), + ("Tab / Shift-Tab", "Next / previous top-level section unconditionally"), + ("Space", "Fold / expand focused section"), + ("? / F1", "Open / close Help"), + ("Esc", "Close the topmost non-confirmation pane or selection"), + ("d", "Detach and leave resources; resume with voxel tui resume"), + ("q", "Quit; destroy deployment first unless already stopped"), +]; +const DEPLOYMENT: &[(&str, &str)] = &[ + ("Sections", "Overall Progress, Phases, Status, Current Phase, Logs"), + ("↑ / ↓", "Move nested; at edge/no vertical content, move section"), + ("PgUp / PgDn", "Page nested content only"), + ("f", "Cycle log filter"), + ("l / r", "Launch / route"), + ("c / x", "Cancel and leave / destroy resources"), +]; +const MONITORING: &[(&str, &str)] = &[ + ("Sections", "Rack Summary, Topology, Top Zones by Traffic"), + ("← / →", "Previous / next rack in Rack Summary"), + ("↑ / ↓", "Move resource; at edge/no vertical content, move section"), + ("PgUp / PgDn", "Page Topology resources or Top Zones"), + ("Enter", "Open selected resource detail"), + ("Top Zones ↑ / ↓", "Browse the rack-wide traffic ranking"), + ("Detail ↑ / ↓", "Browse peers; detail stays open; section unchanged"), + ("Detail Enter/Esc", "Close resource detail"), + ("x", "External monitoring"), +]; +const DIALOGS: &[(&str, &str)] = &[ + ("↑ / ↓", "Choose a confirmation option"), + ("Enter", "Confirm selected safe-default option"), + ("y", "Copy full fallback command from Detach"), + ("n", "Reject confirmation"), + ("Esc", "Reject or close topmost dialog"), + ("Help ↑/↓ / Pg", "Scroll one wrapped line / page"), +]; +const TRAFFIC: &[(&str, &str)] = + &[("Rates", "normal ≤100 KB/s; elevated >100 KB/s–5 MB/s; high >5 MB/s")]; +const HEALTH: &[(&str, &str)] = &[ + ("States", "● healthy; ◌ checking status; ! degraded / failed"), + ("", "? / ◐ / × / ■ checking status"), +]; + +fn key_line(key: &'static str, description: &'static str) -> Line<'static> { + Line::from(vec![ + Span::styled( + format!("{key:<18}"), + Style::default().fg(TUI_YELLOW).add_modifier(Modifier::BOLD), + ), + Span::styled(description, Style::default().fg(OX_OFF_WHITE)), + ]) +} + +fn help_text() -> Text<'static> { + let mut lines = Vec::new(); + for (title, rows) in [ + ("Global", GLOBAL), + ("Deployment", DEPLOYMENT), + ("Monitoring", MONITORING), + ("Dialogs", DIALOGS), + ("Traffic", TRAFFIC), + ("Health", HEALTH), + ] { + if !lines.is_empty() { + lines.push(Line::default()); + } + lines.push(Line::styled( + title, + Style::default().fg(TUI_PURPLE).add_modifier(Modifier::BOLD), + )); + lines.extend( + rows.iter().map(|(key, description)| key_line(key, description)), + ); + } + Text::from(lines) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct HelpMetrics { + total: usize, + capacity: usize, + max_scroll: usize, +} + +fn text_metrics(text: &Text<'_>, area: Rect) -> HelpMetrics { + let inner = Block::bordered().inner(area); + let capacity = usize::from(inner.height).max(1); + let total = Paragraph::new(text.clone()) + .wrap(Wrap { trim: false }) + .line_count(inner.width); + HelpMetrics { total, capacity, max_scroll: total.saturating_sub(capacity) } +} + +fn help_metrics(area: Rect) -> HelpMetrics { + text_metrics(&help_text(), area) +} + +pub(crate) fn help_area(root: Rect) -> Rect { + let horizontal_margin = if root.width >= 52 { 4 } else { 0 }; + let vertical_margin = if root.height >= 12 { 2 } else { 0 }; + let width = root.width.saturating_sub(horizontal_margin).min(76); + let height = root.height.saturating_sub(vertical_margin).min(18); + Rect::new( + root.x + root.width.saturating_sub(width) / 2, + root.y + root.height.saturating_sub(height) / 2, + width, + height, + ) +} + +pub fn draw( + frame: &mut ratatui::Frame<'_>, + app: &App, + _mode: super::renderer::LayoutMode, +) { + let area = help_area(overlay_area(frame.area())); + // The application renderer returns its minimum-size fallback before Help. + // Keep direct helper draws for degenerate test/backend areas empty as well. + if area.width <= 2 || area.height <= 2 { + return; + } + frame.render_widget(Clear, area); + + let body = help_text(); + let metrics = text_metrics(&body, area); + let start = app.session.help_scroll.min(metrics.max_scroll); + let end = (start + metrics.capacity).min(metrics.total); + let title = format!( + " Help {}-{} of {} ", + start.saturating_add(1).min(metrics.total), + end, + metrics.total + ); + let block = Block::bordered() + .border_style(Style::default().fg(TUI_GREY)) + .title(title) + .title_style( + Style::default().fg(TUI_YELLOW).add_modifier(Modifier::BOLD), + ) + .style(Style::default().bg(OX_GREEN_DARKEST)); + frame.render_widget( + Paragraph::new(body) + .wrap(Wrap { trim: false }) + .scroll((start as u16, 0)) + .style(Style::default().bg(OX_GREEN_DARKEST)) + .block(block), + area, + ); +} + +pub(crate) fn page_capacity(app: &App) -> usize { + let terminal = Rect::new( + 0, + 0, + app.session.terminal.width, + app.session.terminal.height, + ); + help_metrics(help_area(overlay_area(terminal))).capacity +} + +#[cfg(test)] +mod tests { + use super::*; + use ratatui::{ + Terminal, + backend::TestBackend, + buffer::{Buffer, Cell}, + widgets::Widget, + }; + + const HELP_SECTIONS: &[(&str, &[(&str, &str)])] = &[ + ("Global", GLOBAL), + ("Deployment", DEPLOYMENT), + ("Monitoring", MONITORING), + ("Dialogs", DIALOGS), + ("Traffic", TRAFFIC), + ("Health", HEALTH), + ]; + + fn rendered_help(app: &App, width: u16, height: u16) -> Buffer { + let mut terminal = + Terminal::new(TestBackend::new(width, height)).unwrap(); + terminal + .draw(|frame| { + super::draw( + frame, + app, + super::super::renderer::LayoutMode::Compact, + ) + }) + .unwrap(); + terminal.backend().buffer().clone() + } + + fn buffer_text(buffer: &Buffer) -> String { + let area = buffer.area; + (area.y..area.bottom()) + .map(|y| { + (area.x..area.right()) + .map(|x| buffer[(x, y)].symbol()) + .collect::() + }) + .collect::>() + .join("\n") + } + + fn panel_rows(buffer: &Buffer, area: Rect) -> Vec> { + let inner = Block::bordered().inner(area); + (inner.y..inner.bottom()) + .map(|y| { + (inner.x..inner.right()) + .map(|x| buffer[(x, y)].clone()) + .collect() + }) + .collect() + } + + fn rendered_lines_at_all_offsets( + width: u16, + height: u16, + ) -> (HelpMetrics, Vec>) { + let area = help_area(overlay_area(Rect::new(0, 0, width, height))); + let metrics = help_metrics(area); + let mut app = App::new(vec![], 8, 8); + app.session.terminal.width = width; + app.session.terminal.height = height; + let mut lines = vec![None; metrics.total]; + + for offset in 0..=metrics.max_scroll { + app.session.help_scroll = offset; + let buffer = rendered_help(&app, width, height); + for (index, row) in + panel_rows(&buffer, area).into_iter().enumerate() + { + let rendered_index = offset + index; + if rendered_index < metrics.total { + if let Some(previous) = &lines[rendered_index] { + assert_eq!( + previous, &row, + "line {rendered_index} changed by offset" + ); + } else { + lines[rendered_index] = Some(row); + } + } + } + } + + ( + metrics, + lines + .into_iter() + .enumerate() + .map(|(index, line)| { + line.unwrap_or_else(|| { + panic!("line {index} was unreachable") + }) + }) + .collect(), + ) + } + + fn normalized_cells(lines: &[Vec]) -> Vec<&Cell> { + lines + .iter() + .flatten() + .filter(|cell| { + !cell.symbol().is_empty() + && !cell.symbol().chars().all(char::is_whitespace) + }) + .collect() + } + + fn normalized_symbols(text: &str) -> Vec { + text.chars() + .filter(|character| !character.is_whitespace()) + .map(|character| character.to_string()) + .collect() + } + + fn find_symbols( + cells: &[&Cell], + expected: &str, + from: usize, + ) -> std::ops::Range { + let expected = normalized_symbols(expected); + let start = (from..=cells.len().saturating_sub(expected.len())) + .find(|&start| { + cells[start..start + expected.len()] + .iter() + .zip(&expected) + .all(|(cell, symbol)| cell.symbol() == symbol) + }) + .unwrap_or_else(|| { + panic!( + "rendered panel did not contain {expected:?} after {from}" + ) + }); + start..start + expected.len() + } + + fn assert_range_style( + cells: &[&Cell], + range: std::ops::Range, + foreground: ratatui::style::Color, + modifier: Option, + label: &str, + ) { + assert!(!range.is_empty(), "empty rendered range for {label}"); + for cell in &cells[range] { + assert_eq!(cell.fg, foreground, "wrong foreground in {label}"); + if let Some(modifier) = modifier { + assert!( + cell.modifier.contains(modifier), + "missing {modifier:?} in {label}" + ); + } + } + } + + #[test] + fn help_is_bounded_and_uses_full_minimum_content_only_when_needed() { + assert_eq!( + help_area(Rect::new(0, 0, 160, 34)), + Rect::new(42, 8, 76, 18) + ); + assert_eq!(help_area(Rect::new(0, 0, 48, 11)), Rect::new(0, 0, 48, 11)); + for area in [ + Rect::new(0, 0, 0, 0), + Rect::new(0, 0, 1, 1), + Rect::new(0, 0, 2, 2), + Rect::new(17, 23, 160, 34), + ] { + let help = help_area(area); + assert!(help.x >= area.x && help.y >= area.y); + assert!( + help.right() <= area.right() && help.bottom() <= area.bottom() + ); + } + assert_eq!( + help_area(Rect::new(17, 23, 160, 34)), + Rect::new(59, 31, 76, 18) + ); + } + + #[test] + fn help_contract_contains_every_exact_category_row_and_legend() { + let rows = HELP_SECTIONS; + assert_eq!( + rows.iter().map(|(category, _)| *category).collect::>(), + [ + "Global", + "Deployment", + "Monitoring", + "Dialogs", + "Traffic", + "Health" + ] + ); + let expected = [ + ("1 / 2", "Deployment / Monitoring"), + ( + "Tab / Shift-Tab", + "Next / previous top-level section unconditionally", + ), + ("Space", "Fold / expand focused section"), + ("? / F1", "Open / close Help"), + ("Esc", "Close the topmost non-confirmation pane or selection"), + ("d", "Detach and leave resources; resume with voxel tui resume"), + ("q", "Quit; destroy deployment first unless already stopped"), + ( + "Sections", + "Overall Progress, Phases, Status, Current Phase, Logs", + ), + ("↑ / ↓", "Move nested; at edge/no vertical content, move section"), + ("PgUp / PgDn", "Page nested content only"), + ("f", "Cycle log filter"), + ("l / r", "Launch / route"), + ("c / x", "Cancel and leave / destroy resources"), + ("Sections", "Rack Summary, Topology, Top Zones by Traffic"), + ("← / →", "Previous / next rack in Rack Summary"), + ( + "↑ / ↓", + "Move resource; at edge/no vertical content, move section", + ), + ("PgUp / PgDn", "Page Topology resources or Top Zones"), + ("Enter", "Open selected resource detail"), + ("Top Zones ↑ / ↓", "Browse the rack-wide traffic ranking"), + ( + "Detail ↑ / ↓", + "Browse peers; detail stays open; section unchanged", + ), + ("Detail Enter/Esc", "Close resource detail"), + ("x", "External monitoring"), + ("↑ / ↓", "Choose a confirmation option"), + ("Enter", "Confirm selected safe-default option"), + ("y", "Copy full fallback command from Detach"), + ("n", "Reject confirmation"), + ("Esc", "Reject or close topmost dialog"), + ("Help ↑/↓ / Pg", "Scroll one wrapped line / page"), + ( + "Rates", + "normal ≤100 KB/s; elevated >100 KB/s–5 MB/s; high >5 MB/s", + ), + ("States", "● healthy; ◌ checking status; ! degraded / failed"), + ("", "? / ◐ / × / ■ checking status"), + ]; + assert_eq!( + rows.iter() + .flat_map(|(_, rows)| rows.iter().copied()) + .collect::>(), + expected + ); + assert!(!DEPLOYMENT.iter().any(|(key, _)| *key == "Enter")); + } + + #[test] + fn tiny_direct_draws_are_empty_and_supported_capacity_contract_stays_one() { + for (width, height) in [(0, 0), (1, 1), (2, 2)] { + let mut app = App::new(vec![], 8, 8); + app.session.terminal.width = width; + app.session.terminal.height = height; + let buffer = rendered_help(&app, width, height); + assert!(buffer_text(&buffer).trim().is_empty()); + assert_eq!(page_capacity(&app), 1); + } + } + + #[test] + fn shared_wrapped_metrics_cover_ranges_and_clamp() { + let mut app = App::new(vec![], 8, 8); + app.session.terminal.width = 48; + app.session.terminal.height = 16; + let area = help_area(overlay_area(Rect::new(0, 0, 48, 16))); + let metrics = help_metrics(area); + assert_eq!(metrics.capacity, page_capacity(&app)); + assert!(metrics.total > metrics.capacity * 3); + + for requested in [0, metrics.max_scroll / 2, usize::MAX] { + app.session.help_scroll = requested; + let buffer = rendered_help(&app, 48, 16); + let text = buffer_text(&buffer); + let start = requested.min(metrics.max_scroll); + let end = (start + metrics.capacity).min(metrics.total); + assert!(text.contains(&format!( + "Help {}-{} of {}", + start + 1, + end, + metrics.total + ))); + } + } + + #[test] + fn every_help_row_is_reachable_and_exactly_styled_at_every_supported_size() + { + for (width, height) in [(160, 38), (100, 22), (80, 20), (48, 16)] { + let (metrics, lines) = rendered_lines_at_all_offsets(width, height); + assert_eq!(lines.len(), metrics.total); + let cells = normalized_cells(&lines); + let mut cursor = 0; + + for (heading, rows) in HELP_SECTIONS { + let heading_range = find_symbols(&cells, heading, cursor); + assert_range_style( + &cells, + heading_range.clone(), + TUI_PURPLE, + Some(Modifier::BOLD), + heading, + ); + cursor = heading_range.end; + + for (key, description) in *rows { + let rendered_row = format!("{key:<18}{description}"); + let row_range = find_symbols(&cells, &rendered_row, cursor); + let key_len = normalized_symbols(key).len(); + if key_len > 0 { + assert_range_style( + &cells, + row_range.start..row_range.start + key_len, + TUI_YELLOW, + Some(Modifier::BOLD), + key, + ); + } + assert_range_style( + &cells, + row_range.start + key_len..row_range.end, + OX_OFF_WHITE, + None, + description, + ); + cursor = row_range.end; + } + } + } + } + + #[test] + fn wrapping_uses_display_width_and_preserves_continuation_styles() { + let text = Text::from(Line::from(vec![ + Span::styled("界e\u{301}界 ", Style::default().fg(TUI_YELLOW)), + Span::styled( + "continuation words", + Style::default().fg(OX_OFF_WHITE), + ), + ])); + let metrics = text_metrics(&text, Rect::new(0, 0, 10, 8)); + assert_eq!(metrics.total, 4); + + let mut buffer = Buffer::empty(Rect::new(0, 0, 10, 8)); + Paragraph::new(text) + .wrap(Wrap { trim: false }) + .render(Rect::new(0, 0, 10, 8), &mut buffer); + assert!( + buffer + .content + .iter() + .any(|cell| cell.symbol() == "界" && cell.fg == TUI_YELLOW) + ); + let rows = (0..buffer.area.height) + .map(|y| { + (0..buffer.area.width) + .map(|x| buffer[(x, y)].clone()) + .collect::>() + }) + .collect::>(); + let word = normalized_symbols("words"); + let (row, start) = rows + .iter() + .enumerate() + .find_map(|(row, cells)| { + (0..=cells.len() - word.len()) + .find(|&start| { + cells[start..start + word.len()] + .iter() + .zip(&word) + .all(|(cell, symbol)| cell.symbol() == symbol) + }) + .map(|start| (row, start)) + }) + .expect("wrapped continuation token must be rendered"); + assert!(row > 0, "words must be on a wrapped continuation row"); + for cell in &rows[row][start..start + word.len()] { + assert_eq!(cell.fg, OX_OFF_WHITE); + } + } +} diff --git a/voxel/src/tui/ui/logs.rs b/voxel/src/tui/ui/logs.rs new file mode 100644 index 0000000..c5fc3af --- /dev/null +++ b/voxel/src/tui/ui/logs.rs @@ -0,0 +1,318 @@ +use super::{ + colors::*, + widgets::{fit_terminal_width, section_block, terminal_width}, +}; +use crate::{ + tui::App, + tui::app::{LogEntry, LogFilter, LogSource}, + tui::event::DeploymentPane, + tui::operation::LogLevel, +}; +use ratatui::{ + layout::Rect, + style::{Modifier, Style}, + text::{Line, Span}, + widgets::Paragraph, +}; +use std::path::Path; + +pub fn draw(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + if area.height == 0 || area.width == 0 { + return; + } + let visible = area.height.saturating_sub(2) as usize; + let title = collapsed_title(app, area.width, visible); + let expanded = app.session.deployment_expanded(DeploymentPane::Logs); + let focused = app.session.deployment_pane == DeploymentPane::Logs; + let block = section_block(title, expanded, focused); + if !expanded || area.height <= 1 { + frame.render_widget(block, area); + return; + } + let entries = filtered_entries(app); + let scroll = effective_scroll(app.logs.scroll, entries.len(), visible); + let end = entries.len().saturating_sub(scroll); + let start = end.saturating_sub(visible); + let lines = if entries.is_empty() { + vec![Line::from("No log records match this filter")] + } else { + entries[start..end].iter().map(log_line).collect() + }; + frame.render_widget(Paragraph::new(lines).block(block), area); +} + +pub fn collapsed_title(app: &App, width: u16, visible: usize) -> String { + // The bordered section consumes two columns and `section_block` owns the + // three-column fold marker. The returned title must fit what remains. + let available = width.saturating_sub(5) as usize; + if available == 0 { + return String::new(); + } + let entries = filtered_entries(app); + let scroll = effective_scroll(app.logs.scroll, entries.len(), visible); + let position = if scroll == 0 { + "Bottom".to_owned() + } else if scroll >= entries.len().saturating_sub(visible) { + "Top".to_owned() + } else { + format!( + "{} above / {} below", + entries.len().saturating_sub(visible + scroll), + scroll + ) + }; + let filter = filter_name(app.logs_filter); + let (counts, compact_counts) = + if app.session.deployment_expanded(DeploymentPane::Logs) { + (String::new(), String::new()) + } else { + let warnings = app + .logs + .entries + .iter() + .filter(|e| e.level == LogLevel::Warning) + .count(); + let errors = app + .logs + .entries + .iter() + .filter(|e| e.level == LogLevel::Error) + .count(); + ( + format!(" · {warnings} warning / {errors} error"), + format!(" · {warnings}W/{errors}E"), + ) + }; + let suffixes = [ + format!(" · {filter} · {position}{counts}"), + format!(" · {filter} · {position}{compact_counts}"), + ]; + let filename = Path::new(&app.durable_log_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&app.durable_log_path); + for path in [app.durable_log_path.as_str(), filename, "…"] { + for suffix in &suffixes { + let candidate = format!("Logs: {path}{suffix}"); + if terminal_width(&candidate) <= available { + return candidate; + } + } + } + // Widths below the supported renderer minimum may not fit even the + // required state. Elide safely rather than indexing or subtracting past 0. + let required = format!("{filter} · {position}{compact_counts}"); + fit_terminal_width(&required, available) +} + +fn filtered_entries(app: &App) -> Vec<&LogEntry> { + app.logs + .entries + .iter() + .filter(|entry| app.logs_filter.accepts(entry.level)) + .collect() +} + +pub(crate) fn filtered_len(app: &App) -> usize { + filtered_entries(app).len() +} + +pub(crate) fn effective_scroll( + requested: usize, + len: usize, + visible: usize, +) -> usize { + requested.min(len.saturating_sub(visible)) +} + +fn log_line(entry: &&LogEntry) -> Line<'static> { + Line::from(vec![ + Span::styled( + source_label(entry.source), + Style::default().fg(TUI_PURPLE), + ), + Span::raw(" "), + Span::styled( + level_label(entry.level), + level_style(entry.level).add_modifier(Modifier::BOLD), + ), + Span::raw(" "), + Span::styled(entry.message.clone(), Style::default().fg(OX_OFF_WHITE)), + ]) +} + +fn source_label(source: LogSource) -> &'static str { + match source { + LogSource::Application => "APP", + LogSource::Operation => "OP", + } +} +fn level_label(level: LogLevel) -> &'static str { + match level { + LogLevel::Info => "INFO", + LogLevel::Warning => "WARN", + LogLevel::Error => "ERROR", + } +} +fn filter_name(filter: LogFilter) -> &'static str { + match filter { + LogFilter::All => "All", + LogFilter::Info => "Info", + LogFilter::Warning => "Warning", + LogFilter::Error => "Error", + } +} +fn level_style(level: LogLevel) -> Style { + Style::default().fg(match level { + LogLevel::Info => TUI_GREEN, + LogLevel::Warning => TUI_YELLOW, + LogLevel::Error => OX_RED, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::app::LogEntry; + use ratatui::{Terminal, backend::TestBackend}; + + fn render(app: &App, width: u16, height: u16) -> String { + let mut terminal = + Terminal::new(TestBackend::new(width, height)).unwrap(); + terminal.draw(|frame| draw(frame, frame.area(), app)).unwrap(); + terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect() + } + + #[test] + fn chronological_window_follows_bottom_and_pauses_above_it() { + let mut app = App::new(vec![], 8, 8); + app.durable_log_path = "/work/voxel-tui.log".into(); + app.session.terminal.height = 5; + app.logs.push(LogEntry::application(LogLevel::Info, "first")); + app.logs.push(LogEntry::operation(LogLevel::Warning, "second")); + app.logs.push(LogEntry::application(LogLevel::Error, "third")); + let bottom = render(&app, 100, 4); + assert!(bottom.find("second").unwrap() < bottom.find("third").unwrap()); + assert!(!bottom.contains("first")); + assert!(bottom.contains("Bottom")); + + app.logs.scroll = 1; + let paused = render(&app, 100, 4); + assert!(paused.find("first").unwrap() < paused.find("second").unwrap()); + assert!(!paused.contains("third")); + app.logs.push(LogEntry::operation(LogLevel::Info, "fourth")); + let still_paused = render(&app, 100, 4); + assert!( + still_paused.contains("first") && still_paused.contains("second") + ); + + app.logs.scroll = 0; + assert!(render(&app, 100, 4).contains("fourth")); + } + + #[test] + fn collapsed_title_keeps_path_and_counts_and_is_width_aware() { + let mut app = App::new(vec![], 8, 8); + app.durable_log_path = "/work/voxel-tui.log".into(); + assert!(collapsed_title(&app, 100, 5).contains("/work/voxel-tui.log")); + let unsupported_narrow = collapsed_title(&app, 32, 5); + assert!(unsupported_narrow.contains("All")); + assert!(unsupported_narrow.contains("Bottom")); + app.session.collapsed_deployment.insert(DeploymentPane::Logs); + app.logs.push(LogEntry::application(LogLevel::Warning, "warn")); + app.logs.push(LogEntry::operation(LogLevel::Error, "error")); + let title = collapsed_title(&app, 100, 0); + assert!(title.contains("1 warning / 1 error")); + } + + #[test] + fn compact_titles_keep_filter_position_and_collapsed_counts_exactly() { + let mut app = App::new(vec![], 8, 8); + app.durable_log_path = + "/very/long/durable/path/to/voxel-tui.log".into(); + app.logs_filter = LogFilter::Error; + for level in + [LogLevel::Error, LogLevel::Error, LogLevel::Error, LogLevel::Error] + { + app.logs.push(LogEntry::application(level, "record")); + } + app.logs.push(LogEntry::application(LogLevel::Warning, "warning")); + app.logs.scroll = 1; + + assert_eq!( + collapsed_title(&app, 48, 1), + "Logs: … · Error · 2 above / 1 below" + ); + + app.session.collapsed_deployment.insert(DeploymentPane::Logs); + app.logs.scroll = 0; + assert_eq!( + collapsed_title(&app, 48, 0), + "Logs: … · Error · Bottom · 1W/4E" + ); + assert_eq!(collapsed_title(&app, 1, 0), ""); + } + + #[test] + fn rendered_48_column_titles_keep_durable_suffixes() { + let mut app = App::new(vec![], 8, 8); + app.durable_log_path = + "/very/long/durable/path/to/voxel-tui.log".into(); + app.logs_filter = LogFilter::Error; + for level in [ + LogLevel::Error, + LogLevel::Error, + LogLevel::Error, + LogLevel::Error, + LogLevel::Warning, + ] { + app.logs.push(LogEntry::application(level, "record")); + } + app.logs.scroll = 1; + let expanded = render(&app, 48, 3); + assert!(expanded.contains("Error · 2 above / 1 below"), "{expanded}"); + + app.session.collapsed_deployment.insert(DeploymentPane::Logs); + app.logs.scroll = 0; + let collapsed = render(&app, 48, 1); + assert!(collapsed.contains("Error · Bottom · 1W/4E"), "{collapsed}"); + } + + #[test] + fn six_entries_can_scroll_in_a_five_row_viewport_and_title_tracks_position() + { + let mut app = App::new(vec![], 8, 8); + for index in 1..=6 { + app.logs.push(LogEntry::application( + LogLevel::Info, + format!("entry-{index}"), + )); + } + assert!(collapsed_title(&app, 100, 5).contains("Bottom")); + app.logs.scroll = 1; + assert!(collapsed_title(&app, 100, 5).contains("Top")); + } + + #[test] + fn filtering_preserves_arrival_order_and_range_uses_filtered_entries() { + let mut app = App::new(vec![], 8, 8); + app.session.terminal.height = 5; + app.logs.push(LogEntry::application(LogLevel::Error, "accepted-one")); + app.logs.push(LogEntry::operation(LogLevel::Info, "filtered")); + app.logs.push(LogEntry::operation(LogLevel::Error, "accepted-two")); + app.logs_filter = LogFilter::Error; + let text = render(&app, 100, 4); + assert!( + text.find("accepted-one").unwrap() + < text.find("accepted-two").unwrap() + ); + assert!(!text.contains("filtered")); + assert!(text.contains("Bottom")); + } +} diff --git a/voxel/src/tui/ui/mod.rs b/voxel/src/tui/ui/mod.rs new file mode 100644 index 0000000..df09443 --- /dev/null +++ b/voxel/src/tui/ui/mod.rs @@ -0,0 +1,39 @@ +pub mod colors; +pub mod confirm_dialog; +pub mod deployment; +pub mod external_monitoring; +pub mod help; +pub mod logs; +pub mod monitor; +pub mod node_detail; +pub mod rack_selector; +pub mod renderer; +#[allow(dead_code)] +// Consumed by the topology renderer and reducer in subsequent tasks. +pub(crate) mod topology; +pub mod widgets; + +#[cfg(test)] +mod tests { + use super::renderer::{ + LayoutMode, MIN_HEIGHT, MIN_WIDTH, WIDE_HEIGHT, WIDE_WIDTH, layout_mode, + }; + use ratatui::layout::Rect; + + #[test] + fn responsive_boundaries_are_explicit() { + assert_eq!(layout_mode(Rect::new(0, 0, 0, 0)), LayoutMode::Minimum); + assert_eq!( + layout_mode(Rect::new(0, 0, MIN_WIDTH, MIN_HEIGHT)), + LayoutMode::Compact + ); + assert_eq!( + layout_mode(Rect::new(0, 0, WIDE_WIDTH, MIN_HEIGHT)), + LayoutMode::Compact + ); + assert_eq!( + layout_mode(Rect::new(0, 0, WIDE_WIDTH, WIDE_HEIGHT)), + LayoutMode::Wide + ); + } +} diff --git a/voxel/src/tui/ui/monitor.rs b/voxel/src/tui/ui/monitor.rs new file mode 100644 index 0000000..54c1c35 --- /dev/null +++ b/voxel/src/tui/ui/monitor.rs @@ -0,0 +1,1335 @@ +use super::{ + colors::{ + OX_GREEN_LIGHT, OX_RED, TUI_GREEN, TUI_GREY, TUI_GREY_DARK, TUI_PURPLE, + TUI_YELLOW, + }, + renderer::LayoutMode, + widgets::{ + fit_terminal_width, format_rate, section_block, section_heights, + section_rects, terminal_width, traffic_style, + }, +}; +use crate::{ + tui::reconcile::ObservedDeploymentState, + tui::{ + App, + event::MonitoringPane, + telemetry::{ + BidirectionalRate, Freshness, HealthContext, HealthState, + LatestSample, ResourceDescriptor, ResourceId, ResourceKind, + TrafficSeverity, derive_health_state, + }, + }, +}; +use ratatui::{ + layout::{Alignment, Constraint, Rect}, + style::{Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Cell, Paragraph, Row, Sparkline, Table}, +}; +use std::time::Duration; + +const TRAFFIC_STALE_AFTER: Duration = Duration::from_secs(15); +const TRAFFIC_UNAVAILABLE_AFTER: Duration = Duration::from_secs(60); +// A sled health probe performs four serial calls, each with a ten-second +// timeout. Keep a successful sample healthy through one slow probe cycle. +const HEALTH_STALE_AFTER: Duration = Duration::from_secs(60); +const HEALTH_UNAVAILABLE_AFTER: Duration = Duration::from_secs(120); + +pub fn draw( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + mode: LayoutMode, +) { + let rows = monitor_rows(area, app, mode); + super::rack_selector::draw( + frame, + rows[0], + app, + app.session.monitoring_pane == MonitoringPane::RackSummary, + app.session.monitoring_expanded(MonitoringPane::RackSummary), + ); + draw_topology(frame, rows[1], app, mode); + draw_top_zones(frame, rows[2], app, app.session.selected_rack); +} + +pub(crate) fn monitor_rows( + area: Rect, + app: &App, + mode: LayoutMode, +) -> Vec { + let preferred = match (mode, area.height) { + (LayoutMode::Wide, 0..=17) => [4, 3, 3], + (LayoutMode::Wide, _) => [5, 12, 7], + (LayoutMode::Compact, 0..=11) => [3, 3, 2], + (LayoutMode::Compact, _) => [4, 8, 3], + (LayoutMode::Minimum, _) => [1, 1, 1], + }; + let expanded = + MonitoringPane::ORDER.map(|pane| app.session.monitoring_expanded(pane)); + let focused = MonitoringPane::ORDER + .iter() + .position(|pane| *pane == app.session.monitoring_pane) + .unwrap_or(1); + let heights = section_heights( + area.height, + &expanded, + focused, + &preferred, + &[0, 2, 1], + 1, + ); + section_rects(area, &heights) +} + +#[derive(Clone, Copy)] +pub(crate) struct MiddleLayout { + pub(crate) topology: Rect, + pub(crate) inspector: Rect, + pub(crate) divider: Rect, +} + +pub(crate) fn middle_layout(area: Rect, mode: LayoutMode) -> MiddleLayout { + let inner = Block::bordered().inner(area); + if mode == LayoutMode::Wide { + let topology_width = ((u32::from(inner.width) * 3 / 5) as u16) + .max(1) + .min(inner.width.saturating_sub(1)); + let divider = Rect::new( + inner.x.saturating_add(topology_width), + inner.y, + 1, + inner.height, + ); + return MiddleLayout { + topology: Rect::new(inner.x, inner.y, topology_width, inner.height), + inspector: Rect::new( + divider.x.saturating_add(1), + inner.y, + inner.right().saturating_sub(divider.x.saturating_add(1)), + inner.height, + ), + divider, + }; + } + let inspector_height: u16 = if inner.height <= 4 { 2 } else { 3 }; + let topology_height = + inner.height.saturating_sub(inspector_height.saturating_add(1)).min(9); + let divider_y = inner.y.saturating_add(topology_height); + MiddleLayout { + topology: Rect::new(inner.x, inner.y, inner.width, topology_height), + divider: Rect::new( + inner.x, + divider_y, + inner.width, + u16::from(inner.height > topology_height), + ), + inspector: Rect::new( + inner.x, + divider_y.saturating_add(1), + inner.width, + inner.bottom().saturating_sub(divider_y.saturating_add(1)), + ), + } +} + +pub(crate) fn page_capacity(app: &App) -> usize { + let (area, mode) = super::widgets::content_area(app); + if !app.session.monitoring_expanded(MonitoringPane::Topology) { + return 1; + } + let rows = monitor_rows(area, app, mode); + if rows[1].height <= 2 { + return 1; + } + let scoped = scoped_descriptors(app, app.session.selected_rack); + let scene = super::topology::layout_scene( + middle_layout(rows[1], mode).topology, + mode, + &scoped, + app.session.selected_resource.as_ref(), + app.session.monitor_scroll, + ); + scene + .tiers + .iter() + .find(|tier| { + tier.visible_ids + .iter() + .any(|id| Some(id) == app.session.selected_resource.as_ref()) + }) + .map_or(1, |tier| tier.visible_ids.len()) + .max(1) +} + +pub(crate) fn resource_health_state(app: &App, id: &ResourceId) -> HealthState { + let kind = resource_kind(app, id); + if kind != Some(ResourceKind::Sled) { + return app + .observability + .traffic_failures + .get(id) + .map_or(HealthState::Unavailable, |sample| { + collection_health_state(app, sample) + }); + } + let Some(sample) = app.observability.health.get(id) else { + return HealthState::Unavailable; + }; + let context = if app.deployment.observed == ObservedDeploymentState::Stopped + { + HealthContext::Stopped + } else if app.now.is_none() + || (sample.good.is_none() && sample.last_attempt.is_none()) + { + HealthContext::Checking + } else { + HealthContext::Active + }; + let freshness = app + .now + .map(|now| { + sample.freshness(now, HEALTH_STALE_AFTER, HEALTH_UNAVAILABLE_AFTER) + }) + .unwrap_or(Freshness::Unavailable); + derive_health_state( + context, + sample.good.as_ref().map(|good| &good.value), + freshness, + ) +} + +fn collection_health_state( + app: &App, + sample: &LatestSample, +) -> HealthState { + if app.deployment.observed == ObservedDeploymentState::Stopped { + return HealthState::Stopped; + } + let Some(now) = app.now else { + return HealthState::Checking; + }; + if sample.good.is_none() && sample.last_attempt.is_none() { + return HealthState::Checking; + } + match sample.freshness(now, TRAFFIC_STALE_AFTER, TRAFFIC_UNAVAILABLE_AFTER) + { + Freshness::Fresh => HealthState::Healthy, + Freshness::Stale => HealthState::Stale, + Freshness::Unavailable => HealthState::Unavailable, + } +} + +fn resource_kind(app: &App, id: &ResourceId) -> Option { + app.deployment + .topology + .iter() + .find(|descriptor| &descriptor.id == id) + .map(|descriptor| descriptor.kind) +} + +fn resource_last_success( + app: &App, + id: &ResourceId, +) -> Option { + if resource_kind(app, id) == Some(ResourceKind::Sled) { + app.observability.health.get(id).and_then(|sample| { + sample.good.as_ref().map(|good| good.captured_at) + }) + } else { + app.observability.traffic_failures.get(id).and_then(|sample| { + sample.good.as_ref().map(|good| good.captured_at) + }) + } +} + +pub(crate) fn health_status_label(state: HealthState) -> &'static str { + match state { + HealthState::Healthy => "Healthy", + HealthState::Degraded => "Degraded", + HealthState::Failed => "Failed", + HealthState::Checking + | HealthState::Unknown + | HealthState::Stale + | HealthState::Unavailable + | HealthState::Stopped => "Checking Status", + } +} + +pub(crate) fn resource_health_summary(app: &App, id: &ResourceId) -> String { + let age = resource_last_success(app, id) + .and_then(|captured_at| { + app.now + .map(|now| now.saturating_duration_since(captured_at).as_secs()) + }) + .map(|seconds| format!("{seconds}s ago")) + .unwrap_or_else(|| "never".into()); + let error = if resource_kind(app, id) == Some(ResourceKind::Sled) { + app.observability + .health + .get(id) + .and_then(|sample| sample.latest_error.as_ref()) + } else { + app.observability + .traffic_failures + .get(id) + .and_then(|sample| sample.latest_error.as_ref()) + } + .map(|error| format!("; latest error: {}", error.message)) + .unwrap_or_default(); + format!( + "{} (last success {age}){error}", + health_status_label(resource_health_state(app, id)) + ) +} + +pub(crate) fn health_style(state: HealthState) -> Style { + Style::default().fg(match state { + HealthState::Healthy => TUI_GREEN, + HealthState::Degraded | HealthState::Failed => OX_RED, + HealthState::Checking => TUI_YELLOW, + HealthState::Stale => TUI_GREY, + HealthState::Unknown + | HealthState::Unavailable + | HealthState::Stopped => TUI_GREY_DARK, + }) +} + +pub(crate) fn sparkline_data( + values: impl DoubleEndedIterator, + width: u16, +) -> Vec { + values + .rev() + .take(width as usize) + .collect::>() + .into_iter() + .rev() + .map(|value| value.max(0.0).min(u64::MAX as f64) as u64) + .collect() +} + +fn draw_topology( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + mode: LayoutMode, +) { + let rack = app.session.selected_rack; + let expanded = app.session.monitoring_expanded(MonitoringPane::Topology); + let focused = app.session.monitoring_pane == MonitoringPane::Topology; + let outer = section_block("Topology", expanded, focused); + frame.render_widget(outer.clone(), area); + if !expanded || outer.inner(area).height == 0 { + return; + } + let layout = middle_layout(area, mode); + let scoped = scoped_descriptors(app, rack); + let scene = super::topology::layout_scene( + layout.topology, + mode, + &scoped, + app.session.selected_resource.as_ref(), + app.session.monitor_scroll, + ); + super::topology::draw(frame, &scene, app); + let edge = + Style::default().fg(if focused { TUI_YELLOW } else { OX_GREEN_LIGHT }); + for y in layout.divider.y..layout.divider.bottom() { + let line = if mode == LayoutMode::Wide { + "│".to_string() + } else { + "─".repeat(layout.divider.width.into()) + }; + frame.render_widget( + Paragraph::new(line).style(edge), + Rect::new(layout.divider.x, y, layout.divider.width, 1), + ); + } + if mode == LayoutMode::Wide && area.height >= 2 { + frame.render_widget( + Paragraph::new("┬").style(edge), + Rect::new(layout.divider.x, area.y, 1, 1), + ); + frame.render_widget( + Paragraph::new("┴").style(edge), + Rect::new(layout.divider.x, area.bottom() - 1, 1, 1), + ); + } else if layout.divider.height > 0 { + frame.render_widget( + Paragraph::new("├").style(edge), + Rect::new(area.x, layout.divider.y, 1, 1), + ); + frame.render_widget( + Paragraph::new("┤").style(edge), + Rect::new(area.right() - 1, layout.divider.y, 1, 1), + ); + } + if layout.inspector.width > 0 { + let title = fit_terminal_width( + " Selected Resource ", + layout.inspector.width.into(), + ); + frame.render_widget( + Paragraph::new(title.clone()).style( + Style::default().fg(TUI_PURPLE).add_modifier(Modifier::BOLD), + ), + Rect::new( + layout.inspector.x, + if mode == LayoutMode::Wide { + area.y + } else { + layout.divider.y + }, + terminal_width(&title) as u16, + 1, + ), + ); + draw_selected_resource_inspector(frame, layout.inspector, app); + } +} + +fn draw_selected_resource_inspector( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, +) { + let Some(id) = app.session.selected_resource.as_ref() else { + frame.render_widget(Paragraph::new("No resource selected"), area); + return; + }; + let Some(descriptor) = + app.deployment.topology.iter().find(|descriptor| &descriptor.id == id) + else { + frame.render_widget(Paragraph::new("No resource selected"), area); + return; + }; + if area.height <= 7 { + let traffic = app.observability.telemetry.resources.get(id); + let sampled = traffic.and_then(|value| value.current_at).is_some(); + let mut lines = Vec::new(); + if area.height >= 3 { + lines.push(Line::from(vec![ + Span::raw(format!( + "{:?} {} · ", + descriptor.kind, descriptor.name + )), + Span::styled( + health_status_label(resource_health_state(app, id)), + health_style(resource_health_state(app, id)), + ), + ])); + } + lines.push(if sampled { + let traffic = traffic.expect("sampled traffic exists"); + Line::from(format!( + "{} · source: {}", + rate_line(traffic.current_rate), + traffic.current_sample.source.label() + )) + } else { + Line::from("RX — TX — Total — · collecting") + }); + let data = sparkline_data( + traffic.into_iter().flat_map(|value| { + value + .history + .points() + .iter() + .map(|point| point.rate.total_bytes_sec()) + }), + area.width.saturating_sub(9), + ); + if data.is_empty() { + lines.push(Line::from("History: collecting (no samples)")); + frame.render_widget(Paragraph::new(lines), area); + } else { + lines.push(Line::from("History ")); + let history_y = + area.y.saturating_add(lines.len().saturating_sub(1) as u16); + frame.render_widget(Paragraph::new(lines), area); + let label_width = terminal_width("History ") as u16; + frame.render_widget( + Sparkline::default() + .data(&data) + .style(health_style(resource_health_state(app, id))), + Rect::new( + area.x.saturating_add(label_width), + history_y, + area.width.saturating_sub(label_width), + 1, + ), + ); + } + return; + } + let age = resource_last_success(app, id) + .and_then(|captured_at| { + app.now + .map(|now| now.saturating_duration_since(captured_at).as_secs()) + }) + .map(|age| format!("{age}s ago")) + .unwrap_or_else(|| "never".into()); + let rack = descriptor + .rack + .map(|rack| format!("Rack {}", rack.0)) + .unwrap_or_else(|| "Fleet".into()); + let host = descriptor + .host + .as_ref() + .map(|host| format!(" · host {host}")) + .unwrap_or_default(); + let mut lines = vec![ + Line::from(format!("{:?} {}", descriptor.kind, descriptor.name)), + Line::from(format!("{rack}{host}")), + Line::from(Span::styled( + health_status_label(resource_health_state(app, id)), + health_style(resource_health_state(app, id)), + )), + Line::from(format!("Last success {age}")), + ]; + let traffic = app.observability.telemetry.resources.get(id); + let sampled = traffic.and_then(|traffic| traffic.current_at).is_some(); + if let Some(traffic) = traffic.filter(|_| sampled) { + lines.push(rate_line(traffic.current_rate)); + lines.push(Line::from(format!( + "Packets RX {:.0}/s · TX {:.0}/s", + traffic.current_rate.rx_packets_sec, + traffic.current_rate.tx_packets_sec + ))); + lines.push(Line::from(format!( + "Link errors RX {:.2}/s · TX {:.2}/s · source: {}", + traffic.current_sample.errors.rx_sec, + traffic.current_sample.errors.tx_sec, + traffic.current_sample.source.label() + ))); + } else { + lines.push(Line::from("Traffic: collecting/unavailable")); + } + if let Some(error) = latest_collection_error(app, id) { + lines.push(Line::from(format!("Latest error: {}", error.message))); + } + if let Some(rack) = descriptor.rack { + if let Some(zfs) = app + .observability + .zfs_headroom + .get(&rack) + .and_then(|sample| sample.good.as_ref()) + { + let pools = zfs.value.iter().filter(|pool| pool.id == *id); + let (available, total, count) = pools.fold( + (0_u64, 0_u64, 0_usize), + |(available, total, count), pool| { + ( + available.saturating_add(pool.available_bytes()), + total.saturating_add(pool.total_bytes), + count + 1, + ) + }, + ); + if count > 0 { + lines.push(Line::from(format!( + "ZFS {:.1}/{:.1} GiB free · {count} pools", + available as f64 / 1024.0_f64.powi(3), + total as f64 / 1024.0_f64.powi(3) + ))); + } + } + if let Some(error) = app + .observability + .zone_cpu + .get(&rack) + .and_then(|sample| sample.latest_error.as_ref()) + { + lines.push(Line::from(format!( + "Zone CPU unavailable: {}", + error.message + ))); + } + if let Some(error) = app + .observability + .zfs_headroom + .get(&rack) + .and_then(|sample| sample.latest_error.as_ref()) + { + lines.push(Line::from(format!( + "ZFS unavailable: {}", + error.message + ))); + } + } + lines.push(Line::from("History (60s)")); + let guidance_y = area.bottom().saturating_sub(1); + let text_height = + lines.len().min(area.height.saturating_sub(1) as usize) as u16; + frame.render_widget( + Paragraph::new(lines), + Rect::new(area.x, area.y, area.width, text_height), + ); + let mut y = area.y.saturating_add(text_height); + if y < guidance_y { + let data = sparkline_data( + traffic.into_iter().flat_map(|traffic| { + traffic + .history + .points() + .iter() + .map(|point| point.rate.total_bytes_sec()) + }), + area.width, + ); + if data.is_empty() { + frame.render_widget( + Paragraph::new("History: collecting (no samples)"), + Rect::new(area.x, y, area.width, 1), + ); + } else { + frame.render_widget( + Sparkline::default() + .data(&data) + .style(health_style(resource_health_state(app, id))), + Rect::new(area.x, y, area.width, 1), + ); + } + y += 1; + } + let mut zones = traffic + .into_iter() + .flat_map(|traffic| traffic.current_sample.zones.iter()) + .collect::>(); + zones.sort_by(|a, b| { + b.rate + .total_bytes_sec() + .total_cmp(&a.rate.total_bytes_sec()) + .then_with(|| a.name.cmp(&b.name)) + }); + if y < guidance_y { + frame.render_widget( + Paragraph::new("Zones for selected resource") + .style(Style::default().add_modifier(Modifier::BOLD)), + Rect::new(area.x, y, area.width, 1), + ); + y += 1; + } + if let Some(rack) = descriptor.rack + && let Some(cpu) = app + .observability + .zone_cpu + .get(&rack) + .and_then(|sample| sample.good.as_ref()) + { + for zone in cpu + .value + .iter() + .filter(|zone| zone.id == *id) + .take(guidance_y.saturating_sub(y) as usize) + { + frame.render_widget( + Paragraph::new(format!( + "CPU {} {:.1}% wait {:.1}%", + zone.name, + zone.total_percent(), + zone.wait_percent + )), + Rect::new(area.x, y, area.width, 1), + ); + y += 1; + } + } + if zones.is_empty() && y < guidance_y { + frame.render_widget( + Paragraph::new("No zone samples"), + Rect::new(area.x, y, area.width, 1), + ); + y += 1; + } + let capacity = guidance_y.saturating_sub(y) as usize; + let shown = if zones.len() > capacity { + capacity.saturating_sub(1) + } else { + capacity + }; + for zone in zones.iter().take(shown) { + frame.render_widget( + Paragraph::new(format!( + "{} {}", + zone.short_name, + format_rate(zone.rate.total_bytes_sec()) + )) + .style(traffic_style( + TrafficSeverity::for_bytes_per_sec(zone.rate.total_bytes_sec()), + )), + Rect::new(area.x, y, area.width, 1), + ); + y += 1; + } + if zones.len() > shown && y < guidance_y { + frame.render_widget( + Paragraph::new(format!("+{} more", zones.len() - shown)), + Rect::new(area.x, y, area.width, 1), + ); + } + if area.height > 0 { + frame.render_widget( + Paragraph::new("Enter opens full detail") + .style(Style::default().fg(TUI_GREY)), + Rect::new(area.x, guidance_y, area.width, 1), + ); + } +} + +fn latest_collection_error<'a>( + app: &'a App, + id: &ResourceId, +) -> Option<&'a crate::tui::telemetry::CollectionError> { + [ + app.observability + .health + .get(id) + .and_then(|sample| sample.latest_error.as_ref()), + app.observability + .addresses + .get(id) + .and_then(|sample| sample.latest_error.as_ref()), + app.observability + .traffic_failures + .get(id) + .and_then(|sample| sample.latest_error.as_ref()), + ] + .into_iter() + .flatten() + .max_by_key(|error| error.attempted_at) +} + +pub(crate) fn scoped_descriptors( + app: &App, + rack: Option, +) -> Vec { + app.deployment + .topology + .iter() + .filter(|descriptor| monitor_scope_contains(app, rack, descriptor)) + .cloned() + .collect() +} + +pub(crate) fn top_zones_len( + app: &App, + rack: Option, +) -> usize { + app.observability + .telemetry + .resources + .values() + .filter(|state| monitor_scope_contains(app, rack, &state.descriptor)) + .map(|state| state.current_sample.zones.len()) + .sum() +} + +pub(crate) fn top_zones_page_capacity(app: &App) -> usize { + let (area, mode) = super::widgets::content_area(app); + let height = monitor_rows(area, app, mode)[2].height; + if height == 3 { 1 } else { usize::from(height.saturating_sub(3)) } +} + +fn monitor_scope_contains( + app: &App, + rack: Option, + descriptor: &ResourceDescriptor, +) -> bool { + descriptor.kind == ResourceKind::Router + || descriptor.rack == rack + || app.session.selected_resource.as_ref() == Some(&descriptor.id) +} + +fn rate_line(rate: BidirectionalRate) -> Line<'static> { + Line::from(vec![ + Span::styled( + format!("RX {} ", format_rate(rate.rx_bytes_sec)), + traffic_style(TrafficSeverity::for_bytes_per_sec( + rate.rx_bytes_sec, + )), + ), + Span::styled( + format!("TX {} ", format_rate(rate.tx_bytes_sec)), + traffic_style(TrafficSeverity::for_bytes_per_sec( + rate.tx_bytes_sec, + )), + ), + Span::styled( + format!("Total {}", format_rate(rate.total_bytes_sec())), + traffic_style(TrafficSeverity::for_bytes_per_sec( + rate.total_bytes_sec(), + )), + ), + ]) +} + +fn draw_top_zones( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + rack: Option, +) { + let expanded = app.session.monitoring_expanded(MonitoringPane::TopZones); + let mut zones = app + .observability + .telemetry + .resources + .values() + .filter(|state| monitor_scope_contains(app, rack, &state.descriptor)) + .flat_map(|state| { + state.current_sample.zones.iter().map(move |zone| { + (&state.descriptor.id, &state.descriptor.name, zone) + }) + }) + .collect::>(); + zones.sort_by(|(_, ra, a), (_, rb, b)| { + b.rate + .total_bytes_sec() + .total_cmp(&a.rate.total_bytes_sec()) + .then_with(|| ra.cmp(rb)) + .then_with(|| a.name.cmp(&b.name)) + }); + let capacity = if area.height == 3 { + 1 + } else { + usize::from(area.height.saturating_sub(3)) + }; + let start = app + .session + .top_zones_scroll + .min(zones.len().saturating_sub(capacity.max(1))); + let end = start.saturating_add(capacity).min(zones.len()); + let title = if capacity > 0 && zones.len() > capacity { + format!( + "Top Zones by Traffic + CPU {}-{end} of {}", + start + 1, + zones.len() + ) + } else { + "Top Zones by Traffic + CPU".into() + }; + let block = section_block( + title, + expanded, + app.session.monitoring_pane == MonitoringPane::TopZones, + ); + frame.render_widget(block.clone(), area); + if !expanded || block.inner(area).height == 0 { + return; + } + if area.height == 3 { + let sample = zones + .get(start) + .map(|(_, resource, zone)| { + format!( + "{resource} · {} · {}", + zone.short_name, + format_rate(zone.rate.total_bytes_sec()) + ) + }) + .unwrap_or_else(|| "No zone samples".into()); + frame.render_widget(Paragraph::new(sample).block(block), area); + return; + } + let header = Row::new(["Resource", "Zone", "Total", "CPU", "Wait"]) + .style(Style::default().add_modifier(Modifier::BOLD)); + let rows = zones + .iter() + .skip(start) + .take(capacity) + .map(|(resource_id, resource, zone)| { + let rates = zone_rate_cells(zone.rate); + let cpu = rack + .and_then(|rack| app.observability.zone_cpu.get(&rack)) + .and_then(|sample| sample.good.as_ref()) + .and_then(|sample| { + sample.value.iter().find(|cpu| { + cpu.id == **resource_id && cpu.name == zone.name + }) + }); + Row::new([ + Cell::from((*resource).clone()), + Cell::from(zone.short_name.clone()), + rates[2].clone(), + Cell::from( + cpu.map(|cpu| format!("{:.1}%", cpu.total_percent())) + .unwrap_or_else(|| "—".into()), + ), + Cell::from( + cpu.map(|cpu| format!("{:.1}%", cpu.wait_percent)) + .unwrap_or_else(|| "—".into()), + ), + ]) + }) + .collect::>(); + if zones.is_empty() { + frame.render_widget( + Paragraph::new("No zone samples").block(block), + area, + ); + } else { + frame.render_widget( + Table::new( + rows, + [ + Constraint::Percentage(22), + Constraint::Percentage(28), + Constraint::Percentage(20), + Constraint::Percentage(15), + Constraint::Percentage(15), + ], + ) + .header(header) + .block(block), + area, + ); + } +} + +pub(crate) fn zone_rate_cells(rate: BidirectionalRate) -> [Cell<'static>; 3] { + let cell = |value| { + Cell::from(Line::from(format_rate(value)).alignment(Alignment::Right)) + .style(traffic_style(TrafficSeverity::for_bytes_per_sec(value))) + }; + [ + cell(rate.rx_bytes_sec), + cell(rate.tx_bytes_sec), + cell(rate.total_bytes_sec()), + ] +} + +#[cfg(test)] +mod height_tests { + use super::*; + use crate::tui::reconcile::ObservedDeploymentState; + use crate::tui::{ + event::AppEvent, + telemetry::{ + HealthDiagnostic, RackId, ResourceKind, ServiceState, + TrafficSample, ZoneTraffic, + }, + }; + use ratatui::{Terminal, backend::TestBackend, widgets::Paragraph}; + use std::time::Instant; + + fn app() -> App { + let descriptor = ResourceDescriptor { + id: ResourceId::rack(RackId(0), ResourceKind::Sled, "seeded"), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "seeded".into(), + host: None, + }; + let mut app = App::new( + vec![ResourceDescriptor { + id: descriptor.id.clone(), + ..descriptor.clone() + }], + 4, + 4, + ); + app.update(AppEvent::Traffic { + id: descriptor.id, + at: Instant::now(), + sample: TrafficSample { + zones: vec![ZoneTraffic { + name: "height-two-zone-traffic".into(), + short_name: "height-two-zone-traffic".into(), + rate: BidirectionalRate { + rx_bytes_sec: 987_654.0, + ..Default::default() + }, + errors: Default::default(), + }], + ..Default::default() + }, + }); + app + } + + fn assert_height_two_chrome( + title: &str, + draw_section: impl FnOnce(&mut ratatui::Frame<'_>, Rect, &App), + ) { + let mut terminal = Terminal::new(TestBackend::new(48, 2)).unwrap(); + let app = app(); + terminal + .draw(|frame| { + let area = frame.area(); + frame.render_widget(Paragraph::new("I".repeat(96)), area); + draw_section(frame, area, &app); + }) + .unwrap(); + let buffer = terminal.backend().buffer(); + let text = buffer + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!(text.contains(title), "missing {title:?}: {text}"); + assert_eq!(buffer[(0, 0)].symbol(), "┌"); + assert_eq!(buffer[(47, 0)].symbol(), "┐"); + assert_eq!(buffer[(0, 1)].symbol(), "└"); + assert_eq!(buffer[(47, 1)].symbol(), "┘"); + assert!(!text.contains('I'), "seeded content survived: {text}"); + } + + #[test] + fn height_two_monitoring_sections_render_only_intact_chrome() { + assert_height_two_chrome("Rack Summary", |frame, area, app| { + super::super::rack_selector::draw(frame, area, app, false, true); + }); + assert_height_two_chrome("Topology", |frame, area, app| { + draw_topology(frame, area, app, LayoutMode::Wide); + }); + assert_height_two_chrome("Top Zones by Traffic", |frame, area, app| { + draw_top_zones(frame, area, app, Some(RackId(0))); + }); + + let mut terminal = Terminal::new(TestBackend::new(48, 2)).unwrap(); + let app = app(); + terminal + .draw(|frame| { + draw_top_zones(frame, frame.area(), &app, Some(RackId(0))) + }) + .unwrap(); + let text = terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!(text.contains("Top Zones by Traffic")); + assert!(!text.contains("height-two-zone-traffic")); + let buffer = terminal.backend().buffer(); + assert_eq!(buffer[(0, 0)].symbol(), "┌"); + assert_eq!(buffer[(47, 0)].symbol(), "┐"); + assert_eq!(buffer[(0, 1)].symbol(), "└"); + assert_eq!(buffer[(47, 1)].symbol(), "┘"); + } + + #[test] + fn topology_divider_follows_focus_without_recoloring_selection() { + let area = Rect::new(0, 0, 160, 20); + let layout = middle_layout(area, LayoutMode::Wide); + let render = |app: &App| { + let mut terminal = + Terminal::new(TestBackend::new(160, 20)).unwrap(); + terminal + .draw(|frame| draw_topology(frame, area, app, LayoutMode::Wide)) + .unwrap(); + terminal.backend().buffer().clone() + }; + + let focused = render(&app()); + assert_eq!(focused[(layout.divider.x, area.y)].fg, TUI_YELLOW); + assert_eq!(focused[(layout.divider.x, area.y + 1)].fg, TUI_YELLOW); + assert_eq!(focused[(layout.inspector.x + 1, area.y)].fg, TUI_PURPLE); + + let mut inactive_app = app(); + inactive_app.session.monitoring_pane = MonitoringPane::TopZones; + let inactive = render(&inactive_app); + assert_eq!(inactive[(layout.divider.x, area.y)].fg, OX_GREEN_LIGHT); + assert_eq!(inactive[(layout.divider.x, area.y + 1)].fg, OX_GREEN_LIGHT); + assert_eq!(inactive[(layout.inspector.x + 1, area.y)].fg, TUI_PURPLE); + } + + #[test] + fn top_zones_capacity_is_zero_when_only_chrome_is_visible() { + let mut app = app(); + app.session.terminal = + crate::tui::app::TerminalSize { width: 48, height: 16 }; + + assert_eq!(top_zones_page_capacity(&app), 0); + } + + #[test] + fn successful_traffic_probe_makes_non_sled_resources_healthy() { + let descriptors = [ + (ResourceKind::SwitchZone, Some(RackId(0)), "switch0"), + (ResourceKind::Router, None, "ce"), + ] + .into_iter() + .map(|(kind, rack, name)| ResourceDescriptor { + id: rack.map_or_else( + || ResourceId::fleet(kind, name), + |rack| ResourceId::rack(rack, kind, name), + ), + rack, + kind, + name: name.into(), + host: None, + }) + .collect::>(); + let mut app = App::new(descriptors.clone(), 4, 4); + let now = Instant::now(); + app.deployment.observed = ObservedDeploymentState::Running; + app.update(AppEvent::Tick { now }); + + for descriptor in &descriptors { + assert_eq!( + resource_health_state(&app, &descriptor.id), + HealthState::Checking + ); + app.update(AppEvent::Traffic { + id: descriptor.id.clone(), + at: now, + sample: TrafficSample::default(), + }); + assert_eq!( + resource_health_state(&app, &descriptor.id), + HealthState::Healthy + ); + } + } + + #[test] + fn sled_health_does_not_expire_between_slow_probe_completions() { + let descriptor = ResourceDescriptor { + id: ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "g0".into(), + host: None, + }; + let mut app = App::new(vec![descriptor.clone()], 4, 4); + let sampled_at = Instant::now(); + app.deployment.observed = ObservedDeploymentState::Running; + app.update(AppEvent::Health { + id: descriptor.id.clone(), + at: sampled_at, + diagnostic: HealthDiagnostic { + sled_agent: Some(ServiceState::Online), + ntp: crate::tui::telemetry::NtpDiagnostic { + synchronized: Some(true), + ..Default::default() + }, + ..Default::default() + }, + }); + app.update(AppEvent::Tick { + now: sampled_at + Duration::from_secs(45), + }); + + assert_eq!( + resource_health_state(&app, &descriptor.id), + HealthState::Healthy + ); + } + + #[test] + fn non_sled_inspector_uses_simple_health_status() { + let descriptor = ResourceDescriptor { + id: ResourceId::fleet(ResourceKind::Router, "ce"), + rack: None, + kind: ResourceKind::Router, + name: "ce".into(), + host: None, + }; + let mut app = App::new(vec![descriptor.clone()], 4, 4); + let now = Instant::now(); + app.deployment.observed = ObservedDeploymentState::Running; + app.session.selected_resource = Some(descriptor.id.clone()); + app.update(AppEvent::Tick { now }); + let mut terminal = Terminal::new(TestBackend::new(80, 12)).unwrap(); + terminal + .draw(|frame| { + draw_selected_resource_inspector(frame, frame.area(), &app) + }) + .unwrap(); + let checking = terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!(checking.contains("Checking Status"), "{checking}"); + + app.update(AppEvent::Traffic { + id: descriptor.id, + at: now, + sample: TrafficSample::default(), + }); + + terminal + .draw(|frame| { + draw_selected_resource_inspector(frame, frame.area(), &app) + }) + .unwrap(); + + let text = terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!(text.contains("Healthy"), "{text}"); + assert!(!text.contains("Freshness"), "{text}"); + assert!(text.contains("Last success 0s ago"), "{text}"); + assert!(text.contains("source: direct probe"), "{text}"); + } + + #[test] + fn sled_inspector_explains_unavailable_oximeter_diagnostics() { + let mut app = app(); + let id = app.deployment.topology[0].id.clone(); + app.session.selected_resource = Some(id); + let now = Instant::now(); + app.update(AppEvent::ZoneCpuFailed { + rack: RackId(0), + at: now, + message: "CPU query timed out".into(), + }); + app.update(AppEvent::ZfsHeadroomFailed { + rack: RackId(0), + at: now, + message: "ZFS response omitted a pool".into(), + }); + let mut terminal = Terminal::new(TestBackend::new(100, 18)).unwrap(); + + terminal + .draw(|frame| { + draw_selected_resource_inspector(frame, frame.area(), &app) + }) + .unwrap(); + + let text = terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!( + text.contains("Zone CPU unavailable: CPU query timed out"), + "{text}" + ); + assert!( + text.contains("ZFS unavailable: ZFS response omitted a pool"), + "{text}" + ); + } + + #[test] + fn rack_summary_names_session_traffic_history() { + let app = app(); + let mut terminal = Terminal::new(TestBackend::new(160, 5)).unwrap(); + + terminal + .draw(|frame| { + super::super::rack_selector::draw( + frame, + frame.area(), + &app, + false, + true, + ) + }) + .unwrap(); + + let text = terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!(text.contains("Rack traffic (this TUI session)"), "{text}"); + } + + #[test] + fn top_zones_renders_the_scrolled_range() { + let descriptor = ResourceDescriptor { + id: ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "g0".into(), + host: None, + }; + let mut app = App::new(vec![descriptor.clone()], 4, 4); + app.session.top_zones_scroll = 1; + app.update(AppEvent::Traffic { + id: descriptor.id, + at: Instant::now(), + sample: TrafficSample { + zones: (0..8) + .map(|index| ZoneTraffic { + name: format!("zone-{index}"), + short_name: format!("zone-{index}"), + rate: BidirectionalRate { + rx_bytes_sec: (8 - index) as f64, + ..Default::default() + }, + errors: Default::default(), + }) + .collect(), + ..Default::default() + }, + }); + let mut terminal = Terminal::new(TestBackend::new(100, 7)).unwrap(); + + terminal + .draw(|frame| { + draw_top_zones(frame, frame.area(), &app, Some(RackId(0))) + }) + .unwrap(); + + let text = terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect::(); + assert!(text.contains("Top Zones by Traffic + CPU 2-5 of 8"), "{text}"); + assert!(!text.contains("zone-0"), "{text}"); + for zone in 1..=4 { + assert!(text.contains(&format!("zone-{zone}")), "{text}"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::telemetry::BidirectionalRate; + use ratatui::{buffer::Buffer, layout::Rect, widgets::Widget}; + + #[test] + fn sparkline_fits_width_and_clamps_values() { + assert_eq!( + sparkline_data([-1.0, 2.0, 3.0, 4.0].into_iter(), 2), + vec![3, 4] + ); + assert_eq!(sparkline_data([-1.0, 2.9].into_iter(), 8), vec![0, 2]); + } + + #[test] + fn zone_rate_cells_keep_independent_severity() { + let cells = zone_rate_cells(BidirectionalRate { + rx_bytes_sec: 100_000.0, + tx_bytes_sec: 5_000_000.0, + ..Default::default() + }); + let row = Row::new(cells); + let table = Table::new( + [row], + [ + Constraint::Length(12), + Constraint::Length(12), + Constraint::Length(12), + ], + ); + let mut buffer = Buffer::empty(Rect::new(0, 0, 36, 1)); + table.render(Rect::new(0, 0, 36, 1), &mut buffer); + assert_eq!(buffer[(5, 0)].fg, super::super::colors::OX_OFF_WHITE); + assert_eq!(buffer[(17, 0)].fg, super::super::colors::TUI_YELLOW); + assert_eq!(buffer[(29, 0)].fg, super::super::colors::OX_RED); + } +} diff --git a/voxel/src/tui/ui/node_detail.rs b/voxel/src/tui/ui/node_detail.rs new file mode 100644 index 0000000..d3226ae --- /dev/null +++ b/voxel/src/tui/ui/node_detail.rs @@ -0,0 +1,241 @@ +use super::{ + monitor::{ + health_style, resource_health_state, resource_health_summary, + sparkline_data, zone_rate_cells, + }, + widgets::{centered_rect, format_rate, overlay_area, selection_style}, +}; +use crate::tui::{App, telemetry::TrafficSeverity}; +use ratatui::{ + layout::{Constraint, Layout}, + style::Modifier, + text::Line, + widgets::{Block, Cell, Clear, Paragraph, Row, Sparkline, Table}, +}; + +pub fn draw(frame: &mut ratatui::Frame<'_>, app: &App) { + let Some(id) = app.session.selected_resource.as_ref() else { + return; + }; + let Some(d) = app.deployment.topology.iter().find(|d| &d.id == id) else { + return; + }; + let Some(t) = app.observability.telemetry.resources.get(id) else { + return; + }; + let area = centered_rect(85, 80, overlay_area(frame.area())); + let rate = t.current_rate; + let health = &app.observability.health[id]; + let addresses = &app.observability.addresses[id]; + let mut lines = vec![ + Line::from(format!( + "Scope {:?} | Rack {:?} | {:?} {} | host {}", + id.scope, + d.rack, + d.kind, + d.name, + d.host.as_deref().unwrap_or("n/a") + )), + Line::from(format!( + "RX {} ({:.0} pkt/s) TX {} ({:.0} pkt/s) Total {} [{:?}] · source: {}", + format_rate(rate.rx_bytes_sec), + rate.rx_packets_sec, + format_rate(rate.tx_bytes_sec), + rate.tx_packets_sec, + format_rate(rate.total_bytes_sec()), + TrafficSeverity::for_bytes_per_sec(rate.total_bytes_sec()), + t.current_sample.source.label() + )), + Line::from(format!( + "Link errors: RX {:.2}/s TX {:.2}/s", + t.current_sample.errors.rx_sec, t.current_sample.errors.tx_sec + )), + Line::from(format!("Health: {}", resource_health_summary(app, id))), + ]; + if let Some(g) = &health.good { + lines.push(Line::from(format!( + "Diagnostics: agent {:?}; NTP sync {:?} stratum {:?}; failed {}; zones {}; notes {}", + g.value.sled_agent, + g.value.ntp.synchronized, + g.value.ntp.stratum, + summarize(&g.value.failed_services, 3), + summarize(&g.value.zones.zones, 3), + summarize(&g.value.notes, 3) + ))); + } else { + lines.push(Line::from("Diagnostics: no successful health sample")); + } + lines.push(Line::from(format!( + "Health latest error: {}", + health + .latest_error + .as_ref() + .map(|e| e.message.as_str()) + .unwrap_or("none") + ))); + if let Some(a) = &addresses.good { + lines.push(Line::from(format!( + "IPv4: {} | IPv6: {}", + summarize(&a.value.ipv4, 3), + summarize(&a.value.ipv6, 3) + ))); + } else { + lines.push(Line::from("Addresses: no successful sample")); + } + lines.push(Line::from(format!( + "Address latest error: {}", + addresses + .latest_error + .as_ref() + .map(|e| e.message.as_str()) + .unwrap_or("none") + ))); + lines.push(Line::from(format!( + "Traffic latest error: {}", + app.observability.traffic_failures[id] + .latest_error + .as_ref() + .map(|e| e.message.as_str()) + .unwrap_or("none") + ))); + if let Some(rack) = d.rack { + if let Some(cpu) = app + .observability + .zone_cpu + .get(&rack) + .and_then(|sample| sample.good.as_ref()) + { + let summary = cpu + .value + .iter() + .take(3) + .map(|zone| { + format!( + "{} {:.1}% wait {:.1}%", + zone.name, + zone.total_percent(), + zone.wait_percent + ) + }) + .collect::>() + .join("; "); + lines.push(Line::from(format!("Zone CPU: {summary}"))); + } + if let Some(pools) = app + .observability + .zfs_headroom + .get(&rack) + .and_then(|sample| sample.good.as_ref()) + { + let summary = pools + .value + .iter() + .filter(|pool| pool.id == *id) + .map(|pool| { + format!( + "{} {:.1}/{:.1} GiB free", + pool.pool, + pool.available_bytes() as f64 / 1024.0_f64.powi(3), + pool.total_bytes as f64 / 1024.0_f64.powi(3) + ) + }) + .collect::>() + .join(", "); + lines.push(Line::from(format!("ZFS: {summary}"))); + } + } + + let mut zones = t.current_sample.zones.iter().collect::>(); + zones.sort_by(|left, right| { + right + .rate + .total_bytes_sec() + .total_cmp(&left.rate.total_bytes_sec()) + .then_with(|| left.name.cmp(&right.name)) + }); + frame.render_widget(Clear, area); + let block = Block::bordered() + .title(Line::styled( + " ▶ Resource detail ", + selection_style().add_modifier(Modifier::REVERSED), + )) + .border_style(health_style(resource_health_state(app, id))); + let inner = block.inner(area); + frame.render_widget(block, area); + let diagnostic_height = + (lines.len() as u16).min(inner.height.saturating_sub(4)); + let rows = Layout::vertical([ + Constraint::Length(diagnostic_height), + Constraint::Length(2), + Constraint::Min(2), + ]) + .split(inner); + frame.render_widget(Paragraph::new(lines), rows[0]); + let data = sparkline_data( + t.history.points().iter().map(|point| point.rate.total_bytes_sec()), + rows[1].width, + ); + if data.is_empty() { + frame.render_widget( + Paragraph::new("History: collecting (no samples)"), + rows[1], + ); + } else { + frame.render_widget( + Sparkline::default() + .data(&data) + .style(health_style(resource_health_state(app, id))) + .block(Block::default().title(" History ")), + rows[1], + ); + } + if zones.is_empty() { + frame.render_widget( + Paragraph::new("No zone samples") + .block(Block::default().title(" Zones ")), + rows[2], + ); + return; + } + let shown = rows[2].height.saturating_sub(2) as usize; + let count = zones.len(); + let table_rows = zones.into_iter().take(shown).map(|zone| { + let rates = zone_rate_cells(zone.rate); + Row::new([ + Cell::from(zone.short_name.clone()), + rates[0].clone(), + rates[1].clone(), + rates[2].clone(), + ]) + }); + frame.render_widget( + Table::new( + table_rows, + [ + Constraint::Percentage(40), + Constraint::Percentage(20), + Constraint::Percentage(20), + Constraint::Percentage(20), + ], + ) + .header(Row::new(["Zone", "RX", "TX", "Total"])) + .block(Block::default().title(format!( + " Zones · showing {} of {count} ", + shown.min(count) + ))), + rows[2], + ); +} + +fn summarize(values: &[String], limit: usize) -> String { + if values.is_empty() { + return "none".into(); + } + let shown = + values.iter().take(limit).cloned().collect::>().join(", "); + if values.len() > limit { + format!("{shown} (+{})", values.len() - limit) + } else { + shown + } +} diff --git a/voxel/src/tui/ui/rack_selector.rs b/voxel/src/tui/ui/rack_selector.rs new file mode 100644 index 0000000..50bf2fd --- /dev/null +++ b/voxel/src/tui/ui/rack_selector.rs @@ -0,0 +1,264 @@ +use super::{ + monitor::{resource_health_state, sparkline_data}, + widgets::{format_rate, section_block, selection_style, traffic_style}, +}; +use crate::{ + tui::reconcile::ObservedDeploymentState, + tui::reconcile::RssObservation, + tui::{ + App, + telemetry::{Freshness, HealthState, LatestSample, TrafficSeverity}, + }, +}; +use ratatui::{ + layout::{Constraint, Layout, Rect}, + style::{Color, Style}, + text::Line, + widgets::{Paragraph, Sparkline}, +}; +use std::{collections::BTreeSet, time::Duration}; + +pub fn draw( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + focused: bool, + expanded: bool, +) { + let block = section_block("Rack Summary", expanded, focused); + frame.render_widget(block.clone(), area); + let inner = block.inner(area); + if !expanded || inner.height == 0 { + return; + } + let racks = app + .deployment + .topology + .iter() + .filter_map(|descriptor| descriptor.rack) + .collect::>() + .into_iter() + .collect::>(); + let selected_index = app + .session + .selected_rack + .and_then(|rack| racks.iter().position(|candidate| *candidate == rack)) + .unwrap_or(0); + let Some(rack) = racks.get(selected_index).copied() else { + frame.render_widget( + Paragraph::new( + "No racks configured · RSS unavailable · no traffic samples", + ) + .block(block), + area, + ); + return; + }; + let rate = app + .observability + .telemetry + .rack_rates + .get(&rack) + .copied() + .unwrap_or_default(); + let (healthy, unhealthy, checking) = app + .deployment + .topology + .iter() + .filter(|descriptor| descriptor.rack == Some(rack)) + .map(|descriptor| resource_health_state(app, &descriptor.id)) + .fold((0, 0, 0), |(healthy, unhealthy, checking), state| match state { + HealthState::Healthy => (healthy + 1, unhealthy, checking), + HealthState::Degraded + | HealthState::Failed + | HealthState::Stale => (healthy, unhealthy + 1, checking), + _ => (healthy, unhealthy, checking + 1), + }); + let rss = app + .deployment + .rss + .get(&rack) + .map(|sample| rss_summary(app, sample)) + .unwrap_or_else(|| "RSS unavailable (no sample)".into()); + let zfs = app + .observability + .zfs_headroom + .get(&rack) + .and_then(|sample| sample.good.as_ref()) + .map(|sample| { + let available: u64 = + sample.value.iter().map(|pool| pool.available_bytes()).sum(); + let total: u64 = + sample.value.iter().map(|pool| pool.total_bytes).sum(); + format!( + "ZFS {:.1}/{:.1} GiB free", + available as f64 / 1024.0_f64.powi(3), + total as f64 / 1024.0_f64.powi(3) + ) + }) + .unwrap_or_else(|| "ZFS —".into()); + let exceptions = app + .observability + .oximeter_exceptions + .get(&rack) + .and_then(|sample| sample.good.as_ref()) + .map(|sample| { + format!( + "Oximeter collection health: {} failures / {} dropped", + sample.value.failed_collections, sample.value.dropped_samples + ) + }) + .unwrap_or_else(|| "Oximeter collection health unavailable".into()); + let compact = area.width < 100; + let title = if compact { + format!(" Rack {} ◀ {}/{} ▶ ", rack.0, selected_index + 1, racks.len()) + } else { + format!( + " Rack {} ◀ {}/{} ▶ · selected rack summary ", + rack.0, + selected_index + 1, + racks.len() + ) + }; + frame.render_widget( + Paragraph::new(Line::from(vec![ + ratatui::text::Span::styled(title, selection_style()), + ratatui::text::Span::raw(format!(" · {rss}")), + ])), + Rect::new(inner.x, inner.y, inner.width, 1), + ); + let rows = Layout::vertical([Constraint::Length(1), Constraint::Min(0)]) + .split(Rect::new( + inner.x, + inner.y.saturating_add(1), + inner.width, + inner.height.saturating_sub(1), + )); + let rate_text = if compact { + format!( + "RX {} TX {} Σ{} · {healthy}/{unhealthy}/{checking} · {zfs} · {exceptions}", + format_rate(rate.rx_bytes_sec), + format_rate(rate.tx_bytes_sec), + format_rate(rate.total_bytes_sec()) + ) + } else { + format!( + "RX {} · TX {} · Total {} · health {healthy} ok / {unhealthy} bad / {checking} checking · {zfs} · {exceptions}", + format_rate(rate.rx_bytes_sec), + format_rate(rate.tx_bytes_sec), + format_rate(rate.total_bytes_sec()) + ) + }; + let history_area = if rows[1].height > 0 { + frame.render_widget( + Paragraph::new(rate_text).style(traffic_style( + TrafficSeverity::for_bytes_per_sec(rate.total_bytes_sec()), + )), + rows[0], + ); + rows[1] + } else { + let columns = Layout::horizontal([ + Constraint::Percentage(68), + Constraint::Percentage(32), + ]) + .split(rows[0]); + frame.render_widget( + Paragraph::new(rate_text).style(traffic_style( + TrafficSeverity::for_bytes_per_sec(rate.total_bytes_sec()), + )), + columns[0], + ); + columns[1] + }; + if history_area.height > 0 { + let data = app + .observability + .telemetry + .rack_histories + .get(&rack) + .map(|history| { + sparkline_data( + history + .points() + .iter() + .map(|point| point.rate.total_bytes_sec()), + history_area.width, + ) + }) + .unwrap_or_default(); + if data.is_empty() { + frame.render_widget( + Paragraph::new("Rack traffic (this TUI session): collecting"), + history_area, + ); + } else { + let label = "Rack traffic (this TUI session) "; + let label_width = label.len().min(history_area.width.into()) as u16; + frame.render_widget(Paragraph::new(label), history_area); + frame.render_widget( + Sparkline::default() + .data(&data) + .style(Style::default().fg(Color::Cyan)), + Rect::new( + history_area.x.saturating_add(label_width), + history_area.y, + history_area.width.saturating_sub(label_width), + history_area.height, + ), + ); + } + } +} + +fn rss_summary(app: &App, sample: &LatestSample) -> String { + if app.deployment.observed == ObservedDeploymentState::Stopped { + return "RSS Stopped".into(); + } + let freshness = app + .now + .map(|now| { + sample.freshness( + now, + Duration::from_secs(15), + Duration::from_secs(60), + ) + }) + .unwrap_or(Freshness::Unavailable); + let state = match freshness { + Freshness::Stale => "Stale", + Freshness::Unavailable => "Unavailable", + Freshness::Fresh => { + match sample.good.as_ref().map(|good| &good.value) { + Some(RssObservation::Initialized { .. }) => { + "Healthy (initialized)" + } + Some(RssObservation::Initializing { .. }) => { + "Checking (initializing)" + } + Some(RssObservation::StaleInitialized) => { + "Degraded (stale initialized)" + } + Some(RssObservation::Failed { .. }) => "Failed", + Some(RssObservation::Unavailable) => "Unavailable", + Some(RssObservation::UnknownResponse) | None => "Unknown", + } + } + }; + let age = sample + .good + .as_ref() + .and_then(|good| { + app.now.map(|now| { + now.saturating_duration_since(good.captured_at).as_secs() + }) + }) + .map(|seconds| format!("{seconds}s ago")) + .unwrap_or_else(|| "never".into()); + let error = sample + .latest_error + .as_ref() + .map(|error| error.message.as_str()) + .unwrap_or("none"); + format!("RSS {state} · success {age} · error {error}") +} diff --git a/voxel/src/tui/ui/renderer.rs b/voxel/src/tui/ui/renderer.rs new file mode 100644 index 0000000..10cdf5f --- /dev/null +++ b/voxel/src/tui/ui/renderer.rs @@ -0,0 +1,496 @@ +use crate::tui::{App, AppEvent, Effect}; +use ratatui::{Terminal, backend::Backend, layout::Rect}; +use std::{ + io::{self, Write}, + time::{Duration, Instant}, +}; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +pub const MIN_WIDTH: u16 = 48; +pub const MIN_HEIGHT: u16 = 16; +pub const WIDE_WIDTH: u16 = 100; +pub const WIDE_HEIGHT: u16 = 22; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LayoutMode { + Minimum, + Compact, + Wide, +} +pub fn layout_mode(area: Rect) -> LayoutMode { + if area.width < MIN_WIDTH || area.height < MIN_HEIGHT { + LayoutMode::Minimum + } else if area.width < WIDE_WIDTH || area.height < WIDE_HEIGHT { + LayoutMode::Compact + } else { + LayoutMode::Wide + } +} + +pub fn render_frame(frame: &mut ratatui::Frame<'_>, app: &App) { + crate::tui::ui::widgets::draw(frame, app); +} + +pub struct Renderer { + minimum_interval: Duration, + last_draw: Option, +} +impl Renderer { + pub fn new(minimum_interval: Duration) -> Self { + Self { minimum_interval, last_draw: None } + } + pub fn draw( + &mut self, + terminal: &mut Terminal, + app: &App, + force: bool, + ) -> io::Result { + self.draw_at(terminal, app, force, Instant::now()) + } + pub fn draw_at( + &mut self, + terminal: &mut Terminal, + app: &App, + force: bool, + now: Instant, + ) -> io::Result { + if !force + && self.last_draw.is_some_and(|last| { + now.saturating_duration_since(last) < self.minimum_interval + }) + { + return Ok(false); + } + terminal.draw(|frame| render_frame(frame, app))?; + self.last_draw = Some(now); + Ok(true) + } +} + +#[derive(Debug, Eq, PartialEq)] +pub enum LoopExit { + ConfirmedExit, + Cancelled { unsent: Vec }, + EventChannelClosed, + EffectChannelClosed { unsent: Vec }, + RenderFailed { kind: io::ErrorKind, message: String, unsent: Vec }, +} + +fn forward_effects( + sender: &mpsc::UnboundedSender, + effects: Vec, +) -> Result<(), Vec> { + let mut effects = effects.into_iter(); + while let Some(current) = effects.next() { + if let Err(error) = sender.send(current) { + return Err(std::iter::once(error.0).chain(effects).collect()); + } + } + Ok(()) +} + +pub async fn run_loop( + terminal: &mut Terminal, + clipboard: &mut impl Write, + renderer: &mut Renderer, + app: &mut App, + events: &mut mpsc::Receiver, + effects: &mpsc::UnboundedSender, + cancel: CancellationToken, +) -> io::Result { + if let Err(error) = renderer.draw(terminal, app, true) { + return Ok(LoopExit::RenderFailed { + kind: error.kind(), + message: error.to_string(), + unsent: vec![], + }); + } + loop { + let event = tokio::select! { + biased; + _ = cancel.cancelled() => return Ok(LoopExit::Cancelled { unsent: vec![] }), + event = events.recv() => match event { + Some(event) => event, + None => return Ok(LoopExit::EventChannelClosed), + } + }; + let pending_effects = app.update(event); + let force_draw = !pending_effects.is_empty(); + let mut forwarded = Vec::with_capacity(pending_effects.len()); + for effect in pending_effects { + match effect { + Effect::CopyToClipboard(command) => { + crate::tui::terminal::write_osc52( + &mut *clipboard, + &command, + )?; + } + effect => forwarded.push(effect), + } + } + if let Err(error) = renderer.draw(terminal, app, force_draw) { + return Ok(LoopExit::RenderFailed { + kind: error.kind(), + message: error.to_string(), + unsent: forwarded, + }); + } + let confirmed_exit = + forwarded.iter().any(|effect| matches!(effect, Effect::Quit)); + if let Err(unsent) = forward_effects(effects, forwarded) { + return Ok(LoopExit::EffectChannelClosed { unsent }); + } + if confirmed_exit { + return Ok(LoopExit::ConfirmedExit); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::{Action, AppEvent}; + use ratatui::{ + backend::{ClearType, TestBackend, WindowSize}, + buffer::Cell, + layout::{Position, Size}, + }; + + struct FailAfterDraw { + inner: TestBackend, + draws_remaining: usize, + } + + impl Backend for FailAfterDraw { + fn draw<'a, I>(&mut self, content: I) -> io::Result<()> + where + I: Iterator, + { + if self.draws_remaining == 0 { + return Err(io::Error::other("draw failed")); + } + self.draws_remaining -= 1; + self.inner.draw(content) + } + + fn hide_cursor(&mut self) -> io::Result<()> { + self.inner.hide_cursor() + } + + fn show_cursor(&mut self) -> io::Result<()> { + self.inner.show_cursor() + } + + fn get_cursor_position(&mut self) -> io::Result { + self.inner.get_cursor_position() + } + + fn set_cursor_position>( + &mut self, + position: P, + ) -> io::Result<()> { + self.inner.set_cursor_position(position) + } + + fn clear(&mut self) -> io::Result<()> { + self.inner.clear() + } + + fn clear_region(&mut self, clear_type: ClearType) -> io::Result<()> { + self.inner.clear_region(clear_type) + } + + fn size(&self) -> io::Result { + self.inner.size() + } + + fn window_size(&mut self) -> io::Result { + self.inner.window_size() + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } + } + + #[test] + fn deterministic_throttle_boundaries_and_force() { + let mut t = Terminal::new(TestBackend::new(80, 20)).unwrap(); + let app = App::new(vec![], 2, 2); + let start = Instant::now(); + let mut r = Renderer::new(Duration::from_secs(1)); + assert!(r.draw_at(&mut t, &app, false, start).unwrap()); + assert!( + !r.draw_at(&mut t, &app, false, start + Duration::from_millis(999)) + .unwrap() + ); + assert!( + r.draw_at(&mut t, &app, false, start + Duration::from_secs(1)) + .unwrap() + ); + assert!( + r.draw_at(&mut t, &app, true, start + Duration::from_secs(1)) + .unwrap() + ); + assert!( + !r.draw_at(&mut t, &app, false, start - Duration::from_secs(1)) + .unwrap() + ); + } + #[tokio::test] + async fn loop_forces_draw_forwards_exact_effect_and_closes() { + let mut t = Terminal::new(TestBackend::new(80, 20)).unwrap(); + let mut app = App::new(vec![], 2, 2); + app.deployment.observed = + crate::tui::reconcile::ObservedDeploymentState::Stopped; + let (tx, mut rx) = { + let (a, b) = mpsc::channel(4); + (a, b) + }; + tx.send(AppEvent::Action(Action::RequestLaunch)).await.unwrap(); + tx.send(AppEvent::Action(Action::Scroll { delta: -1, page: false })) + .await + .unwrap(); + tx.send(AppEvent::Action(Action::Activate)).await.unwrap(); + drop(tx); + let (etx, mut erx) = mpsc::unbounded_channel(); + let mut r = Renderer::new(Duration::ZERO); + let mut clipboard = Vec::new(); + let exit = run_loop( + &mut t, + &mut clipboard, + &mut r, + &mut app, + &mut rx, + &etx, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(exit, LoopExit::EventChannelClosed); + assert!(matches!( + erx.recv().await, + Some(Effect::Start { + kind: crate::tui::operation::OperationKind::Launch, + .. + }) + )); + assert!(erx.try_recv().is_err()); + assert!( + t.backend() + .buffer() + .content() + .iter() + .any(|c| !c.symbol().is_empty()) + ); + } + + #[tokio::test] + async fn loop_consumes_clipboard_effect_without_forwarding_it() { + let mut terminal = Terminal::new(TestBackend::new(80, 20)).unwrap(); + let mut app = App::new(vec![], 2, 2); + app.reattach_command = Some("pfexec '/opt/voxel' tui".into()); + let (tx, mut rx) = mpsc::channel(2); + tx.send(Action::RequestDetach.into()).await.unwrap(); + tx.send(Action::CopyReattachCommand.into()).await.unwrap(); + drop(tx); + let (effects_tx, mut effects_rx) = mpsc::unbounded_channel(); + let mut clipboard = Vec::new(); + let exit = run_loop( + &mut terminal, + &mut clipboard, + &mut Renderer::new(Duration::ZERO), + &mut app, + &mut rx, + &effects_tx, + CancellationToken::new(), + ) + .await + .unwrap(); + + assert_eq!(exit, LoopExit::EventChannelClosed); + assert_eq!( + String::from_utf8(clipboard).unwrap(), + crate::tui::terminal::osc52("pfexec '/opt/voxel' tui") + ); + assert!(effects_rx.try_recv().is_err()); + assert!(app.clipboard_copied); + } + #[tokio::test] + async fn effect_forwarding_is_unbounded_and_never_blocks_control_delivery() + { + let mut t = Terminal::new(TestBackend::new(80, 20)).unwrap(); + let mut app = App::new(vec![], 2, 2); + app.deployment.observed = + crate::tui::reconcile::ObservedDeploymentState::Stopped; + let (tx, mut rx) = mpsc::channel(3); + tx.send(Action::RequestLaunch.into()).await.unwrap(); + tx.send(Action::Scroll { delta: -1, page: false }.into()) + .await + .unwrap(); + tx.send(Action::Activate.into()).await.unwrap(); + let (etx, mut erx) = mpsc::unbounded_channel(); + etx.send(Effect::Quit).unwrap(); + let c = CancellationToken::new(); + let cc = c.clone(); + let mut r = Renderer::new(Duration::ZERO); + let mut clipboard = Vec::new(); + let loop_future = tokio::time::timeout( + Duration::from_millis(50), + run_loop( + &mut t, + &mut clipboard, + &mut r, + &mut app, + &mut rx, + &etx, + cc, + ), + ); + let cancel_future = async move { + tokio::time::sleep(Duration::from_millis(10)).await; + c.cancel(); + }; + let (result, ()) = tokio::join!(loop_future, cancel_future); + let exit = result.unwrap().unwrap(); + assert!( + matches!(exit, LoopExit::Cancelled { unsent } if unsent.is_empty()) + ); + assert_eq!(erx.recv().await, Some(Effect::Quit)); + assert!(matches!(erx.recv().await, Some(Effect::Start { .. }))); + assert!( + app.operation.pending.is_some(), + "launch and confirm were processed before cancellation" + ); + let rendered = t.backend().buffer().content().iter().fold( + String::new(), + |mut text, cell| { + text.push_str(cell.symbol()); + text + }, + ); + assert!( + rendered.contains("Launch pending"), + "pending launch must render before effect forwarding" + ); + } + + #[tokio::test] + async fn closed_effect_receiver_returns_exact_unsent_effect() { + let mut t = Terminal::new(TestBackend::new(80, 20)).unwrap(); + let mut app = App::new(vec![], 2, 2); + app.deployment.observed = + crate::tui::reconcile::ObservedDeploymentState::Stopped; + let (tx, mut rx) = mpsc::channel(3); + tx.send(Action::RequestLaunch.into()).await.unwrap(); + tx.send(Action::Scroll { delta: -1, page: false }.into()) + .await + .unwrap(); + tx.send(Action::Activate.into()).await.unwrap(); + let (etx, erx) = mpsc::unbounded_channel(); + drop(erx); + let mut clipboard = Vec::new(); + let exit = run_loop( + &mut t, + &mut clipboard, + &mut Renderer::new(Duration::ZERO), + &mut app, + &mut rx, + &etx, + CancellationToken::new(), + ) + .await + .unwrap(); + assert!( + matches!(exit, LoopExit::EffectChannelClosed { unsent } if matches!(unsent.as_slice(), [Effect::Start { kind: crate::tui::operation::OperationKind::Launch, .. }])) + ); + } + + #[tokio::test] + async fn confirmed_quit_is_forwarded_before_loop_exits() { + let mut terminal = Terminal::new(TestBackend::new(80, 20)).unwrap(); + let mut app = App::new(vec![], 2, 2); + app.deployment.observed = + crate::tui::reconcile::ObservedDeploymentState::Stopped; + let (tx, mut rx) = mpsc::channel(3); + tx.send(Action::RequestQuit.into()).await.unwrap(); + tx.send(Action::Scroll { delta: -1, page: false }.into()) + .await + .unwrap(); + tx.send(Action::Activate.into()).await.unwrap(); + let (effects_tx, mut effects_rx) = mpsc::unbounded_channel(); + let exit = run_loop( + &mut terminal, + &mut Vec::new(), + &mut Renderer::new(Duration::ZERO), + &mut app, + &mut rx, + &effects_tx, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(exit, LoopExit::ConfirmedExit); + assert_eq!(effects_rx.recv().await, Some(Effect::Quit)); + } + + #[tokio::test] + async fn render_failure_returns_reducer_effect_without_forwarding_it() { + let mut t = Terminal::new(FailAfterDraw { + inner: TestBackend::new(80, 20), + draws_remaining: 1, + }) + .unwrap(); + let mut app = App::new(vec![], 2, 2); + app.deployment.observed = + crate::tui::reconcile::ObservedDeploymentState::Stopped; + app.session.confirmation = + Some(crate::tui::event::Confirmation::Launch); + let (tx, mut rx) = mpsc::channel(1); + tx.send(Action::Activate.into()).await.unwrap(); + let (etx, mut erx) = mpsc::unbounded_channel(); + let mut clipboard = Vec::new(); + + let exit = run_loop( + &mut t, + &mut clipboard, + &mut Renderer::new(Duration::ZERO), + &mut app, + &mut rx, + &etx, + CancellationToken::new(), + ) + .await + .unwrap(); + + assert!(matches!( + exit, + LoopExit::RenderFailed { + kind: io::ErrorKind::Other, + message, + unsent, + } if message == "draw failed" + && matches!(unsent.as_slice(), [Effect::Start { + kind: crate::tui::operation::OperationKind::Launch, + .. + }]) + )); + assert!(erx.try_recv().is_err()); + assert!(app.operation.pending.is_some()); + } + + #[tokio::test] + async fn closed_forwarding_preserves_unsent_current_and_tail_order() { + let (tx, rx) = mpsc::unbounded_channel(); + drop(rx); + let effects = vec![ + Effect::Quit, + Effect::Cancel { + request_id: crate::tui::event::OperationRequestId::FIRST, + choice: crate::tui::event::CancelChoice::Leave, + }, + ]; + let unsent = forward_effects(&tx, effects.clone()).unwrap_err(); + assert_eq!(unsent, effects); + } +} diff --git a/voxel/src/tui/ui/topology.rs b/voxel/src/tui/ui/topology.rs new file mode 100644 index 0000000..34132b4 --- /dev/null +++ b/voxel/src/tui/ui/topology.rs @@ -0,0 +1,1110 @@ +use std::collections::BTreeMap; + +use ratatui::{ + layout::Rect, + style::{Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph}, +}; + +use super::{ + colors::OX_GREEN_LIGHT, + monitor::{health_style, resource_health_state}, + renderer::LayoutMode, + widgets::{fit_terminal_width, selection_style, terminal_width}, +}; +use crate::tui::{ + App, + telemetry::{HealthState, ResourceDescriptor, ResourceId, ResourceKind}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum TopologyTier { + Routers, + SwitchZones, + Sleds, +} + +impl TopologyTier { + const ALL: [Self; 3] = [Self::Routers, Self::SwitchZones, Self::Sleds]; + + fn kind(self) -> ResourceKind { + match self { + Self::Routers => ResourceKind::Router, + Self::SwitchZones => ResourceKind::SwitchZone, + Self::Sleds => ResourceKind::Sled, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OverflowRange { + /// Half-open range in the tier's stable-ID ordered resources. + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) total: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TierScene { + pub(crate) tier: TopologyTier, + pub(crate) label_area: Rect, + pub(crate) node_area: Rect, + pub(crate) range_area: Option, + pub(crate) visible_ids: Vec, + pub(crate) overflow: OverflowRange, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ConnectorCell { + pub(crate) x: u16, + pub(crate) y: u16, + pub(crate) symbol: &'static str, + /// The tier whose membership in the shared fabric this cell describes. + pub(crate) tier: TopologyTier, + /// Junctions are the only connector cells that intentionally occupy bus cells. + pub(crate) fabric_junction: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FabricBus { + pub(crate) area: Rect, + pub(crate) symbol: &'static str, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TopologyScene { + pub(crate) node_rects: BTreeMap, + pub(crate) tiers: Vec, + pub(crate) connectors: Vec, + pub(crate) fabric_buses: Vec, + pub(crate) dividers: Vec, + pub(crate) visible_ids: Vec, + pub(crate) navigation_order: Vec, +} + +pub(crate) fn draw( + frame: &mut ratatui::Frame<'_>, + scene: &TopologyScene, + app: &App, +) { + let structural = Style::default().fg(OX_GREEN_LIGHT); + for tier in &scene.tiers { + let (label, empty) = match tier.tier { + TopologyTier::Routers => ("Routers", "No routers"), + TopologyTier::SwitchZones => ("Switch zones", "No switch zones"), + TopologyTier::Sleds => ("Sleds", "No sleds"), + }; + if tier.label_area.area() > 0 { + frame.render_widget( + Paragraph::new(if tier.overflow.total == 0 { + empty + } else { + label + }) + .style(Style::default().add_modifier(Modifier::BOLD)), + tier.label_area, + ); + } + if let Some(area) = tier.range_area { + let (visible_start, visible_end) = + if tier.overflow.start == tier.overflow.end { + (0, 0) + } else { + (tier.overflow.start + 1, tier.overflow.end) + }; + frame.render_widget( + Paragraph::new(format!( + "{visible_start}-{visible_end} of {}", + tier.overflow.total + )) + .right_aligned(), + area, + ); + } + } + for bus in &scene.fabric_buses { + frame.render_widget( + Paragraph::new(bus.symbol.repeat(bus.area.width.into())) + .style(structural), + bus.area, + ); + let label = " Switch fabric "; + let label_width = terminal_width(label); + if usize::from(bus.area.width) >= label_width + 2 { + frame.render_widget( + Paragraph::new(label).style(structural), + Rect::new(bus.area.x + 2, bus.area.y, label_width as u16, 1), + ); + } + } + for connector in &scene.connectors { + frame.render_widget( + Paragraph::new(connector.symbol).style(structural), + Rect::new(connector.x, connector.y, 1, 1), + ); + } + for id in &scene.visible_ids { + let Some(area) = scene.node_rects.get(id).copied() else { + continue; + }; + let Some(descriptor) = + app.deployment.topology.iter().find(|item| item.id == *id) + else { + continue; + }; + match descriptor.kind { + ResourceKind::Router => { + draw_router_node(frame, area, app, descriptor) + } + ResourceKind::SwitchZone => { + draw_switch_zone_node(frame, area, app, descriptor) + } + ResourceKind::Sled => draw_sled_node(frame, area, app, descriptor), + } + } +} + +fn draw_router_node( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + descriptor: &ResourceDescriptor, +) { + draw_node_primitive(frame, area, app, descriptor, "RTR"); +} + +fn draw_switch_zone_node( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + descriptor: &ResourceDescriptor, +) { + draw_node_primitive(frame, area, app, descriptor, "SWZ"); +} + +fn draw_sled_node( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + descriptor: &ResourceDescriptor, +) { + draw_node_primitive(frame, area, app, descriptor, "SLD"); +} + +fn draw_node_primitive( + frame: &mut ratatui::Frame<'_>, + area: Rect, + app: &App, + descriptor: &ResourceDescriptor, + kind: &str, +) { + let selected = + app.session.selected_resource.as_ref() == Some(&descriptor.id); + let state = resource_health_state(app, &descriptor.id); + let glyph = health_glyph(state); + let pointer = if selected { "▶ " } else { "" }; + let bordered = area.height >= 3 && area.width >= 3; + let title_width = + usize::from(area.width.saturating_sub(u16::from(bordered) * 2)); + let reserved = + terminal_width(pointer) + terminal_width(" ") + terminal_width(glyph); + let identity = fit_terminal_width( + &format!("{kind} {}", descriptor.name), + title_width.saturating_sub(reserved), + ); + let title = Line::from(vec![ + Span::styled( + pointer, + selection_style().add_modifier(Modifier::REVERSED), + ), + Span::styled( + identity, + if selected { + selection_style().add_modifier(Modifier::REVERSED) + } else { + Style::default().add_modifier(Modifier::BOLD) + }, + ), + Span::raw(" "), + Span::styled(glyph, health_style(state)), + ]); + if bordered { + frame.render_widget( + Block::default() + .borders(Borders::ALL) + .border_style(health_style(state)) + .title(title), + area, + ); + } else { + frame.render_widget(Paragraph::new(title), area); + } +} + +fn health_glyph(state: HealthState) -> &'static str { + match state { + HealthState::Healthy => "●", + HealthState::Checking => "◌", + HealthState::Degraded | HealthState::Failed => "!", + HealthState::Stale => "◐", + HealthState::Unknown => "?", + HealthState::Unavailable => "×", + HealthState::Stopped => "■", + } +} + +pub(crate) fn semantic_order( + descriptors: &[ResourceDescriptor], +) -> Vec { + TopologyTier::ALL + .into_iter() + .flat_map(|tier| { + let mut ids = tier_ids(descriptors, tier); + ids.sort(); + ids + }) + .collect() +} + +fn empty_scene(descriptors: &[ResourceDescriptor]) -> TopologyScene { + TopologyScene { + node_rects: BTreeMap::new(), + tiers: TopologyTier::ALL + .into_iter() + .map(|tier| TierScene { + tier, + label_area: Rect::default(), + node_area: Rect::default(), + range_area: None, + visible_ids: Vec::new(), + overflow: OverflowRange { + start: 0, + end: 0, + total: tier_ids(descriptors, tier).len(), + }, + }) + .collect(), + connectors: Vec::new(), + fabric_buses: Vec::new(), + dividers: Vec::new(), + visible_ids: Vec::new(), + navigation_order: semantic_order(descriptors), + } +} + +/// Builds render-ready topology geometry without reading or mutating application state. +pub(crate) fn layout_scene( + area: Rect, + mode: LayoutMode, + descriptors: &[ResourceDescriptor], + selected: Option<&ResourceId>, + overflow_position: usize, +) -> TopologyScene { + let mut scene = empty_scene(descriptors); + // Rect's right/bottom are saturating. Rebuild from those effective bounds so + // a raw overflowing origin/extent can never wrap subsequent coordinate math. + let area = Rect::new( + area.x, + area.y, + area.right().saturating_sub(area.x), + area.bottom().saturating_sub(area.y), + ); + if mode == LayoutMode::Minimum || area.width == 0 || area.height == 0 { + return scene; + } + + let selected_tier = selected.and_then(|id| { + descriptors + .iter() + .any(|descriptor| descriptor.id == *id) + .then(|| { + TopologyTier::ALL + .into_iter() + .find(|tier| tier.kind() == id.kind) + }) + .flatten() + }); + let node_height = match mode { + LayoutMode::Wide => 3_u16, + LayoutMode::Compact => 1_u16, + LayoutMode::Minimum => unreachable!(), + }; + let full_height = 6_u32 + 3 * u32::from(node_height); + let has_spine = area.width >= 3; + let minimum_width = if mode == LayoutMode::Wide { 12 } else { 8 }; + let has_readable_content = has_spine && area.width - 2 >= minimum_width; + + // Reduced scenes spend every available cell on one truthful tier. In + // particular a selected node wins over labels and decorative structure. + if u32::from(area.height) < full_height || !has_readable_content { + let tier = selected_tier + .or_else(|| { + TopologyTier::ALL.into_iter().find(|tier| { + descriptors.iter().any(|item| item.kind == tier.kind()) + }) + }) + .unwrap_or(TopologyTier::Routers); + let mut ids = tier_ids(descriptors, tier); + ids.sort(); + let chosen = + selected.and_then(|id| ids.binary_search(id).ok()).or_else(|| { + (!ids.is_empty()).then(|| overflow_position.min(ids.len() - 1)) + }); + let visible_ids = + chosen.map(|index| vec![ids[index].clone()]).unwrap_or_default(); + place_nodes(area, &visible_ids, &mut scene.node_rects); + scene.visible_ids.clone_from(&visible_ids); + let metadata = + scene.tiers.iter_mut().find(|item| item.tier == tier).unwrap(); + metadata.node_area = area; + metadata.visible_ids = visible_ids; + metadata.overflow = OverflowRange { + start: chosen.unwrap_or(0), + end: chosen.map_or(0, |index| index + 1), + total: ids.len(), + }; + return scene; + } + + let spine_x = area.x; + let content = Rect::new(area.x + 2, area.y, area.width - 2, area.height); + let mut y = area.y; + let mut lower_rails = Vec::new(); + for tier in TopologyTier::ALL { + let label_row = Rect::new(content.x, y, content.width, 1); + y += 1; + let node_area = Rect::new(content.x, y, content.width, node_height); + y += node_height; + + let mut ids = tier_ids(descriptors, tier); + ids.sort(); + let capacity = + usize::from(exact_capacity(content.width, minimum_width)); + let visible_count = ids.len().min(capacity); + let max_start = ids.len().saturating_sub(visible_count); + let start = selected + .and_then(|id| ids.binary_search(id).ok()) + .map(|index| index.saturating_sub(visible_count / 2).min(max_start)) + .unwrap_or_else(|| overflow_position.min(max_start)); + let end = start + visible_count; + let visible_ids = ids[start..end].to_vec(); + + let (label_area, range_area) = + split_label_row(label_row, ids.len() > visible_count); + place_nodes(node_area, &visible_ids, &mut scene.node_rects); + scene.visible_ids.extend(visible_ids.iter().cloned()); + + if tier == TopologyTier::Routers { + let bus_y = y; + scene.fabric_buses.push(FabricBus { + area: Rect::new(spine_x, bus_y, area.width, 1), + symbol: "─", + }); + for id in &visible_ids { + let rect = scene.node_rects[id]; + scene.connectors.push(ConnectorCell { + x: rect.x + rect.width / 2, + y: bus_y, + symbol: "┴", + tier, + fabric_junction: true, + }); + } + y += 1; + } else if !visible_ids.is_empty() { + lower_rails.push((tier, y, visible_ids.clone())); + } + if tier != TopologyTier::Routers { + y += 1; + } + + let tier_scene = scene + .tiers + .iter_mut() + .find(|metadata| metadata.tier == tier) + .expect("all tiers have metadata"); + *tier_scene = TierScene { + tier, + label_area, + node_area, + range_area, + visible_ids, + overflow: OverflowRange { start, end, total: ids.len() }, + }; + } + + let bus_y = scene.fabric_buses[0].area.y; + let last_rail = lower_rails.last().map_or(bus_y, |(_, rail_y, _)| *rail_y); + if let Some((tier, _, _)) = lower_rails.first() { + scene.connectors.push(ConnectorCell { + x: spine_x, + y: bus_y, + symbol: "┬", + tier: *tier, + fabric_junction: true, + }); + } + for connector_y in bus_y + 1..=last_rail { + let at_rail = + lower_rails.iter().find(|(_, rail_y, _)| *rail_y == connector_y); + let tier = at_rail.map_or(TopologyTier::Sleds, |(tier, _, _)| *tier); + let is_last_rail = connector_y == last_rail; + scene.connectors.push(ConnectorCell { + x: spine_x, + y: connector_y, + symbol: if at_rail.is_some() { + if is_last_rail { "└" } else { "├" } + } else { + "│" + }, + tier, + fabric_junction: false, + }); + if let Some((tier, _, ids)) = at_rail { + let centers: Vec = ids + .iter() + .map(|id| { + let rect = scene.node_rects[id]; + rect.x + rect.width / 2 + }) + .collect(); + let rightmost = centers.iter().copied().max().unwrap_or(spine_x); + for x in spine_x + 1..=rightmost { + scene.connectors.push(ConnectorCell { + x, + y: connector_y, + symbol: if centers.contains(&x) { "┴" } else { "─" }, + tier: *tier, + fabric_junction: false, + }); + } + } + } + scene +} + +fn tier_ids( + descriptors: &[ResourceDescriptor], + tier: TopologyTier, +) -> Vec { + descriptors + .iter() + .filter(|descriptor| descriptor.kind == tier.kind()) + .map(|descriptor| descriptor.id.clone()) + .collect() +} + +fn exact_capacity(width: u16, minimum_width: u16) -> u16 { + if width < minimum_width { + 0 + } else { + ((u32::from(width) + 1) / (u32::from(minimum_width) + 1)) as u16 + } +} + +fn split_label_row(row: Rect, overflowing: bool) -> (Rect, Option) { + if !overflowing || row.width < 2 || row.height == 0 { + return (row, None); + } + let range_width = row.width.min(12).min(row.width - 1); + ( + Rect::new(row.x, row.y, row.width - range_width, 1), + Some(Rect::new(row.right() - range_width, row.y, range_width, 1)), + ) +} + +fn place_nodes( + area: Rect, + ids: &[ResourceId], + output: &mut BTreeMap, +) { + if ids.is_empty() || area.area() == 0 { + return; + } + let count = ids.len() as u16; + let gaps = count.saturating_sub(1); + let gap = + u16::from(area.width >= count.saturating_mul(2).saturating_sub(1)); + let usable = area.width.saturating_sub(gaps * gap); + let base = usable / count; + let extra = usable % count; + let mut x = area.x; + for (index, id) in ids.iter().enumerate() { + let width = base + u16::from((index as u16) < extra); + if width > 0 { + output.insert(id.clone(), Rect::new(x, area.y, width, area.height)); + } + x = x.saturating_add(width + gap); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + tui::reconcile::ObservedDeploymentState, + tui::{ + event::AppEvent, + telemetry::{ + HealthDiagnostic, NtpDiagnostic, RackId, ServiceState, + }, + }, + }; + use ratatui::{Terminal, backend::TestBackend, style::Color}; + use std::time::{Duration, Instant}; + + fn descriptor(kind: ResourceKind, name: &str) -> ResourceDescriptor { + let rack = (kind != ResourceKind::Router).then_some(RackId(0)); + ResourceDescriptor { + id: rack.map_or_else( + || ResourceId::fleet(kind, name), + |rack| ResourceId::rack(rack, kind, name), + ), + rack, + kind, + name: name.into(), + host: None, + } + } + + fn resources() -> Vec { + [ + (ResourceKind::Sled, "sled界"), + (ResourceKind::Router, "routere\u{301}"), + (ResourceKind::SwitchZone, "switch"), + ] + .into_iter() + .flat_map(|(kind, prefix)| { + (0..12) + .rev() + .map(move |i| descriptor(kind, &format!("{prefix}-{i:02}"))) + }) + .collect() + } + + fn cells(rect: Rect) -> impl Iterator { + (rect.y..rect.bottom()) + .flat_map(move |y| (rect.x..rect.right()).map(move |x| (x, y))) + } + + #[test] + fn every_selected_health_state_keeps_interaction_and_semantic_styles_local() + { + let cases = [ + (HealthState::Healthy, "●", super::super::colors::TUI_GREEN), + (HealthState::Checking, "◌", super::super::colors::TUI_YELLOW), + (HealthState::Degraded, "!", super::super::colors::OX_RED), + (HealthState::Failed, "!", super::super::colors::OX_RED), + (HealthState::Stale, "◐", super::super::colors::TUI_GREY), + (HealthState::Unknown, "?", super::super::colors::TUI_GREY_DARK), + ( + HealthState::Unavailable, + "×", + super::super::colors::TUI_GREY_DARK, + ), + (HealthState::Stopped, "■", super::super::colors::TUI_GREY_DARK), + ]; + for (wanted, glyph, color) in cases { + let selected = descriptor(ResourceKind::Sled, "selected"); + let mut app = App::new(vec![selected.clone()], 4, 4); + app.session.selected_resource = Some(selected.id.clone()); + app.deployment.observed = if wanted == HealthState::Stopped { + ObservedDeploymentState::Stopped + } else { + ObservedDeploymentState::Running + }; + let now = Instant::now(); + app.update(AppEvent::Tick { now }); + let diagnostic = match wanted { + HealthState::Healthy | HealthState::Stale => { + Some(HealthDiagnostic { + sled_agent: Some(ServiceState::Online), + ntp: NtpDiagnostic { + synchronized: Some(true), + ..Default::default() + }, + ..Default::default() + }) + } + HealthState::Degraded => Some(HealthDiagnostic { + failed_services: vec!["nexus".into()], + ..Default::default() + }), + HealthState::Failed => Some(HealthDiagnostic { + sled_agent: Some(ServiceState::Offline), + ..Default::default() + }), + HealthState::Unknown => Some(HealthDiagnostic::default()), + _ => None, + }; + if let Some(diagnostic) = diagnostic { + app.update(AppEvent::Health { + id: selected.id.clone(), + at: if wanted == HealthState::Stale { + now - Duration::from_secs(61) + } else { + now + }, + diagnostic, + }); + } + if wanted == HealthState::Unavailable { + app.observability.health.remove(&selected.id); + } + assert_eq!(resource_health_state(&app, &selected.id), wanted); + let scene = layout_scene( + Rect::new(0, 0, 40, 15), + LayoutMode::Wide, + std::slice::from_ref(&selected), + Some(&selected.id), + 0, + ); + let rect = scene.node_rects[&selected.id]; + let mut terminal = Terminal::new(TestBackend::new(40, 15)).unwrap(); + terminal.draw(|frame| draw(frame, &scene, &app)).unwrap(); + let buffer = terminal.backend().buffer(); + let selected_cells = cells(rect) + .filter_map(|position| { + let cell = &buffer[position]; + (cell.fg == super::super::colors::TUI_PURPLE) + .then_some(cell) + }) + .collect::>(); + assert!(selected_cells.iter().any(|cell| cell.symbol() == "▶")); + assert!(selected_cells.iter().any(|cell| cell.symbol() == "S")); + assert!(selected_cells.iter().all(|cell| { + cell.modifier.contains(Modifier::BOLD) + && cell.modifier.contains(Modifier::REVERSED) + })); + assert!(cells(rect).any(|position| { + buffer[position].symbol() == glyph + && buffer[position].fg == color + })); + assert_eq!(buffer[(rect.x, rect.bottom() - 1)].fg, color); + assert_ne!(color, Color::Reset); + } + } + + #[test] + fn every_artifact_is_bounded_and_disjoint_except_explicit_bus_junctions() { + for (area, mode) in [ + (Rect::new(3, 2, 160, 20), LayoutMode::Wide), + (Rect::new(3, 2, 60, 10), LayoutMode::Compact), + (Rect::new(3, 2, 48, 6), LayoutMode::Compact), + (Rect::new(9, 11, 7, 3), LayoutMode::Compact), + (Rect::new(65_500, 65_520, 35, 15), LayoutMode::Compact), + ] { + let scene = layout_scene(area, mode, &resources(), None, 4); + assert!(scene.node_rects.values().all(|rect| rect.area() > 0)); + assert_geometry(&scene, area); + assert_connector_continuity(&scene); + assert!(scene.fabric_buses.len() <= 1); + } + } + + fn assert_geometry(scene: &TopologyScene, area: Rect) { + let mut occupied: BTreeMap<(u16, u16), &'static str> = BTreeMap::new(); + let mut claim = |rect: Rect, kind: &'static str| { + assert!(rect.x >= area.x && rect.y >= area.y); + assert!( + rect.right() <= area.right() && rect.bottom() <= area.bottom() + ); + for cell in cells(rect) { + assert!( + occupied.insert(cell, kind).is_none(), + "{kind} collision {cell:?}" + ); + } + }; + for tier in &scene.tiers { + if tier.label_area.area() > 0 { + claim(tier.label_area, "label"); + } + if let Some(rect) = tier.range_area { + claim(rect, "range"); + } + } + for rect in scene.node_rects.values() { + claim(*rect, "node"); + } + for rect in &scene.dividers { + claim(*rect, "divider"); + } + for cell in scene.connectors.iter().filter(|cell| !cell.fabric_junction) + { + claim(Rect::new(cell.x, cell.y, 1, 1), "connector"); + } + for bus in &scene.fabric_buses { + claim(bus.area, "bus"); + } + let mut junctions = BTreeMap::new(); + for cell in scene.connectors.iter().filter(|cell| cell.fabric_junction) + { + assert_eq!(occupied.get(&(cell.x, cell.y)), Some(&"bus")); + assert!(junctions.insert((cell.x, cell.y), cell.tier).is_none()); + } + } + + fn assert_connector_continuity(scene: &TopologyScene) { + let Some(bus) = scene.fabric_buses.first() else { + assert!(scene.connectors.is_empty()); + return; + }; + let router = &scene.tiers[0]; + for id in &router.visible_ids { + let node = scene.node_rects[id]; + let center = node.x + node.width / 2; + assert_eq!(node.bottom(), bus.area.y); + assert!(scene.connectors.iter().any(|cell| { + cell.x == center + && cell.y == bus.area.y + && cell.tier == TopologyTier::Routers + && cell.fabric_junction + })); + } + + let populated_lower: Vec<&TierScene> = scene + .tiers + .iter() + .skip(1) + .filter(|tier| !tier.visible_ids.is_empty()) + .collect(); + if populated_lower.is_empty() { + assert!(!scene.connectors.iter().any(|cell| cell.x == bus.area.x + && cell.tier != TopologyTier::Routers)); + return; + } + + assert!(scene.connectors.iter().any(|cell| { + cell.x == bus.area.x && cell.y == bus.area.y && cell.fabric_junction + })); + let last_rail_y = + populated_lower.last().expect("nonempty").node_area.bottom(); + for y in bus.area.y + 1..=last_rail_y { + assert!( + scene + .connectors + .iter() + .any(|cell| cell.x == bus.area.x && cell.y == y) + ); + } + for tier in populated_lower { + let rail_y = tier.node_area.bottom(); + assert!(bus.area.y < tier.label_area.y); + for id in &tier.visible_ids { + let node = scene.node_rects[id]; + let center = node.x + node.width / 2; + assert_eq!(node.bottom(), rail_y); + assert!(scene.connectors.iter().any(|cell| { + cell.x == center + && cell.y == rail_y + && cell.tier == tier.tier + && cell.symbol == "┴" + && !cell.fabric_junction + })); + for x in bus.area.x..=center { + assert!( + scene + .connectors + .iter() + .any(|cell| cell.x == x && cell.y == rail_y) + ); + } + } + } + } + + #[test] + fn maximal_dimensions_do_not_overflow_geometry_arithmetic() { + for area in + [Rect::new(0, 0, u16::MAX, 20), Rect::new(0, 0, 40, u16::MAX)] + { + let scene = layout_scene( + area, + LayoutMode::Wide, + &resources(), + None, + usize::MAX, + ); + assert!(scene.fabric_buses.len() <= 1); + assert_geometry(&scene, area); + assert_connector_continuity(&scene); + } + } + + #[test] + fn overflowing_raw_rect_extents_are_normalized_before_layout() { + let area = Rect::new(u16::MAX - 5, u16::MAX - 5, 20, 20); + let descriptors = resources(); + let selected = descriptors + .iter() + .find(|item| item.kind == ResourceKind::Sled) + .unwrap() + .id + .clone(); + let scene = layout_scene( + area, + LayoutMode::Compact, + &descriptors, + Some(&selected), + 0, + ); + assert!(scene.visible_ids.contains(&selected)); + assert_geometry(&scene, area); + + let effectively_empty = layout_scene( + Rect::new(u16::MAX, 3, 1, 10), + LayoutMode::Compact, + &descriptors, + Some(&selected), + 0, + ); + assert!(effectively_empty.visible_ids.is_empty()); + assert!(effectively_empty.node_rects.is_empty()); + } + + #[test] + fn near_maximum_origin_still_allocates_the_complete_scene() { + let area = Rect::new(65_500, 65_520, 35, 15); + let scene = + layout_scene(area, LayoutMode::Compact, &resources(), None, 0); + assert_eq!(scene.fabric_buses.len(), 1); + assert!(scene.dividers.is_empty()); + assert!(scene.tiers.iter().all(|tier| tier.label_area.area() > 0)); + assert!(scene.node_rects.len() >= 3); + assert!(scene.connectors.iter().any(|cell| !cell.fabric_junction)); + assert_eq!( + scene.connectors.iter().filter(|cell| cell.fabric_junction).count(), + scene.tiers[0].visible_ids.len() + + usize::from( + scene + .tiers + .iter() + .skip(1) + .any(|tier| !tier.visible_ids.is_empty()) + ) + ); + assert_geometry(&scene, area); + assert_connector_continuity(&scene); + } + + #[test] + fn empty_and_partial_tiers_keep_explicit_metadata() { + let cases = [ + Vec::new(), + vec![descriptor(ResourceKind::Router, "router")], + vec![descriptor(ResourceKind::SwitchZone, "switch")], + vec![descriptor(ResourceKind::Sled, "sled")], + vec![ + descriptor(ResourceKind::Router, "router"), + descriptor(ResourceKind::Sled, "sled"), + ], + vec![ + descriptor(ResourceKind::SwitchZone, "switch"), + descriptor(ResourceKind::Sled, "sled"), + ], + ]; + for descriptors in cases { + let scene = layout_scene( + Rect::new(2, 3, 80, 20), + LayoutMode::Wide, + &descriptors, + None, + 0, + ); + assert_eq!(scene.tiers.len(), 3); + for tier in TopologyTier::ALL { + let metadata = + scene.tiers.iter().find(|item| item.tier == tier).unwrap(); + let expected_ids = tier_ids(&descriptors, tier); + assert_eq!(metadata.overflow.total, expected_ids.len()); + assert_eq!(metadata.overflow.start, 0); + assert_eq!(metadata.overflow.end, expected_ids.len()); + assert_eq!(metadata.visible_ids, expected_ids); + for id in expected_ids { + assert!(scene.node_rects.contains_key(&id)); + } + } + assert_eq!(scene.fabric_buses.len(), 1); + assert_geometry(&scene, Rect::new(2, 3, 80, 20)); + assert_connector_continuity(&scene); + } + } + + #[test] + fn nonexistent_selection_does_not_displace_semantic_tier_priority() { + let descriptors = resources(); + let missing = + ResourceId::rack(RackId(0), ResourceKind::Sled, "missing"); + let scene = layout_scene( + Rect::new(0, 0, 48, 3), + LayoutMode::Compact, + &descriptors, + Some(&missing), + 0, + ); + assert!(!scene.tiers[0].visible_ids.is_empty()); + assert!(scene.tiers[2].visible_ids.is_empty()); + } + + #[test] + fn selected_tier_is_prioritized_and_zero_area_is_truthful() { + let descriptors = resources(); + let selected = descriptors + .iter() + .find(|d| d.kind == ResourceKind::Sled) + .unwrap() + .id + .clone(); + for area in [Rect::new(0, 0, 48, 6), Rect::new(4, 7, 2, 1)] { + let scene = layout_scene( + area, + LayoutMode::Compact, + &descriptors, + Some(&selected), + usize::MAX, + ); + assert!(scene.visible_ids.contains(&selected)); + assert!(scene.node_rects.contains_key(&selected)); + } + for area in [Rect::new(0, 0, 0, 8), Rect::new(0, 0, 8, 0)] { + let scene = layout_scene( + area, + LayoutMode::Compact, + &descriptors, + Some(&selected), + 0, + ); + assert!( + scene.visible_ids.is_empty() && scene.node_rects.is_empty() + ); + } + } + + #[test] + fn minimum_is_empty_but_retains_safe_metadata() { + let scene = layout_scene( + Rect::new(5, 8, 160, 20), + LayoutMode::Minimum, + &resources(), + None, + 0, + ); + assert!(scene.visible_ids.is_empty()); + assert!( + scene.node_rects.is_empty() + && scene.connectors.is_empty() + && scene.fabric_buses.is_empty() + ); + assert_eq!(scene.tiers.len(), 3); + assert!(scene.tiers.iter().all(|tier| tier.overflow.end == 0)); + } + + #[test] + fn reduced_empty_scene_has_no_selection_or_overflow() { + let scene = layout_scene( + Rect::new(0, 0, 48, 3), + LayoutMode::Compact, + &[], + None, + 7, + ); + + assert!(scene.visible_ids.is_empty()); + assert!(scene.node_rects.is_empty()); + assert!(scene.tiers.iter().all(|tier| { + tier.overflow.start == 0 + && tier.overflow.end == 0 + && tier.overflow.total == 0 + })); + } + + #[test] + fn wide_and_compact_are_distinct_and_exact_fit_is_used() { + let descriptors = resources(); + let wide = layout_scene( + Rect::new(0, 0, 27, 20), // spine + gap + (12 + gap + 12) + LayoutMode::Wide, + &descriptors, + None, + 0, + ); + let compact = layout_scene( + Rect::new(0, 0, 19, 20), // spine + gap + (8 + gap + 8) + LayoutMode::Compact, + &descriptors, + None, + 0, + ); + assert_eq!(wide.tiers[0].visible_ids.len(), 2); // 12 + one gap + 12 + assert_eq!(compact.tiers[0].visible_ids.len(), 2); + assert!( + wide.tiers[0].node_area.height > compact.tiers[0].node_area.height + ); + } + + #[test] + fn capacity_respects_minimum_width_and_gap_boundaries() { + for minimum_width in [8, 12] { + assert_eq!(exact_capacity(minimum_width - 1, minimum_width), 0); + assert_eq!(exact_capacity(minimum_width, minimum_width), 1); + assert_eq!(exact_capacity(2 * minimum_width, minimum_width), 1); + assert_eq!(exact_capacity(2 * minimum_width + 1, minimum_width), 2); + } + + let descriptors = resources(); + let selected = descriptors + .iter() + .find(|item| item.kind == ResourceKind::SwitchZone) + .unwrap() + .id + .clone(); + let scene = layout_scene( + Rect::new(0, 0, 9, 20), // 7 content columns: below compact minimum. + LayoutMode::Compact, + &descriptors, + Some(&selected), + 0, + ); + assert_eq!(scene.visible_ids, vec![selected.clone()]); + assert_eq!(scene.node_rects[&selected], Rect::new(0, 0, 9, 20)); + assert!(scene.fabric_buses.is_empty() && scene.connectors.is_empty()); + } + + #[test] + fn overflow_ranges_are_truthful_and_semantic_order_is_stable() { + let descriptors = resources(); + let scene = layout_scene( + Rect::new(7, 9, 60, 20), + LayoutMode::Wide, + &descriptors, + None, + 99, + ); + for tier in &scene.tiers { + assert_eq!( + tier.overflow.end - tier.overflow.start, + tier.visible_ids.len() + ); + assert!( + tier.overflow.start <= tier.overflow.end + && tier.overflow.end <= tier.overflow.total + ); + } + let mut expected = Vec::new(); + for kind in + [ResourceKind::Router, ResourceKind::SwitchZone, ResourceKind::Sled] + { + let mut ids: Vec<_> = descriptors + .iter() + .filter(|descriptor| descriptor.kind == kind) + .map(|descriptor| descriptor.id.clone()) + .collect(); + ids.sort(); + expected.extend(ids); + } + assert_eq!(scene.navigation_order, expected); + // Unicode names deliberately participate only as stable IDs; Task 3 fits display text. + } +} diff --git a/voxel/src/tui/ui/widgets.rs b/voxel/src/tui/ui/widgets.rs new file mode 100644 index 0000000..109cb4b --- /dev/null +++ b/voxel/src/tui/ui/widgets.rs @@ -0,0 +1,534 @@ +use super::{ + colors::*, + deployment, + renderer::{LayoutMode, MIN_HEIGHT, MIN_WIDTH, layout_mode}, +}; +use crate::tui::{app::App, event::View, telemetry::TrafficSeverity}; +use ratatui::{ + layout::{Constraint, Layout, Rect}, + style::{Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Borders, Paragraph, Tabs}, +}; + +pub(crate) fn terminal_width(text: &str) -> usize { + Line::from(text).width() +} + +pub(crate) fn fit_terminal_width(text: &str, width: usize) -> String { + if terminal_width(text) <= width { + return text.to_owned(); + } + if width == 0 { + return String::new(); + } + + let content_width = width - 1; + let mut fitted = String::new(); + let mut used = 0; + for character in text.chars() { + let character_width = terminal_width(&character.to_string()); + if used + character_width > content_width { + break; + } + fitted.push(character); + used += character_width; + } + fitted.push('…'); + fitted +} + +pub fn draw(frame: &mut ratatui::Frame<'_>, app: &App) { + let area = frame.area(); + let root = root_layout(area); + if root.mode == LayoutMode::Minimum { + minimum(frame, area); + return; + } + let alerts = app.operation.retained_warnings.len() + + usize::from(app.operation.start_failure.is_some()) + + usize::from(app.deployment.reconciliation_failure.is_some()); + let titles = ["Deployment", "Monitoring"] + .into_iter() + .map(Line::from) + .collect::>(); + let tabs = Tabs::new(titles) + .select(usize::from(app.session.view == View::Monitor)) + .style(Style::default().fg(TUI_GREY)) + .highlight_style(active_tab_style()) + .divider(Span::styled(" │ ", Style::default().fg(TUI_GREY_DARK))); + let state = format!("{:?} │ alerts {alerts}", app.deployment.observed); + if area.width < 80 { + let header = + Layout::vertical([Constraint::Length(1), Constraint::Length(2)]) + .split(root.header); + frame.render_widget( + Paragraph::new(format!(" VOXEL · {state}")) + .style(Style::default().fg(OX_YELLOW)), + header[0], + ); + frame.render_widget( + tabs.block( + Block::default() + .borders(Borders::BOTTOM) + .border_style(primary_edge_style()), + ), + header[1], + ); + } else { + let header = + Layout::horizontal([Constraint::Min(1), Constraint::Length(32)]) + .split(root.header); + frame.render_widget( + tabs.block( + Block::default() + .borders(Borders::BOTTOM) + .title(" VOXEL ") + .border_style(primary_edge_style()), + ), + header[0], + ); + frame.render_widget( + Paragraph::new(state) + .style(Style::default().fg(OX_YELLOW)) + .right_aligned() + .block( + Block::default() + .borders(Borders::BOTTOM) + .border_style(primary_edge_style()), + ), + header[1], + ); + } + match app.session.view { + View::Deployment => { + deployment::draw(frame, root.content, app, root.mode) + } + View::Monitor => { + super::monitor::draw(frame, root.content, app, root.mode) + } + } + frame.render_widget( + Paragraph::new(guidance_lines(app, root.footer.width)), + root.footer, + ); + if app.session.view == View::Monitor && app.session.detail_open { + super::node_detail::draw(frame, app); + } + if app.session.external_monitoring_open { + super::external_monitoring::draw(frame, app); + } + if app.session.help_open { + super::help::draw(frame, app, root.mode); + } + if app.session.confirmation.is_some() { + super::confirm_dialog::draw(frame, app); + } +} + +pub(crate) fn content_area(app: &App) -> (Rect, LayoutMode) { + let terminal = Rect::new( + 0, + 0, + app.session.terminal.width, + app.session.terminal.height, + ); + let root = root_layout(terminal); + (root.content, root.mode) +} + +pub(crate) fn overlay_area(area: Rect) -> Rect { + root_layout(area).content +} + +struct RootLayout { + header: Rect, + content: Rect, + footer: Rect, + mode: LayoutMode, +} + +fn root_layout(area: Rect) -> RootLayout { + let mode = layout_mode(area); + if mode == LayoutMode::Minimum { + return RootLayout { + header: Rect::default(), + content: Rect::default(), + footer: Rect::default(), + mode, + }; + } + let footer_height = if area.width >= 160 { + 1 + } else if area.width >= 100 { + 2 + } else { + 3 + }; + let rows = Layout::vertical([ + Constraint::Length(3), + Constraint::Min(1), + Constraint::Length(footer_height), + ]) + .split(area); + RootLayout { header: rows[0], content: rows[1], footer: rows[2], mode } +} + +fn grow_section( + heights: &mut [u16], + index: usize, + preferred: &[u16], + remaining: &mut u16, +) { + if index >= heights.len() || index >= preferred.len() || *remaining == 0 { + return; + } + let wanted = preferred[index].saturating_sub(heights[index]); + let granted = wanted.min(*remaining); + heights[index] = heights[index].saturating_add(granted); + *remaining = remaining.saturating_sub(granted); +} + +pub(crate) fn section_heights( + total: u16, + expanded: &[bool], + focused: usize, + preferred: &[u16], + priority: &[usize], + flexible: usize, +) -> Vec { + let count = expanded.len().min(preferred.len()); + let mut heights = vec![0; count]; + for height in heights.iter_mut().take(total as usize) { + *height = 1; + } + let mut remaining = total.saturating_sub(count as u16); + if expanded.get(focused).copied().unwrap_or(false) { + grow_section(&mut heights, focused, preferred, &mut remaining); + } + for &index in priority { + if expanded.get(index).copied().unwrap_or(false) && index != focused { + grow_section(&mut heights, index, preferred, &mut remaining); + } + } + if expanded.get(flexible).copied().unwrap_or(false) + && flexible < heights.len() + { + heights[flexible] = heights[flexible].saturating_add(remaining); + } + heights +} + +pub(crate) fn section_rects(area: Rect, heights: &[u16]) -> Vec { + let mut y = area.y; + heights + .iter() + .map(|height| { + let available = area.bottom().saturating_sub(y); + let height = (*height).min(available); + let rect = Rect::new(area.x, y, area.width, height); + y = y.saturating_add(height); + rect + }) + .collect() +} + +#[derive(Clone, Copy)] +struct ActionGroup { + key: &'static str, + description: &'static str, +} + +fn guidance_lines(app: &App, width: u16) -> Vec> { + let groups = action_groups(app, width < 80); + let row_count = if width >= 160 { + 1 + } else if width >= 100 { + 2 + } else { + 3 + }; + let mut rows = vec![Vec::new(); row_count]; + let mut row = 0; + for group in groups { + let mut candidate = rows[row].clone(); + candidate.push(group); + if row + 1 < row_count + && styled_groups(candidate).width() > width as usize + { + row += 1; + } + rows[row].push(group); + } + rows.into_iter().map(styled_groups).collect() +} + +fn styled_groups(groups: Vec) -> Line<'static> { + let mut spans = Vec::new(); + for (index, group) in groups.into_iter().enumerate() { + if index > 0 { + spans.push(Span::raw(" ")); + } + spans.push(Span::styled( + format!("[{}]", group.key), + Style::default().fg(TUI_YELLOW).add_modifier(Modifier::BOLD), + )); + if !group.description.is_empty() { + spans.push(Span::styled( + format!(" {}", group.description), + Style::default().fg(OX_OFF_WHITE), + )); + } + } + Line::from(spans) +} + +fn action_groups(app: &App, narrow: bool) -> Vec { + let group = |key, description| ActionGroup { key, description }; + let lifecycle = || { + vec![ + group("d", "detach"), + group( + "q", + if app.resources_may_exist() { "destroy+quit" } else { "quit" }, + ), + ] + }; + let contextual_navigation = if narrow { "Tab/⇧" } else { "Tab/S-Tab" }; + let item_or_section = if narrow { "item/sect" } else { "item/section" }; + let log_or_section = if narrow { "log/sec" } else { "log/section" }; + let resource_or_section = + if narrow { "res/sect" } else { "resource/section" }; + if app.session.confirmation.is_some() { + return vec![ + group("↑/↓", "move"), + group("Enter", "select"), + group("y", "confirm"), + group("n/Esc", "back"), + ]; + } + if app.session.external_monitoring_open { + return vec![ + group("s", "copy rack YAML"), + group("a", "copy all YAML"), + group("u", "copy guide"), + group("Esc", "close"), + ]; + } + if app.session.help_open { + let mut groups = + vec![group("↑/↓/Pg", "scroll"), group("?/Esc", "close")]; + groups.extend(lifecycle()); + return groups; + } + if app.session.detail_open { + let mut groups = vec![ + group("↑/↓", "resource"), + group("Enter/Esc", "close"), + group("?", "help"), + ]; + groups.extend(lifecycle()); + return groups; + } + if app.session.view == View::Monitor { + let pane = app.session.monitoring_pane; + let expanded = app.session.monitoring_expanded(pane); + let mut groups = + vec![group("Space", if expanded { "fold" } else { "expand" })]; + if expanded { + match pane { + crate::tui::event::MonitoringPane::RackSummary => { + groups.push(group("←/→", "rack")); + groups.push(group("↑/↓", "section")); + } + crate::tui::event::MonitoringPane::Topology => { + groups.push(group("↑/↓", resource_or_section)); + groups.push(group("Pg", if narrow { "" } else { "page" })); + if app.session.selected_resource.is_some() { + groups.push(group("Enter", "open")); + } + } + crate::tui::event::MonitoringPane::TopZones => { + groups.push(group("↑/↓", "zones/section")); + groups.push(group("Pg", if narrow { "" } else { "page" })); + } + } + } else { + groups.push(group("↑/↓", "section")); + } + if app.session.selected_resource.is_some() { + groups.push(group("Esc", if narrow { "" } else { "clear" })); + } + groups.extend([ + group(contextual_navigation, "section"), + group("1/2", "view"), + ]); + groups.extend(lifecycle()); + groups.push(group("?", "help")); + return groups; + } + let mut groups = Vec::new(); + if app.can_cancel() { + if narrow { + groups.push(group("c/x", "")); + } else { + groups.extend([group("c", "leave"), group("x", "destroy")]); + } + } else { + for (kind, key, description) in [ + (crate::tui::operation::OperationKind::Launch, "l", "launch"), + (crate::tui::operation::OperationKind::Route, "r", "route"), + ] { + if app.can_start(kind) { + groups.push(group(key, if narrow { "" } else { description })); + } + } + } + let pane = app.session.deployment_pane; + let expanded = app.session.deployment_expanded(pane); + groups.push(group("Space", if expanded { "fold" } else { "expand" })); + if expanded { + match pane { + crate::tui::event::DeploymentPane::Phases + | crate::tui::event::DeploymentPane::CurrentPhase => { + groups.push(group("↑/↓", item_or_section)); + groups.push(group("Pg", if narrow { "" } else { "page" })); + } + crate::tui::event::DeploymentPane::Logs => { + groups.push(group("f", "filter")); + groups.push(group("↑/↓", log_or_section)); + groups.push(group("Pg", if narrow { "" } else { "page" })); + } + crate::tui::event::DeploymentPane::OverallProgress + | crate::tui::event::DeploymentPane::Status => { + groups.push(group("↑/↓", "section")); + } + } + } else { + groups.push(group("↑/↓", "section")); + } + groups.extend([ + group(contextual_navigation, "section"), + group("1/2", "view"), + ]); + groups.extend(lifecycle()); + groups.push(group("?", "help")); + groups +} + +fn minimum(frame: &mut ratatui::Frame<'_>, area: ratatui::layout::Rect) { + frame.render_widget( + Paragraph::new(format!( + "Terminal too small: {}x{}; required at least {}x{}", + area.width, area.height, MIN_WIDTH, MIN_HEIGHT + )) + .block(primary_block(" VOXEL ")), + area, + ); +} + +pub fn format_rate(value: f64) -> String { + if value >= 1_000_000.0 { + format!("{:.1} MB/s", value / 1_000_000.0) + } else if value >= 1_000.0 { + format!("{:.1} KB/s", value / 1_000.0) + } else { + format!("{value:.0} B/s") + } +} + +pub(crate) fn traffic_style(severity: TrafficSeverity) -> Style { + Style::default().fg(match severity { + TrafficSeverity::Normal => OX_OFF_WHITE, + TrafficSeverity::Elevated => TUI_YELLOW, + TrafficSeverity::High => OX_RED, + }) +} + +pub(crate) fn selection_style() -> Style { + Style::default().fg(TUI_PURPLE).add_modifier(Modifier::BOLD) +} + +pub(crate) fn primary_edge_style() -> Style { + Style::default().fg(OX_GREEN_LIGHT) +} + +pub(crate) fn primary_block<'a>(title: impl Into>) -> Block<'a> { + Block::bordered().title(title).border_style(primary_edge_style()) +} + +pub(crate) fn section_block<'a>( + title: impl Into>, + expanded: bool, + focused: bool, +) -> Block<'a> { + let mut spans = vec![Span::raw(if expanded { " ▾ " } else { " ▸ " })]; + spans.extend(title.into().spans); + let style = if focused { active_tab_style() } else { primary_edge_style() }; + Block::bordered() + .title(Line::from(spans)) + .border_style(style) + .title_style(style) +} + +pub(crate) fn active_tab_style() -> Style { + Style::default().fg(TUI_YELLOW).add_modifier(Modifier::BOLD) +} + +pub fn centered_rect(percent_x: u16, percent_y: u16, area: Rect) -> Rect { + let vertical = Layout::vertical([ + Constraint::Percentage((100 - percent_y) / 2), + Constraint::Percentage(percent_y), + Constraint::Percentage((100 - percent_y) / 2), + ]) + .split(area); + Layout::horizontal([ + Constraint::Percentage((100 - percent_x) / 2), + Constraint::Percentage(percent_x), + Constraint::Percentage((100 - percent_x) / 2), + ]) + .split(vertical[1])[1] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tui::telemetry::{ + RackId, ResourceDescriptor, ResourceId, ResourceKind, + }; + use ratatui::{Terminal, backend::TestBackend}; + + fn screen_text(app: &App) -> String { + let mut terminal = Terminal::new(TestBackend::new(160, 50)).unwrap(); + terminal.draw(|frame| draw(frame, app)).unwrap(); + terminal + .backend() + .buffer() + .content() + .iter() + .map(|cell| cell.symbol()) + .collect() + } + + #[test] + fn resource_detail_title_omits_redundant_close_instruction() { + let descriptor = ResourceDescriptor { + id: ResourceId::rack(RackId(0), ResourceKind::Sled, "g0"), + rack: Some(RackId(0)), + kind: ResourceKind::Sled, + name: "g0".into(), + host: None, + }; + let mut app = App::new(vec![descriptor.clone()], 8, 8); + app.session.view = View::Monitor; + app.session.selected_resource = Some(descriptor.id); + app.session.detail_open = true; + + let text = screen_text(&app); + + assert!(text.contains("Resource detail")); + assert!(!text.contains("Enter/Esc closes")); + assert!(text.contains("Enter/Esc")); + assert!(text.contains("close")); + } +}