{ "title": "Sharing telemetry while one Engine runs a normal workload", "generated": "2026-10-30T01:13:45.771Z", "build_configuration": "Windows Clang Release", "host": { "cpu": "Intel Core i7-9700K", "logical_processors": 8, "memory_configuration_gb": 32, "os": "Windows 11 build 26100" }, "workload": { "modules": 25, "target_hz": 100, "device_io_channels": 250, "periodic_tasks": 75, "input_calculations": 125, "generated_event_calculations": 50, "events": 25, "task_groups": 8, "recording": "on_value_change plus baseline recording", "channel_snapshot_hz": 5, "status_catalog_hz": 2 }, "methods": { "windows": { "web": { "warmup_seconds": 30, "screen_seconds": 120, "boundary_repeats": 3, "soak_seconds": 600 }, "zmq": { "warmup_seconds": 15, "screen_seconds": 60 }, "peers": { "warmup_seconds": 15, "screen_seconds": 60 }, "combined": { "warmup_seconds": 15, "screen_seconds": 60 } }, "clients": "Independent protocol sessions, additional to one operator measurement connection; all load sessions share one Node process.", "heartbeats": "Web fixture: one-second HTTP heartbeat, 30-second request deadline. Node ZeroMQ SDK: five-second heartbeat after success, one-second heartbeat request deadline. Channel data remains 5 Hz for both.", "peers": "Protocol-only Release simulator; generated channels and tiny empty ARGUS boards at 5 Hz, no Engine, control workload or recording store.", "cpu": "Percent of the complete eight-core host. Existing Engine/Interface and other background processes remained running.", "ambient_preflight": "Before each new run, whole-host CPU must remain at or below 45% for 20 seconds with no benchmark Engine or load generator running. Activity during measurement remains recorded.", "excluded_series": "The earlier verified-capacity series encountered sharply increased unrelated background load. It and a subsequent workload-alone control are retained separately in context_runs; they do not select boundaries. Main runs use fresh WebSocket qualification and the subsequent short ZeroMQ/peer/combined screens.", "timing": "Counter deltas bracket measurement. Diagnostic callback/late-release distributions are sampled observations, not every callback.", "recording_counters": "Recorder admission uses enqueued_rows_total. The legacy accepted_rows_total counts RAPID operations and is not recorder admission. Commit deltas can include rows queued before the window; drain/loss checks are separate.", "freshness": "All selected channels checked for complete delivery. Source-age sentinels are the first and last subscribed channel. UI freshness tracks source timestamp advancement.", "scope": "Loopback test on one shared machine. Cross-machine throughput, clock synchronization, physical network links and remote control execution were not qualified." }, "boundaries": { "web": { "last_qualified_level": null, "highest_healthy_screen_level": 32, "highest_consistently_healthy_tested_level": 16, "first_degraded_level": 32, "screened_cap": 32, "qualified_run": null, "selected_run": "verified-capacity-stable-release-25-web-16", "evidence": "120-second screen", "qualification": "Qualification requires a screen, two boundary repeats and a 600-second soak. Screening results are reported separately; a capped sweep is a tested lower bound." }, "zmq": { "last_qualified_level": null, "highest_healthy_screen_level": 8, "highest_consistently_healthy_tested_level": 8, "first_degraded_level": 16, "screened_cap": 32, "qualified_run": null, "selected_run": "verified-capacity-final-release-25-zmq-8", "evidence": "61-second screen", "qualification": "Qualification requires a screen, two boundary repeats and a 600-second soak. Screening results are reported separately; a capped sweep is a tested lower bound." }, "peer": { "last_qualified_level": null, "highest_healthy_screen_level": 8, "highest_consistently_healthy_tested_level": 8, "first_degraded_level": null, "screened_cap": 8, "qualified_run": null, "selected_run": "verified-capacity-final-release-25-peer-8", "evidence": "61-second screen", "qualification": "Qualification requires a screen, two boundary repeats and a 600-second soak. Screening results are reported separately; a capped sweep is a tested lower bound." } }, "runs": [ { "id": "verified-capacity-final-release-25-alone", "healthy": true, "failure_reasons": [], "seconds": 60.82974500000001, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.014917299999997, "wait_seconds": 21.0153984, "completed_at": "2026-10-30T00:28:45.454Z", "accepted_cpu": { "count": 20, "mean": 23.199449264933666, "p95": 37.03657780533168, "max": 39.15416098226466 } }, "requested_client_count": 0, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 0, "transport": "web", "actual_connections": { "loadClients": 0, "actualClients": 1, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 0, "recipient_channel_subscriptions": 0, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 54, "mean": 35.43377955439559, "p50": 35.71783322907758, "p95": 51.823103287387525, "p99": 56.9157299113895, "min": 18.74585720479113, "max": 56.9157299113895 }, "engine_cpu_percent": { "count": 54, "mean": 9.23038578729715, "p50": 6.6303924549115445, "p95": 20.031048844063154, "p99": 20.80675432298883, "min": 2.7829848041905247, "max": 20.80675432298883 }, "client_generator_cpu_percent": { "count": 54, "mean": 0.5212914437270588, "p50": 0.34923146726388044, "p95": 1.1838412675322156, "p99": 1.220971956861216, "min": 0.16333665477331905, "max": 1.220971956861216 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 54, "mean": 999143499.8518518, "p50": 999493632, "p95": 1004662784, "p99": 1005187072, "min": 989118464, "max": 1005187072 }, "system_samples": 54, "system_sample_interval_seconds": { "count": 54, "mean": 1.1327581962962963, "p95": 1.1957665, "max": 1.2133324, "p50": 1.1250998, "p99": 1.2133324, "min": 1.1092773 }, "achieved_read_hz": { "minimum": 99.02251409156155, "maximum": 99.49882517312393 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24256, "hz": 99.44887610765285 }, { "group": "stress_group_1", "controls": 3, "completed": 18195, "hz": 99.46527598827747 }, { "group": "stress_group_2", "controls": 3, "completed": 18198, "hz": 99.48167586890209 }, { "group": "stress_group_3", "controls": 3, "completed": 18126, "hz": 99.08807873391136 }, { "group": "stress_group_4", "controls": 3, "completed": 18114, "hz": 99.02247921141293 }, { "group": "stress_group_5", "controls": 3, "completed": 18120, "hz": 99.05527897266215 }, { "group": "stress_group_6", "controls": 3, "completed": 18117, "hz": 99.03887909203753 }, { "group": "stress_group_7", "controls": 3, "completed": 18201, "hz": 99.4980757495267 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 3369, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 348523628, "member_service_ns_total": 356613700, "members_executed_total": 24256 }, "example_device_plugin.stress_read": { "callback_ns_total": 2040779618, "member_service_ns_total": 2056057100, "members_executed_total": 24256 }, "example_device_plugin.stress_write": { "callback_ns_total": 96388690, "member_service_ns_total": 104184800, "members_executed_total": 24256 } }, "callback_ns_total": 2485691936, "cycles_total": 6064, "member_service_ns_total": 2516855600, "members_executed_total": 72768, "thread_cpu_ns_total": 781250000, "wall_ns_total": 2554930000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 271887873, "member_service_ns_total": 277522900, "members_executed_total": 18195 }, "example_device_plugin.stress_read": { "callback_ns_total": 1578090839, "member_service_ns_total": 1590206600, "members_executed_total": 18195 }, "example_device_plugin.stress_write": { "callback_ns_total": 81365598, "member_service_ns_total": 87353700, "members_executed_total": 18195 } }, "callback_ns_total": 1931344310, "cycles_total": 6065, "member_service_ns_total": 1955083200, "members_executed_total": 54585, "thread_cpu_ns_total": 625000000, "wall_ns_total": 1993802500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 248306395, "member_service_ns_total": 254430500, "members_executed_total": 18198 }, "example_device_plugin.stress_read": { "callback_ns_total": 1560434363, "member_service_ns_total": 1571372100, "members_executed_total": 18198 }, "example_device_plugin.stress_write": { "callback_ns_total": 75210097, "member_service_ns_total": 80251500, "members_executed_total": 18198 } }, "callback_ns_total": 1883950855, "cycles_total": 6066, "member_service_ns_total": 1906054100, "members_executed_total": 54594, "thread_cpu_ns_total": 640625000, "wall_ns_total": 1938113700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 242792504, "member_service_ns_total": 249401300, "members_executed_total": 18126 }, "example_device_plugin.stress_read": { "callback_ns_total": 1626450749, "member_service_ns_total": 1639826300, "members_executed_total": 18126 }, "example_device_plugin.stress_write": { "callback_ns_total": 77804795, "member_service_ns_total": 83643600, "members_executed_total": 18126 } }, "callback_ns_total": 1947048048, "cycles_total": 6042, "member_service_ns_total": 1972871200, "members_executed_total": 54378, "thread_cpu_ns_total": 656250000, "wall_ns_total": 2009960100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 240109578, "member_service_ns_total": 247110700, "members_executed_total": 18114 }, "example_device_plugin.stress_read": { "callback_ns_total": 1645437516, "member_service_ns_total": 1657926300, "members_executed_total": 18114 }, "example_device_plugin.stress_write": { "callback_ns_total": 79966097, "member_service_ns_total": 84984800, "members_executed_total": 18114 } }, "callback_ns_total": 1965513191, "cycles_total": 6038, "member_service_ns_total": 1990021800, "members_executed_total": 54342, "thread_cpu_ns_total": 437500000, "wall_ns_total": 2027855200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 244870890, "member_service_ns_total": 250753800, "members_executed_total": 18120 }, "example_device_plugin.stress_read": { "callback_ns_total": 1615054568, "member_service_ns_total": 1626425000, "members_executed_total": 18120 }, "example_device_plugin.stress_write": { "callback_ns_total": 80218997, "member_service_ns_total": 87279000, "members_executed_total": 18120 } }, "callback_ns_total": 1940144455, "cycles_total": 6040, "member_service_ns_total": 1964457800, "members_executed_total": 54360, "thread_cpu_ns_total": 390625000, "wall_ns_total": 1996837300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 246900159, "member_service_ns_total": 262103400, "members_executed_total": 18117 }, "example_device_plugin.stress_read": { "callback_ns_total": 1667378571, "member_service_ns_total": 1678105900, "members_executed_total": 18117 }, "example_device_plugin.stress_write": { "callback_ns_total": 86111992, "member_service_ns_total": 92744800, "members_executed_total": 18117 } }, "callback_ns_total": 2000390722, "cycles_total": 6039, "member_service_ns_total": 2032954100, "members_executed_total": 54351, "thread_cpu_ns_total": 453125000, "wall_ns_total": 2068300300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 276808989, "member_service_ns_total": 283335300, "members_executed_total": 18201 }, "example_device_plugin.stress_read": { "callback_ns_total": 1620578386, "member_service_ns_total": 1632066800, "members_executed_total": 18201 }, "example_device_plugin.stress_write": { "callback_ns_total": 76930298, "member_service_ns_total": 82392100, "members_executed_total": 18201 } }, "callback_ns_total": 1974317673, "cycles_total": 6067, "member_service_ns_total": 1997794200, "members_executed_total": 54603, "thread_cpu_ns_total": 640625000, "wall_ns_total": 2036196800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 60.9760536, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 0.9996141178150016, "failed": 0, "missed": 0, "callback_last_ms": 0.0041, "callback_ewma_ms": 0.004574, "release_lateness_last_ms": 0.3271 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 0.9996141178150016, "failed": 0, "missed": 0, "callback_last_ms": 8.3238, "callback_ewma_ms": 7.412406, "release_lateness_last_ms": 0.1803 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 305, "hz": 4.998070589075009, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.001112, "release_lateness_last_ms": 0.3558 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 305, "hz": 4.998070589075009, "failed": 0, "missed": 0, "callback_last_ms": 0.0106, "callback_ewma_ms": 0.013026, "release_lateness_last_ms": 0.5344 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 0.9996141178150016, "failed": 0, "missed": 0, "callback_last_ms": 0.01, "callback_ewma_ms": 0.011267, "release_lateness_last_ms": 0.4075 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 0.9996141178150016, "failed": 0, "missed": 0, "callback_last_ms": 0.0057, "callback_ewma_ms": 0.016904, "release_lateness_last_ms": 0.5257 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 122, "hz": 1.9992282356300033, "failed": 0, "missed": 0, "callback_last_ms": 0.023, "callback_ewma_ms": 0.003264, "release_lateness_last_ms": 0.037 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 0.9996141178150016, "failed": 0, "missed": 0, "callback_last_ms": 2.7985, "callback_ewma_ms": 2.154818, "release_lateness_last_ms": 0.2206 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 122, "hz": 1.9992282356300033, "failed": 0, "missed": 0, "callback_last_ms": 3.5785, "callback_ewma_ms": 4.624699, "release_lateness_last_ms": 0.3943 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 305, "hz": 4.998070589075009, "failed": 0, "missed": 0, "callback_last_ms": 3.2817, "callback_ewma_ms": 4.910282, "release_lateness_last_ms": 0.2018 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 610, "hz": 9.996141178150017, "failed": 0, "missed": 0, "callback_last_ms": 0.0181, "callback_ewma_ms": 0.31612, "release_lateness_last_ms": 0.2664 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 305, "hz": 4.998070589075009, "failed": 0, "missed": 0, "callback_last_ms": 0.3184, "callback_ewma_ms": 0.38917, "release_lateness_last_ms": 0.4451 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 122, "hz": 1.9992282356300033, "failed": 0, "missed": 0, "callback_last_ms": 4.7651, "callback_ewma_ms": 6.530581, "release_lateness_last_ms": 0.0985 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 305, "hz": 4.998070589075009, "failed": 0, "missed": 0, "callback_last_ms": 0.0019, "callback_ewma_ms": 0.002282, "release_lateness_last_ms": 0.3196 }, { "name": "general_program_checks_thread", "running": true, "completed": 610, "hz": 9.996141178150017, "failed": 0, "missed": 0, "callback_last_ms": 0.1388, "callback_ewma_ms": 0.189929, "release_lateness_last_ms": 0.912 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 0.9996141178150016, "failed": 0, "missed": 0, "callback_last_ms": 0.09, "callback_ewma_ms": 0.126333, "release_lateness_last_ms": 0.1312 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 610, "hz": 9.996141178150017, "failed": 0, "missed": 0, "callback_last_ms": 0.0051, "callback_ewma_ms": 0.006028, "release_lateness_last_ms": 0.5792 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 1.9992282356300033, "failed": 0, "missed": 0, "callback_last_ms": 39.0638, "callback_ewma_ms": 43.186102, "release_lateness_last_ms": 0.5118 } ], "peer_counter_seconds": 61.01262609999999, "peer_counter_deltas": [], "observations": [], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 56, "mean": 72.95791785714229, "p95": 90.92119999999704, "max": 92.67119999999704, "p50": 71.9435000000085, "p99": 92.67119999999704, "min": 55.71049999999741 }, { "operation": "rapid/get-persistence-diagnostics", "count": 56, "mean": 11.478210714286108, "p95": 17.38520000000426, "max": 18.594499999999243, "p50": 10.74529999999504, "p99": 18.594499999999243, "min": 5.213400000000547 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1110457, "accepted_rows_total": 7012676, "committed_rows_total": 1110417, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 56, "minimum": 17.96875, "maximum": 78.125, "mean": 36.39048126352814 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 56, "minimum": 3.4395591494107802, "maximum": 21.4828840878443, "mean": 8.87708843396557 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 56, "minimum": 989126656, "maximum": 1005187072, "mean": 998923556.5714285 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0.836181640625, "mean": 0.31863621303013395 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0.01220703125, "mean": 0.0004359654017857143 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0.0244140625, "mean": 0.0008719308035714286 }, { "channel": "engine_tempest_tx_mbps", "observations": 56, "minimum": 3.512420085608601, "maximum": 6.794552886716534, "mean": 4.348692019397718 }, { "channel": "engine_tempest_rx_mbps", "observations": 56, "minimum": 0, "maximum": 0.002738097930634452, "mean": 0.00029311572745747326 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 56, "minimum": 10518.10253430281, "maximum": 39900.70100580311, "mean": 18488.664446669063 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 56, "minimum": 11205.978441489155, "maximum": 40919.29360587805, "mean": 18415.841178085848 } ], "graceful_shutdown": true, "shutdown_seconds": 1.3959167999999917 }, { "id": "verified-capacity-final-release-25-combined", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_3: control callbacks below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz" ], "seconds": 61.0879938, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.066373799999997, "wait_seconds": 37.1951529, "completed_at": "2026-10-30T00:52:31.368Z", "accepted_cpu": { "count": 20, "mean": 27.514619482022965, "p95": 44.40187839841819, "max": 44.46790016143052 } }, "requested_client_count": 8, "requested_peer_count": 4, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": "17bf8cb763b8c97152f26fbeb8cce42f75b63b9c2e08fd968d2d26b1b4fc44b4", "clients": 8, "transport": "web", "actual_connections": { "loadClients": 8, "actualClients": 9, "actualPeers": 4, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 947, "imported_channels": 1000, "total_channels": 1947, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 2000, "imported_channels_per_peer": 250, "peers": [ { "name": "SIM_0", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 304, "complete_export_hz": 4.983574131466019, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_1", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 304, "complete_export_hz": 4.983574131466019, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_2", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 304, "complete_export_hz": 4.983574131466019, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_3", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 304, "complete_export_hz": 4.983574131466019, "dropped_telemetry": 0, "operation_timeouts": 0 } ], "peer_simulator_offered": { "counter_seconds": 61.000396900000005, "send_cycles": 305, "offered_snapshots": 1220, "offered_bytes": 100254020, "offered_argus_snapshots": 1220, "offered_argus_bytes": 61000, "received_snapshots": 1216, "late_cycles": 0 }, "host_cpu_percent": { "count": 48, "mean": 59.87035313418864, "p50": 57.41281415033863, "p95": 85.49756827735415, "p99": 91.32750441145693, "min": 36.39346345720701, "max": 91.32750441145693 }, "engine_cpu_percent": { "count": 48, "mean": 15.210249482398803, "p50": 12.50863743096246, "p95": 28.353344262267775, "p99": 31.18054019574368, "min": 7.597422703537777, "max": 31.18054019574368 }, "client_generator_cpu_percent": { "count": 48, "mean": 2.832446724307424, "p50": 2.6612228408370733, "p95": 4.271312728099104, "p99": 4.860539079667424, "min": 1.1746212645164393, "max": 4.860539079667424 }, "peer_generator_cpu_percent": { "count": 48, "mean": 6.14457278825331, "p50": 6.077938162830222, "p95": 7.959981866578705, "p99": 8.761527042396603, "min": 4.071082003707095, "max": 8.761527042396603 }, "engine_resident_bytes": { "count": 48, "mean": 1045281962.6666666, "p50": 1047515136, "p95": 1054040064, "p99": 1055862784, "min": 1017991168, "max": 1055862784 }, "system_samples": 48, "system_sample_interval_seconds": { "count": 48, "mean": 1.2598498499999995, "p95": 1.6916053, "max": 2.6978679, "p50": 1.1591872, "p99": 2.6978679, "min": 1.1260655 }, "achieved_read_hz": { "minimum": 93.75374846261647, "maximum": 96.43710481975698 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 23332, "hz": 95.31077833366561 }, { "group": "stress_group_1", "controls": 3, "completed": 17637, "hz": 96.06241485061206 }, { "group": "stress_group_2", "controls": 3, "completed": 17583, "hz": 95.76829621354607 }, { "group": "stress_group_3", "controls": 3, "completed": 17259, "hz": 94.00358439115006 }, { "group": "stress_group_4", "controls": 3, "completed": 17259, "hz": 94.00358439115006 }, { "group": "stress_group_5", "controls": 3, "completed": 17286, "hz": 94.15064370968305 }, { "group": "stress_group_6", "controls": 3, "completed": 17217, "hz": 93.77482545120984 }, { "group": "stress_group_7", "controls": 3, "completed": 17709, "hz": 96.45457303336673 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 23220, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 560918018, "member_service_ns_total": 572328400, "members_executed_total": 23332 }, "example_device_plugin.stress_read": { "callback_ns_total": 4760912596, "member_service_ns_total": 4777432700, "members_executed_total": 23332 }, "example_device_plugin.stress_write": { "callback_ns_total": 130778295, "member_service_ns_total": 140031900, "members_executed_total": 23332 } }, "callback_ns_total": 5452608909, "cycles_total": 5833, "member_service_ns_total": 5489793000, "members_executed_total": 69996, "thread_cpu_ns_total": 1984375000, "wall_ns_total": 5536177500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 447387448, "member_service_ns_total": 455266500, "members_executed_total": 17637 }, "example_device_plugin.stress_read": { "callback_ns_total": 3897210345, "member_service_ns_total": 3917552000, "members_executed_total": 17637 }, "example_device_plugin.stress_write": { "callback_ns_total": 131148390, "member_service_ns_total": 137690300, "members_executed_total": 17637 } }, "callback_ns_total": 4475746183, "cycles_total": 5879, "member_service_ns_total": 4510508800, "members_executed_total": 52911, "thread_cpu_ns_total": 1312500000, "wall_ns_total": 4552561800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 423409993, "member_service_ns_total": 430278200, "members_executed_total": 17583 }, "example_device_plugin.stress_read": { "callback_ns_total": 4031470263, "member_service_ns_total": 4043083100, "members_executed_total": 17583 }, "example_device_plugin.stress_write": { "callback_ns_total": 99360390, "member_service_ns_total": 104913700, "members_executed_total": 17583 } }, "callback_ns_total": 4554240646, "cycles_total": 5861, "member_service_ns_total": 4578275000, "members_executed_total": 52749, "thread_cpu_ns_total": 1375000000, "wall_ns_total": 4616852400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 377675455, "member_service_ns_total": 463994700, "members_executed_total": 17259 }, "example_device_plugin.stress_read": { "callback_ns_total": 4410464922, "member_service_ns_total": 4429411700, "members_executed_total": 17259 }, "example_device_plugin.stress_write": { "callback_ns_total": 117083194, "member_service_ns_total": 125113900, "members_executed_total": 17259 } }, "callback_ns_total": 4905223571, "cycles_total": 5753, "member_service_ns_total": 5018520300, "members_executed_total": 51777, "thread_cpu_ns_total": 1281250000, "wall_ns_total": 5065899000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 329680087, "member_service_ns_total": 340832800, "members_executed_total": 17259 }, "example_device_plugin.stress_read": { "callback_ns_total": 4126680833, "member_service_ns_total": 4147322000, "members_executed_total": 17259 }, "example_device_plugin.stress_write": { "callback_ns_total": 102741993, "member_service_ns_total": 109103100, "members_executed_total": 17259 } }, "callback_ns_total": 4559102913, "cycles_total": 5753, "member_service_ns_total": 4597257900, "members_executed_total": 51777, "thread_cpu_ns_total": 1046875000, "wall_ns_total": 4644355200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 336746061, "member_service_ns_total": 348136300, "members_executed_total": 17286 }, "example_device_plugin.stress_read": { "callback_ns_total": 4130858944, "member_service_ns_total": 4145598700, "members_executed_total": 17286 }, "example_device_plugin.stress_write": { "callback_ns_total": 102734892, "member_service_ns_total": 110518600, "members_executed_total": 17286 } }, "callback_ns_total": 4570339897, "cycles_total": 5762, "member_service_ns_total": 4604253600, "members_executed_total": 51858, "thread_cpu_ns_total": 1046875000, "wall_ns_total": 4691734100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 326615075, "member_service_ns_total": 339114700, "members_executed_total": 17217 }, "example_device_plugin.stress_read": { "callback_ns_total": 4453386441, "member_service_ns_total": 4468604000, "members_executed_total": 17217 }, "example_device_plugin.stress_write": { "callback_ns_total": 92472295, "member_service_ns_total": 98350300, "members_executed_total": 17217 } }, "callback_ns_total": 4872473811, "cycles_total": 5739, "member_service_ns_total": 4906069000, "members_executed_total": 51651, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 4951047200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 506990232, "member_service_ns_total": 516062400, "members_executed_total": 17709 }, "example_device_plugin.stress_read": { "callback_ns_total": 3636547323, "member_service_ns_total": 3659661900, "members_executed_total": 17709 }, "example_device_plugin.stress_write": { "callback_ns_total": 108904988, "member_service_ns_total": 115035800, "members_executed_total": 17709 } }, "callback_ns_total": 4252442543, "cycles_total": 5903, "member_service_ns_total": 4290760100, "members_executed_total": 53127, "thread_cpu_ns_total": 1203125000, "wall_ns_total": 4336408700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 61.19979400000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 0.9952657671149014, "failed": 0, "missed": 0, "callback_last_ms": 0.1232, "callback_ewma_ms": 0.156479, "release_lateness_last_ms": 0.3963 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 0.9952657671149014, "failed": 0, "missed": 0, "callback_last_ms": 6.4319, "callback_ewma_ms": 15.420952, "release_lateness_last_ms": 0.3464 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 307, "hz": 5.008960500070078, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001472, "release_lateness_last_ms": 1.2536 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 307, "hz": 5.008960500070078, "failed": 0, "missed": 0, "callback_last_ms": 0.9707, "callback_ewma_ms": 0.746287, "release_lateness_last_ms": 0.6459 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 0.9952657671149014, "failed": 0, "missed": 0, "callback_last_ms": 0.9829, "callback_ewma_ms": 1.119089, "release_lateness_last_ms": 0.4411 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 0.9952657671149014, "failed": 0, "missed": 0, "callback_last_ms": 1.1885, "callback_ewma_ms": 0.853263, "release_lateness_last_ms": 0.4956 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 123, "hz": 2.0068473664775883, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001687, "release_lateness_last_ms": 0.4149 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 0.9952657671149014, "failed": 0, "missed": 0, "callback_last_ms": 3.2485, "callback_ewma_ms": 4.639415, "release_lateness_last_ms": 0.6711 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 123, "hz": 2.0068473664775883, "failed": 0, "missed": 0, "callback_last_ms": 6.431, "callback_ewma_ms": 10.659206, "release_lateness_last_ms": 0.2755 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 304, "hz": 4.960013003326722, "failed": 0, "missed": 3, "callback_last_ms": 11.1345, "callback_ewma_ms": 10.05357, "release_lateness_last_ms": 0.9513 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 610, "hz": 9.952657671149014, "failed": 0, "missed": 3, "callback_last_ms": 0.1629, "callback_ewma_ms": 0.696394, "release_lateness_last_ms": 0.4303 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 306, "hz": 4.992644667822293, "failed": 0, "missed": 1, "callback_last_ms": 9.4849, "callback_ewma_ms": 7.981063, "release_lateness_last_ms": 0.1355 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 123, "hz": 2.0068473664775883, "failed": 0, "missed": 0, "callback_last_ms": 8.5928, "callback_ewma_ms": 14.295263, "release_lateness_last_ms": 0.212 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 306, "hz": 4.992644667822293, "failed": 0, "missed": 1, "callback_last_ms": 25.6854, "callback_ewma_ms": 27.390104, "release_lateness_last_ms": 0.3742 }, { "name": "general_program_checks_thread", "running": true, "completed": 609, "hz": 9.93634183890123, "failed": 0, "missed": 4, "callback_last_ms": 0.1357, "callback_ewma_ms": 0.18005, "release_lateness_last_ms": 0.1795 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 0.9952657671149014, "failed": 0, "missed": 0, "callback_last_ms": 0.1394, "callback_ewma_ms": 0.166416, "release_lateness_last_ms": 0.615 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 613, "hz": 10.00160516789237, "failed": 0, "missed": 0, "callback_last_ms": 0.0046, "callback_ewma_ms": 0.006043, "release_lateness_last_ms": 1.9464 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 1.990531534229803, "failed": 0, "missed": 0, "callback_last_ms": 45.1621, "callback_ewma_ms": 61.291718, "release_lateness_last_ms": 0.4586 } ], "peer_counter_seconds": 61.2926355, "peer_counter_deltas": [ { "name": "SIM_0", "connected": true, "channels_received": 306, "argus_received": 306, "frames_received": 612, "frames_sent": 610, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.56213096580691, "downlink_mbps": 3.3143556226418713 }, { "name": "SIM_1", "connected": true, "channels_received": 306, "argus_received": 306, "frames_received": 612, "frames_sent": 610, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.56213096580691, "downlink_mbps": 3.3143556226418713 }, { "name": "SIM_2", "connected": true, "channels_received": 306, "argus_received": 305, "frames_received": 611, "frames_sent": 610, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.56213096580691, "downlink_mbps": 3.3143556226418713 }, { "name": "SIM_3", "connected": true, "channels_received": 307, "argus_received": 306, "frames_received": 613, "frames_sent": 610, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.56213096580691, "downlink_mbps": 3.3143556226418713 } ], "observations": [ { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.69520000000193, "gap_p95_ms": 236.3992999999973, "gap_p99_ms": 340.98659999998927, "gap_max_ms": 870.1088000000018, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 20, "p95": 136, "p99": 614, "max": 940, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.23570000000473, "gap_p95_ms": 244.38639999999577, "gap_p99_ms": 470.1001000000033, "gap_max_ms": 1018.1906000000017, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 27, "p95": 186, "p99": 660, "max": 1051, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.07660000000033, "gap_p95_ms": 253.68759999999747, "gap_p99_ms": 372.5, "gap_max_ms": 1003.7735999999968, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 33, "p95": 161, "p99": 601, "max": 809, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.29370000000927, "gap_p95_ms": 247.00619999999617, "gap_p99_ms": 339.69400000000314, "gap_max_ms": 1450.222600000001, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 39, "p95": 166, "p99": 767, "max": 1259, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.6079000000027, "gap_p95_ms": 237.8784999999989, "gap_p99_ms": 385.44710000000487, "gap_max_ms": 1208.4703000000009, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 36, "p95": 180, "p99": 893, "max": 1392, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.8920999999973, "gap_p95_ms": 243.7256999999954, "gap_p99_ms": 397.1915000000008, "gap_max_ms": 1190.3518999999942, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 30, "p95": 158, "p99": 854, "max": 1361, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.52560000000085, "gap_p95_ms": 249.1206999999995, "gap_p99_ms": 415.7887000000046, "gap_max_ms": 1563.0962000000036, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 38, "p95": 159, "p99": 872, "max": 1392, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992846601344354, "complete_hz": 4.992846601344354, "catalog_hz": 1.9971386405377416, "bytes_per_second": 1549528.5349081785, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.42950000000565, "gap_p95_ms": 236.61710000000312, "gap_p99_ms": 336.9323000000004, "gap_max_ms": 1608.499499999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 38, "p95": 151, "p99": 918, "max": 1446, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 52, "mean": 118.48364038461554, "p95": 297.0020000000004, "max": 833.9526999999944, "p50": 85.57780000000275, "p99": 833.9526999999944, "min": 63.11490000000049 }, { "operation": "rapid/get-persistence-diagnostics", "count": 52, "mean": 17.02980961538532, "p95": 38.88739999999234, "max": 376.9846000000034, "p50": 7.113900000003923, "p99": 376.9846000000034, "min": 4.666199999999662 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1385971, "accepted_rows_total": 7388543, "committed_rows_total": 1385526, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 52, "minimum": 36.328125, "maximum": 99.60159362549801, "mean": 60.17761336412666 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 52, "minimum": 7.0276012694209165, "maximum": 31.61302051049024, "mean": 15.259723033525926 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 52, "minimum": 1022300160, "maximum": 1055547392, "mean": 1044872507.0769231 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 52, "minimum": 0, "maximum": 0.2899169921875, "mean": 0.033510648287259616 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 52, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 52, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 52, "minimum": 0, "maximum": 0.1220703125, "mean": 0.007042518028846154 }, { "channel": "engine_tempest_tx_mbps", "observations": 52, "minimum": 334.73926993662894, "maximum": 547.9563245374139, "mean": 438.6822943022841 }, { "channel": "engine_tempest_rx_mbps", "observations": 52, "minimum": 10.466861635144486, "maximum": 15.953615843108638, "mean": 13.66593128792958 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 52, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 52, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 52, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 52, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 52, "minimum": 16077.32482919218, "maximum": 41615.31377299889, "mean": 22675.620290368235 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 52, "minimum": 16654.090815964504, "maximum": 40345.57901301177, "mean": 22686.90362419212 } ], "graceful_shutdown": true, "shutdown_seconds": 2.484257599999997 }, { "id": "verified-capacity-final-release-25-peer-1", "healthy": true, "failure_reasons": [], "seconds": 60.9902785, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.038885499999996, "wait_seconds": 42.0621556, "completed_at": "2026-10-30T00:43:51.102Z", "accepted_cpu": { "count": 20, "mean": 27.08820980579295, "p95": 33.78361783126612, "max": 40.77263779527559 } }, "requested_client_count": 1, "requested_peer_count": 1, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": "17bf8cb763b8c97152f26fbeb8cce42f75b63b9c2e08fd968d2d26b1b4fc44b4", "clients": 1, "transport": "web", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 1, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 938, "imported_channels": 250, "total_channels": 1188, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 250, "peers": [ { "name": "SIM_0", "ready": true, "exported_channels_at_end": 938, "complete_received_snapshots": 305, "complete_export_hz": 4.9998967644266425, "dropped_telemetry": 0, "operation_timeouts": 0 } ], "peer_simulator_offered": { "counter_seconds": 61.0012595, "send_cycles": 305, "offered_snapshots": 305, "offered_bytes": 25062255, "offered_argus_snapshots": 305, "offered_argus_bytes": 15250, "received_snapshots": 305, "late_cycles": 0 }, "host_cpu_percent": { "count": 54, "mean": 42.188066520669324, "p50": 39.46560459245836, "p95": 58.997067874671785, "p99": 72.94979878360037, "min": 29.458757229872745, "max": 72.94979878360037 }, "engine_cpu_percent": { "count": 54, "mean": 9.732244506999063, "p50": 7.228357180615327, "p95": 20.247655936971242, "p99": 21.190421637892864, "min": 4.394102909468306, "max": 21.190421637892864 }, "client_generator_cpu_percent": { "count": 54, "mean": 0.9850726704977987, "p50": 0.8605187119780151, "p95": 1.8713006767276503, "p99": 2.031787692294889, "min": 0.3412439616744484, "max": 2.031787692294889 }, "peer_generator_cpu_percent": { "count": 54, "mean": 1.30291236494964, "p50": 1.3582399921834918, "p95": 1.701182433388773, "p99": 1.885274650869621, "min": 0.6819554458103145, "max": 1.885274650869621 }, "engine_resident_bytes": { "count": 54, "mean": 1010178730.6666666, "p50": 1009856512, "p95": 1018327040, "p99": 1018560512, "min": 996524032, "max": 1018560512 }, "system_samples": 54, "system_sample_interval_seconds": { "count": 54, "mean": 1.1458459444444447, "p95": 1.1704616, "max": 1.2208056, "p50": 1.145066, "p99": 1.2208056, "min": 1.1215831 }, "achieved_read_hz": { "minimum": 99.51782638415531, "maximum": 99.76355545910107 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24384, "hz": 99.73496684157155 }, { "group": "stress_group_1", "controls": 3, "completed": 18294, "hz": 99.76768828738572 }, { "group": "stress_group_2", "controls": 3, "completed": 18285, "hz": 99.71860611866447 }, { "group": "stress_group_3", "controls": 3, "completed": 18258, "hz": 99.57135961250073 }, { "group": "stress_group_4", "controls": 3, "completed": 18249, "hz": 99.52227744377949 }, { "group": "stress_group_5", "controls": 3, "completed": 18258, "hz": 99.57135961250073 }, { "group": "stress_group_6", "controls": 3, "completed": 18261, "hz": 99.58772033540781 }, { "group": "stress_group_7", "controls": 3, "completed": 18285, "hz": 99.71860611866447 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 1578, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 354182240, "member_service_ns_total": 361807500, "members_executed_total": 24384 }, "example_device_plugin.stress_read": { "callback_ns_total": 2159507185, "member_service_ns_total": 2175817100, "members_executed_total": 24384 }, "example_device_plugin.stress_write": { "callback_ns_total": 114168791, "member_service_ns_total": 120934500, "members_executed_total": 24384 } }, "callback_ns_total": 2627858216, "cycles_total": 6096, "member_service_ns_total": 2658559100, "members_executed_total": 73152, "thread_cpu_ns_total": 750000000, "wall_ns_total": 2699437700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 275905465, "member_service_ns_total": 282537000, "members_executed_total": 18294 }, "example_device_plugin.stress_read": { "callback_ns_total": 1751524536, "member_service_ns_total": 1764849600, "members_executed_total": 18294 }, "example_device_plugin.stress_write": { "callback_ns_total": 85634093, "member_service_ns_total": 91767900, "members_executed_total": 18294 } }, "callback_ns_total": 2113064094, "cycles_total": 6098, "member_service_ns_total": 2139154500, "members_executed_total": 54882, "thread_cpu_ns_total": 734375000, "wall_ns_total": 2181271900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 276659573, "member_service_ns_total": 282620900, "members_executed_total": 18285 }, "example_device_plugin.stress_read": { "callback_ns_total": 1759926941, "member_service_ns_total": 1773890300, "members_executed_total": 18285 }, "example_device_plugin.stress_write": { "callback_ns_total": 86086398, "member_service_ns_total": 92005100, "members_executed_total": 18285 } }, "callback_ns_total": 2122672912, "cycles_total": 6095, "member_service_ns_total": 2148516300, "members_executed_total": 54855, "thread_cpu_ns_total": 828125000, "wall_ns_total": 2184261200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 265189870, "member_service_ns_total": 271668600, "members_executed_total": 18258 }, "example_device_plugin.stress_read": { "callback_ns_total": 1870731134, "member_service_ns_total": 1883054500, "members_executed_total": 18258 }, "example_device_plugin.stress_write": { "callback_ns_total": 86220599, "member_service_ns_total": 92273400, "members_executed_total": 18258 } }, "callback_ns_total": 2222141603, "cycles_total": 6086, "member_service_ns_total": 2246996500, "members_executed_total": 54774, "thread_cpu_ns_total": 531250000, "wall_ns_total": 2286525700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 264668396, "member_service_ns_total": 271997800, "members_executed_total": 18249 }, "example_device_plugin.stress_read": { "callback_ns_total": 1864467096, "member_service_ns_total": 1876610800, "members_executed_total": 18249 }, "example_device_plugin.stress_write": { "callback_ns_total": 79383091, "member_service_ns_total": 85085600, "members_executed_total": 18249 } }, "callback_ns_total": 2208518583, "cycles_total": 6083, "member_service_ns_total": 2233694200, "members_executed_total": 54747, "thread_cpu_ns_total": 312500000, "wall_ns_total": 2271757200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 272282474, "member_service_ns_total": 279732400, "members_executed_total": 18258 }, "example_device_plugin.stress_read": { "callback_ns_total": 1792908597, "member_service_ns_total": 1805346000, "members_executed_total": 18258 }, "example_device_plugin.stress_write": { "callback_ns_total": 87070593, "member_service_ns_total": 92788900, "members_executed_total": 18258 } }, "callback_ns_total": 2152261664, "cycles_total": 6086, "member_service_ns_total": 2177867300, "members_executed_total": 54774, "thread_cpu_ns_total": 390625000, "wall_ns_total": 2216908500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 255520478, "member_service_ns_total": 264552100, "members_executed_total": 18261 }, "example_device_plugin.stress_read": { "callback_ns_total": 1778523981, "member_service_ns_total": 1794674400, "members_executed_total": 18261 }, "example_device_plugin.stress_write": { "callback_ns_total": 94969095, "member_service_ns_total": 100429500, "members_executed_total": 18261 } }, "callback_ns_total": 2129013554, "cycles_total": 6087, "member_service_ns_total": 2159656000, "members_executed_total": 54783, "thread_cpu_ns_total": 546875000, "wall_ns_total": 2197566300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 322614260, "member_service_ns_total": 329910100, "members_executed_total": 18285 }, "example_device_plugin.stress_read": { "callback_ns_total": 1820903279, "member_service_ns_total": 1837293400, "members_executed_total": 18285 }, "example_device_plugin.stress_write": { "callback_ns_total": 90779991, "member_service_ns_total": 96336000, "members_executed_total": 18285 } }, "callback_ns_total": 2234297530, "cycles_total": 6095, "member_service_ns_total": 2263539500, "members_executed_total": 54855, "thread_cpu_ns_total": 593750000, "wall_ns_total": 2303216300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 61.12199355, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 0.9970438581270464, "failed": 0, "missed": 0, "callback_last_ms": 0.0555, "callback_ewma_ms": 0.081641, "release_lateness_last_ms": 0.4911 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 0.9970438581270464, "failed": 0, "missed": 0, "callback_last_ms": 11.6017, "callback_ewma_ms": 9.995559, "release_lateness_last_ms": 0.59 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 306, "hz": 5.001564271916004, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.001716, "release_lateness_last_ms": 0.5249 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 306, "hz": 5.001564271916004, "failed": 0, "missed": 0, "callback_last_ms": 0.1793, "callback_ewma_ms": 0.146682, "release_lateness_last_ms": 0.6215 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 0.9970438581270464, "failed": 0, "missed": 0, "callback_last_ms": 0.2168, "callback_ewma_ms": 0.160646, "release_lateness_last_ms": 0.1263 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 0.9970438581270464, "failed": 0, "missed": 0, "callback_last_ms": 0.2271, "callback_ewma_ms": 0.178564, "release_lateness_last_ms": 0.2763 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 122, "hz": 1.9940877162540929, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001165, "release_lateness_last_ms": 0.589 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 62, "hz": 1.0133888394078177, "failed": 0, "missed": 0, "callback_last_ms": 2.8386, "callback_ewma_ms": 3.26903, "release_lateness_last_ms": 0.4612 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 122, "hz": 1.9940877162540929, "failed": 0, "missed": 0, "callback_last_ms": 7.2951, "callback_ewma_ms": 6.107446, "release_lateness_last_ms": 0.4405 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 306, "hz": 5.001564271916004, "failed": 0, "missed": 0, "callback_last_ms": 5.549, "callback_ewma_ms": 7.290812, "release_lateness_last_ms": 0.2876 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 611, "hz": 9.986783562551237, "failed": 0, "missed": 0, "callback_last_ms": 0.0276, "callback_ewma_ms": 0.663291, "release_lateness_last_ms": 0.3091 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 306, "hz": 5.001564271916004, "failed": 0, "missed": 0, "callback_last_ms": 3.5468, "callback_ewma_ms": 3.576663, "release_lateness_last_ms": 0.1693 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 122, "hz": 1.9940877162540929, "failed": 0, "missed": 0, "callback_last_ms": 14.5901, "callback_ewma_ms": 10.038191, "release_lateness_last_ms": 0.3611 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 306, "hz": 5.001564271916004, "failed": 0, "missed": 0, "callback_last_ms": 5.3615, "callback_ewma_ms": 6.719167, "release_lateness_last_ms": 0.5492 }, { "name": "general_program_checks_thread", "running": true, "completed": 612, "hz": 10.003128543832007, "failed": 0, "missed": 0, "callback_last_ms": 0.1911, "callback_ewma_ms": 0.1992, "release_lateness_last_ms": 0.2068 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 62, "hz": 1.0133888394078177, "failed": 0, "missed": 0, "callback_last_ms": 0.1093, "callback_ewma_ms": 0.137281, "release_lateness_last_ms": 0.3763 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 611, "hz": 9.986783562551237, "failed": 0, "missed": 0, "callback_last_ms": 0.0042, "callback_ewma_ms": 0.00502, "release_lateness_last_ms": 0.5294 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 1.9940877162540929, "failed": 0, "missed": 0, "callback_last_ms": 44.7234, "callback_ewma_ms": 46.151947, "release_lateness_last_ms": 0.4985 } ], "peer_counter_seconds": 61.1714825, "peer_counter_deltas": [ { "name": "SIM_0", "connected": true, "channels_received": 306, "argus_received": 306, "frames_received": 612, "frames_sent": 612, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 41.836957353667266, "downlink_mbps": 3.3154435941957647 } ], "observations": [ { "snapshot_hz": 5.000831072539587, "complete_hz": 5.000831072539587, "catalog_hz": 2.0003324290158346, "bytes_per_second": 1550843.2052116152, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.07919999999285, "gap_p95_ms": 205.39990000000398, "gap_p99_ms": 214.16139999999723, "gap_max_ms": 266.0457000000024, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 6, "p95": 14, "p99": 23, "max": 50, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 56, "mean": 76.90889107142831, "p95": 93.08600000000297, "max": 188.982600000003, "p50": 74.24210000000312, "p99": 188.982600000003, "min": 55.61750000000029 }, { "operation": "rapid/get-persistence-diagnostics", "count": 56, "mean": 8.723624999999888, "p95": 17.69340000000375, "max": 19.25390000000334, "p50": 7.021600000000035, "p99": 19.25390000000334, "min": 4.371500000001106 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1191533, "accepted_rows_total": 7208423, "committed_rows_total": 1192214, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 56, "minimum": 22.65625, "maximum": 71.21212121212122, "mean": 41.97977761485826 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 56, "minimum": 2.34422728467516, "maximum": 23.82008810227429, "mean": 9.485645575941104 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 56, "minimum": 998391808, "maximum": 1018437632, "mean": 1009987218.2857143 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0.1953125, "mean": 0.006866455078125 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 56, "minimum": 60.3810541501878, "maximum": 97.43007109401287, "mean": 77.24552970642301 }, { "channel": "engine_tempest_rx_mbps", "observations": 56, "minimum": 2.630907523567231, "maximum": 3.9912747928766206, "mean": 3.290646805391128 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 56, "minimum": 13010.884444665664, "maximum": 40942.293762059526, "mean": 19379.242411762683 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 56, "minimum": 11874.961871911624, "maximum": 41923.48209203238, "mean": 19547.78895387351 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7904110999999976 }, { "id": "verified-capacity-final-release-25-peer-2", "healthy": true, "failure_reasons": [], "seconds": 60.079299899999995, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.01223389999999, "wait_seconds": 20.01223389999999, "completed_at": "2026-10-30T00:45:52.303Z", "accepted_cpu": { "count": 20, "mean": 25.971163713890093, "p95": 33.77672799602188, "max": 34.30332922318126 } }, "requested_client_count": 1, "requested_peer_count": 2, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": "17bf8cb763b8c97152f26fbeb8cce42f75b63b9c2e08fd968d2d26b1b4fc44b4", "clients": 1, "transport": "web", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 2, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 941, "imported_channels": 500, "total_channels": 1441, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 250, "peers": [ { "name": "SIM_0", "ready": true, "exported_channels_at_end": 941, "complete_received_snapshots": 305, "complete_export_hz": 5.000017729571064, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_1", "ready": true, "exported_channels_at_end": 941, "complete_received_snapshots": 305, "complete_export_hz": 5.000017729571064, "dropped_telemetry": 0, "operation_timeouts": 0 } ], "peer_simulator_offered": { "counter_seconds": 60.9997837, "send_cycles": 305, "offered_snapshots": 610, "offered_bytes": 50124510, "offered_argus_snapshots": 610, "offered_argus_bytes": 30500, "received_snapshots": 610, "late_cycles": 0 }, "host_cpu_percent": { "count": 53, "mean": 46.09990760290009, "p50": 43.032847507907135, "p95": 61.135188313935465, "p99": 78.56766910380534, "min": 32.84181016106288, "max": 78.56766910380534 }, "engine_cpu_percent": { "count": 53, "mean": 11.493669699404697, "p50": 8.629780992561896, "p95": 23.82174406066346, "p99": 26.238039575710964, "min": 4.65794955398953, "max": 26.238039575710964 }, "client_generator_cpu_percent": { "count": 53, "mean": 1.0054412615219561, "p50": 0.8593514331462975, "p95": 1.932488120402894, "p99": 2.234681651955008, "min": 0.33806294024449235, "max": 2.234681651955008 }, "peer_generator_cpu_percent": { "count": 53, "mean": 2.9540006930740303, "p50": 2.9237399250536353, "p95": 3.82303367837431, "p99": 3.991834941137999, "min": 1.8790192152686627, "max": 3.991834941137999 }, "engine_resident_bytes": { "count": 53, "mean": 1015930570.8679246, "p50": 1015508992, "p95": 1023348736, "p99": 1023504384, "min": 1001660416, "max": 1023504384 }, "system_samples": 53, "system_sample_interval_seconds": { "count": 53, "mean": 1.1496897622641506, "p95": 1.2059121, "max": 1.2282382, "p50": 1.1466691, "p99": 1.2282382, "min": 1.1211641 }, "achieved_read_hz": { "minimum": 98.77401108588732, "maximum": 99.3562113399667 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 23868, "hz": 99.1444636971314 }, { "group": "stress_group_1", "controls": 3, "completed": 17928, "hz": 99.29400285806223 }, { "group": "stress_group_2", "controls": 3, "completed": 17940, "hz": 99.36046470736481 }, { "group": "stress_group_3", "controls": 3, "completed": 17838, "hz": 98.79553898829283 }, { "group": "stress_group_4", "controls": 3, "completed": 17856, "hz": 98.8952317622467 }, { "group": "stress_group_5", "controls": 3, "completed": 17886, "hz": 99.06138638550317 }, { "group": "stress_group_6", "controls": 3, "completed": 17835, "hz": 98.77892352596719 }, { "group": "stress_group_7", "controls": 3, "completed": 17913, "hz": 99.21092554643398 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 4233, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 365928311, "member_service_ns_total": 373707800, "members_executed_total": 23868 }, "example_device_plugin.stress_read": { "callback_ns_total": 2499233581, "member_service_ns_total": 2514023900, "members_executed_total": 23868 }, "example_device_plugin.stress_write": { "callback_ns_total": 129406593, "member_service_ns_total": 137055500, "members_executed_total": 23868 } }, "callback_ns_total": 2994568485, "cycles_total": 5967, "member_service_ns_total": 3024787200, "members_executed_total": 71604, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3067350000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 300989453, "member_service_ns_total": 308001400, "members_executed_total": 17928 }, "example_device_plugin.stress_read": { "callback_ns_total": 2134685326, "member_service_ns_total": 2147130400, "members_executed_total": 17928 }, "example_device_plugin.stress_write": { "callback_ns_total": 87959399, "member_service_ns_total": 93587500, "members_executed_total": 17928 } }, "callback_ns_total": 2523634178, "cycles_total": 5976, "member_service_ns_total": 2548719300, "members_executed_total": 53784, "thread_cpu_ns_total": 859375000, "wall_ns_total": 2587647200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 277624753, "member_service_ns_total": 284530100, "members_executed_total": 17940 }, "example_device_plugin.stress_read": { "callback_ns_total": 2059479421, "member_service_ns_total": 2072039700, "members_executed_total": 17940 }, "example_device_plugin.stress_write": { "callback_ns_total": 82412396, "member_service_ns_total": 87716000, "members_executed_total": 17940 } }, "callback_ns_total": 2419516570, "cycles_total": 5980, "member_service_ns_total": 2444285800, "members_executed_total": 53820, "thread_cpu_ns_total": 921875000, "wall_ns_total": 2478095200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 275549465, "member_service_ns_total": 281630900, "members_executed_total": 17838 }, "example_device_plugin.stress_read": { "callback_ns_total": 2247827689, "member_service_ns_total": 2263966600, "members_executed_total": 17838 }, "example_device_plugin.stress_write": { "callback_ns_total": 91077794, "member_service_ns_total": 96596900, "members_executed_total": 17838 } }, "callback_ns_total": 2614454948, "cycles_total": 5946, "member_service_ns_total": 2642194400, "members_executed_total": 53514, "thread_cpu_ns_total": 937500000, "wall_ns_total": 2688812700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 262674845, "member_service_ns_total": 271466200, "members_executed_total": 17856 }, "example_device_plugin.stress_read": { "callback_ns_total": 2082075026, "member_service_ns_total": 2096792700, "members_executed_total": 17856 }, "example_device_plugin.stress_write": { "callback_ns_total": 86492799, "member_service_ns_total": 93118200, "members_executed_total": 17856 } }, "callback_ns_total": 2431242670, "cycles_total": 5952, "member_service_ns_total": 2461377100, "members_executed_total": 53568, "thread_cpu_ns_total": 625000000, "wall_ns_total": 2495753800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 288377363, "member_service_ns_total": 301760000, "members_executed_total": 17886 }, "example_device_plugin.stress_read": { "callback_ns_total": 1978333840, "member_service_ns_total": 1989161700, "members_executed_total": 17886 }, "example_device_plugin.stress_write": { "callback_ns_total": 91828795, "member_service_ns_total": 98087100, "members_executed_total": 17886 } }, "callback_ns_total": 2358539998, "cycles_total": 5962, "member_service_ns_total": 2389008800, "members_executed_total": 53658, "thread_cpu_ns_total": 578125000, "wall_ns_total": 2426276300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 298987685, "member_service_ns_total": 306270400, "members_executed_total": 17835 }, "example_device_plugin.stress_read": { "callback_ns_total": 2214633813, "member_service_ns_total": 2227562900, "members_executed_total": 17835 }, "example_device_plugin.stress_write": { "callback_ns_total": 82064696, "member_service_ns_total": 88253200, "members_executed_total": 17835 } }, "callback_ns_total": 2595686194, "cycles_total": 5945, "member_service_ns_total": 2622086500, "members_executed_total": 53505, "thread_cpu_ns_total": 609375000, "wall_ns_total": 2660438700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 300198152, "member_service_ns_total": 307096000, "members_executed_total": 17913 }, "example_device_plugin.stress_read": { "callback_ns_total": 2137966045, "member_service_ns_total": 2152972200, "members_executed_total": 17913 }, "example_device_plugin.stress_write": { "callback_ns_total": 91083193, "member_service_ns_total": 96948500, "members_executed_total": 17913 } }, "callback_ns_total": 2529247390, "cycles_total": 5971, "member_service_ns_total": 2557016700, "members_executed_total": 53739, "thread_cpu_ns_total": 859375000, "wall_ns_total": 2595520000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 60.18490370000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 60, "hz": 0.9958499232456139, "failed": 0, "missed": 0, "callback_last_ms": 0.0799, "callback_ewma_ms": 0.09988, "release_lateness_last_ms": 0.1304 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 60, "hz": 0.9958499232456139, "failed": 0, "missed": 0, "callback_last_ms": 9.5372, "callback_ewma_ms": 16.661718, "release_lateness_last_ms": 0.5082 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 301, "hz": 4.99584711494883, "failed": 0, "missed": 0, "callback_last_ms": 0.0016, "callback_ewma_ms": 0.002119, "release_lateness_last_ms": 0.3722 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 301, "hz": 4.99584711494883, "failed": 0, "missed": 0, "callback_last_ms": 0.0984, "callback_ewma_ms": 0.132964, "release_lateness_last_ms": 0.4859 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 60, "hz": 0.9958499232456139, "failed": 0, "missed": 0, "callback_last_ms": 0.1378, "callback_ewma_ms": 0.158957, "release_lateness_last_ms": 0.0447 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 60, "hz": 0.9958499232456139, "failed": 0, "missed": 0, "callback_last_ms": 0.0755, "callback_ewma_ms": 0.163623, "release_lateness_last_ms": 0.1488 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 121, "hz": 2.008297345211988, "failed": 0, "missed": 0, "callback_last_ms": 0.0014, "callback_ewma_ms": 0.002166, "release_lateness_last_ms": 0.0256 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 60, "hz": 0.9958499232456139, "failed": 0, "missed": 0, "callback_last_ms": 2.7417, "callback_ewma_ms": 2.657388, "release_lateness_last_ms": 0.1967 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 121, "hz": 2.008297345211988, "failed": 0, "missed": 0, "callback_last_ms": 4.9483, "callback_ewma_ms": 7.140644, "release_lateness_last_ms": 0.2366 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 301, "hz": 4.99584711494883, "failed": 0, "missed": 0, "callback_last_ms": 9.8025, "callback_ewma_ms": 9.343256, "release_lateness_last_ms": 0.5969 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 602, "hz": 9.99169422989766, "failed": 0, "missed": 0, "callback_last_ms": 0.0322, "callback_ewma_ms": 1.058555, "release_lateness_last_ms": 0.3035 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 301, "hz": 4.99584711494883, "failed": 0, "missed": 0, "callback_last_ms": 2.9974, "callback_ewma_ms": 2.977004, "release_lateness_last_ms": 0.5025 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 121, "hz": 2.008297345211988, "failed": 0, "missed": 0, "callback_last_ms": 6.0536, "callback_ewma_ms": 11.001838, "release_lateness_last_ms": 0.183 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 301, "hz": 4.99584711494883, "failed": 0, "missed": 0, "callback_last_ms": 11.8908, "callback_ewma_ms": 11.781679, "release_lateness_last_ms": 0.547 }, { "name": "general_program_checks_thread", "running": true, "completed": 603, "hz": 10.00829172861842, "failed": 0, "missed": 0, "callback_last_ms": 0.149, "callback_ewma_ms": 0.303941, "release_lateness_last_ms": 0.3771 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 60, "hz": 0.9958499232456139, "failed": 0, "missed": 0, "callback_last_ms": 0.1235, "callback_ewma_ms": 0.121971, "release_lateness_last_ms": 0.0872 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 602, "hz": 9.99169422989766, "failed": 0, "missed": 0, "callback_last_ms": 0.0054, "callback_ewma_ms": 0.009543, "release_lateness_last_ms": 0.2254 }, { "name": "engine_diagnostics", "running": true, "completed": 120, "hz": 1.9916998464912279, "failed": 0, "missed": 0, "callback_last_ms": 44.326, "callback_ewma_ms": 52.657156, "release_lateness_last_ms": 0.5062 } ], "peer_counter_seconds": 60.248847600000005, "peer_counter_deltas": [ { "name": "SIM_0", "connected": true, "channels_received": 301, "argus_received": 301, "frames_received": 602, "frames_sent": 602, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 41.974608449810894, "downlink_mbps": 3.31520146620921 }, { "name": "SIM_1", "connected": true, "channels_received": 301, "argus_received": 301, "frames_received": 602, "frames_sent": 602, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 41.974608449810894, "downlink_mbps": 3.31520146620921 } ], "observations": [ { "snapshot_hz": 5.010087219726233, "complete_hz": 5.010087219726233, "catalog_hz": 1.9973769646749104, "bytes_per_second": 1553856.3137345512, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.97329999999783, "gap_p95_ms": 207.5951000000059, "gap_p99_ms": 223.9048999999941, "gap_max_ms": 270.833000000006, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 602, "p50": 11, "p95": 20, "p99": 36, "max": 72, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 55, "mean": 77.94889636363594, "p95": 98.87860000000364, "max": 104.42359999999826, "p50": 77.28530000000319, "p99": 104.42359999999826, "min": 59.04409999999916 }, { "operation": "rapid/get-persistence-diagnostics", "count": 55, "mean": 10.839225454544927, "p95": 18.020099999994272, "max": 27.97019999999975, "p50": 8.535399999993388, "p99": 27.97019999999975, "min": 4.912100000001374 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1249232, "accepted_rows_total": 7224953, "committed_rows_total": 1248629, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 55, "minimum": 30.46875, "maximum": 77.34375, "mean": 46.61284076016343 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 55, "minimum": 4.297755180260918, "maximum": 26.53943722904796, "mean": 11.747028321352378 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 55, "minimum": 1002721280, "maximum": 1023578112, "mean": 1015900048.290909 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0.0335693359375, "mean": 0.0008322975852272727 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0.048828125, "mean": 0.0008877840909090909 }, { "channel": "engine_tempest_tx_mbps", "observations": 55, "minimum": 94.67628982022141, "maximum": 155.85138154374832, "mean": 123.90424260744877 }, { "channel": "engine_tempest_rx_mbps", "observations": 55, "minimum": 5.279888254301465, "maximum": 7.980917583041155, "mean": 6.771062584196826 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 55, "minimum": 14471.464512208637, "maximum": 41484.80256343941, "mean": 20949.107850419376 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 55, "minimum": 13583.374637513316, "maximum": 43013.800152619944, "mean": 20690.835871833424 } ], "graceful_shutdown": true, "shutdown_seconds": 1.440854800000001 }, { "id": "verified-capacity-final-release-25-peer-4", "healthy": true, "failure_reasons": [], "seconds": 60.3722296, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.014706799999985, "wait_seconds": 21.014915500000004, "completed_at": "2026-10-30T00:47:53.511Z", "accepted_cpu": { "count": 20, "mean": 28.375847812456072, "p95": 39.04832898496707, "max": 40.9710180827347 } }, "requested_client_count": 1, "requested_peer_count": 4, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": "17bf8cb763b8c97152f26fbeb8cce42f75b63b9c2e08fd968d2d26b1b4fc44b4", "clients": 1, "transport": "web", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 4, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 947, "imported_channels": 1000, "total_channels": 1947, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 250, "peers": [ { "name": "SIM_0", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 305, "complete_export_hz": 5.000058353140026, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_1", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 305, "complete_export_hz": 5.000058353140026, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_2", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 305, "complete_export_hz": 5.000058353140026, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_3", "ready": true, "exported_channels_at_end": 947, "complete_received_snapshots": 305, "complete_export_hz": 5.000058353140026, "dropped_telemetry": 0, "operation_timeouts": 0 } ], "peer_simulator_offered": { "counter_seconds": 60.9992881, "send_cycles": 305, "offered_snapshots": 1220, "offered_bytes": 100249020, "offered_argus_snapshots": 1220, "offered_argus_bytes": 61000, "received_snapshots": 1220, "late_cycles": 0 }, "host_cpu_percent": { "count": 52, "mean": 49.06681212875172, "p50": 46.23375445391883, "p95": 64.89856371629656, "p99": 69.74457489912497, "min": 36.92113959905588, "max": 69.74457489912497 }, "engine_cpu_percent": { "count": 52, "mean": 12.467453822265211, "p50": 8.915652856000943, "p95": 26.233463040218368, "p99": 29.009032064479747, "min": 6.397155319888196, "max": 29.009032064479747 }, "client_generator_cpu_percent": { "count": 52, "mean": 0.9673998315428023, "p50": 0.9778325414993173, "p95": 1.8263612503980537, "p99": 2.042764217551796, "min": 0.3455106698559642, "max": 2.042764217551796 }, "peer_generator_cpu_percent": { "count": 52, "mean": 5.68235311829215, "p50": 5.736051908457082, "p95": 7.197769997593175, "p99": 7.529840071673124, "min": 3.046155741706268, "max": 7.529840071673124 }, "engine_resident_bytes": { "count": 52, "mean": 1033249240.6153846, "p50": 1032286208, "p95": 1044664320, "p99": 1045225472, "min": 1018421248, "max": 1045225472 }, "system_samples": 52, "system_sample_interval_seconds": { "count": 52, "mean": 1.1626000596153845, "p95": 1.2087451, "max": 1.4220268, "p50": 1.150579, "p99": 1.4220268, "min": 1.1272517 }, "achieved_read_hz": { "minimum": 99.34101602412794, "maximum": 99.58929503826519 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24076, "hz": 99.50596890042696 }, { "group": "stress_group_1", "controls": 3, "completed": 18078, "hz": 99.6216927386564 }, { "group": "stress_group_2", "controls": 3, "completed": 18072, "hz": 99.58862878487656 }, { "group": "stress_group_3", "controls": 3, "completed": 18036, "hz": 99.39024506219754 }, { "group": "stress_group_4", "controls": 3, "completed": 18045, "hz": 99.43984099286729 }, { "group": "stress_group_5", "controls": 3, "completed": 18033, "hz": 99.37371308530761 }, { "group": "stress_group_6", "controls": 3, "completed": 18054, "hz": 99.48943692353704 }, { "group": "stress_group_7", "controls": 3, "completed": 18078, "hz": 99.6216927386564 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 2334, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 431685800, "member_service_ns_total": 442965300, "members_executed_total": 24076 }, "example_device_plugin.stress_read": { "callback_ns_total": 2528039109, "member_service_ns_total": 2545838600, "members_executed_total": 24076 }, "example_device_plugin.stress_write": { "callback_ns_total": 117952395, "member_service_ns_total": 124561400, "members_executed_total": 24076 } }, "callback_ns_total": 3077677304, "cycles_total": 6019, "member_service_ns_total": 3113365300, "members_executed_total": 72228, "thread_cpu_ns_total": 1343750000, "wall_ns_total": 3156375900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 322785954, "member_service_ns_total": 329726600, "members_executed_total": 18078 }, "example_device_plugin.stress_read": { "callback_ns_total": 2080403052, "member_service_ns_total": 2094891200, "members_executed_total": 18078 }, "example_device_plugin.stress_write": { "callback_ns_total": 88016295, "member_service_ns_total": 95724800, "members_executed_total": 18078 } }, "callback_ns_total": 2491205301, "cycles_total": 6026, "member_service_ns_total": 2520342600, "members_executed_total": 54234, "thread_cpu_ns_total": 734375000, "wall_ns_total": 2563079500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 320572829, "member_service_ns_total": 326692100, "members_executed_total": 18072 }, "example_device_plugin.stress_read": { "callback_ns_total": 2117157819, "member_service_ns_total": 2128762900, "members_executed_total": 18072 }, "example_device_plugin.stress_write": { "callback_ns_total": 87892995, "member_service_ns_total": 93998200, "members_executed_total": 18072 } }, "callback_ns_total": 2525623643, "cycles_total": 6024, "member_service_ns_total": 2549453200, "members_executed_total": 54216, "thread_cpu_ns_total": 687500000, "wall_ns_total": 2592570100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 284576473, "member_service_ns_total": 291317500, "members_executed_total": 18036 }, "example_device_plugin.stress_read": { "callback_ns_total": 2113432211, "member_service_ns_total": 2128953100, "members_executed_total": 18036 }, "example_device_plugin.stress_write": { "callback_ns_total": 96622097, "member_service_ns_total": 102465300, "members_executed_total": 18036 } }, "callback_ns_total": 2494630781, "cycles_total": 6012, "member_service_ns_total": 2522735900, "members_executed_total": 54108, "thread_cpu_ns_total": 937500000, "wall_ns_total": 2561706000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 270091363, "member_service_ns_total": 276286100, "members_executed_total": 18045 }, "example_device_plugin.stress_read": { "callback_ns_total": 1988580488, "member_service_ns_total": 2000902400, "members_executed_total": 18045 }, "example_device_plugin.stress_write": { "callback_ns_total": 88236394, "member_service_ns_total": 94506000, "members_executed_total": 18045 } }, "callback_ns_total": 2346908245, "cycles_total": 6015, "member_service_ns_total": 2371694500, "members_executed_total": 54135, "thread_cpu_ns_total": 765625000, "wall_ns_total": 2411484500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 277384042, "member_service_ns_total": 286335500, "members_executed_total": 18033 }, "example_device_plugin.stress_read": { "callback_ns_total": 2104235612, "member_service_ns_total": 2116354300, "members_executed_total": 18033 }, "example_device_plugin.stress_write": { "callback_ns_total": 85311891, "member_service_ns_total": 89683800, "members_executed_total": 18033 } }, "callback_ns_total": 2466931545, "cycles_total": 6011, "member_service_ns_total": 2492373600, "members_executed_total": 54099, "thread_cpu_ns_total": 750000000, "wall_ns_total": 2532323700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 278755569, "member_service_ns_total": 285134800, "members_executed_total": 18054 }, "example_device_plugin.stress_read": { "callback_ns_total": 2057801569, "member_service_ns_total": 2072570600, "members_executed_total": 18054 }, "example_device_plugin.stress_write": { "callback_ns_total": 94595791, "member_service_ns_total": 100476700, "members_executed_total": 18054 } }, "callback_ns_total": 2431152929, "cycles_total": 6018, "member_service_ns_total": 2458182100, "members_executed_total": 54162, "thread_cpu_ns_total": 765625000, "wall_ns_total": 2497111500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 352939761, "member_service_ns_total": 358519800, "members_executed_total": 18078 }, "example_device_plugin.stress_read": { "callback_ns_total": 2088431571, "member_service_ns_total": 2102558100, "members_executed_total": 18078 }, "example_device_plugin.stress_write": { "callback_ns_total": 94405892, "member_service_ns_total": 99768900, "members_executed_total": 18078 } }, "callback_ns_total": 2535777224, "cycles_total": 6026, "member_service_ns_total": 2560846800, "members_executed_total": 54234, "thread_cpu_ns_total": 781250000, "wall_ns_total": 2596977200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 60.48883365, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 60, "hz": 0.9908911636160778, "failed": 0, "missed": 0, "callback_last_ms": 0.1217, "callback_ewma_ms": 0.133467, "release_lateness_last_ms": 0.1467 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 60, "hz": 0.9908911636160778, "failed": 0, "missed": 0, "callback_last_ms": 7.1524, "callback_ewma_ms": 8.199946, "release_lateness_last_ms": 0.3498 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 302, "hz": 4.987485523534258, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.000935, "release_lateness_last_ms": 0.175 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 302, "hz": 4.987485523534258, "failed": 0, "missed": 0, "callback_last_ms": 0.0643, "callback_ewma_ms": 0.102416, "release_lateness_last_ms": 0.2107 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 60, "hz": 0.9908911636160778, "failed": 0, "missed": 0, "callback_last_ms": 0.074, "callback_ewma_ms": 0.083406, "release_lateness_last_ms": 0.4511 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 60, "hz": 0.9908911636160778, "failed": 0, "missed": 0, "callback_last_ms": 0.0703, "callback_ewma_ms": 0.104384, "release_lateness_last_ms": 0.1524 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 121, "hz": 1.9982971799590903, "failed": 0, "missed": 0, "callback_last_ms": 0.0014, "callback_ewma_ms": 0.001617, "release_lateness_last_ms": 0.4217 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 1.0074060163430123, "failed": 0, "missed": 0, "callback_last_ms": 3.2563, "callback_ewma_ms": 3.934983, "release_lateness_last_ms": 0.6577 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 121, "hz": 1.9982971799590903, "failed": 0, "missed": 0, "callback_last_ms": 5.9223, "callback_ewma_ms": 8.006905, "release_lateness_last_ms": 0.0365 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 303, "hz": 5.004000376261192, "failed": 0, "missed": 0, "callback_last_ms": 5.2886, "callback_ewma_ms": 5.657801, "release_lateness_last_ms": 0.1179 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 605, "hz": 9.991485899795451, "failed": 0, "missed": 0, "callback_last_ms": 0.025, "callback_ewma_ms": 0.697909, "release_lateness_last_ms": 0.6088 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 302, "hz": 4.987485523534258, "failed": 0, "missed": 0, "callback_last_ms": 2.7334, "callback_ewma_ms": 2.98134, "release_lateness_last_ms": 0.3232 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 121, "hz": 1.9982971799590903, "failed": 0, "missed": 0, "callback_last_ms": 6.286, "callback_ewma_ms": 9.506057, "release_lateness_last_ms": 0.518 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 303, "hz": 5.004000376261192, "failed": 0, "missed": 0, "callback_last_ms": 20.0065, "callback_ewma_ms": 22.156107, "release_lateness_last_ms": 0.1758 }, { "name": "general_program_checks_thread", "running": true, "completed": 604, "hz": 9.974971047068516, "failed": 0, "missed": 1, "callback_last_ms": 0.1288, "callback_ewma_ms": 0.196624, "release_lateness_last_ms": 0.478 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 1.0074060163430123, "failed": 0, "missed": 0, "callback_last_ms": 0.1451, "callback_ewma_ms": 0.138727, "release_lateness_last_ms": 0.601 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 606, "hz": 10.008000752522385, "failed": 0, "missed": 0, "callback_last_ms": 0.0071, "callback_ewma_ms": 0.005881, "release_lateness_last_ms": 0.3877 }, { "name": "engine_diagnostics", "running": true, "completed": 121, "hz": 1.9982971799590903, "failed": 0, "missed": 0, "callback_last_ms": 43.2191, "callback_ewma_ms": 55.900969, "release_lateness_last_ms": 0.4079 } ], "peer_counter_seconds": 60.5537472, "peer_counter_deltas": [ { "name": "SIM_0", "connected": true, "channels_received": 303, "argus_received": 303, "frames_received": 606, "frames_sent": 604, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.28939012774712, "downlink_mbps": 3.3151245233505104 }, { "name": "SIM_1", "connected": true, "channels_received": 303, "argus_received": 303, "frames_received": 606, "frames_sent": 605, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.28939012774712, "downlink_mbps": 3.3151245233505104 }, { "name": "SIM_2", "connected": true, "channels_received": 303, "argus_received": 303, "frames_received": 606, "frames_sent": 605, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.28939012774712, "downlink_mbps": 3.3151245233505104 }, { "name": "SIM_3", "connected": true, "channels_received": 303, "argus_received": 303, "frames_received": 606, "frames_sent": 605, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.28741335225809, "downlink_mbps": 3.3151245233505104 } ], "observations": [ { "snapshot_hz": 4.985773872763612, "complete_hz": 4.985773872763612, "catalog_hz": 2.004247968785372, "bytes_per_second": 1546291.8842667784, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.11549999999988, "gap_p95_ms": 206.23349999999846, "gap_p99_ms": 240.95839999999953, "gap_max_ms": 248.82480000000214, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 602, "p50": 7, "p95": 17, "p99": 47, "max": 58, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 55, "mean": 84.46724545454576, "p95": 110.78439999999682, "max": 224.41969999999856, "p50": 79.46100000000297, "p99": 224.41969999999856, "min": 59.52690000000439 }, { "operation": "rapid/get-persistence-diagnostics", "count": 55, "mean": 9.01383454545536, "p95": 17.211300000009942, "max": 17.938099999999395, "p50": 7.061500000003434, "p99": 17.938099999999395, "min": 5.709100000000035 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1407989, "accepted_rows_total": 7583850, "committed_rows_total": 1407967, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 55, "minimum": 30.859375, "maximum": 77.34375, "mean": 50.24076281340469 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 55, "minimum": 5.46493001392027, "maximum": 27.472734488423146, "mean": 11.582812840909888 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 55, "minimum": 1018994688, "maximum": 1046319104, "mean": 1033484697.6 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0.0003814697265625, "mean": 0.0000069358132102272725 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 55, "minimum": 164.51428496139346, "maximum": 259.27833304963565, "mean": 207.58648380299786 }, { "channel": "engine_tempest_rx_mbps", "observations": 55, "minimum": 10.584609046779143, "maximum": 15.972727073826908, "mean": 13.205917917443506 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 55, "minimum": 16488.474556285157, "maximum": 43898.802528996675, "mean": 23029.120598227943 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 55, "minimum": 15907.861983349436, "maximum": 46261.89075472617, "mean": 23139.310570438472 } ], "graceful_shutdown": true, "shutdown_seconds": 1.9142679999999963 }, { "id": "verified-capacity-final-release-25-peer-8", "healthy": true, "failure_reasons": [], "seconds": 60.66931300000001, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.0110503, "wait_seconds": 38.02320810000005, "completed_at": "2026-10-30T00:50:11.717Z", "accepted_cpu": { "count": 20, "mean": 26.82097307634003, "p95": 36.842757936507944, "max": 40.72884496602841 } }, "requested_client_count": 1, "requested_peer_count": 8, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": "17bf8cb763b8c97152f26fbeb8cce42f75b63b9c2e08fd968d2d26b1b4fc44b4", "clients": 1, "transport": "web", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 8, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 959, "imported_channels": 2000, "total_channels": 2959, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 250, "peers": [ { "name": "SIM_0", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_1", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 302, "complete_export_hz": 4.950725381102496, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_2", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_3", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_4", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_5", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_6", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 }, { "name": "SIM_7", "ready": true, "exported_channels_at_end": 959, "complete_received_snapshots": 303, "complete_export_hz": 4.967118511503497, "dropped_telemetry": 0, "operation_timeouts": 0 } ], "peer_simulator_offered": { "counter_seconds": 61.001161800000006, "send_cycles": 305, "offered_snapshots": 2440, "offered_bytes": 200508040, "offered_argus_snapshots": 2440, "offered_argus_bytes": 122000, "received_snapshots": 2423, "late_cycles": 0 }, "host_cpu_percent": { "count": 51, "mean": 64.22439378297611, "p50": 59.7130510038448, "p95": 84.44913669825664, "p99": 98.45093074029549, "min": 45.928518568426504, "max": 98.45093074029549 }, "engine_cpu_percent": { "count": 51, "mean": 18.961647051250043, "p50": 15.207082342417317, "p95": 33.63196780931184, "p99": 34.18661805984304, "min": 11.031705386040409, "max": 34.18661805984304 }, "client_generator_cpu_percent": { "count": 51, "mean": 1.1100418131049148, "p50": 0.9954168565833548, "p95": 2.036111120281152, "p99": 2.371328413949755, "min": 0.3219372691065906, "max": 2.371328413949755 }, "peer_generator_cpu_percent": { "count": 51, "mean": 11.292419706774442, "p50": 11.106812380547725, "p95": 13.612173688605266, "p99": 14.086943392619338, "min": 8.70357532627175, "max": 14.086943392619338 }, "engine_resident_bytes": { "count": 51, "mean": 1068256396.5490196, "p50": 1068593152, "p95": 1089945600, "p99": 1090199552, "min": 1046560768, "max": 1090199552 }, "system_samples": 51, "system_sample_interval_seconds": { "count": 51, "mean": 1.1967559490196078, "p95": 1.2770049, "max": 1.7659059, "p50": 1.1754797, "p99": 1.7659059, "min": 1.1330977 }, "achieved_read_hz": { "minimum": 96.77526098120576, "maximum": 98.30747073891808 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 23755, "hz": 97.69814942694693 }, { "group": "stress_group_1", "controls": 3, "completed": 17856, "hz": 97.91612467087991 }, { "group": "stress_group_2", "controls": 3, "completed": 17924, "hz": 98.28901313848854 }, { "group": "stress_group_3", "controls": 3, "completed": 17680, "hz": 96.9510015782458 }, { "group": "stress_group_4", "controls": 3, "completed": 17763, "hz": 97.40614485488574 }, { "group": "stress_group_5", "controls": 3, "completed": 17646, "hz": 96.76455734444147 }, { "group": "stress_group_6", "controls": 3, "completed": 17805, "hz": 97.63645832017345 }, { "group": "stress_group_7", "controls": 3, "completed": 17910, "hz": 98.21224198339264 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 10785, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 493830999, "member_service_ns_total": 505037000, "members_executed_total": 23755 }, "example_device_plugin.stress_read": { "callback_ns_total": 3673825790, "member_service_ns_total": 3688084600, "members_executed_total": 23755 }, "example_device_plugin.stress_write": { "callback_ns_total": 125763393, "member_service_ns_total": 134301600, "members_executed_total": 23755 } }, "callback_ns_total": 4293473282, "cycles_total": 5939, "member_service_ns_total": 4327477800, "members_executed_total": 71268, "thread_cpu_ns_total": 1671875000, "wall_ns_total": 4382027900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 429159640, "member_service_ns_total": 437041500, "members_executed_total": 17856 }, "example_device_plugin.stress_read": { "callback_ns_total": 3228754534, "member_service_ns_total": 3240767100, "members_executed_total": 17856 }, "example_device_plugin.stress_write": { "callback_ns_total": 116889992, "member_service_ns_total": 122082300, "members_executed_total": 17856 } }, "callback_ns_total": 3774804166, "cycles_total": 5952, "member_service_ns_total": 3799890900, "members_executed_total": 53568, "thread_cpu_ns_total": 1218750000, "wall_ns_total": 3844315600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 407822249, "member_service_ns_total": 415239600, "members_executed_total": 17924 }, "example_device_plugin.stress_read": { "callback_ns_total": 2925064877, "member_service_ns_total": 2937031500, "members_executed_total": 17924 }, "example_device_plugin.stress_write": { "callback_ns_total": 101276088, "member_service_ns_total": 108348700, "members_executed_total": 17924 } }, "callback_ns_total": 3434205114, "cycles_total": 5975, "member_service_ns_total": 3460662400, "members_executed_total": 53775, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 3505014400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 323567555, "member_service_ns_total": 330302400, "members_executed_total": 17680 }, "example_device_plugin.stress_read": { "callback_ns_total": 3635603804, "member_service_ns_total": 3653266400, "members_executed_total": 17680 }, "example_device_plugin.stress_write": { "callback_ns_total": 119536292, "member_service_ns_total": 124802500, "members_executed_total": 17680 } }, "callback_ns_total": 4078799651, "cycles_total": 5894, "member_service_ns_total": 4108465600, "members_executed_total": 53046, "thread_cpu_ns_total": 1156250000, "wall_ns_total": 4156957600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 341835336, "member_service_ns_total": 348961000, "members_executed_total": 17763 }, "example_device_plugin.stress_read": { "callback_ns_total": 3224361863, "member_service_ns_total": 3241527100, "members_executed_total": 17763 }, "example_device_plugin.stress_write": { "callback_ns_total": 106484992, "member_service_ns_total": 111726300, "members_executed_total": 17763 } }, "callback_ns_total": 3672682191, "cycles_total": 5921, "member_service_ns_total": 3702214400, "members_executed_total": 53289, "thread_cpu_ns_total": 1000000000, "wall_ns_total": 3753267700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 348945666, "member_service_ns_total": 356001300, "members_executed_total": 17646 }, "example_device_plugin.stress_read": { "callback_ns_total": 3542918651, "member_service_ns_total": 3555444200, "members_executed_total": 17646 }, "example_device_plugin.stress_write": { "callback_ns_total": 104276592, "member_service_ns_total": 110109100, "members_executed_total": 17646 } }, "callback_ns_total": 3996140909, "cycles_total": 5882, "member_service_ns_total": 4021554600, "members_executed_total": 52938, "thread_cpu_ns_total": 1359375000, "wall_ns_total": 4070980500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 337932954, "member_service_ns_total": 347494200, "members_executed_total": 17805 }, "example_device_plugin.stress_read": { "callback_ns_total": 3143031061, "member_service_ns_total": 3156046100, "members_executed_total": 17805 }, "example_device_plugin.stress_write": { "callback_ns_total": 118795691, "member_service_ns_total": 126106000, "members_executed_total": 17805 } }, "callback_ns_total": 3599759706, "cycles_total": 5935, "member_service_ns_total": 3629646300, "members_executed_total": 53415, "thread_cpu_ns_total": 1015625000, "wall_ns_total": 3671780600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 419719436, "member_service_ns_total": 427402300, "members_executed_total": 17910 }, "example_device_plugin.stress_read": { "callback_ns_total": 3073329880, "member_service_ns_total": 3087348300, "members_executed_total": 17910 }, "example_device_plugin.stress_write": { "callback_ns_total": 108109992, "member_service_ns_total": 116655800, "members_executed_total": 17910 } }, "callback_ns_total": 3601159308, "cycles_total": 5970, "member_service_ns_total": 3631406400, "members_executed_total": 53730, "thread_cpu_ns_total": 968750000, "wall_ns_total": 3679382600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 60.78671944999999, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 1.000254940387872, "failed": 0, "missed": 0, "callback_last_ms": 0.2303, "callback_ewma_ms": 0.270861, "release_lateness_last_ms": 0.3318 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 1.000254940387872, "failed": 0, "missed": 0, "callback_last_ms": 15.1158, "callback_ewma_ms": 12.843511, "release_lateness_last_ms": 0.095 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 305, "hz": 5.00127470193936, "failed": 0, "missed": 0, "callback_last_ms": 0.0007, "callback_ewma_ms": 0.000978, "release_lateness_last_ms": 0.4604 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 305, "hz": 5.00127470193936, "failed": 0, "missed": 0, "callback_last_ms": 0.0675, "callback_ewma_ms": 0.124866, "release_lateness_last_ms": 0.1594 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 1.000254940387872, "failed": 0, "missed": 0, "callback_last_ms": 0.0784, "callback_ewma_ms": 0.153127, "release_lateness_last_ms": 0.3207 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 1.000254940387872, "failed": 0, "missed": 0, "callback_last_ms": 0.1291, "callback_ewma_ms": 0.116426, "release_lateness_last_ms": 0.2174 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 122, "hz": 2.000509880775744, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001489, "release_lateness_last_ms": 3.2567 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 1.000254940387872, "failed": 0, "missed": 0, "callback_last_ms": 5.8596, "callback_ewma_ms": 6.703209, "release_lateness_last_ms": 0.2271 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 122, "hz": 2.000509880775744, "failed": 0, "missed": 0, "callback_last_ms": 20.3246, "callback_ewma_ms": 25.564826, "release_lateness_last_ms": 2.2285 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 305, "hz": 5.00127470193936, "failed": 0, "missed": 0, "callback_last_ms": 13.2476, "callback_ewma_ms": 10.147982, "release_lateness_last_ms": 0.2379 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 610, "hz": 10.00254940387872, "failed": 0, "missed": 0, "callback_last_ms": 0.0273, "callback_ewma_ms": 2.437614, "release_lateness_last_ms": 0.3359 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 305, "hz": 5.00127470193936, "failed": 0, "missed": 0, "callback_last_ms": 16.6422, "callback_ewma_ms": 7.198858, "release_lateness_last_ms": 0.8912 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 122, "hz": 2.000509880775744, "failed": 0, "missed": 0, "callback_last_ms": 20.8531, "callback_ewma_ms": 23.327865, "release_lateness_last_ms": 2.3458 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 302, "hz": 4.952081836018645, "failed": 0, "missed": 2, "callback_last_ms": 120.1426, "callback_ewma_ms": 79.652208, "release_lateness_last_ms": 0.6001 }, { "name": "general_program_checks_thread", "running": true, "completed": 608, "hz": 9.969754159931577, "failed": 0, "missed": 2, "callback_last_ms": 0.7679, "callback_ewma_ms": 1.034642, "release_lateness_last_ms": 13.7766 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 1.000254940387872, "failed": 0, "missed": 0, "callback_last_ms": 0.0994, "callback_ewma_ms": 0.096589, "release_lateness_last_ms": 4.1266 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 610, "hz": 10.00254940387872, "failed": 0, "missed": 0, "callback_last_ms": 0.0042, "callback_ewma_ms": 0.005172, "release_lateness_last_ms": 0.8755 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 2.000509880775744, "failed": 0, "missed": 0, "callback_last_ms": 72.6878, "callback_ewma_ms": 73.306729, "release_lateness_last_ms": 0.7226 } ], "peer_counter_seconds": 60.98728010000001, "peer_counter_deltas": [ { "name": "SIM_0", "connected": true, "channels_received": 305, "argus_received": 305, "frames_received": 610, "frames_sent": 608, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.963859931264565, "downlink_mbps": 3.3146649304928664 }, { "name": "SIM_1", "connected": true, "channels_received": 304, "argus_received": 304, "frames_received": 608, "frames_sent": 608, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.963859931264565, "downlink_mbps": 3.3146649304928664 }, { "name": "SIM_2", "connected": true, "channels_received": 305, "argus_received": 305, "frames_received": 610, "frames_sent": 608, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.963859931264565, "downlink_mbps": 3.3146649304928664 }, { "name": "SIM_3", "connected": true, "channels_received": 304, "argus_received": 305, "frames_received": 609, "frames_sent": 607, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.96386793329468, "downlink_mbps": 3.3146649304928664 }, { "name": "SIM_4", "connected": true, "channels_received": 304, "argus_received": 305, "frames_received": 609, "frames_sent": 607, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.96388393735491, "downlink_mbps": 3.3146809345530963 }, { "name": "SIM_5", "connected": true, "channels_received": 305, "argus_received": 305, "frames_received": 610, "frames_sent": 607, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.96388393735491, "downlink_mbps": 3.3146809345530963 }, { "name": "SIM_6", "connected": true, "channels_received": 305, "argus_received": 305, "frames_received": 610, "frames_sent": 607, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.96387593532479, "downlink_mbps": 3.3146809345530963 }, { "name": "SIM_7", "connected": true, "channels_received": 305, "argus_received": 305, "frames_received": 610, "frames_sent": 607, "operation_timeouts_total": 0, "rejected_snapshots_total": 0, "uplink_mbps": 42.96384392720434, "downlink_mbps": 3.3146809345530963 } ], "observations": [ { "snapshot_hz": 4.994363160643727, "complete_hz": 4.994363160643727, "catalog_hz": 1.9944486549105314, "bytes_per_second": 1549289.1656426133, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.03510000000097, "gap_p95_ms": 214.28130000000237, "gap_p99_ms": 245.21259999999893, "gap_max_ms": 296.0813000000053, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 606, "p50": 14, "p95": 28, "p99": 58, "max": 103, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 54, "mean": 108.60450370370482, "p95": 194.25330000001122, "max": 306.1313000000009, "p50": 92.82559999999648, "p99": 306.1313000000009, "min": 71.29349999999977 }, { "operation": "rapid/get-persistence-diagnostics", "count": 54, "mean": 8.690375925926169, "p95": 16.07600000000093, "max": 26.26140000000305, "p50": 7.5679999999993015, "p99": 26.26140000000305, "min": 4.936600000000908 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1705098, "accepted_rows_total": 8121484, "committed_rows_total": 1704688, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 54, "minimum": 44.921875, "maximum": 96.7741935483871, "mean": 63.57510289951487 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 54, "minimum": 10.932940963618172, "maximum": 36.27526468430204, "mean": 18.22359277033076 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 54, "minimum": 1045274624, "maximum": 1089200128, "mean": 1067921180.4444444 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 54, "minimum": 0, "maximum": 0.3631591796875, "mean": 0.10156984682436343 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 54, "minimum": 0, "maximum": 0.01220703125, "mean": 0.0004521122685185185 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 54, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 54, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 54, "minimum": 304.1150766898698, "maximum": 476.0663561058825, "mean": 387.25334174791794 }, { "channel": "engine_tempest_rx_mbps", "observations": 54, "minimum": 21.133215097074665, "maximum": 31.992517831120782, "mean": 26.70402540720552 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 54, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 54, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 54, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 54, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 54, "minimum": 21636.17227169608, "maximum": 48219.66982767839, "mean": 27979.905351446338 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 54, "minimum": 19847.99273847094, "maximum": 48132.27059328982, "mean": 27835.108292458994 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7546270000000077 }, { "id": "verified-capacity-final-release-25-zmq-1", "healthy": true, "failure_reasons": [], "seconds": 60.9044222, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.046859899999994, "wait_seconds": 20.046859899999994, "completed_at": "2026-10-30T00:30:45.670Z", "accepted_cpu": { "count": 20, "mean": 24.020632688309696, "p95": 35.59405940594059, "max": 35.93132411067194 } }, "requested_client_count": 1, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 1, "transport": "zmq", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 54, "mean": 35.42900183101346, "p50": 34.844277575192436, "p95": 47.00301460787733, "p99": 64.35985994453873, "min": 19.757579945485826, "max": 64.35985994453873 }, "engine_cpu_percent": { "count": 54, "mean": 8.453652447866196, "p50": 6.0692078022173055, "p95": 18.26352207466289, "p99": 20.021576687068315, "min": 2.800061789523522, "max": 20.021576687068315 }, "client_generator_cpu_percent": { "count": 54, "mean": 1.4028011535049199, "p50": 1.3796486765482772, "p95": 2.1824415169065, "p99": 2.7615967844232157, "min": 0.3484685937132296, "max": 2.7615967844232157 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 54, "mean": 996441505.1851852, "p50": 997650432, "p95": 1002328064, "p99": 1002700800, "min": 987820032, "max": 1002700800 }, "system_samples": 54, "system_sample_interval_seconds": { "count": 54, "mean": 1.1336722444444445, "p95": 1.1993315, "max": 1.3522104, "p50": 1.1263311, "p99": 1.3522104, "min": 1.1103399 }, "achieved_read_hz": { "minimum": 99.32563015144774, "maximum": 99.71934311576426 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24332, "hz": 99.63750147271615 }, { "group": "stress_group_1", "controls": 3, "completed": 18255, "hz": 99.67026080247867 }, { "group": "stress_group_2", "controls": 3, "completed": 18261, "hz": 99.70302013224119 }, { "group": "stress_group_3", "controls": 3, "completed": 18204, "hz": 99.39180649949724 }, { "group": "stress_group_4", "controls": 3, "completed": 18192, "hz": 99.32628783997217 }, { "group": "stress_group_5", "controls": 3, "completed": 18189, "hz": 99.30990817509091 }, { "group": "stress_group_6", "controls": 3, "completed": 18201, "hz": 99.37542683461596 }, { "group": "stress_group_7", "controls": 3, "completed": 18255, "hz": 99.67026080247867 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 2133, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 318114147, "member_service_ns_total": 327607600, "members_executed_total": 24332 }, "example_device_plugin.stress_read": { "callback_ns_total": 1947985107, "member_service_ns_total": 1961141600, "members_executed_total": 24332 }, "example_device_plugin.stress_write": { "callback_ns_total": 98695298, "member_service_ns_total": 106625600, "members_executed_total": 24332 } }, "callback_ns_total": 2364794552, "cycles_total": 6083, "member_service_ns_total": 2395374800, "members_executed_total": 72996, "thread_cpu_ns_total": 531250000, "wall_ns_total": 2448616800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 286895282, "member_service_ns_total": 294479300, "members_executed_total": 18255 }, "example_device_plugin.stress_read": { "callback_ns_total": 1540564234, "member_service_ns_total": 1552122400, "members_executed_total": 18255 }, "example_device_plugin.stress_write": { "callback_ns_total": 80574591, "member_service_ns_total": 85211300, "members_executed_total": 18255 } }, "callback_ns_total": 1908034107, "cycles_total": 6085, "member_service_ns_total": 1931813000, "members_executed_total": 54765, "thread_cpu_ns_total": 453125000, "wall_ns_total": 1965847400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 246048687, "member_service_ns_total": 252417400, "members_executed_total": 18261 }, "example_device_plugin.stress_read": { "callback_ns_total": 1494130053, "member_service_ns_total": 1504053700, "members_executed_total": 18261 }, "example_device_plugin.stress_write": { "callback_ns_total": 72612393, "member_service_ns_total": 78060100, "members_executed_total": 18261 } }, "callback_ns_total": 1812791133, "cycles_total": 6087, "member_service_ns_total": 1834531200, "members_executed_total": 54783, "thread_cpu_ns_total": 578125000, "wall_ns_total": 1868412200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 249975800, "member_service_ns_total": 255968300, "members_executed_total": 18204 }, "example_device_plugin.stress_read": { "callback_ns_total": 1610019720, "member_service_ns_total": 1620913500, "members_executed_total": 18204 }, "example_device_plugin.stress_write": { "callback_ns_total": 78722796, "member_service_ns_total": 84076500, "members_executed_total": 18204 } }, "callback_ns_total": 1938718316, "cycles_total": 6068, "member_service_ns_total": 1960958300, "members_executed_total": 54612, "thread_cpu_ns_total": 453125000, "wall_ns_total": 1995862800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 250483085, "member_service_ns_total": 256395600, "members_executed_total": 18192 }, "example_device_plugin.stress_read": { "callback_ns_total": 1556905663, "member_service_ns_total": 1567128700, "members_executed_total": 18192 }, "example_device_plugin.stress_write": { "callback_ns_total": 108057293, "member_service_ns_total": 113214700, "members_executed_total": 18192 } }, "callback_ns_total": 1915446041, "cycles_total": 6064, "member_service_ns_total": 1936739000, "members_executed_total": 54576, "thread_cpu_ns_total": 500000000, "wall_ns_total": 1969507900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 238134986, "member_service_ns_total": 243971400, "members_executed_total": 18189 }, "example_device_plugin.stress_read": { "callback_ns_total": 1569898192, "member_service_ns_total": 1579854000, "members_executed_total": 18189 }, "example_device_plugin.stress_write": { "callback_ns_total": 75554197, "member_service_ns_total": 81270600, "members_executed_total": 18189 } }, "callback_ns_total": 1883587375, "cycles_total": 6063, "member_service_ns_total": 1905096000, "members_executed_total": 54567, "thread_cpu_ns_total": 375000000, "wall_ns_total": 1936493800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 255254391, "member_service_ns_total": 261469500, "members_executed_total": 18201 }, "example_device_plugin.stress_read": { "callback_ns_total": 1614330772, "member_service_ns_total": 1625664200, "members_executed_total": 18201 }, "example_device_plugin.stress_write": { "callback_ns_total": 78681396, "member_service_ns_total": 84880200, "members_executed_total": 18201 } }, "callback_ns_total": 1948266559, "cycles_total": 6067, "member_service_ns_total": 1972013900, "members_executed_total": 54603, "thread_cpu_ns_total": 343750000, "wall_ns_total": 2008637800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 276213984, "member_service_ns_total": 282390500, "members_executed_total": 18255 }, "example_device_plugin.stress_read": { "callback_ns_total": 1593326738, "member_service_ns_total": 1603945700, "members_executed_total": 18255 }, "example_device_plugin.stress_write": { "callback_ns_total": 79853394, "member_service_ns_total": 85490200, "members_executed_total": 18255 } }, "callback_ns_total": 1949394116, "cycles_total": 6085, "member_service_ns_total": 1971826400, "members_executed_total": 54765, "thread_cpu_ns_total": 656250000, "wall_ns_total": 2019302400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 61.051310099999995, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 0.9984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 0.005, "callback_ewma_ms": 0.005396, "release_lateness_last_ms": 0.3036 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 0.9984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 4.9068, "callback_ewma_ms": 6.751763, "release_lateness_last_ms": 0.2731 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 305, "hz": 4.992293176053772, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.001279, "release_lateness_last_ms": 0.5254 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 305, "hz": 4.992293176053772, "failed": 0, "missed": 0, "callback_last_ms": 0.0824, "callback_ewma_ms": 0.077032, "release_lateness_last_ms": 0.1988 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 0.9984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 0.0659, "callback_ewma_ms": 0.098886, "release_lateness_last_ms": 0.1741 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 0.9984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 0.1137, "callback_ewma_ms": 0.126085, "release_lateness_last_ms": 0.4304 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 122, "hz": 1.9969172704215086, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.001029, "release_lateness_last_ms": 0.393 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 0.9984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 2.6686, "callback_ewma_ms": 2.508059, "release_lateness_last_ms": 0.1224 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 122, "hz": 1.9969172704215086, "failed": 0, "missed": 0, "callback_last_ms": 3.6226, "callback_ewma_ms": 4.130785, "release_lateness_last_ms": 0.5915 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 305, "hz": 4.992293176053772, "failed": 0, "missed": 0, "callback_last_ms": 3.5885, "callback_ewma_ms": 4.266019, "release_lateness_last_ms": 0.4438 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 610, "hz": 9.984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 0.0166, "callback_ewma_ms": 0.63373, "release_lateness_last_ms": 0.3143 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 305, "hz": 4.992293176053772, "failed": 0, "missed": 0, "callback_last_ms": 2.5504, "callback_ewma_ms": 2.730211, "release_lateness_last_ms": 0.5682 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 122, "hz": 1.9969172704215086, "failed": 0, "missed": 0, "callback_last_ms": 6.4201, "callback_ewma_ms": 6.852379, "release_lateness_last_ms": 0.5475 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 305, "hz": 4.992293176053772, "failed": 0, "missed": 0, "callback_last_ms": 0.0018, "callback_ewma_ms": 0.002508, "release_lateness_last_ms": 0.3224 }, { "name": "general_program_checks_thread", "running": true, "completed": 611, "hz": 10.00095452645526, "failed": 0, "missed": 0, "callback_last_ms": 0.1494, "callback_ewma_ms": 0.159588, "release_lateness_last_ms": 0.4063 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 0.9984586352107543, "failed": 0, "missed": 0, "callback_last_ms": 0.0684, "callback_ewma_ms": 0.12518, "release_lateness_last_ms": 0.11 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 611, "hz": 10.00095452645526, "failed": 0, "missed": 0, "callback_last_ms": 0.0042, "callback_ewma_ms": 0.005449, "release_lateness_last_ms": 0.4654 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 1.9969172704215086, "failed": 0, "missed": 0, "callback_last_ms": 38.2305, "callback_ewma_ms": 41.859001, "release_lateness_last_ms": 0.5854 } ], "peer_counter_seconds": 61.094046799999994, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.991421150878838, "complete_hz": 4.991421150878838, "catalog_hz": 2.0031361197605864, "bytes_per_second": 1547880.7467588342, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.97200000000885, "gap_p95_ms": 208.92230000000563, "gap_p99_ms": 226.29039999999804, "gap_max_ms": 254.63969999999972, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 608, "p50": 15, "p95": 27, "p99": 42, "max": 71, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 56, "mean": 72.80132857142962, "p95": 93.03360000000248, "max": 133.4383999999991, "p50": 70.91509999999835, "p99": 133.4383999999991, "min": 52.944400000000314 }, { "operation": "rapid/get-persistence-diagnostics", "count": 56, "mean": 10.682819642857535, "p95": 17.139100000000326, "max": 24.254300000000512, "p50": 7.378799999998591, "p99": 24.254300000000512, "min": 4.607900000002701 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1113517, "accepted_rows_total": 7034904, "committed_rows_total": 1113192, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 56, "minimum": 15.625, "maximum": 87.09677419354838, "mean": 37.24425438175996 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 56, "minimum": 1.9526868170782476, "maximum": 21.893850605371224, "mean": 7.883284195505026 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 56, "minimum": 985780224, "maximum": 1002356736, "mean": 996220562.2857143 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0.0003814697265625, "mean": 0.000006811959402901786 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 56, "minimum": 13.278303301082344, "maximum": 21.69827877907465, "mean": 16.729686076824215 }, { "channel": "engine_tempest_rx_mbps", "observations": 56, "minimum": 0, "maximum": 0.0027422644288612906, "mean": 0.0005374402515303633 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 56, "minimum": 10137.538605626552, "maximum": 39385.47927570256, "mean": 17974.0952592237 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 56, "minimum": 9847.609000534669, "maximum": 41049.01847475772, "mean": 18002.083569114886 } ], "graceful_shutdown": true, "shutdown_seconds": 1.9408829000000114 }, { "id": "verified-capacity-final-release-25-zmq-2", "healthy": true, "failure_reasons": [], "seconds": 61.05524239999999, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.046966400000006, "wait_seconds": 20.046966400000006, "completed_at": "2026-10-30T00:32:44.840Z", "accepted_cpu": { "count": 20, "mean": 22.76507409307034, "p95": 33.703429558351836, "max": 38.469111493969976 } }, "requested_client_count": 2, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 2, "transport": "zmq", "actual_connections": { "loadClients": 2, "actualClients": 3, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 500, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 54, "mean": 36.07171327253775, "p50": 34.43002632607484, "p95": 55.79870383560795, "p99": 62.71021655172373, "min": 23.459387974450497, "max": 62.71021655172373 }, "engine_cpu_percent": { "count": 54, "mean": 9.11666528281111, "p50": 6.663407943528023, "p95": 17.578657529919287, "p99": 18.867521200158738, "min": 4.001980149153953, "max": 18.867521200158738 }, "client_generator_cpu_percent": { "count": 54, "mean": 2.1715268480403984, "p50": 2.06764623915991, "p95": 2.9792219722423656, "p99": 3.7893465907587696, "min": 1.0476285578248052, "max": 3.7893465907587696 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 54, "mean": 1001093650.962963, "p50": 1000865792, "p95": 1007079424, "p99": 1007161344, "min": 991653888, "max": 1007161344 }, "system_samples": 54, "system_sample_interval_seconds": { "count": 54, "mean": 1.1301607092592594, "p95": 1.1538379, "max": 1.1747608, "p50": 1.1281157, "p99": 1.1747608, "min": 1.1106343 }, "achieved_read_hz": { "minimum": 99.45454925053198, "maximum": 99.61823561857642 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24360, "hz": 99.53776582027201 }, { "group": "stress_group_1", "controls": 3, "completed": 18282, "hz": 99.60314366317532 }, { "group": "stress_group_2", "controls": 3, "completed": 18288, "hz": 99.63583258462695 }, { "group": "stress_group_3", "controls": 3, "completed": 18258, "hz": 99.47238797736873 }, { "group": "stress_group_4", "controls": 3, "completed": 18261, "hz": 99.48873243809454 }, { "group": "stress_group_5", "controls": 3, "completed": 18270, "hz": 99.53776582027201 }, { "group": "stress_group_6", "controls": 3, "completed": 18258, "hz": 99.47238797736873 }, { "group": "stress_group_7", "controls": 3, "completed": 18279, "hz": 99.58679920244948 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 2157, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 331777257, "member_service_ns_total": 340391300, "members_executed_total": 24360 }, "example_device_plugin.stress_read": { "callback_ns_total": 2148187207, "member_service_ns_total": 2162721600, "members_executed_total": 24360 }, "example_device_plugin.stress_write": { "callback_ns_total": 100084497, "member_service_ns_total": 106823000, "members_executed_total": 24360 } }, "callback_ns_total": 2580048961, "cycles_total": 6090, "member_service_ns_total": 2609935900, "members_executed_total": 73080, "thread_cpu_ns_total": 984375000, "wall_ns_total": 2653738000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 269784652, "member_service_ns_total": 277211500, "members_executed_total": 18282 }, "example_device_plugin.stress_read": { "callback_ns_total": 1757410884, "member_service_ns_total": 1769700900, "members_executed_total": 18282 }, "example_device_plugin.stress_write": { "callback_ns_total": 81996699, "member_service_ns_total": 87573800, "members_executed_total": 18282 } }, "callback_ns_total": 2109192235, "cycles_total": 6094, "member_service_ns_total": 2134486200, "members_executed_total": 54846, "thread_cpu_ns_total": 890625000, "wall_ns_total": 2169947400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 253093498, "member_service_ns_total": 259626500, "members_executed_total": 18288 }, "example_device_plugin.stress_read": { "callback_ns_total": 1678203465, "member_service_ns_total": 1689309400, "members_executed_total": 18288 }, "example_device_plugin.stress_write": { "callback_ns_total": 78100398, "member_service_ns_total": 83642900, "members_executed_total": 18288 } }, "callback_ns_total": 2009397361, "cycles_total": 6096, "member_service_ns_total": 2032578800, "members_executed_total": 54864, "thread_cpu_ns_total": 828125000, "wall_ns_total": 2066609400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 245530260, "member_service_ns_total": 251510500, "members_executed_total": 18258 }, "example_device_plugin.stress_read": { "callback_ns_total": 1800092045, "member_service_ns_total": 1811752500, "members_executed_total": 18258 }, "example_device_plugin.stress_write": { "callback_ns_total": 84107896, "member_service_ns_total": 90086500, "members_executed_total": 18258 } }, "callback_ns_total": 2129730201, "cycles_total": 6086, "member_service_ns_total": 2153349500, "members_executed_total": 54774, "thread_cpu_ns_total": 703125000, "wall_ns_total": 2189358800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 241911097, "member_service_ns_total": 247915800, "members_executed_total": 18261 }, "example_device_plugin.stress_read": { "callback_ns_total": 1731606034, "member_service_ns_total": 1742582800, "members_executed_total": 18261 }, "example_device_plugin.stress_write": { "callback_ns_total": 74866299, "member_service_ns_total": 79955000, "members_executed_total": 18261 } }, "callback_ns_total": 2048383430, "cycles_total": 6087, "member_service_ns_total": 2070453600, "members_executed_total": 54783, "thread_cpu_ns_total": 531250000, "wall_ns_total": 2105669600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 235054092, "member_service_ns_total": 240943200, "members_executed_total": 18270 }, "example_device_plugin.stress_read": { "callback_ns_total": 1696540492, "member_service_ns_total": 1707883800, "members_executed_total": 18270 }, "example_device_plugin.stress_write": { "callback_ns_total": 80809794, "member_service_ns_total": 86953400, "members_executed_total": 18270 } }, "callback_ns_total": 2012404378, "cycles_total": 6090, "member_service_ns_total": 2035780400, "members_executed_total": 54810, "thread_cpu_ns_total": 375000000, "wall_ns_total": 2067710400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 242302377, "member_service_ns_total": 247855700, "members_executed_total": 18258 }, "example_device_plugin.stress_read": { "callback_ns_total": 1811308596, "member_service_ns_total": 1823332400, "members_executed_total": 18258 }, "example_device_plugin.stress_write": { "callback_ns_total": 80151893, "member_service_ns_total": 86374200, "members_executed_total": 18258 } }, "callback_ns_total": 2133762866, "cycles_total": 6086, "member_service_ns_total": 2157562300, "members_executed_total": 54774, "thread_cpu_ns_total": 718750000, "wall_ns_total": 2192615000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 270082980, "member_service_ns_total": 276269000, "members_executed_total": 18279 }, "example_device_plugin.stress_read": { "callback_ns_total": 1744176336, "member_service_ns_total": 1757455500, "members_executed_total": 18279 }, "example_device_plugin.stress_write": { "callback_ns_total": 80417291, "member_service_ns_total": 85412700, "members_executed_total": 18279 } }, "callback_ns_total": 2094676607, "cycles_total": 6093, "member_service_ns_total": 2119137200, "members_executed_total": 54837, "thread_cpu_ns_total": 640625000, "wall_ns_total": 2153582200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 61.182807849999996, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 0.9960299535989976, "failed": 0, "missed": 0, "callback_last_ms": 0.0034, "callback_ewma_ms": 0.005532, "release_lateness_last_ms": 0.283 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 0.9960299535989976, "failed": 0, "missed": 0, "callback_last_ms": 4.497, "callback_ewma_ms": 8.642733, "release_lateness_last_ms": 0.2348 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 306, "hz": 4.996478127890054, "failed": 0, "missed": 0, "callback_last_ms": 0.0007, "callback_ewma_ms": 0.000747, "release_lateness_last_ms": 0.3805 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 306, "hz": 4.996478127890054, "failed": 0, "missed": 0, "callback_last_ms": 0.1467, "callback_ewma_ms": 0.145716, "release_lateness_last_ms": 0.2667 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 0.9960299535989976, "failed": 0, "missed": 0, "callback_last_ms": 0.1719, "callback_ewma_ms": 0.12344, "release_lateness_last_ms": 0.0596 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 0.9960299535989976, "failed": 0, "missed": 0, "callback_last_ms": 0.1225, "callback_ewma_ms": 0.160703, "release_lateness_last_ms": 0.2068 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 122, "hz": 1.9920599071979952, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.000992, "release_lateness_last_ms": 0.5043 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 0.9960299535989976, "failed": 0, "missed": 0, "callback_last_ms": 2.4384, "callback_ewma_ms": 2.29171, "release_lateness_last_ms": 0.4678 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 122, "hz": 1.9920599071979952, "failed": 0, "missed": 0, "callback_last_ms": 4.8098, "callback_ewma_ms": 4.695117, "release_lateness_last_ms": 0.1008 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 306, "hz": 4.996478127890054, "failed": 0, "missed": 0, "callback_last_ms": 3.5839, "callback_ewma_ms": 4.746958, "release_lateness_last_ms": 0.3851 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 612, "hz": 9.992956255780108, "failed": 0, "missed": 0, "callback_last_ms": 0.0217, "callback_ewma_ms": 0.440675, "release_lateness_last_ms": 0.4501 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 306, "hz": 4.996478127890054, "failed": 0, "missed": 0, "callback_last_ms": 2.531, "callback_ewma_ms": 3.10412, "release_lateness_last_ms": 0.262 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 122, "hz": 1.9920599071979952, "failed": 0, "missed": 0, "callback_last_ms": 4.8568, "callback_ewma_ms": 8.079835, "release_lateness_last_ms": 0.0305 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 306, "hz": 4.996478127890054, "failed": 0, "missed": 0, "callback_last_ms": 0.0014, "callback_ewma_ms": 0.002003, "release_lateness_last_ms": 0.4486 }, { "name": "general_program_checks_thread", "running": true, "completed": 612, "hz": 9.992956255780108, "failed": 0, "missed": 0, "callback_last_ms": 0.1398, "callback_ewma_ms": 0.166638, "release_lateness_last_ms": 0.5147 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 0.9960299535989976, "failed": 0, "missed": 0, "callback_last_ms": 0.1421, "callback_ewma_ms": 0.169874, "release_lateness_last_ms": 0.3969 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 612, "hz": 9.992956255780108, "failed": 0, "missed": 0, "callback_last_ms": 0.0055, "callback_ewma_ms": 0.005692, "release_lateness_last_ms": 0.3744 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 1.9920599071979952, "failed": 0, "missed": 0, "callback_last_ms": 38.281, "callback_ewma_ms": 43.1039, "release_lateness_last_ms": 0.4429 } ], "peer_counter_seconds": 61.23991560000001, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.995471744082054, "complete_hz": 4.995471744082054, "catalog_hz": 1.9981886976328218, "bytes_per_second": 1549157.2075744034, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0338999999949, "gap_p95_ms": 210.62189999999828, "gap_p99_ms": 230.0306999999957, "gap_max_ms": 344.2378000000026, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 34, "p95": 46, "p99": 68, "max": 177, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.995471744082054, "complete_hz": 4.995471744082054, "catalog_hz": 1.9981886976328218, "bytes_per_second": 1549157.2075744034, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.87029999999504, "gap_p95_ms": 209.23600000000442, "gap_p99_ms": 221.48619999999937, "gap_max_ms": 275.48459999999614, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 610, "p50": 23, "p95": 32, "p99": 47, "max": 98, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 56, "mean": 74.37139821428556, "p95": 89.07069999999658, "max": 127.73179999999411, "p50": 72.81390000000101, "p99": 127.73179999999411, "min": 55.55140000000392 }, { "operation": "rapid/get-persistence-diagnostics", "count": 56, "mean": 10.54122857142868, "p95": 18.647799999998824, "max": 25.602599999998347, "p50": 7.449300000000221, "p99": 25.602599999998347, "min": 4.894700000004377 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1114988, "accepted_rows_total": 7048994, "committed_rows_total": 1114887, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 56, "minimum": 17.1875, "maximum": 66.66666666666667, "mean": 36.924200680176945 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 56, "minimum": 1.5625853171583168, "maximum": 21.09257303442468, "mean": 8.83123129809375 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 56, "minimum": 993722368, "maximum": 1007099904, "mean": 1001180672 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 56, "minimum": 23.240741499414153, "maximum": 37.29623886987107, "mean": 29.941560071363785 }, { "channel": "engine_tempest_rx_mbps", "observations": 56, "minimum": 0, "maximum": 0.008591520593150903, "mean": 0.001032758324527718 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 56, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 56, "minimum": 11587.1379769197, "maximum": 39392.386045383246, "mean": 18239.454610663175 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 56, "minimum": 11167.241691302248, "maximum": 48737.273372978954, "mean": 18537.834194792857 } ], "graceful_shutdown": true, "shutdown_seconds": 1.9353551000000007 }, { "id": "verified-capacity-final-release-25-zmq-4", "healthy": true, "failure_reasons": [], "seconds": 60.726699200000006, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.015712700000034, "wait_seconds": 22.01781310000003, "completed_at": "2026-10-30T00:34:45.966Z", "accepted_cpu": { "count": 20, "mean": 23.565688631241777, "p95": 31.989214364505457, "max": 33.148353681095074 } }, "requested_client_count": 4, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 4, "transport": "zmq", "actual_connections": { "loadClients": 4, "actualClients": 5, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 1000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 54, "mean": 37.443113246759346, "p50": 38.346098199550816, "p95": 51.83221071929835, "p99": 56.53289694680114, "min": 23.53440697457083, "max": 56.53289694680114 }, "engine_cpu_percent": { "count": 54, "mean": 9.074017662003108, "p50": 6.605395885877982, "p95": 18.910563107726063, "p99": 20.45020127562403, "min": 3.814542841660529, "max": 20.45020127562403 }, "client_generator_cpu_percent": { "count": 54, "mean": 4.1591968146437965, "p50": 3.963760357578521, "p95": 5.727578218307456, "p99": 6.186615511953487, "min": 2.9423067073139393, "max": 6.186615511953487 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 54, "mean": 1001226391.7037038, "p50": 1000988672, "p95": 1006436352, "p99": 1007333376, "min": 992325632, "max": 1007333376 }, "system_samples": 54, "system_sample_interval_seconds": { "count": 54, "mean": 1.1320519666666664, "p95": 1.14968, "max": 1.2318637, "p50": 1.1280833, "p99": 1.2318637, "min": 1.1107024 }, "achieved_read_hz": { "minimum": 99.71763927588516, "maximum": 99.86576546755389 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24300, "hz": 99.87032253717419 }, { "group": "stress_group_1", "controls": 3, "completed": 18228, "hz": 99.88676209643957 }, { "group": "stress_group_2", "controls": 3, "completed": 18219, "hz": 99.83744341864343 }, { "group": "stress_group_3", "controls": 3, "completed": 18201, "hz": 99.73880606305117 }, { "group": "stress_group_4", "controls": 3, "completed": 18207, "hz": 99.77168518158192 }, { "group": "stress_group_5", "controls": 3, "completed": 18210, "hz": 99.7881247408473 }, { "group": "stress_group_6", "controls": 3, "completed": 18210, "hz": 99.7881247408473 }, { "group": "stress_group_7", "controls": 3, "completed": 18228, "hz": 99.88676209643957 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 891, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 312407940, "member_service_ns_total": 319680400, "members_executed_total": 24300 }, "example_device_plugin.stress_read": { "callback_ns_total": 1962304763, "member_service_ns_total": 1974122700, "members_executed_total": 24300 }, "example_device_plugin.stress_write": { "callback_ns_total": 101236495, "member_service_ns_total": 108492200, "members_executed_total": 24300 } }, "callback_ns_total": 2375949198, "cycles_total": 6075, "member_service_ns_total": 2402295300, "members_executed_total": 72900, "thread_cpu_ns_total": 921875000, "wall_ns_total": 2441081400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 268231856, "member_service_ns_total": 275154900, "members_executed_total": 18228 }, "example_device_plugin.stress_read": { "callback_ns_total": 1586196985, "member_service_ns_total": 1598472100, "members_executed_total": 18228 }, "example_device_plugin.stress_write": { "callback_ns_total": 86543695, "member_service_ns_total": 91486700, "members_executed_total": 18228 } }, "callback_ns_total": 1940972536, "cycles_total": 6076, "member_service_ns_total": 1965113700, "members_executed_total": 54684, "thread_cpu_ns_total": 734375000, "wall_ns_total": 2001225500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 256931480, "member_service_ns_total": 262963100, "members_executed_total": 18219 }, "example_device_plugin.stress_read": { "callback_ns_total": 1553557862, "member_service_ns_total": 1563642400, "members_executed_total": 18219 }, "example_device_plugin.stress_write": { "callback_ns_total": 74918590, "member_service_ns_total": 80391200, "members_executed_total": 18219 } }, "callback_ns_total": 1885407932, "cycles_total": 6073, "member_service_ns_total": 1906996700, "members_executed_total": 54657, "thread_cpu_ns_total": 718750000, "wall_ns_total": 1939350600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 252617675, "member_service_ns_total": 258856600, "members_executed_total": 18201 }, "example_device_plugin.stress_read": { "callback_ns_total": 1653264640, "member_service_ns_total": 1665293300, "members_executed_total": 18201 }, "example_device_plugin.stress_write": { "callback_ns_total": 79465996, "member_service_ns_total": 85834100, "members_executed_total": 18201 } }, "callback_ns_total": 1985348311, "cycles_total": 6067, "member_service_ns_total": 2009984000, "members_executed_total": 54603, "thread_cpu_ns_total": 906250000, "wall_ns_total": 2048133700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 239038897, "member_service_ns_total": 248196500, "members_executed_total": 18207 }, "example_device_plugin.stress_read": { "callback_ns_total": 1588727912, "member_service_ns_total": 1600865000, "members_executed_total": 18207 }, "example_device_plugin.stress_write": { "callback_ns_total": 82768694, "member_service_ns_total": 89128700, "members_executed_total": 18207 } }, "callback_ns_total": 1910535503, "cycles_total": 6069, "member_service_ns_total": 1938190200, "members_executed_total": 54621, "thread_cpu_ns_total": 421875000, "wall_ns_total": 1970722000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 234651187, "member_service_ns_total": 241334000, "members_executed_total": 18210 }, "example_device_plugin.stress_read": { "callback_ns_total": 1507824638, "member_service_ns_total": 1517917000, "members_executed_total": 18210 }, "example_device_plugin.stress_write": { "callback_ns_total": 81227492, "member_service_ns_total": 86469400, "members_executed_total": 18210 } }, "callback_ns_total": 1823703317, "cycles_total": 6070, "member_service_ns_total": 1845720400, "members_executed_total": 54630, "thread_cpu_ns_total": 296875000, "wall_ns_total": 1879181000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 251709476, "member_service_ns_total": 258110600, "members_executed_total": 18210 }, "example_device_plugin.stress_read": { "callback_ns_total": 1616521338, "member_service_ns_total": 1628744700, "members_executed_total": 18210 }, "example_device_plugin.stress_write": { "callback_ns_total": 80272293, "member_service_ns_total": 85993600, "members_executed_total": 18210 } }, "callback_ns_total": 1948503107, "cycles_total": 6070, "member_service_ns_total": 1972848900, "members_executed_total": 54630, "thread_cpu_ns_total": 812500000, "wall_ns_total": 2007589800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 257005674, "member_service_ns_total": 262718500, "members_executed_total": 18228 }, "example_device_plugin.stress_read": { "callback_ns_total": 1560781124, "member_service_ns_total": 1573114400, "members_executed_total": 18228 }, "example_device_plugin.stress_write": { "callback_ns_total": 77450991, "member_service_ns_total": 83359100, "members_executed_total": 18228 } }, "callback_ns_total": 1895237789, "cycles_total": 6076, "member_service_ns_total": 1919192000, "members_executed_total": 54684, "thread_cpu_ns_total": 703125000, "wall_ns_total": 1953566900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 60.82888135, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 1.0009621182243513, "failed": 0, "missed": 0, "callback_last_ms": 0.0051, "callback_ewma_ms": 0.006126, "release_lateness_last_ms": 0.3757 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 1.0009621182243513, "failed": 0, "missed": 0, "callback_last_ms": 6.3047, "callback_ewma_ms": 6.291156, "release_lateness_last_ms": 0.3585 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 305, "hz": 5.004810591121757, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.001062, "release_lateness_last_ms": 0.2159 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 305, "hz": 5.004810591121757, "failed": 0, "missed": 0, "callback_last_ms": 0.2737, "callback_ewma_ms": 0.437161, "release_lateness_last_ms": 0.3694 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 1.0009621182243513, "failed": 0, "missed": 0, "callback_last_ms": 0.4703, "callback_ewma_ms": 0.385774, "release_lateness_last_ms": 0.4166 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 1.0009621182243513, "failed": 0, "missed": 0, "callback_last_ms": 0.3217, "callback_ewma_ms": 0.525226, "release_lateness_last_ms": 0.5338 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 122, "hz": 2.0019242364487027, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.001624, "release_lateness_last_ms": 0.1866 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 61, "hz": 1.0009621182243513, "failed": 0, "missed": 0, "callback_last_ms": 2.5841, "callback_ewma_ms": 2.089188, "release_lateness_last_ms": 0.3905 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 122, "hz": 2.0019242364487027, "failed": 0, "missed": 0, "callback_last_ms": 4.0906, "callback_ewma_ms": 5.09892, "release_lateness_last_ms": 0.3313 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 305, "hz": 5.004810591121757, "failed": 0, "missed": 0, "callback_last_ms": 3.8457, "callback_ewma_ms": 6.190227, "release_lateness_last_ms": 0.5171 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 609, "hz": 9.993211967190655, "failed": 0, "missed": 0, "callback_last_ms": 0.0229, "callback_ewma_ms": 1.077901, "release_lateness_last_ms": 0.4515 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 305, "hz": 5.004810591121757, "failed": 0, "missed": 0, "callback_last_ms": 3.2076, "callback_ewma_ms": 4.940572, "release_lateness_last_ms": 0.0941 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 122, "hz": 2.0019242364487027, "failed": 0, "missed": 0, "callback_last_ms": 6.7442, "callback_ewma_ms": 7.783549, "release_lateness_last_ms": 0.2793 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 305, "hz": 5.004810591121757, "failed": 0, "missed": 0, "callback_last_ms": 0.0302, "callback_ewma_ms": 0.005806, "release_lateness_last_ms": 0.2084 }, { "name": "general_program_checks_thread", "running": true, "completed": 610, "hz": 10.009621182243514, "failed": 0, "missed": 0, "callback_last_ms": 0.1137, "callback_ewma_ms": 0.163639, "release_lateness_last_ms": 0.4438 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 61, "hz": 1.0009621182243513, "failed": 0, "missed": 0, "callback_last_ms": 0.1004, "callback_ewma_ms": 0.114233, "release_lateness_last_ms": 0.2963 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 609, "hz": 9.993211967190655, "failed": 0, "missed": 0, "callback_last_ms": 0.0035, "callback_ewma_ms": 0.007621, "release_lateness_last_ms": 0.4451 }, { "name": "engine_diagnostics", "running": true, "completed": 122, "hz": 2.0019242364487027, "failed": 0, "missed": 0, "callback_last_ms": 37.7282, "callback_ewma_ms": 42.872105, "release_lateness_last_ms": 0.3393 } ], "peer_counter_seconds": 60.942404700000004, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.00603193914246, "complete_hz": 5.00603193914246, "catalog_hz": 1.9925324494613081, "bytes_per_second": 1552392.8173980687, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.53489999999874, "gap_p95_ms": 228.2146000000066, "gap_p99_ms": 252.44109999999637, "gap_max_ms": 294.9470000000001, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 608, "p50": 26, "p95": 59, "p99": 80, "max": 134, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.00603193914246, "complete_hz": 5.00603193914246, "catalog_hz": 1.9925324494613081, "bytes_per_second": 1552392.8173980687, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.72149999999965, "gap_p95_ms": 225.61959999999817, "gap_p99_ms": 234.7448000000004, "gap_max_ms": 293.7544999999991, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 608, "p50": 48, "p95": 62, "p99": 70, "max": 122, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.00603193914246, "complete_hz": 5.00603193914246, "catalog_hz": 1.9925324494613081, "bytes_per_second": 1552392.8173980687, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.15170000000217, "gap_p95_ms": 227.50049999999464, "gap_p99_ms": 238.84829999999783, "gap_max_ms": 290.7926000000007, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 608, "p50": 40, "p95": 56, "p99": 69, "max": 108, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.00603193914246, "complete_hz": 5.00603193914246, "catalog_hz": 1.9925324494613081, "bytes_per_second": 1552392.8173980687, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.48199999999633, "gap_p95_ms": 224.191899999998, "gap_p99_ms": 241.300900000002, "gap_max_ms": 259.0222999999969, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 608, "p50": 42, "p95": 64, "p99": 79, "max": 92, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 55, "mean": 72.99708000000028, "p95": 95.90260000000126, "max": 182.59519999999975, "p50": 68.71570000000065, "p99": 182.59519999999975, "min": 55.343699999997625 }, { "operation": "rapid/get-persistence-diagnostics", "count": 55, "mean": 15.798043636363184, "p95": 43.70679999999993, "max": 68.44969999999739, "p50": 16.224699999998847, "p99": 68.44969999999739, "min": 4.439099999999598 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1112907, "accepted_rows_total": 7029637, "committed_rows_total": 1111878, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 55, "minimum": 23.046875, "maximum": 64.453125, "mean": 38.63032428213201 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 55, "minimum": 1.953571977268399, "maximum": 21.10487226768507, "mean": 8.707822062436838 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 55, "minimum": 991465472, "maximum": 1007230976, "mean": 1000804575.4181818 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0.1312255859375, "mean": 0.0029407848011363635 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 55, "minimum": 42.79196601312404, "maximum": 66.51067056049648, "mean": 54.2389474250698 }, { "channel": "engine_tempest_rx_mbps", "observations": 55, "minimum": 0, "maximum": 0.021223434221948022, "mean": 0.001729181425538392 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 55, "minimum": 12492.24980821898, "maximum": 39388.48989244205, "mean": 18106.54309961656 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 55, "minimum": 11569.136089163956, "maximum": 39601.446595323156, "mean": 18037.79573083423 } ], "graceful_shutdown": true, "shutdown_seconds": 1.9261641000000091 }, { "id": "verified-capacity-final-release-25-zmq-8", "healthy": true, "failure_reasons": [], "seconds": 61.323848399999996, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.01394, "wait_seconds": 32.021804, "completed_at": "2026-10-30T00:36:57.353Z", "accepted_cpu": { "count": 20, "mean": 22.160970580968673, "p95": 30.568280302092365, "max": 32.890201870999505 } }, "requested_client_count": 8, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 8, "transport": "zmq", "actual_connections": { "loadClients": 8, "actualClients": 9, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 2000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 54, "mean": 43.866476572177866, "p50": 41.60650338541685, "p95": 63.623793820728345, "p99": 67.12097173323679, "min": 30.902289763337887, "max": 67.12097173323679 }, "engine_cpu_percent": { "count": 54, "mean": 10.765544159108423, "p50": 8.119301093657556, "p95": 23.713946882342665, "p99": 24.175274606234208, "min": 4.631732703125646, "max": 24.175274606234208 }, "client_generator_cpu_percent": { "count": 54, "mean": 7.761584268103148, "p50": 7.692106566731547, "p95": 9.621326665066526, "p99": 10.283246300536463, "min": 5.9100121579935765, "max": 10.283246300536463 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 54, "mean": 1005274908.4444444, "p50": 1006919680, "p95": 1010790400, "p99": 1011146752, "min": 993247232, "max": 1011146752 }, "system_samples": 54, "system_sample_interval_seconds": { "count": 54, "mean": 1.1378683166666665, "p95": 1.1645578, "max": 1.1775829, "p50": 1.1365938, "p99": 1.1775829, "min": 1.1133672 }, "achieved_read_hz": { "minimum": 99.34627770305254, "maximum": 99.76995521800384 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24568, "hz": 99.69060035337525 }, { "group": "stress_group_1", "controls": 3, "completed": 18441, "hz": 99.77175518922137 }, { "group": "stress_group_2", "controls": 3, "completed": 18429, "hz": 99.70683132054448 }, { "group": "stress_group_3", "controls": 3, "completed": 18363, "hz": 99.34975004282154 }, { "group": "stress_group_4", "controls": 3, "completed": 18363, "hz": 99.34975004282154 }, { "group": "stress_group_5", "controls": 3, "completed": 18369, "hz": 99.38221197716 }, { "group": "stress_group_6", "controls": 3, "completed": 18360, "hz": 99.33351907565232 }, { "group": "stress_group_7", "controls": 3, "completed": 18429, "hz": 99.70683132054448 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 2073, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 338157307, "member_service_ns_total": 346435000, "members_executed_total": 24568 }, "example_device_plugin.stress_read": { "callback_ns_total": 2073693725, "member_service_ns_total": 2087987600, "members_executed_total": 24568 }, "example_device_plugin.stress_write": { "callback_ns_total": 105849097, "member_service_ns_total": 114401900, "members_executed_total": 24568 } }, "callback_ns_total": 2517700129, "cycles_total": 6142, "member_service_ns_total": 2548824500, "members_executed_total": 73704, "thread_cpu_ns_total": 906250000, "wall_ns_total": 2588173600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 283748551, "member_service_ns_total": 291222500, "members_executed_total": 18441 }, "example_device_plugin.stress_read": { "callback_ns_total": 1662478450, "member_service_ns_total": 1680583400, "members_executed_total": 18441 }, "example_device_plugin.stress_write": { "callback_ns_total": 82926697, "member_service_ns_total": 88728700, "members_executed_total": 18441 } }, "callback_ns_total": 2029153698, "cycles_total": 6147, "member_service_ns_total": 2060534600, "members_executed_total": 55323, "thread_cpu_ns_total": 656250000, "wall_ns_total": 2098212700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 265191783, "member_service_ns_total": 271965300, "members_executed_total": 18429 }, "example_device_plugin.stress_read": { "callback_ns_total": 1639913134, "member_service_ns_total": 1651576100, "members_executed_total": 18429 }, "example_device_plugin.stress_write": { "callback_ns_total": 79196096, "member_service_ns_total": 85902400, "members_executed_total": 18429 } }, "callback_ns_total": 1984301013, "cycles_total": 6143, "member_service_ns_total": 2009443800, "members_executed_total": 55287, "thread_cpu_ns_total": 671875000, "wall_ns_total": 2047424300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 289870572, "member_service_ns_total": 297508700, "members_executed_total": 18363 }, "example_device_plugin.stress_read": { "callback_ns_total": 1740164601, "member_service_ns_total": 1751567200, "members_executed_total": 18363 }, "example_device_plugin.stress_write": { "callback_ns_total": 89770897, "member_service_ns_total": 95448800, "members_executed_total": 18363 } }, "callback_ns_total": 2119806070, "cycles_total": 6121, "member_service_ns_total": 2144524700, "members_executed_total": 55089, "thread_cpu_ns_total": 640625000, "wall_ns_total": 2179071900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 255798888, "member_service_ns_total": 262304000, "members_executed_total": 18363 }, "example_device_plugin.stress_read": { "callback_ns_total": 1688445104, "member_service_ns_total": 1699846200, "members_executed_total": 18363 }, "example_device_plugin.stress_write": { "callback_ns_total": 82272096, "member_service_ns_total": 87950100, "members_executed_total": 18363 } }, "callback_ns_total": 2026516088, "cycles_total": 6121, "member_service_ns_total": 2050100300, "members_executed_total": 55089, "thread_cpu_ns_total": 515625000, "wall_ns_total": 2085276100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 283273600, "member_service_ns_total": 289214200, "members_executed_total": 18369 }, "example_device_plugin.stress_read": { "callback_ns_total": 1650799521, "member_service_ns_total": 1666523700, "members_executed_total": 18369 }, "example_device_plugin.stress_write": { "callback_ns_total": 85675893, "member_service_ns_total": 92714800, "members_executed_total": 18369 } }, "callback_ns_total": 2019749014, "cycles_total": 6123, "member_service_ns_total": 2048452700, "members_executed_total": 55107, "thread_cpu_ns_total": 640625000, "wall_ns_total": 2085230200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 264228264, "member_service_ns_total": 271027000, "members_executed_total": 18360 }, "example_device_plugin.stress_read": { "callback_ns_total": 1702950154, "member_service_ns_total": 1715350500, "members_executed_total": 18360 }, "example_device_plugin.stress_write": { "callback_ns_total": 86026797, "member_service_ns_total": 91800100, "members_executed_total": 18360 } }, "callback_ns_total": 2053205215, "cycles_total": 6120, "member_service_ns_total": 2078177600, "members_executed_total": 55080, "thread_cpu_ns_total": 718750000, "wall_ns_total": 2116585600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 298164170, "member_service_ns_total": 306839500, "members_executed_total": 18429 }, "example_device_plugin.stress_read": { "callback_ns_total": 1742025814, "member_service_ns_total": 1754913600, "members_executed_total": 18429 }, "example_device_plugin.stress_write": { "callback_ns_total": 81604199, "member_service_ns_total": 87221700, "members_executed_total": 18429 } }, "callback_ns_total": 2121794183, "cycles_total": 6143, "member_service_ns_total": 2148974800, "members_executed_total": 55287, "thread_cpu_ns_total": 734375000, "wall_ns_total": 2184740200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 61.61062305000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 62, "hz": 1.0054862133430884, "failed": 0, "missed": 0, "callback_last_ms": 0.0065, "callback_ewma_ms": 0.005823, "release_lateness_last_ms": 0.881 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 62, "hz": 1.0054862133430884, "failed": 0, "missed": 0, "callback_last_ms": 13.657, "callback_ewma_ms": 9.085768, "release_lateness_last_ms": 0.6944 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 308, "hz": 4.9949960275753424, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.000877, "release_lateness_last_ms": 0.0794 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 308, "hz": 4.9949960275753424, "failed": 0, "missed": 0, "callback_last_ms": 0.4631, "callback_ewma_ms": 0.67923, "release_lateness_last_ms": 0.3097 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 62, "hz": 1.0054862133430884, "failed": 0, "missed": 0, "callback_last_ms": 2.0782, "callback_ewma_ms": 0.996496, "release_lateness_last_ms": 0.4405 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 62, "hz": 1.0054862133430884, "failed": 0, "missed": 0, "callback_last_ms": 1.6316, "callback_ewma_ms": 1.101498, "release_lateness_last_ms": 0.1219 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 124, "hz": 2.010972426686177, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.001165, "release_lateness_last_ms": 0.4377 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 62, "hz": 1.0054862133430884, "failed": 0, "missed": 0, "callback_last_ms": 20.6249, "callback_ewma_ms": 3.97504, "release_lateness_last_ms": 9.6793 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 124, "hz": 2.010972426686177, "failed": 0, "missed": 0, "callback_last_ms": 4.4118, "callback_ewma_ms": 5.441645, "release_lateness_last_ms": 0.5046 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 308, "hz": 4.9949960275753424, "failed": 0, "missed": 0, "callback_last_ms": 5.7561, "callback_ewma_ms": 6.044298, "release_lateness_last_ms": 0.8212 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 616, "hz": 9.989992055150685, "failed": 0, "missed": 0, "callback_last_ms": 0.0213, "callback_ewma_ms": 5.583893, "release_lateness_last_ms": 0.1423 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 308, "hz": 4.9949960275753424, "failed": 0, "missed": 0, "callback_last_ms": 9.4127, "callback_ewma_ms": 7.818856, "release_lateness_last_ms": 0.5772 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 124, "hz": 2.010972426686177, "failed": 0, "missed": 0, "callback_last_ms": 6.1773, "callback_ewma_ms": 8.068702, "release_lateness_last_ms": 0.4461 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 308, "hz": 4.9949960275753424, "failed": 0, "missed": 0, "callback_last_ms": 0.0023, "callback_ewma_ms": 0.002021, "release_lateness_last_ms": 0.0819 }, { "name": "general_program_checks_thread", "running": true, "completed": 617, "hz": 10.006209574720733, "failed": 0, "missed": 0, "callback_last_ms": 0.141, "callback_ewma_ms": 0.240678, "release_lateness_last_ms": 0.2296 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 62, "hz": 1.0054862133430884, "failed": 0, "missed": 0, "callback_last_ms": 0.8804, "callback_ewma_ms": 0.200674, "release_lateness_last_ms": 8.2888 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 617, "hz": 10.006209574720733, "failed": 0, "missed": 0, "callback_last_ms": 0.0064, "callback_ewma_ms": 0.006383, "release_lateness_last_ms": 0.2687 }, { "name": "engine_diagnostics", "running": true, "completed": 123, "hz": 1.9947549071161268, "failed": 0, "missed": 0, "callback_last_ms": 65.3305, "callback_ewma_ms": 48.057532, "release_lateness_last_ms": 0.3611 } ], "peer_counter_seconds": 61.661454500000005, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.7479999999996, "gap_p95_ms": 245.2667999999976, "gap_p99_ms": 267.4459999999999, "gap_max_ms": 313.83400000000256, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 49, "p95": 89, "p99": 107, "max": 143, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.0269999999946, "gap_p95_ms": 245.9953999999998, "gap_p99_ms": 277.5671000000002, "gap_max_ms": 309.77410000000236, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 79, "p95": 115, "p99": 145, "max": 169, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.60810000000492, "gap_p95_ms": 249.0578999999998, "gap_p99_ms": 275.568299999999, "gap_max_ms": 310.93559999999707, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 68, "p95": 108, "p99": 126, "max": 164, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.40639999999985, "gap_p95_ms": 256.2137000000075, "gap_p99_ms": 286.0433000000048, "gap_max_ms": 312.65030000000115, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 73, "p95": 114, "p99": 135, "max": 192, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.40099999999802, "gap_p95_ms": 256.1539000000048, "gap_p99_ms": 278.10059999999794, "gap_max_ms": 403.41160000000673, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 35, "p95": 94, "p99": 124, "max": 230, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.89860000000044, "gap_p95_ms": 263.3698000000004, "gap_p99_ms": 285.14340000000084, "gap_max_ms": 387.5512000000017, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 89, "p95": 134, "p99": 156, "max": 285, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.80709999999817, "gap_p95_ms": 261.53310000000056, "gap_p99_ms": 288.3993000000046, "gap_max_ms": 383.8475999999937, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 95, "p95": 134, "p99": 172, "max": 273, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.989887837382725, "complete_hz": 4.989887837382725, "catalog_hz": 2.00573922875188, "bytes_per_second": 1547371.4749086173, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.9615999999951, "gap_p95_ms": 260.6860000000015, "gap_p99_ms": 284.20820000000094, "gap_max_ms": 426.84739999999874, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 612, "p50": 47, "p95": 110, "p99": 132, "max": 258, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 53, "mean": 85.51784150943354, "p95": 136.9340000000011, "max": 152.82949999999983, "p50": 75.88709999999264, "p99": 152.82949999999983, "min": 58.48210000000108 }, { "operation": "rapid/get-persistence-diagnostics", "count": 53, "mean": 30.631111320754226, "p95": 94.16279999999097, "max": 106.37139999999636, "p50": 17.080999999998312, "p99": 106.37139999999636, "min": 4.906599999994796 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1120924, "accepted_rows_total": 7089973, "committed_rows_total": 1120881, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 53, "minimum": 26.953125, "maximum": 74.609375, "mean": 44.37746867257481 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 53, "minimum": 3.5166279422891407, "maximum": 26.191502912282466, "mean": 10.341397972478136 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 53, "minimum": 993394688, "maximum": 1010794496, "mean": 1005013880.754717 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 53, "minimum": 0, "maximum": 0.3814697265625, "mean": 0.007197542010613208 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 53, "minimum": 82.40563828968682, "maximum": 127.10122570857996, "mean": 104.72689303100472 }, { "channel": "engine_tempest_rx_mbps", "observations": 53, "minimum": 0, "maximum": 0.021874630226005866, "mean": 0.002588107157669359 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 53, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 53, "minimum": 12445.33911848915, "maximum": 39397.52450486395, "mean": 18016.431186666392 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 53, "minimum": 11959.985648017222, "maximum": 40930.67344668094, "mean": 18169.494611980263 } ], "graceful_shutdown": true, "shutdown_seconds": 2.123991099999999 }, { "id": "verified-capacity-final-release-25-zmq-16", "healthy": false, "failure_reasons": [ "client 0: complete snapshots below 4.75 Hz", "client 1: complete snapshots below 4.75 Hz", "client 2: complete snapshots below 4.75 Hz", "client 3: complete snapshots below 4.75 Hz", "client 4: complete snapshots below 4.75 Hz", "client 5: complete snapshots below 4.75 Hz", "client 6: complete snapshots below 4.75 Hz", "client 7: complete snapshots below 4.75 Hz", "client 8: complete snapshots below 4.75 Hz", "client 9: complete snapshots below 4.75 Hz", "client 10: complete snapshots below 4.75 Hz", "client 11: complete snapshots below 4.75 Hz", "client 12: complete snapshots below 4.75 Hz", "client 13: complete snapshots below 4.75 Hz", "client 14: complete snapshots below 4.75 Hz", "client 15: complete snapshots below 4.75 Hz", "ARGUS events deferred_updates: sustained queue growth" ], "seconds": 61.9188533, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.020023600000073, "wait_seconds": 22.021372700000065, "completed_at": "2026-10-30T00:39:00.455Z", "accepted_cpu": { "count": 20, "mean": 25.642058068922598, "p95": 36.67820069204152, "max": 37.064492216456635 } }, "requested_client_count": 16, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 16, "transport": "zmq", "actual_connections": { "loadClients": 16, "actualClients": 17, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 4000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 52, "mean": 70.09686962115036, "p50": 69.16486615299071, "p95": 91.06380174287061, "p99": 94.67321001836848, "min": 44.96466973642473, "max": 94.67321001836848 }, "engine_cpu_percent": { "count": 52, "mean": 17.059420004085098, "p50": 15.919059492774782, "p95": 31.83987753130194, "p99": 32.745129580678, "min": 5.759480370303202, "max": 32.745129580678 }, "client_generator_cpu_percent": { "count": 52, "mean": 16.031687801289554, "p50": 16.081809604531525, "p95": 19.08139369915522, "p99": 19.973163801996236, "min": 10.066447110284047, "max": 19.973163801996236 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 52, "mean": 1220730958.7692308, "p50": 1251905536, "p95": 1340370944, "p99": 1351643136, "min": 1009741824, "max": 1351643136 }, "system_samples": 52, "system_sample_interval_seconds": { "count": 52, "mean": 1.195195051923077, "p95": 1.3581629, "max": 1.47061, "p50": 1.1711171, "p99": 1.47061, "min": 1.1361612 }, "achieved_read_hz": { "minimum": 97.98281767443362, "maximum": 99.24207649810704 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 25256, "hz": 99.09504145001665 }, { "group": "stress_group_1", "controls": 3, "completed": 18966, "hz": 99.22059741004203 }, { "group": "stress_group_2", "controls": 3, "completed": 18945, "hz": 99.11073594501983 }, { "group": "stress_group_3", "controls": 3, "completed": 18726, "hz": 97.9650378097884 }, { "group": "stress_group_4", "controls": 3, "completed": 18735, "hz": 98.01212129479791 }, { "group": "stress_group_5", "controls": 3, "completed": 18738, "hz": 98.02781578980108 }, { "group": "stress_group_6", "controls": 3, "completed": 18723, "hz": 97.94934331478522 }, { "group": "stress_group_7", "controls": 3, "completed": 18948, "hz": 99.126430440023 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 6859, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 485162291, "member_service_ns_total": 501371000, "members_executed_total": 25256 }, "example_device_plugin.stress_read": { "callback_ns_total": 2809995823, "member_service_ns_total": 2825609100, "members_executed_total": 25256 }, "example_device_plugin.stress_write": { "callback_ns_total": 143778493, "member_service_ns_total": 153451700, "members_executed_total": 25256 } }, "callback_ns_total": 3438936607, "cycles_total": 6314, "member_service_ns_total": 3480431800, "members_executed_total": 75768, "thread_cpu_ns_total": 1156250000, "wall_ns_total": 3531332800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 400581530, "member_service_ns_total": 407776200, "members_executed_total": 18966 }, "example_device_plugin.stress_read": { "callback_ns_total": 2299196432, "member_service_ns_total": 2310790300, "members_executed_total": 18966 }, "example_device_plugin.stress_write": { "callback_ns_total": 111887592, "member_service_ns_total": 121024000, "members_executed_total": 18966 } }, "callback_ns_total": 2811665554, "cycles_total": 6322, "member_service_ns_total": 2839590500, "members_executed_total": 56898, "thread_cpu_ns_total": 859375000, "wall_ns_total": 2889017800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 388294934, "member_service_ns_total": 395985600, "members_executed_total": 18945 }, "example_device_plugin.stress_read": { "callback_ns_total": 2230277919, "member_service_ns_total": 2243395500, "members_executed_total": 18945 }, "example_device_plugin.stress_write": { "callback_ns_total": 97040893, "member_service_ns_total": 103112900, "members_executed_total": 18945 } }, "callback_ns_total": 2715613746, "cycles_total": 6315, "member_service_ns_total": 2742494000, "members_executed_total": 56835, "thread_cpu_ns_total": 828125000, "wall_ns_total": 2785462700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 505633558, "member_service_ns_total": 514770400, "members_executed_total": 18726 }, "example_device_plugin.stress_read": { "callback_ns_total": 2785876351, "member_service_ns_total": 2804577500, "members_executed_total": 18726 }, "example_device_plugin.stress_write": { "callback_ns_total": 118441187, "member_service_ns_total": 127853600, "members_executed_total": 18726 } }, "callback_ns_total": 3409951096, "cycles_total": 6242, "member_service_ns_total": 3447201500, "members_executed_total": 56178, "thread_cpu_ns_total": 1312500000, "wall_ns_total": 3508211100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 379654350, "member_service_ns_total": 394423900, "members_executed_total": 18735 }, "example_device_plugin.stress_read": { "callback_ns_total": 2534956803, "member_service_ns_total": 2552930400, "members_executed_total": 18735 }, "example_device_plugin.stress_write": { "callback_ns_total": 108420592, "member_service_ns_total": 116749100, "members_executed_total": 18735 } }, "callback_ns_total": 3023031745, "cycles_total": 6245, "member_service_ns_total": 3064103400, "members_executed_total": 56205, "thread_cpu_ns_total": 859375000, "wall_ns_total": 3136175200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 401160063, "member_service_ns_total": 409826400, "members_executed_total": 18738 }, "example_device_plugin.stress_read": { "callback_ns_total": 2620064639, "member_service_ns_total": 2638194600, "members_executed_total": 18738 }, "example_device_plugin.stress_write": { "callback_ns_total": 121681890, "member_service_ns_total": 130128900, "members_executed_total": 18738 } }, "callback_ns_total": 3142906592, "cycles_total": 6246, "member_service_ns_total": 3178149900, "members_executed_total": 56214, "thread_cpu_ns_total": 703125000, "wall_ns_total": 3234334400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 354697750, "member_service_ns_total": 366123200, "members_executed_total": 18723 }, "example_device_plugin.stress_read": { "callback_ns_total": 2787586576, "member_service_ns_total": 2805910100, "members_executed_total": 18723 }, "example_device_plugin.stress_write": { "callback_ns_total": 117941988, "member_service_ns_total": 127751200, "members_executed_total": 18723 } }, "callback_ns_total": 3260226314, "cycles_total": 6241, "member_service_ns_total": 3299784500, "members_executed_total": 56169, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3352784500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 399505622, "member_service_ns_total": 405536700, "members_executed_total": 18948 }, "example_device_plugin.stress_read": { "callback_ns_total": 2391156189, "member_service_ns_total": 2403666700, "members_executed_total": 18948 }, "example_device_plugin.stress_write": { "callback_ns_total": 115080796, "member_service_ns_total": 122996800, "members_executed_total": 18948 } }, "callback_ns_total": 2905742607, "cycles_total": 6316, "member_service_ns_total": 2932200200, "members_executed_total": 56844, "thread_cpu_ns_total": 750000000, "wall_ns_total": 2975285100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 63.7166089, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 64, "hz": 1.0042355013091528, "failed": 0, "missed": 0, "callback_last_ms": 0.0065, "callback_ewma_ms": 0.005214, "release_lateness_last_ms": 0.2132 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 64, "hz": 1.0042355013091528, "failed": 0, "missed": 0, "callback_last_ms": 8.1887, "callback_ewma_ms": 9.993364, "release_lateness_last_ms": 0.5532 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 317, "hz": 4.9741039674218985, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001184, "release_lateness_last_ms": 0.3135 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 317, "hz": 4.9741039674218985, "failed": 0, "missed": 0, "callback_last_ms": 1.265, "callback_ewma_ms": 1.678655, "release_lateness_last_ms": 0.4036 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 64, "hz": 1.0042355013091528, "failed": 0, "missed": 0, "callback_last_ms": 2.2974, "callback_ewma_ms": 2.255021, "release_lateness_last_ms": 0.2412 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 64, "hz": 1.0042355013091528, "failed": 0, "missed": 0, "callback_last_ms": 2.072, "callback_ewma_ms": 2.017537, "release_lateness_last_ms": 0.3751 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 127, "hz": 1.9927798229103504, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.002591, "release_lateness_last_ms": 0.1868 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 63, "hz": 0.9885443216011974, "failed": 0, "missed": 0, "callback_last_ms": 2.7041, "callback_ewma_ms": 2.374526, "release_lateness_last_ms": 0.3191 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 127, "hz": 1.9927798229103504, "failed": 0, "missed": 0, "callback_last_ms": 6.7179, "callback_ewma_ms": 5.866042, "release_lateness_last_ms": 1.0002 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 317, "hz": 4.9741039674218985, "failed": 0, "missed": 0, "callback_last_ms": 6.0496, "callback_ewma_ms": 6.44313, "release_lateness_last_ms": 0.1835 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 635, "hz": 9.963899114551753, "failed": 0, "missed": 0, "callback_last_ms": 0.024, "callback_ewma_ms": 1.603833, "release_lateness_last_ms": 0.4151 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 316, "hz": 4.958412787713943, "failed": 0, "missed": 1, "callback_last_ms": 13.5324, "callback_ewma_ms": 10.772759, "release_lateness_last_ms": 2.6081 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 127, "hz": 1.9927798229103504, "failed": 0, "missed": 0, "callback_last_ms": 9.3712, "callback_ewma_ms": 9.373063, "release_lateness_last_ms": 0.3391 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 317, "hz": 4.9741039674218985, "failed": 0, "missed": 0, "callback_last_ms": 0.0025, "callback_ewma_ms": 0.002648, "release_lateness_last_ms": 1.8494 }, { "name": "general_program_checks_thread", "running": true, "completed": 636, "hz": 9.979590294259708, "failed": 0, "missed": 0, "callback_last_ms": 0.2059, "callback_ewma_ms": 0.249264, "release_lateness_last_ms": 0.5016 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 63, "hz": 0.9885443216011974, "failed": 0, "missed": 0, "callback_last_ms": 0.1322, "callback_ewma_ms": 0.112511, "release_lateness_last_ms": 0.1326 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 635, "hz": 9.963899114551753, "failed": 0, "missed": 0, "callback_last_ms": 0.0027, "callback_ewma_ms": 0.005612, "release_lateness_last_ms": 0.9424 }, { "name": "engine_diagnostics", "running": true, "completed": 127, "hz": 1.9927798229103504, "failed": 0, "missed": 0, "callback_last_ms": 110.141, "callback_ewma_ms": 56.835908, "release_lateness_last_ms": 2.9913 } ], "peer_counter_seconds": 63.712514299999995, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 3.4238317416935495, "complete_hz": 3.4238317416935495, "catalog_hz": 1.3566125768974442, "bytes_per_second": 1060956.0333718038, "stale_episodes": 0, "sequence_gaps": 41, "gap_p50_ms": 239.21889999999985, "gap_p95_ms": 595.5268999999971, "gap_p99_ms": 670.2749000000003, "gap_max_ms": 714.1448999999993, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 424, "p50": 15567, "p95": 17913, "p99": 18179, "max": 18506, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.4238317416935495, "complete_hz": 3.4238317416935495, "catalog_hz": 1.3889128763473833, "bytes_per_second": 1061134.476376115, "stale_episodes": 0, "sequence_gaps": 42, "gap_p50_ms": 237.95620000000054, "gap_p95_ms": 582.9929999999877, "gap_p99_ms": 735.178899999999, "gap_max_ms": 993.1598000000013, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 424, "p50": 14232, "p95": 18302, "p99": 18865, "max": 18921, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.4399818914185194, "complete_hz": 3.4399818914185194, "catalog_hz": 1.3889128763473833, "bytes_per_second": 1066135.7578420453, "stale_episodes": 0, "sequence_gaps": 43, "gap_p50_ms": 239.28579999999783, "gap_p95_ms": 571.7444000000105, "gap_p99_ms": 790.9760999999999, "gap_max_ms": 1122.2869999999966, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 426, "p50": 15107, "p95": 17038, "p99": 17282, "max": 17328, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.456132041143489, "complete_hz": 3.456132041143489, "catalog_hz": 1.3889128763473833, "bytes_per_second": 1071221.0523868448, "stale_episodes": 0, "sequence_gaps": 31, "gap_p50_ms": 243.5679999999993, "gap_p95_ms": 597.2994000000035, "gap_p99_ms": 711.1820000000007, "gap_max_ms": 930.4182999999975, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 428, "p50": 14125, "p95": 18841, "p99": 19734, "max": 19741, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.4722821908684582, "complete_hz": 3.4722821908684582, "catalog_hz": 1.4212131757973225, "bytes_per_second": 1075922.910076874, "stale_episodes": 0, "sequence_gaps": 37, "gap_p50_ms": 244.5335999999952, "gap_p95_ms": 581.3653999999951, "gap_p99_ms": 657.3425000000061, "gap_max_ms": 907.6803999999975, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 430, "p50": 14801, "p95": 18269, "p99": 18557, "max": 18577, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.456132041143489, "complete_hz": 3.456132041143489, "catalog_hz": 1.4212131757973225, "bytes_per_second": 1071017.0921459682, "stale_episodes": 0, "sequence_gaps": 33, "gap_p50_ms": 241.25769999999466, "gap_p95_ms": 559.3401999999987, "gap_p99_ms": 879.0075999999972, "gap_max_ms": 1578.4133999999976, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 428, "p50": 14513, "p95": 16724, "p99": 16914, "max": 17022, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.488432340593428, "complete_hz": 3.488432340593428, "catalog_hz": 1.405063026072353, "bytes_per_second": 1081682.5702735893, "stale_episodes": 0, "sequence_gaps": 36, "gap_p50_ms": 240.99150000000373, "gap_p95_ms": 574.9832999999999, "gap_p99_ms": 664.4972999999991, "gap_max_ms": 1117.0031999999992, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 432, "p50": 13628, "p95": 17129, "p99": 18421, "max": 18540, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.456132041143489, "complete_hz": 3.456132041143489, "catalog_hz": 1.405063026072353, "bytes_per_second": 1071298.492354776, "stale_episodes": 0, "sequence_gaps": 46, "gap_p50_ms": 239.6244999999908, "gap_p95_ms": 608.0832000000009, "gap_p99_ms": 728.1325999999972, "gap_max_ms": 981.700499999999, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 428, "p50": 14867, "p95": 16506, "p99": 16588, "max": 16677, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.4238317416935495, "complete_hz": 3.4238317416935495, "catalog_hz": 1.405063026072353, "bytes_per_second": 1061134.4602259654, "stale_episodes": 0, "sequence_gaps": 29, "gap_p50_ms": 238.07560000000376, "gap_p95_ms": 578.1212000000014, "gap_p99_ms": 779.0515000000014, "gap_max_ms": 1162.0266999999985, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 424, "p50": 14643, "p95": 17513, "p99": 18049, "max": 18111, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.5853332389432455, "complete_hz": 3.5853332389432455, "catalog_hz": 1.4535134752472616, "bytes_per_second": 1110983.8192199012, "stale_episodes": 0, "sequence_gaps": 28, "gap_p50_ms": 243.5470000000023, "gap_p95_ms": 581.397100000002, "gap_p99_ms": 749.4435999999987, "gap_max_ms": 954.329700000002, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 444, "p50": 13950, "p95": 16970, "p99": 17278, "max": 17331, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.4399818914185194, "complete_hz": 3.4399818914185194, "catalog_hz": 1.3889128763473833, "bytes_per_second": 1066243.4954908607, "stale_episodes": 0, "sequence_gaps": 47, "gap_p50_ms": 238.83299999999872, "gap_p95_ms": 582.8821000000025, "gap_p99_ms": 717.2176000000036, "gap_max_ms": 872.0921000000089, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 426, "p50": 14857, "p95": 17244, "p99": 17406, "max": 17654, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.456132041143489, "complete_hz": 3.456132041143489, "catalog_hz": 1.437363325522292, "bytes_per_second": 1071016.7206925245, "stale_episodes": 0, "sequence_gaps": 40, "gap_p50_ms": 238.1221000000005, "gap_p95_ms": 581.7168999999994, "gap_p99_ms": 726.7383000000045, "gap_max_ms": 1128.3534, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 428, "p50": 14369, "p95": 17197, "p99": 17662, "max": 17768, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.536882789768337, "complete_hz": 3.536882789768337, "catalog_hz": 1.437363325522292, "bytes_per_second": 1095963.905423134, "stale_episodes": 0, "sequence_gaps": 32, "gap_p50_ms": 238.52009999999427, "gap_p95_ms": 577.8064000000013, "gap_p99_ms": 714.8591000000015, "gap_max_ms": 1378.6903999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 438, "p50": 14146, "p95": 15511, "p99": 16132, "max": 16301, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.536882789768337, "complete_hz": 3.536882789768337, "catalog_hz": 1.4535134752472616, "bytes_per_second": 1095824.626531906, "stale_episodes": 0, "sequence_gaps": 27, "gap_p50_ms": 238.9198000000033, "gap_p95_ms": 608.1756999999998, "gap_p99_ms": 694.5314999999973, "gap_max_ms": 1246.0515000000014, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 438, "p50": 14670, "p95": 17857, "p99": 18012, "max": 18072, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.4238317416935495, "complete_hz": 3.4238317416935495, "catalog_hz": 1.405063026072353, "bytes_per_second": 1061037.9953816582, "stale_episodes": 0, "sequence_gaps": 27, "gap_p50_ms": 239.83130000000529, "gap_p95_ms": 594.0285000000003, "gap_p99_ms": 814.9069999999992, "gap_max_ms": 1062.955500000011, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 424, "p50": 14953, "p95": 17537, "p99": 18022, "max": 18082, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 3.3915314422436107, "complete_hz": 3.3915314422436107, "catalog_hz": 1.3889128763473833, "bytes_per_second": 1051258.6275189968, "stale_episodes": 0, "sequence_gaps": 31, "gap_p50_ms": 243.30909999999858, "gap_p95_ms": 600.8358999999909, "gap_p99_ms": 764.063900000001, "gap_max_ms": 1347.782699999996, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 420, "p50": 15151, "p95": 18077, "p99": 18777, "max": 19100, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 28, "mean": 292.46403928571465, "p95": 355.22490000000107, "max": 501.8439999999973, "p50": 281.0677999999971, "p99": 501.8439999999973, "min": 220.22980000000098 }, { "operation": "rapid/get-persistence-diagnostics", "count": 28, "mean": 241.70341428571373, "p95": 332.90810000000056, "max": 350.26320000000123, "p50": 227.42250000000058, "p99": 350.26320000000123, "min": 196.07650000000285 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1135217, "accepted_rows_total": 7205174, "committed_rows_total": 1135158, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 28, "minimum": 43.93939393939394, "maximum": 97.97297297297297, "mean": 71.72462456340625 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 28, "minimum": 3.1268586047546663, "maximum": 33.189052841595164, "mean": 16.745465526516544 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 28, "minimum": 1012154368, "maximum": 1346162688, "mean": 1223869878.857143 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 28, "minimum": 0, "maximum": 0.543212890625, "mean": 0.03880092075892857 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 28, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 28, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 28, "minimum": 0, "maximum": 0.1220703125, "mean": 0.004359654017857143 }, { "channel": "engine_tempest_tx_mbps", "observations": 28, "minimum": 160.76243675652634, "maximum": 248.56214909016103, "mean": 197.10544229743385 }, { "channel": "engine_tempest_rx_mbps", "observations": 28, "minimum": 0, "maximum": 0.021912327065508126, "mean": 0.004787378774435379 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 28, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 28, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 28, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 28, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 28, "minimum": 12500.18500273804, "maximum": 39149.11100103434, "mean": 17635.14514060932 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 28, "minimum": 12125.207223173455, "maximum": 40384.47137988353, "mean": 17806.18603041317 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7178181000000041 }, { "id": "verified-capacity-final-release-25-zmq-32", "healthy": false, "failure_reasons": [ "client 0: complete snapshots below 4.75 Hz", "client 1: complete snapshots below 4.75 Hz", "client 2: complete snapshots below 4.75 Hz", "client 3: complete snapshots below 4.75 Hz", "client 4: complete snapshots below 4.75 Hz", "client 5: complete snapshots below 4.75 Hz", "client 6: complete snapshots below 4.75 Hz", "client 7: complete snapshots below 4.75 Hz", "client 8: complete snapshots below 4.75 Hz", "client 9: complete snapshots below 4.75 Hz", "client 10: complete snapshots below 4.75 Hz", "client 11: complete snapshots below 4.75 Hz", "client 12: complete snapshots below 4.75 Hz", "client 13: complete snapshots below 4.75 Hz", "client 14: complete snapshots below 4.75 Hz", "client 15: complete snapshots below 4.75 Hz", "client 16: complete snapshots below 4.75 Hz", "client 17: complete snapshots below 4.75 Hz", "client 18: complete snapshots below 4.75 Hz", "client 19: complete snapshots below 4.75 Hz", "client 20: complete snapshots below 4.75 Hz", "client 21: complete snapshots below 4.75 Hz", "client 22: complete snapshots below 4.75 Hz", "client 23: complete snapshots below 4.75 Hz", "client 24: complete snapshots below 4.75 Hz", "client 25: complete snapshots below 4.75 Hz", "client 26: complete snapshots below 4.75 Hz", "client 27: complete snapshots below 4.75 Hz", "client 28: complete snapshots below 4.75 Hz", "client 29: complete snapshots below 4.75 Hz", "client 30: complete snapshots below 4.75 Hz", "client 31: complete snapshots below 4.75 Hz" ], "seconds": 60.619982300000004, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.014988899999995, "wait_seconds": 20.014988899999995, "completed_at": "2026-10-30T00:41:09.591Z", "accepted_cpu": { "count": 20, "mean": 29.59325601269423, "p95": 39.066369489846466, "max": 40.022064231429276 } }, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "zmq", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 50, "mean": 73.7662159937911, "p50": 71.54816098165277, "p95": 90.57977027392687, "p99": 97.92713659957933, "min": 62.42078393187016, "max": 97.92713659957933 }, "engine_cpu_percent": { "count": 50, "mean": 16.98408866892446, "p50": 13.867848914817657, "p95": 29.23562480263608, "p99": 33.035662121742625, "min": 10.556833260814926, "max": 33.035662121742625 }, "client_generator_cpu_percent": { "count": 50, "mean": 21.927872286915296, "p50": 22.56299169271497, "p95": 24.905777511184006, "p99": 26.291324137003873, "min": 13.600009478943381, "max": 26.291324137003873 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 50, "mean": 1826936832, "p50": 1847558144, "p95": 1937350656, "p99": 1951064064, "min": 1709101056, "max": 1951064064 }, "system_samples": 50, "system_sample_interval_seconds": { "count": 50, "mean": 1.193880058, "p95": 1.309486, "max": 1.3366725, "p50": 1.1801131, "p99": 1.3366725, "min": 1.1485913 }, "achieved_read_hz": { "minimum": 98.8101979989856, "maximum": 99.38325307308818 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 24496, "hz": 99.31616965528346 }, { "group": "stress_group_1", "controls": 3, "completed": 18378, "hz": 99.34860472048767 }, { "group": "stress_group_2", "controls": 3, "completed": 18375, "hz": 99.33238718788556 }, { "group": "stress_group_3", "controls": 3, "completed": 18291, "hz": 98.87829627502664 }, { "group": "stress_group_4", "controls": 3, "completed": 18282, "hz": 98.82964367722035 }, { "group": "stress_group_5", "controls": 3, "completed": 18288, "hz": 98.86207874242456 }, { "group": "stress_group_6", "controls": 3, "completed": 18279, "hz": 98.81342614461823 }, { "group": "stress_group_7", "controls": 3, "completed": 18384, "hz": 99.38103978569187 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 4131, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 502011449, "member_service_ns_total": 513325400, "members_executed_total": 24496 }, "example_device_plugin.stress_read": { "callback_ns_total": 2744582946, "member_service_ns_total": 2759421300, "members_executed_total": 24496 }, "example_device_plugin.stress_write": { "callback_ns_total": 164199885, "member_service_ns_total": 172317600, "members_executed_total": 24496 } }, "callback_ns_total": 3410794280, "cycles_total": 6124, "member_service_ns_total": 3445064300, "members_executed_total": 73488, "thread_cpu_ns_total": 1421875000, "wall_ns_total": 3495989800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 428280119, "member_service_ns_total": 435635300, "members_executed_total": 18378 }, "example_device_plugin.stress_read": { "callback_ns_total": 2289421917, "member_service_ns_total": 2306436800, "members_executed_total": 18378 }, "example_device_plugin.stress_write": { "callback_ns_total": 109196592, "member_service_ns_total": 115115400, "members_executed_total": 18378 } }, "callback_ns_total": 2826898628, "cycles_total": 6126, "member_service_ns_total": 2857187500, "members_executed_total": 55134, "thread_cpu_ns_total": 906250000, "wall_ns_total": 2909574400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 438568668, "member_service_ns_total": 446925600, "members_executed_total": 18375 }, "example_device_plugin.stress_read": { "callback_ns_total": 2248159907, "member_service_ns_total": 2261569500, "members_executed_total": 18375 }, "example_device_plugin.stress_write": { "callback_ns_total": 114387691, "member_service_ns_total": 120945500, "members_executed_total": 18375 } }, "callback_ns_total": 2801116266, "cycles_total": 6125, "member_service_ns_total": 2829440600, "members_executed_total": 55125, "thread_cpu_ns_total": 937500000, "wall_ns_total": 2874376600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 367711717, "member_service_ns_total": 376579300, "members_executed_total": 18291 }, "example_device_plugin.stress_read": { "callback_ns_total": 2662607034, "member_service_ns_total": 2676764400, "members_executed_total": 18291 }, "example_device_plugin.stress_write": { "callback_ns_total": 129835593, "member_service_ns_total": 137908900, "members_executed_total": 18291 } }, "callback_ns_total": 3160154344, "cycles_total": 6097, "member_service_ns_total": 3191252600, "members_executed_total": 54873, "thread_cpu_ns_total": 812500000, "wall_ns_total": 3244776000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 344509453, "member_service_ns_total": 351612300, "members_executed_total": 18282 }, "example_device_plugin.stress_read": { "callback_ns_total": 2423545835, "member_service_ns_total": 2445568100, "members_executed_total": 18282 }, "example_device_plugin.stress_write": { "callback_ns_total": 114837889, "member_service_ns_total": 122123700, "members_executed_total": 18282 } }, "callback_ns_total": 2882893177, "cycles_total": 6094, "member_service_ns_total": 2919304100, "members_executed_total": 54846, "thread_cpu_ns_total": 656250000, "wall_ns_total": 2961237600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 378500122, "member_service_ns_total": 390590300, "members_executed_total": 18288 }, "example_device_plugin.stress_read": { "callback_ns_total": 2437393068, "member_service_ns_total": 2452500300, "members_executed_total": 18288 }, "example_device_plugin.stress_write": { "callback_ns_total": 150872496, "member_service_ns_total": 163904000, "members_executed_total": 18288 } }, "callback_ns_total": 2966765686, "cycles_total": 6096, "member_service_ns_total": 3006994600, "members_executed_total": 54864, "thread_cpu_ns_total": 781250000, "wall_ns_total": 3056998000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 349700623, "member_service_ns_total": 359033500, "members_executed_total": 18279 }, "example_device_plugin.stress_read": { "callback_ns_total": 2469792826, "member_service_ns_total": 2488705400, "members_executed_total": 18279 }, "example_device_plugin.stress_write": { "callback_ns_total": 115455591, "member_service_ns_total": 125946300, "members_executed_total": 18279 } }, "callback_ns_total": 2934949040, "cycles_total": 6093, "member_service_ns_total": 2973685200, "members_executed_total": 54837, "thread_cpu_ns_total": 781250000, "wall_ns_total": 3020185400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 431145633, "member_service_ns_total": 440305800, "members_executed_total": 18384 }, "example_device_plugin.stress_read": { "callback_ns_total": 2329812046, "member_service_ns_total": 2345499300, "members_executed_total": 18384 }, "example_device_plugin.stress_write": { "callback_ns_total": 109427892, "member_service_ns_total": 116577900, "members_executed_total": 18384 } }, "callback_ns_total": 2870385571, "cycles_total": 6128, "member_service_ns_total": 2902383000, "members_executed_total": 55152, "thread_cpu_ns_total": 671875000, "wall_ns_total": 2949372800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 61.66166114999999, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 61, "hz": 0.9883285606920134, "failed": 0, "missed": 0, "callback_last_ms": 0.0038, "callback_ewma_ms": 0.009677, "release_lateness_last_ms": 0.155 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 61, "hz": 0.9883285606920134, "failed": 0, "missed": 0, "callback_last_ms": 8.1528, "callback_ewma_ms": 10.954164, "release_lateness_last_ms": 0.2254 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 308, "hz": 4.990249126117051, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.001815, "release_lateness_last_ms": 0.2852 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 308, "hz": 4.990249126117051, "failed": 0, "missed": 0, "callback_last_ms": 4.2591, "callback_ewma_ms": 5.098429, "release_lateness_last_ms": 0.4128 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 61, "hz": 0.9883285606920134, "failed": 0, "missed": 0, "callback_last_ms": 4.8718, "callback_ewma_ms": 5.86425, "release_lateness_last_ms": 0.2394 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 61, "hz": 0.9883285606920134, "failed": 0, "missed": 0, "callback_last_ms": 5.7026, "callback_ewma_ms": 6.376439, "release_lateness_last_ms": 0.3668 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 123, "hz": 1.992859228936355, "failed": 0, "missed": 0, "callback_last_ms": 0.0027, "callback_ewma_ms": 0.002585, "release_lateness_last_ms": 0.0702 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 62, "hz": 1.0045306682443416, "failed": 0, "missed": 0, "callback_last_ms": 3.8542, "callback_ewma_ms": 4.211882, "release_lateness_last_ms": 2.4541 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 123, "hz": 1.992859228936355, "failed": 0, "missed": 0, "callback_last_ms": 5.3517, "callback_ewma_ms": 10.488316, "release_lateness_last_ms": 0.2058 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 308, "hz": 4.990249126117051, "failed": 0, "missed": 0, "callback_last_ms": 9.1787, "callback_ewma_ms": 12.207686, "release_lateness_last_ms": 0.9005 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 616, "hz": 9.980498252234103, "failed": 0, "missed": 0, "callback_last_ms": 0.0331, "callback_ewma_ms": 1.176561, "release_lateness_last_ms": 0.5703 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 308, "hz": 4.990249126117051, "failed": 0, "missed": 0, "callback_last_ms": 18.2683, "callback_ewma_ms": 25.729724, "release_lateness_last_ms": 0.6526 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 123, "hz": 1.992859228936355, "failed": 0, "missed": 0, "callback_last_ms": 7.3664, "callback_ewma_ms": 12.722876, "release_lateness_last_ms": 0.1518 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 308, "hz": 4.990249126117051, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.009626, "release_lateness_last_ms": 0.4669 }, { "name": "general_program_checks_thread", "running": true, "completed": 616, "hz": 9.980498252234103, "failed": 0, "missed": 0, "callback_last_ms": 0.4225, "callback_ewma_ms": 0.356939, "release_lateness_last_ms": 0.3707 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 62, "hz": 1.0045306682443416, "failed": 0, "missed": 0, "callback_last_ms": 0.0987, "callback_ewma_ms": 0.147977, "release_lateness_last_ms": 2.6348 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 616, "hz": 9.980498252234103, "failed": 0, "missed": 0, "callback_last_ms": 0.0059, "callback_ewma_ms": 0.006782, "release_lateness_last_ms": 0.1811 }, { "name": "engine_diagnostics", "running": true, "completed": 123, "hz": 1.992859228936355, "failed": 0, "missed": 0, "callback_last_ms": 59.054, "callback_ewma_ms": 74.8753, "release_lateness_last_ms": 0.7413 } ], "peer_counter_seconds": 61.732821900000005, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 1.4021763241412526, "complete_hz": 1.4021763241412526, "catalog_hz": 0.5278781455590598, "bytes_per_second": 433842.95547307585, "stale_episodes": 0, "sequence_gaps": 227, "gap_p50_ms": 695.5585999999894, "gap_p95_ms": 958.883600000001, "gap_p99_ms": 1470.494599999991, "gap_max_ms": 1470.494599999991, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 170, "p50": 13858, "p95": 17493, "p99": 18345, "max": 18345, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.4021763241412526, "complete_hz": 1.4021763241412526, "catalog_hz": 0.5443743376077804, "bytes_per_second": 433842.95547307585, "stale_episodes": 0, "sequence_gaps": 227, "gap_p50_ms": 691.9162999999971, "gap_p95_ms": 1002.0092000000004, "gap_p99_ms": 1424.6347999999998, "gap_max_ms": 1424.6347999999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 170, "p50": 13899, "p95": 17789, "p99": 18335, "max": 18335, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3856801320925318, "complete_hz": 1.3856801320925318, "catalog_hz": 0.560870529656501, "bytes_per_second": 428148.98135981103, "stale_episodes": 0, "sequence_gaps": 193, "gap_p50_ms": 684.9762000000046, "gap_p95_ms": 933.7834999999977, "gap_p99_ms": 1473.025800000003, "gap_max_ms": 1473.025800000003, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 168, "p50": 14191, "p95": 18305, "p99": 19075, "max": 19075, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3856801320925318, "complete_hz": 1.3856801320925318, "catalog_hz": 0.5443743376077804, "bytes_per_second": 428148.98135981103, "stale_episodes": 0, "sequence_gaps": 194, "gap_p50_ms": 677.770199999999, "gap_p95_ms": 1017.4934999999969, "gap_p99_ms": 1519.1041000000114, "gap_max_ms": 1519.1041000000114, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 168, "p50": 14377, "p95": 18319, "p99": 19206, "max": 19206, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3856801320925318, "complete_hz": 1.3856801320925318, "catalog_hz": 0.560870529656501, "bytes_per_second": 428145.0387699114, "stale_episodes": 0, "sequence_gaps": 195, "gap_p50_ms": 701.3329000000085, "gap_p95_ms": 1038.9369000000006, "gap_p99_ms": 1328.577999999994, "gap_max_ms": 1328.577999999994, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 168, "p50": 14499, "p95": 18365, "p99": 19341, "max": 19341, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5443743376077804, "bytes_per_second": 423001.4601043521, "stale_episodes": 0, "sequence_gaps": 195, "gap_p50_ms": 690.2888000000021, "gap_p95_ms": 995.270399999994, "gap_p99_ms": 1409.4138000000094, "gap_max_ms": 1409.4138000000094, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14488, "p95": 18386, "p99": 19437, "max": 19437, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5443743376077804, "bytes_per_second": 423001.4601043521, "stale_episodes": 0, "sequence_gaps": 195, "gap_p50_ms": 689.3260000000009, "gap_p95_ms": 996.5506000000023, "gap_p99_ms": 1390.2378999999928, "gap_max_ms": 1390.2378999999928, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14211, "p95": 18410, "p99": 19423, "max": 19423, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5443743376077804, "bytes_per_second": 423048.77118314785, "stale_episodes": 0, "sequence_gaps": 195, "gap_p50_ms": 693.2480999999971, "gap_p95_ms": 1025.5479999999952, "gap_p99_ms": 1375.5060000000085, "gap_max_ms": 1375.5060000000085, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14522, "p95": 18414, "p99": 19532, "max": 19532, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5278781455590598, "bytes_per_second": 423001.4601043521, "stale_episodes": 0, "sequence_gaps": 196, "gap_p50_ms": 700.1132999999973, "gap_p95_ms": 1020.7660000000033, "gap_p99_ms": 1335.806700000001, "gap_max_ms": 1335.806700000001, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14516, "p95": 18484, "p99": 19375, "max": 19375, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5113819535103391, "bytes_per_second": 423001.4601043521, "stale_episodes": 0, "sequence_gaps": 196, "gap_p50_ms": 682.4453999999969, "gap_p95_ms": 1014.7116000000096, "gap_p99_ms": 1405.7209999999905, "gap_max_ms": 1405.7209999999905, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14466, "p95": 18449, "p99": 19535, "max": 19535, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3856801320925318, "complete_hz": 1.3856801320925318, "catalog_hz": 0.5443743376077804, "bytes_per_second": 428086.4772881384, "stale_episodes": 0, "sequence_gaps": 196, "gap_p50_ms": 691.0466000000015, "gap_p95_ms": 1027.5328000000009, "gap_p99_ms": 1452.7691999999952, "gap_max_ms": 1452.7691999999952, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 168, "p50": 14392, "p95": 18255, "p99": 19355, "max": 19355, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3856801320925318, "complete_hz": 1.3856801320925318, "catalog_hz": 0.5443743376077804, "bytes_per_second": 428086.4772881384, "stale_episodes": 0, "sequence_gaps": 196, "gap_p50_ms": 688.1228000000119, "gap_p95_ms": 1007.7260999999999, "gap_p99_ms": 1366.1036000000022, "gap_max_ms": 1366.1036000000022, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 168, "p50": 14590, "p95": 18432, "p99": 19743, "max": 19743, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5443743376077804, "bytes_per_second": 422990.2097013749, "stale_episodes": 0, "sequence_gaps": 196, "gap_p50_ms": 690.6754000000001, "gap_p95_ms": 989.0276999999915, "gap_p99_ms": 1456.4706000000006, "gap_max_ms": 1456.4706000000006, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14493, "p95": 18270, "p99": 19481, "max": 19481, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3856801320925318, "complete_hz": 1.3856801320925318, "catalog_hz": 0.5278781455590598, "bytes_per_second": 428086.4772881384, "stale_episodes": 0, "sequence_gaps": 196, "gap_p50_ms": 683.1814999999988, "gap_p95_ms": 962.9853000000003, "gap_p99_ms": 1443.1205000000045, "gap_max_ms": 1443.1205000000045, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 168, "p50": 14564, "p95": 18486, "p99": 19736, "max": 19736, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5443743376077804, "bytes_per_second": 422990.2097013749, "stale_episodes": 0, "sequence_gaps": 198, "gap_p50_ms": 698.2972000000009, "gap_p95_ms": 1089.6712000000116, "gap_p99_ms": 1465.9038, "gap_max_ms": 1465.9038, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14652, "p95": 18381, "p99": 19318, "max": 19319, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3526877479950907, "complete_hz": 1.3526877479950907, "catalog_hz": 0.560870529656501, "bytes_per_second": 417893.94211461133, "stale_episodes": 0, "sequence_gaps": 199, "gap_p50_ms": 706.8575000000055, "gap_p95_ms": 991.8852000000043, "gap_p99_ms": 1519.2286000000022, "gap_max_ms": 1519.2286000000022, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 164, "p50": 14548, "p95": 18355, "p99": 19572, "max": 19572, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3196953638976494, "complete_hz": 1.3196953638976494, "catalog_hz": 0.5443743376077804, "bytes_per_second": 407701.40694108425, "stale_episodes": 0, "sequence_gaps": 200, "gap_p50_ms": 695.8793000000005, "gap_p95_ms": 1133.6340000000055, "gap_p99_ms": 1458.5687000000034, "gap_max_ms": 1458.5687000000034, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 160, "p50": 14538, "p95": 18268, "p99": 19381, "max": 19381, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3196953638976494, "complete_hz": 1.3196953638976494, "catalog_hz": 0.5278781455590598, "bytes_per_second": 407701.40694108425, "stale_episodes": 0, "sequence_gaps": 220, "gap_p50_ms": 717.5567999999985, "gap_p95_ms": 1123.5412999999971, "gap_p99_ms": 1463.6056000000099, "gap_max_ms": 1463.6056000000099, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 160, "p50": 14662, "p95": 18304, "p99": 19274, "max": 19274, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.286702979800208, "complete_hz": 1.286702979800208, "catalog_hz": 0.560870529656501, "bytes_per_second": 397519.29736093193, "stale_episodes": 0, "sequence_gaps": 221, "gap_p50_ms": 711.0480000000025, "gap_p95_ms": 1143.4147000000085, "gap_p99_ms": 1346.3135000000038, "gap_max_ms": 1346.3135000000038, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 156, "p50": 14712, "p95": 18663, "p99": 19241, "max": 19241, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3196953638976494, "complete_hz": 1.3196953638976494, "catalog_hz": 0.5443743376077804, "bytes_per_second": 408215.4282853224, "stale_episodes": 0, "sequence_gaps": 225, "gap_p50_ms": 726.709299999995, "gap_p95_ms": 1059.3466999999946, "gap_p99_ms": 1428.1068999999989, "gap_max_ms": 1428.1068999999989, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 160, "p50": 14331, "p95": 17919, "p99": 19086, "max": 19086, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3031991718489289, "complete_hz": 1.3031991718489289, "catalog_hz": 0.5443743376077804, "bytes_per_second": 403377.6725241543, "stale_episodes": 0, "sequence_gaps": 219, "gap_p50_ms": 710.2167000000045, "gap_p95_ms": 1118.4646999999968, "gap_p99_ms": 1339.7194000000018, "gap_max_ms": 1339.7194000000018, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 158, "p50": 13436, "p95": 18307, "p99": 19515, "max": 19515, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.286702979800208, "complete_hz": 1.286702979800208, "catalog_hz": 0.5443743376077804, "bytes_per_second": 398277.1159274581, "stale_episodes": 0, "sequence_gaps": 220, "gap_p50_ms": 725.1971999999951, "gap_p95_ms": 1204.9663999999902, "gap_p99_ms": 1313.4019999999873, "gap_max_ms": 1313.4019999999873, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 156, "p50": 13578, "p95": 18557, "p99": 19407, "max": 19407, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3196953638976494, "complete_hz": 1.3196953638976494, "catalog_hz": 0.5113819535103391, "bytes_per_second": 408606.09110542014, "stale_episodes": 0, "sequence_gaps": 216, "gap_p50_ms": 713.8776000000071, "gap_p95_ms": 1167.0448000000033, "gap_p99_ms": 1393.7966000000015, "gap_max_ms": 1393.7966000000015, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 160, "p50": 13491, "p95": 18063, "p99": 19308, "max": 19308, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3031991718489289, "complete_hz": 1.3031991718489289, "catalog_hz": 0.5443743376077804, "bytes_per_second": 403744.5643315099, "stale_episodes": 0, "sequence_gaps": 212, "gap_p50_ms": 714.5255999999936, "gap_p95_ms": 1092.8531999999977, "gap_p99_ms": 1430.3366000000096, "gap_max_ms": 1430.3366000000096, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 158, "p50": 13338, "p95": 17745, "p99": 18648, "max": 18648, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3196953638976494, "complete_hz": 1.3196953638976494, "catalog_hz": 0.560870529656501, "bytes_per_second": 408813.13481182366, "stale_episodes": 0, "sequence_gaps": 211, "gap_p50_ms": 717.6389999999956, "gap_p95_ms": 1075.8084999999992, "gap_p99_ms": 1556.8447000000015, "gap_max_ms": 1556.8447000000015, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 160, "p50": 13355, "p95": 17360, "p99": 18491, "max": 18491, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3031991718489289, "complete_hz": 1.3031991718489289, "catalog_hz": 0.5278781455590598, "bytes_per_second": 404216.586370792, "stale_episodes": 0, "sequence_gaps": 227, "gap_p50_ms": 700.3151999999973, "gap_p95_ms": 1146.8222999999998, "gap_p99_ms": 1411.4997000000003, "gap_max_ms": 1411.4997000000003, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 158, "p50": 13571, "p95": 17361, "p99": 18416, "max": 18416, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3031991718489289, "complete_hz": 1.3031991718489289, "catalog_hz": 0.5443743376077804, "bytes_per_second": 402605.13935432065, "stale_episodes": 0, "sequence_gaps": 201, "gap_p50_ms": 710.1377999999968, "gap_p95_ms": 1121.5486999999994, "gap_p99_ms": 1439.5705999999918, "gap_max_ms": 1439.5705999999918, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 158, "p50": 14645, "p95": 18395, "p99": 19456, "max": 19456, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.4021763241412526, "complete_hz": 1.4021763241412526, "catalog_hz": 0.5113819535103391, "bytes_per_second": 433406.1528038178, "stale_episodes": 0, "sequence_gaps": 223, "gap_p50_ms": 687.7592000000004, "gap_p95_ms": 1028.7464000000036, "gap_p99_ms": 1369.2163, "gap_max_ms": 1369.2163, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 170, "p50": 14515, "p95": 18355, "p99": 19336, "max": 19336, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.286702979800208, "complete_hz": 1.286702979800208, "catalog_hz": 0.5443743376077804, "bytes_per_second": 397995.55892157054, "stale_episodes": 0, "sequence_gaps": 229, "gap_p50_ms": 717.6061000000045, "gap_p95_ms": 1193.755999999994, "gap_p99_ms": 1378.5215000000026, "gap_max_ms": 1378.5215000000026, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 156, "p50": 14188, "p95": 18458, "p99": 19373, "max": 19374, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3031991718489289, "complete_hz": 1.3031991718489289, "catalog_hz": 0.5278781455590598, "bytes_per_second": 403430.9717206637, "stale_episodes": 0, "sequence_gaps": 217, "gap_p50_ms": 710.2517000000007, "gap_p95_ms": 1141.279600000009, "gap_p99_ms": 1441.9593999999997, "gap_max_ms": 1441.9593999999997, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 158, "p50": 13567, "p95": 18479, "p99": 19403, "max": 19403, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3031991718489289, "complete_hz": 1.3031991718489289, "catalog_hz": 0.5443743376077804, "bytes_per_second": 402605.13935432065, "stale_episodes": 0, "sequence_gaps": 201, "gap_p50_ms": 710.2885999999999, "gap_p95_ms": 1137.421000000002, "gap_p99_ms": 1381.1321999999927, "gap_max_ms": 1381.1321999999927, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 158, "p50": 14664, "p95": 18598, "p99": 19794, "max": 19794, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 1.3691839400438113, "complete_hz": 1.3691839400438113, "catalog_hz": 0.5443743376077804, "bytes_per_second": 423002.0374710738, "stale_episodes": 0, "sequence_gaps": 221, "gap_p50_ms": 689.5359000000026, "gap_p95_ms": 934.488400000002, "gap_p99_ms": 1464.389599999995, "gap_max_ms": 1464.389599999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 166, "p50": 14545, "p95": 18338, "p99": 19421, "max": 19421, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 42, "mean": 209.3987500000008, "p95": 293.1247999999978, "max": 351.1802000000025, "p50": 197.80329999999958, "p99": 351.1802000000025, "min": 150.45270000000892 }, { "operation": "rapid/get-persistence-diagnostics", "count": 42, "mean": 143.44399761904597, "p95": 218.7109000000055, "max": 271.43610000000626, "p50": 125.43359999998938, "p99": 271.43610000000626, "min": 94.78009999998903 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1115062, "accepted_rows_total": 7054973, "committed_rows_total": 1115525, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 42, "minimum": 56.640625, "maximum": 99.30313588850174, "mean": 75.43000593241197 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 42, "minimum": 6.636962723968914, "maximum": 33.620552304296986, "mean": 16.872414700141185 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 42, "minimum": 1716879360, "maximum": 1948348416, "mean": 1826711454.4761906 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 42, "minimum": 0, "maximum": 0.5950927734375, "mean": 0.13870965866815477 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 42, "minimum": 0, "maximum": 0.01220703125, "mean": 0.0002906436011904762 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 42, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 42, "minimum": 0, "maximum": 0.29296875, "mean": 0.012788318452380952 }, { "channel": "engine_tempest_tx_mbps", "observations": 42, "minimum": 319.39559853351045, "maximum": 489.40549610538835, "mean": 402.6661535511281 }, { "channel": "engine_tempest_rx_mbps", "observations": 42, "minimum": 0, "maximum": 0.03707204448645338, "mean": 0.00947817744270496 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 42, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 42, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 42, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 42, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 42, "minimum": 12249.037225674063, "maximum": 39746.85128948521, "mean": 18031.43288602207 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 42, "minimum": 8301.534444270052, "maximum": 40372.78595546135, "mean": 17983.13845142441 } ], "graceful_shutdown": true, "shutdown_seconds": 2.087868300000002 }, { "id": "verified-capacity-stable-release-25-alone", "healthy": true, "failure_reasons": [], "seconds": 120.3393315, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.037842299999998, "wait_seconds": 20.0378423, "completed_at": "2026-10-29T23:46:30.136Z", "accepted_cpu": { "count": 20, "mean": 27.794983777340445, "p95": 40.066839955440024, "max": 43.01628106255355 } }, "requested_client_count": 0, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 0, "transport": "web", "actual_connections": { "loadClients": 0, "actualClients": 1, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 0, "recipient_channel_subscriptions": 0, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 32.61562253293163, "p50": 31.56126398878383, "p95": 47.3462682083202, "p99": 54.970300024754536, "min": 21.332063400125822, "max": 55.19601227618334 }, "engine_cpu_percent": { "count": 109, "mean": 7.177354960533695, "p50": 4.716965731891055, "p95": 16.533623203911002, "p99": 18.429749101262505, "min": 1.929345812856543, "max": 19.527042010406404 }, "client_generator_cpu_percent": { "count": 109, "mean": 0.4389745243610078, "p50": 0.3525208455446554, "p95": 0.8839071135984637, "p99": 1.2343856284129422, "min": 0, "max": 1.238936958447204 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1000889729.1743119, "p50": 1001533440, "p95": 1007091712, "p99": 1008521216, "min": 990318592, "max": 1009336320 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1101821211009175, "p95": 1.1235253, "max": 1.1550438, "p50": 1.1093266, "p99": 1.1336208, "min": 1.0976964 }, "achieved_read_hz": { "minimum": 99.97360668537681, "maximum": 99.99022669471665 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48170, "hz": 99.98565669741573 }, { "group": "stress_group_1", "controls": 3, "completed": 36128, "hz": 99.98704048617357 }, { "group": "stress_group_2", "controls": 3, "completed": 36129, "hz": 99.98980806368924 }, { "group": "stress_group_3", "controls": 3, "completed": 36126, "hz": 99.98150533114222 }, { "group": "stress_group_4", "controls": 3, "completed": 36124, "hz": 99.97597017611088 }, { "group": "stress_group_5", "controls": 3, "completed": 36129, "hz": 99.98980806368924 }, { "group": "stress_group_6", "controls": 3, "completed": 36128, "hz": 99.98704048617357 }, { "group": "stress_group_7", "controls": 3, "completed": 36129, "hz": 99.98980806368924 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 132, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 565341333, "member_service_ns_total": 579861700, "members_executed_total": 48170 }, "example_device_plugin.stress_read": { "callback_ns_total": 3473552281, "member_service_ns_total": 3496258400, "members_executed_total": 48170 }, "example_device_plugin.stress_write": { "callback_ns_total": 175351592, "member_service_ns_total": 188575500, "members_executed_total": 48170 } }, "callback_ns_total": 4214681106, "cycles_total": 12043, "member_service_ns_total": 4265133500, "members_executed_total": 144516, "thread_cpu_ns_total": 1109375000, "wall_ns_total": 4335007000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 459535093, "member_service_ns_total": 469966900, "members_executed_total": 36128 }, "example_device_plugin.stress_read": { "callback_ns_total": 2858067169, "member_service_ns_total": 2878562400, "members_executed_total": 36128 }, "example_device_plugin.stress_write": { "callback_ns_total": 136648697, "member_service_ns_total": 146683800, "members_executed_total": 36128 } }, "callback_ns_total": 3454664959, "cycles_total": 12043, "member_service_ns_total": 3495628800, "members_executed_total": 108387, "thread_cpu_ns_total": 1109375000, "wall_ns_total": 3557780500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 440934385, "member_service_ns_total": 451489400, "members_executed_total": 36129 }, "example_device_plugin.stress_read": { "callback_ns_total": 2721829508, "member_service_ns_total": 2740620400, "members_executed_total": 36129 }, "example_device_plugin.stress_write": { "callback_ns_total": 139578994, "member_service_ns_total": 148463100, "members_executed_total": 36129 } }, "callback_ns_total": 3302342887, "cycles_total": 12043, "member_service_ns_total": 3340572900, "members_executed_total": 108387, "thread_cpu_ns_total": 1109375000, "wall_ns_total": 3400853700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 420639475, "member_service_ns_total": 431440400, "members_executed_total": 36126 }, "example_device_plugin.stress_read": { "callback_ns_total": 2699776474, "member_service_ns_total": 2720067300, "members_executed_total": 36126 }, "example_device_plugin.stress_write": { "callback_ns_total": 138954295, "member_service_ns_total": 149240400, "members_executed_total": 36126 } }, "callback_ns_total": 3259370244, "cycles_total": 12042, "member_service_ns_total": 3300748100, "members_executed_total": 108378, "thread_cpu_ns_total": 1109375000, "wall_ns_total": 3360813800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 408768583, "member_service_ns_total": 418550700, "members_executed_total": 36124 }, "example_device_plugin.stress_read": { "callback_ns_total": 2547297593, "member_service_ns_total": 2566143100, "members_executed_total": 36124 }, "example_device_plugin.stress_write": { "callback_ns_total": 139380297, "member_service_ns_total": 149490400, "members_executed_total": 36124 } }, "callback_ns_total": 3095809073, "cycles_total": 12042, "member_service_ns_total": 3134548600, "members_executed_total": 108378, "thread_cpu_ns_total": 500000000, "wall_ns_total": 3190320500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 396117027, "member_service_ns_total": 405658000, "members_executed_total": 36129 }, "example_device_plugin.stress_read": { "callback_ns_total": 2522410166, "member_service_ns_total": 2539910900, "members_executed_total": 36129 }, "example_device_plugin.stress_write": { "callback_ns_total": 132744794, "member_service_ns_total": 141990300, "members_executed_total": 36129 } }, "callback_ns_total": 3051271987, "cycles_total": 12043, "member_service_ns_total": 3087559200, "members_executed_total": 108387, "thread_cpu_ns_total": 562500000, "wall_ns_total": 3141815000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 408755185, "member_service_ns_total": 418994600, "members_executed_total": 36128 }, "example_device_plugin.stress_read": { "callback_ns_total": 2657524551, "member_service_ns_total": 2676748000, "members_executed_total": 36127 }, "example_device_plugin.stress_write": { "callback_ns_total": 134645795, "member_service_ns_total": 145055600, "members_executed_total": 36128 } }, "callback_ns_total": 3201381131, "cycles_total": 12043, "member_service_ns_total": 3241255100, "members_executed_total": 108387, "thread_cpu_ns_total": 812500000, "wall_ns_total": 3299687800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 439588503, "member_service_ns_total": 450216600, "members_executed_total": 36129 }, "example_device_plugin.stress_read": { "callback_ns_total": 2779841355, "member_service_ns_total": 2800644300, "members_executed_total": 36129 }, "example_device_plugin.stress_write": { "callback_ns_total": 138467489, "member_service_ns_total": 149980700, "members_executed_total": 36129 } }, "callback_ns_total": 3357897347, "cycles_total": 12043, "member_service_ns_total": 3400841600, "members_executed_total": 108387, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3462772100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.4422754, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0042394741253682, "failed": 0, "missed": 0, "callback_last_ms": 0.0117, "callback_ewma_ms": 0.005562, "release_lateness_last_ms": 0.5671 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0042394741253682, "failed": 0, "missed": 0, "callback_last_ms": 7.4802, "callback_ewma_ms": 5.932937, "release_lateness_last_ms": 0.0772 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 603, "hz": 5.004598371054521, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.000896, "release_lateness_last_ms": 0.4942 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 603, "hz": 5.004598371054521, "failed": 0, "missed": 0, "callback_last_ms": 0.0094, "callback_ewma_ms": 0.01256, "release_lateness_last_ms": 0.0909 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0042394741253682, "failed": 0, "missed": 0, "callback_last_ms": 0.0076, "callback_ewma_ms": 0.009742, "release_lateness_last_ms": 0.467 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0042394741253682, "failed": 0, "missed": 0, "callback_last_ms": 0.0098, "callback_ewma_ms": 0.01142, "release_lateness_last_ms": 0.1024 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 2.0001794484645763, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.002958, "release_lateness_last_ms": 0.1293 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0042394741253682, "failed": 0, "missed": 0, "callback_last_ms": 2.3868, "callback_ewma_ms": 2.15096, "release_lateness_last_ms": 0.2556 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.0001794484645763, "failed": 0, "missed": 0, "callback_last_ms": 3.7727, "callback_ewma_ms": 3.908724, "release_lateness_last_ms": 0.3966 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 603, "hz": 5.004598371054521, "failed": 0, "missed": 0, "callback_last_ms": 3.42, "callback_ewma_ms": 4.41441, "release_lateness_last_ms": 0.2102 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1205, "hz": 10.000897242322882, "failed": 0, "missed": 0, "callback_last_ms": 0.0257, "callback_ewma_ms": 0.353545, "release_lateness_last_ms": 0.3942 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 603, "hz": 5.004598371054521, "failed": 0, "missed": 0, "callback_last_ms": 0.3947, "callback_ewma_ms": 0.325015, "release_lateness_last_ms": 0.3792 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.0001794484645763, "failed": 0, "missed": 0, "callback_last_ms": 6.4962, "callback_ewma_ms": 6.169572, "release_lateness_last_ms": 0.6117 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 603, "hz": 5.004598371054521, "failed": 0, "missed": 0, "callback_last_ms": 0.0016, "callback_ewma_ms": 0.00194, "release_lateness_last_ms": 0.1789 }, { "name": "general_program_checks_thread", "running": true, "completed": 1205, "hz": 10.000897242322882, "failed": 0, "missed": 0, "callback_last_ms": 0.17, "callback_ewma_ms": 0.164439, "release_lateness_last_ms": 0.2955 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0042394741253682, "failed": 0, "missed": 0, "callback_last_ms": 0.1567, "callback_ewma_ms": 0.129231, "release_lateness_last_ms": 0.1863 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1205, "hz": 10.000897242322882, "failed": 0, "missed": 0, "callback_last_ms": 0.0059, "callback_ewma_ms": 0.00779, "release_lateness_last_ms": 0.0907 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 2.0001794484645763, "failed": 0, "missed": 0, "callback_last_ms": 43.6868, "callback_ewma_ms": 40.148961, "release_lateness_last_ms": 0.3302 } ], "peer_counter_seconds": 120.49907129999998, "peer_counter_deltas": [], "observations": [], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 112, "mean": 61.87916071428591, "p95": 74.37220000001253, "max": 89.50049999999464, "p50": 62.42750000000524, "p99": 77.46300000000338, "min": 50.6820000000007 }, { "operation": "rapid/get-persistence-diagnostics", "count": 112, "mean": 10.999194642856894, "p95": 16.78469999999652, "max": 17.758499999996275, "p50": 7.854599999998754, "p99": 17.741399999998976, "min": 4.402100000006612 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2223508, "accepted_rows_total": 13986630, "committed_rows_total": 2222747, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 112, "minimum": 19.53125, "maximum": 62.890625, "mean": 32.91713169642857 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 112, "minimum": 0.7815532426581514, "maximum": 19.549172681514413, "mean": 6.620479331666076 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 112, "minimum": 985366528, "maximum": 1008521216, "mean": 1000309174.8571428 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 112, "minimum": 3.115167982915779, "maximum": 6.696715579871844, "mean": 4.092918039906491 }, { "channel": "engine_tempest_rx_mbps", "observations": 112, "minimum": 0, "maximum": 0.0031207233836803372, "mean": 0.0003210226605342165 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 112, "minimum": 12490.022969651842, "maximum": 39487.53387273769, "mean": 18717.303232580296 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 112, "minimum": 0, "maximum": 41061.196479474806, "mean": 18730.738189941265 } ], "graceful_shutdown": true, "shutdown_seconds": 1.8930334000000149 }, { "id": "verified-capacity-stable-release-25-web-1", "healthy": true, "failure_reasons": [], "seconds": 120.2886786, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.037030700000003, "wait_seconds": 22.038172900000006, "completed_at": "2026-10-29T23:49:50.294Z", "accepted_cpu": { "count": 20, "mean": 24.42004140115743, "p95": 33.05105853051059, "max": 37.78051787916152 } }, "requested_client_count": 1, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 1, "transport": "web", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 36.876021560255296, "p50": 35.30061178811241, "p95": 52.983772875160994, "p99": 57.94835353655819, "min": 22.760334608896937, "max": 62.75409528654565 }, "engine_cpu_percent": { "count": 108, "mean": 8.42458656726363, "p50": 6.046139916527835, "p95": 19.557857496040334, "p99": 20.16831052663405, "min": 2.4788144370504193, "max": 20.493129172947583 }, "client_generator_cpu_percent": { "count": 108, "mean": 0.8420673382431431, "p50": 0.7077480954895089, "p95": 1.5686324465450219, "p99": 1.7584555320189488, "min": 0.17512123208596658, "max": 2.639716015549856 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1003022070.5185186, "p50": 1003233280, "p95": 1007824896, "p99": 1010515968, "min": 996220928, "max": 1010819072 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1146361231481483, "p95": 1.1321066, "max": 1.1581262, "p50": 1.1138754, "p99": 1.1427271, "min": 1.1001621 }, "achieved_read_hz": { "minimum": 99.9654178833784, "maximum": 99.99035870187201 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48152, "hz": 99.99226260237465 }, { "group": "stress_group_1", "controls": 3, "completed": 36114, "hz": 99.99226260237465 }, { "group": "stress_group_2", "controls": 3, "completed": 36114, "hz": 99.99226260237465 }, { "group": "stress_group_3", "controls": 3, "completed": 36111, "hz": 99.98395621737694 }, { "group": "stress_group_4", "controls": 3, "completed": 36105, "hz": 99.96734344738154 }, { "group": "stress_group_5", "controls": 3, "completed": 36114, "hz": 99.99226260237465 }, { "group": "stress_group_6", "controls": 3, "completed": 36111, "hz": 99.98395621737694 }, { "group": "stress_group_7", "controls": 3, "completed": 36114, "hz": 99.99226260237465 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 120, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 581087967, "member_service_ns_total": 594870000, "members_executed_total": 48152 }, "example_device_plugin.stress_read": { "callback_ns_total": 3475855550, "member_service_ns_total": 3499053200, "members_executed_total": 48152 }, "example_device_plugin.stress_write": { "callback_ns_total": 177138395, "member_service_ns_total": 189571900, "members_executed_total": 48152 } }, "callback_ns_total": 4234081912, "cycles_total": 12038, "member_service_ns_total": 4283495100, "members_executed_total": 144456, "thread_cpu_ns_total": 1578125000, "wall_ns_total": 4351957200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 478410464, "member_service_ns_total": 489015300, "members_executed_total": 36114 }, "example_device_plugin.stress_read": { "callback_ns_total": 2823113076, "member_service_ns_total": 2844847300, "members_executed_total": 36114 }, "example_device_plugin.stress_write": { "callback_ns_total": 149319897, "member_service_ns_total": 160782300, "members_executed_total": 36114 } }, "callback_ns_total": 3450843437, "cycles_total": 12038, "member_service_ns_total": 3494644900, "members_executed_total": 108342, "thread_cpu_ns_total": 1296875000, "wall_ns_total": 3560378600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 447311484, "member_service_ns_total": 458700000, "members_executed_total": 36114 }, "example_device_plugin.stress_read": { "callback_ns_total": 2696503524, "member_service_ns_total": 2717203300, "members_executed_total": 36114 }, "example_device_plugin.stress_write": { "callback_ns_total": 137666996, "member_service_ns_total": 147844800, "members_executed_total": 36114 } }, "callback_ns_total": 3281482004, "cycles_total": 12038, "member_service_ns_total": 3323748100, "members_executed_total": 108342, "thread_cpu_ns_total": 1250000000, "wall_ns_total": 3384220500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 443313040, "member_service_ns_total": 454005300, "members_executed_total": 36111 }, "example_device_plugin.stress_read": { "callback_ns_total": 2701646108, "member_service_ns_total": 2721849200, "members_executed_total": 36111 }, "example_device_plugin.stress_write": { "callback_ns_total": 141131394, "member_service_ns_total": 150904500, "members_executed_total": 36111 } }, "callback_ns_total": 3286090542, "cycles_total": 12037, "member_service_ns_total": 3326759000, "members_executed_total": 108333, "thread_cpu_ns_total": 1265625000, "wall_ns_total": 3387993100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 417622969, "member_service_ns_total": 428430200, "members_executed_total": 36105 }, "example_device_plugin.stress_read": { "callback_ns_total": 2569912027, "member_service_ns_total": 2587394800, "members_executed_total": 36105 }, "example_device_plugin.stress_write": { "callback_ns_total": 140022894, "member_service_ns_total": 149731000, "members_executed_total": 36105 } }, "callback_ns_total": 3127557890, "cycles_total": 12035, "member_service_ns_total": 3165556000, "members_executed_total": 108315, "thread_cpu_ns_total": 421875000, "wall_ns_total": 3225420500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 412459954, "member_service_ns_total": 422109400, "members_executed_total": 36114 }, "example_device_plugin.stress_read": { "callback_ns_total": 2549284406, "member_service_ns_total": 2566578500, "members_executed_total": 36114 }, "example_device_plugin.stress_write": { "callback_ns_total": 137026494, "member_service_ns_total": 146339000, "members_executed_total": 36114 } }, "callback_ns_total": 3098770854, "cycles_total": 12038, "member_service_ns_total": 3135026900, "members_executed_total": 108342, "thread_cpu_ns_total": 500000000, "wall_ns_total": 3191224200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 436084748, "member_service_ns_total": 447287900, "members_executed_total": 36111 }, "example_device_plugin.stress_read": { "callback_ns_total": 2695746558, "member_service_ns_total": 2714298900, "members_executed_total": 36111 }, "example_device_plugin.stress_write": { "callback_ns_total": 139853190, "member_service_ns_total": 150397200, "members_executed_total": 36111 } }, "callback_ns_total": 3271684496, "cycles_total": 12037, "member_service_ns_total": 3311984000, "members_executed_total": 108333, "thread_cpu_ns_total": 1109375000, "wall_ns_total": 3369790700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 465651048, "member_service_ns_total": 475525300, "members_executed_total": 36114 }, "example_device_plugin.stress_read": { "callback_ns_total": 2726368751, "member_service_ns_total": 2745564400, "members_executed_total": 36114 }, "example_device_plugin.stress_write": { "callback_ns_total": 138940898, "member_service_ns_total": 149617300, "members_executed_total": 36114 } }, "callback_ns_total": 3330960697, "cycles_total": 12038, "member_service_ns_total": 3370707000, "members_executed_total": 108342, "thread_cpu_ns_total": 984375000, "wall_ns_total": 3433819500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.38931499999998, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 120, "hz": 0.9962371690422501, "failed": 0, "missed": 0, "callback_last_ms": 0.0042, "callback_ewma_ms": 0.004587, "release_lateness_last_ms": 0.0736 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 120, "hz": 0.9962371690422501, "failed": 0, "missed": 0, "callback_last_ms": 5.9683, "callback_ewma_ms": 5.446023, "release_lateness_last_ms": 0.4169 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 602, "hz": 4.997789798028621, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001132, "release_lateness_last_ms": 0.0484 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 602, "hz": 4.997789798028621, "failed": 0, "missed": 0, "callback_last_ms": 0.0709, "callback_ewma_ms": 0.078186, "release_lateness_last_ms": 0.233 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 120, "hz": 0.9962371690422501, "failed": 0, "missed": 0, "callback_last_ms": 0.1312, "callback_ewma_ms": 0.113673, "release_lateness_last_ms": 0.4849 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 120, "hz": 0.9962371690422501, "failed": 0, "missed": 0, "callback_last_ms": 0.0986, "callback_ewma_ms": 0.095247, "release_lateness_last_ms": 0.2621 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 2.000776314493186, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.00167, "release_lateness_last_ms": 0.3833 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.9962371690422501, "failed": 0, "missed": 0, "callback_last_ms": 4.1582, "callback_ewma_ms": 2.176874, "release_lateness_last_ms": 0.496 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.000776314493186, "failed": 0, "missed": 0, "callback_last_ms": 4.2717, "callback_ewma_ms": 3.994242, "release_lateness_last_ms": 0.4061 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 4.997789798028621, "failed": 0, "missed": 0, "callback_last_ms": 3.3974, "callback_ewma_ms": 4.327645, "release_lateness_last_ms": 0.1144 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1204, "hz": 9.995579596057242, "failed": 0, "missed": 0, "callback_last_ms": 0.0247, "callback_ewma_ms": 0.523432, "release_lateness_last_ms": 1.5476 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 602, "hz": 4.997789798028621, "failed": 0, "missed": 0, "callback_last_ms": 2.3271, "callback_ewma_ms": 2.461107, "release_lateness_last_ms": 0.1641 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.000776314493186, "failed": 0, "missed": 0, "callback_last_ms": 6.2923, "callback_ewma_ms": 6.431581, "release_lateness_last_ms": 0.3103 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 602, "hz": 4.997789798028621, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.002383, "release_lateness_last_ms": 0.2923 }, { "name": "general_program_checks_thread", "running": true, "completed": 1205, "hz": 10.003881572465929, "failed": 0, "missed": 0, "callback_last_ms": 0.3141, "callback_ewma_ms": 0.187064, "release_lateness_last_ms": 0.2628 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.9962371690422501, "failed": 0, "missed": 0, "callback_last_ms": 0.1085, "callback_ewma_ms": 0.115292, "release_lateness_last_ms": 0.4084 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1204, "hz": 9.995579596057242, "failed": 0, "missed": 0, "callback_last_ms": 0.0051, "callback_ewma_ms": 0.007499, "release_lateness_last_ms": 0.0567 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 2.000776314493186, "failed": 0, "missed": 0, "callback_last_ms": 48.3092, "callback_ewma_ms": 40.412521, "release_lateness_last_ms": 0.517 } ], "peer_counter_seconds": 120.4636446, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.004625400326284, "complete_hz": 5.004625400326284, "catalog_hz": 2.003512826376469, "bytes_per_second": 1552124.3045731122, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.00319999999192, "gap_p95_ms": 201.512999999999, "gap_p99_ms": 213.01940000000468, "gap_max_ms": 222.72669999999925, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 11, "p95": 12, "p99": 23, "max": 34, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 112, "mean": 61.72975982142874, "p95": 74.86049999999523, "max": 86.70240000000922, "p50": 62.938099999999395, "p99": 80.30909999999858, "min": 51.22469999999885 }, { "operation": "rapid/get-persistence-diagnostics", "count": 112, "mean": 10.174756250000298, "p95": 16.81470000000263, "max": 17.368699999991804, "p50": 6.933599999989383, "p99": 17.168100000009872, "min": 4.720399999991059 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2222676, "accepted_rows_total": 13978232, "committed_rows_total": 2222009, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 112, "minimum": 21.875, "maximum": 80.078125, "mean": 37.831333705357146 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 112, "minimum": 1.172132165797176, "maximum": 24.21735508034737, "mean": 8.217511392394064 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 112, "minimum": 994078720, "maximum": 1010520064, "mean": 1002926518.8571428 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0.1373291015625, "mean": 0.0012261526925223215 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 112, "minimum": 26.489535858858027, "maximum": 43.94369309944592, "mean": 33.52960572068386 }, { "channel": "engine_tempest_rx_mbps", "observations": 112, "minimum": 0, "maximum": 0.005477625521410488, "mean": 0.0016123393612162786 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 112, "minimum": 12256.910446109516, "maximum": 39408.02278794344, "mean": 18238.827454153125 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 112, "minimum": 11191.66950513357, "maximum": 41235.42237602768, "mean": 18262.004553275438 } ], "graceful_shutdown": true, "shutdown_seconds": 1.2801727000000247 }, { "id": "verified-capacity-stable-release-25-web-2", "healthy": true, "failure_reasons": [], "seconds": 120.7519443, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.01095529999997, "wait_seconds": 44.026695999999994, "completed_at": "2026-10-29T23:53:31.465Z", "accepted_cpu": { "count": 20, "mean": 26.53403784567255, "p95": 35.62833395360377, "max": 40.28016711722782 } }, "requested_client_count": 2, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 2, "transport": "web", "actual_connections": { "loadClients": 2, "actualClients": 3, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 500, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 24.085271967376727, "p50": 23.841641942865987, "p95": 38.4168018901649, "p99": 42.120929129894634, "min": 9.306511697509857, "max": 47.234176957450636 }, "engine_cpu_percent": { "count": 109, "mean": 7.541214122842524, "p50": 5.277876986226965, "p95": 17.32907315210346, "p99": 18.89081330085402, "min": 1.770035854299394, "max": 20.094848814536476 }, "client_generator_cpu_percent": { "count": 109, "mean": 0.8804266797816781, "p50": 0.7091402252292894, "p95": 1.7531334918981334, "p99": 1.9422122391226169, "min": 0.17667591208416653, "max": 2.4835936358966126 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1005338849.4678899, "p50": 1005133824, "p95": 1011761152, "p99": 1011867648, "min": 993918976, "max": 1011867648 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1109027880733948, "p95": 1.1257764, "max": 1.1370648, "p50": 1.1090238, "p99": 1.1322101, "min": 1.0999029 }, "achieved_read_hz": { "minimum": 99.92416367723703, "maximum": 99.94899902410242 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48320, "hz": 99.94483434002643 }, { "group": "stress_group_1", "controls": 3, "completed": 36243, "hz": 99.95310791902808 }, { "group": "stress_group_2", "controls": 3, "completed": 36246, "hz": 99.96138149802975 }, { "group": "stress_group_3", "controls": 3, "completed": 36234, "hz": 99.92828718202311 }, { "group": "stress_group_4", "controls": 3, "completed": 36239, "hz": 99.94207648035922 }, { "group": "stress_group_5", "controls": 3, "completed": 36240, "hz": 99.94483434002643 }, { "group": "stress_group_6", "controls": 3, "completed": 36242, "hz": 99.95035005936087 }, { "group": "stress_group_7", "controls": 3, "completed": 36235, "hz": 99.93104504169033 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 564, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 532999456, "member_service_ns_total": 545823400, "members_executed_total": 48320 }, "example_device_plugin.stress_read": { "callback_ns_total": 3638013987, "member_service_ns_total": 3660331000, "members_executed_total": 48320 }, "example_device_plugin.stress_write": { "callback_ns_total": 163147697, "member_service_ns_total": 173497300, "members_executed_total": 48320 } }, "callback_ns_total": 4334161140, "cycles_total": 12080, "member_service_ns_total": 4379651700, "members_executed_total": 144960, "thread_cpu_ns_total": 1406250000, "wall_ns_total": 4449104800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 420751690, "member_service_ns_total": 430725100, "members_executed_total": 36243 }, "example_device_plugin.stress_read": { "callback_ns_total": 2998256351, "member_service_ns_total": 3018463300, "members_executed_total": 36243 }, "example_device_plugin.stress_write": { "callback_ns_total": 128807597, "member_service_ns_total": 139435200, "members_executed_total": 36243 } }, "callback_ns_total": 3547815638, "cycles_total": 12081, "member_service_ns_total": 3588623600, "members_executed_total": 108729, "thread_cpu_ns_total": 1234375000, "wall_ns_total": 3651213600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 418934282, "member_service_ns_total": 429285500, "members_executed_total": 36246 }, "example_device_plugin.stress_read": { "callback_ns_total": 2813682592, "member_service_ns_total": 2831525300, "members_executed_total": 36246 }, "example_device_plugin.stress_write": { "callback_ns_total": 123560495, "member_service_ns_total": 132515900, "members_executed_total": 36246 } }, "callback_ns_total": 3356177369, "cycles_total": 12082, "member_service_ns_total": 3393326700, "members_executed_total": 108738, "thread_cpu_ns_total": 1015625000, "wall_ns_total": 3455491500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 400350702, "member_service_ns_total": 410411300, "members_executed_total": 36234 }, "example_device_plugin.stress_read": { "callback_ns_total": 2874567064, "member_service_ns_total": 2894094400, "members_executed_total": 36234 }, "example_device_plugin.stress_write": { "callback_ns_total": 125429897, "member_service_ns_total": 132964400, "members_executed_total": 36234 } }, "callback_ns_total": 3400347663, "cycles_total": 12078, "member_service_ns_total": 3437470100, "members_executed_total": 108702, "thread_cpu_ns_total": 1125000000, "wall_ns_total": 3497393700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 383625814, "member_service_ns_total": 393062600, "members_executed_total": 36239 }, "example_device_plugin.stress_read": { "callback_ns_total": 2741746409, "member_service_ns_total": 2757809800, "members_executed_total": 36240 }, "example_device_plugin.stress_write": { "callback_ns_total": 126465298, "member_service_ns_total": 134810300, "members_executed_total": 36239 } }, "callback_ns_total": 3251277021, "cycles_total": 12079, "member_service_ns_total": 3285120000, "members_executed_total": 108711, "thread_cpu_ns_total": 531250000, "wall_ns_total": 3340784100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 383375808, "member_service_ns_total": 393313000, "members_executed_total": 36240 }, "example_device_plugin.stress_read": { "callback_ns_total": 2750519628, "member_service_ns_total": 2767395700, "members_executed_total": 36240 }, "example_device_plugin.stress_write": { "callback_ns_total": 123767997, "member_service_ns_total": 132164900, "members_executed_total": 36240 } }, "callback_ns_total": 3257663433, "cycles_total": 12080, "member_service_ns_total": 3292873600, "members_executed_total": 108720, "thread_cpu_ns_total": 609375000, "wall_ns_total": 3348019200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 397251003, "member_service_ns_total": 407065800, "members_executed_total": 36242 }, "example_device_plugin.stress_read": { "callback_ns_total": 2853416490, "member_service_ns_total": 2871146400, "members_executed_total": 36242 }, "example_device_plugin.stress_write": { "callback_ns_total": 128412994, "member_service_ns_total": 138166600, "members_executed_total": 36242 } }, "callback_ns_total": 3378733787, "cycles_total": 12080, "member_service_ns_total": 3416030400, "members_executed_total": 108720, "thread_cpu_ns_total": 875000000, "wall_ns_total": 3474071700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 422440913, "member_service_ns_total": 432668900, "members_executed_total": 36235 }, "example_device_plugin.stress_read": { "callback_ns_total": 2955149971, "member_service_ns_total": 2974045300, "members_executed_total": 36235 }, "example_device_plugin.stress_write": { "callback_ns_total": 131958794, "member_service_ns_total": 140236600, "members_executed_total": 36235 } }, "callback_ns_total": 3509500478, "cycles_total": 12078, "member_service_ns_total": 3546902200, "members_executed_total": 108702, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 3606175500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.8666769, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0005552097396948, "failed": 0, "missed": 0, "callback_last_ms": 0.0127, "callback_ewma_ms": 0.005461, "release_lateness_last_ms": 0.501 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0005552097396948, "failed": 0, "missed": 0, "callback_last_ms": 9.2069, "callback_ewma_ms": 7.228328, "release_lateness_last_ms": 0.4948 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 604, "hz": 4.994506997378312, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.00076, "release_lateness_last_ms": 0.3611 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 604, "hz": 4.994506997378312, "failed": 0, "missed": 0, "callback_last_ms": 0.1141, "callback_ewma_ms": 0.119083, "release_lateness_last_ms": 0.2712 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0005552097396948, "failed": 0, "missed": 0, "callback_last_ms": 0.1079, "callback_ewma_ms": 0.123053, "release_lateness_last_ms": 0.0452 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0005552097396948, "failed": 0, "missed": 0, "callback_last_ms": 0.1733, "callback_ewma_ms": 0.160107, "release_lateness_last_ms": 0.3499 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.0011104194793896, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001231, "release_lateness_last_ms": 0.491 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0005552097396948, "failed": 0, "missed": 0, "callback_last_ms": 2.3426, "callback_ewma_ms": 2.041064, "release_lateness_last_ms": 0.3793 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.0011104194793896, "failed": 0, "missed": 0, "callback_last_ms": 6.1579, "callback_ewma_ms": 4.586676, "release_lateness_last_ms": 2.4374 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 604, "hz": 4.994506997378312, "failed": 0, "missed": 0, "callback_last_ms": 3.3153, "callback_ewma_ms": 4.644772, "release_lateness_last_ms": 0.6284 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1209, "hz": 9.997283046076786, "failed": 0, "missed": 0, "callback_last_ms": 0.02, "callback_ewma_ms": 0.537628, "release_lateness_last_ms": 0.6022 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.994506997378312, "failed": 0, "missed": 0, "callback_last_ms": 2.4465, "callback_ewma_ms": 2.929853, "release_lateness_last_ms": 0.0815 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.0011104194793896, "failed": 0, "missed": 0, "callback_last_ms": 8.5519, "callback_ewma_ms": 7.045562, "release_lateness_last_ms": 2.3637 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.994506997378312, "failed": 0, "missed": 0, "callback_last_ms": 0.0196, "callback_ewma_ms": 0.00476, "release_lateness_last_ms": 0.2849 }, { "name": "general_program_checks_thread", "running": true, "completed": 1210, "hz": 10.005552097396949, "failed": 0, "missed": 0, "callback_last_ms": 0.224, "callback_ewma_ms": 0.163024, "release_lateness_last_ms": 0.3836 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0005552097396948, "failed": 0, "missed": 0, "callback_last_ms": 0.1272, "callback_ewma_ms": 0.126794, "release_lateness_last_ms": 0.2888 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1209, "hz": 9.997283046076786, "failed": 0, "missed": 0, "callback_last_ms": 0.0048, "callback_ewma_ms": 0.004575, "release_lateness_last_ms": 0.4244 }, { "name": "engine_diagnostics", "running": true, "completed": 242, "hz": 2.0011104194793896, "failed": 0, "missed": 0, "callback_last_ms": 39.5394, "callback_ewma_ms": 38.963019, "release_lateness_last_ms": 0.2661 } ], "peer_counter_seconds": 120.93626509999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.0019868115214114, "complete_hz": 5.0019868115214114, "catalog_hz": 2.0041072986559296, "bytes_per_second": 1551392.875814447, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.01350000000093, "gap_p95_ms": 208.32480000000214, "gap_p99_ms": 214.20200000000477, "gap_max_ms": 224.90660000001662, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 6, "p95": 17, "p99": 25, "max": 39, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.0019868115214114, "complete_hz": 5.0019868115214114, "catalog_hz": 2.0041072986559296, "bytes_per_second": 1551392.875814447, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.07850000000326, "gap_p95_ms": 208.734300000011, "gap_p99_ms": 217.6252999999997, "gap_max_ms": 225.5020000000004, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 6, "p95": 18, "p99": 25, "max": 42, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 112, "mean": 63.76647767857191, "p95": 75.9198000000033, "max": 86.94770000000426, "p50": 63.642399999997, "p99": 84.13430000000517, "min": 50.061799999995856 }, { "operation": "rapid/get-persistence-diagnostics", "count": 112, "mean": 11.923246428570565, "p95": 16.85099999999511, "max": 18.862200000003213, "p50": 15.180099999997765, "p99": 17.933300000004238, "min": 4.302299999995739 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2238433, "accepted_rows_total": 14050917, "committed_rows_total": 2238833, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 112, "minimum": 7.421875, "maximum": 50.390625, "mean": 24.082728794642858 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 112, "minimum": 0.7811147109320665, "maximum": 21.885237914296308, "mean": 6.982900768185297 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 112, "minimum": 994033664, "maximum": 1011867648, "mean": 1005221083.4285715 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0.0244140625, "mean": 0.00021798270089285715 }, { "channel": "engine_tempest_tx_mbps", "observations": 112, "minimum": 49.605290970196336, "maximum": 81.15845206353524, "mean": 62.77738767880912 }, { "channel": "engine_tempest_rx_mbps", "observations": 112, "minimum": 0.002732888879299805, "maximum": 0.0058579272580679045, "mean": 0.003032516144802815 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 112, "minimum": 12486.619338716631, "maximum": 39568.232979293345, "mean": 18384.73591233158 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 112, "minimum": 11162.54157681458, "maximum": 41648.34804028737, "mean": 18343.995131707936 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7643066999999863 }, { "id": "verified-capacity-stable-release-25-web-4", "healthy": true, "failure_reasons": [], "seconds": 120.20924440000002, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.013064699999987, "wait_seconds": 20.013064699999987, "completed_at": "2026-10-29T23:56:49.707Z", "accepted_cpu": { "count": 20, "mean": 20.77750610252738, "p95": 33.333333333333336, "max": 34.733480717268485 } }, "requested_client_count": 4, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 4, "transport": "web", "actual_connections": { "loadClients": 4, "actualClients": 5, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 1000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 35.092644703104014, "p50": 32.580988485433224, "p95": 55.9199650959482, "p99": 62.289773519446044, "min": 15.496790800692251, "max": 62.97001215284895 }, "engine_cpu_percent": { "count": 108, "mean": 7.990207573704199, "p50": 5.773609603954063, "p95": 17.963480343950796, "p99": 19.348072998346023, "min": 2.1138614383781995, "max": 21.26036770746893 }, "client_generator_cpu_percent": { "count": 108, "mean": 1.4499623941024604, "p50": 1.3860583728836773, "p95": 2.4550839170690497, "p99": 2.801750049288387, "min": 0.5323758087342205, "max": 2.9924468557210377 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1006720606.8148148, "p50": 1006317568, "p95": 1015078912, "p99": 1015283712, "min": 995069952, "max": 1015291904 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1201096703703706, "p95": 1.1391501, "max": 1.4592445, "p50": 1.1116202, "p99": 1.2700608, "min": 1.0991099 }, "achieved_read_hz": { "minimum": 99.24663711355993, "maximum": 99.34643136908034 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47764, "hz": 99.23808384174932 }, { "group": "stress_group_1", "controls": 3, "completed": 35841, "hz": 99.28794804935761 }, { "group": "stress_group_2", "controls": 3, "completed": 35859, "hz": 99.33781225696589 }, { "group": "stress_group_3", "controls": 3, "completed": 35850, "hz": 99.31288015316174 }, { "group": "stress_group_4", "controls": 3, "completed": 35839, "hz": 99.28240758184558 }, { "group": "stress_group_5", "controls": 3, "completed": 35857, "hz": 99.33227178945386 }, { "group": "stress_group_6", "controls": 3, "completed": 35841, "hz": 99.28794804935761 }, { "group": "stress_group_7", "controls": 3, "completed": 35839, "hz": 99.28240758184558 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 6348, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 576784045, "member_service_ns_total": 590212400, "members_executed_total": 47764 }, "example_device_plugin.stress_read": { "callback_ns_total": 3526808690, "member_service_ns_total": 3547359100, "members_executed_total": 47764 }, "example_device_plugin.stress_write": { "callback_ns_total": 194575196, "member_service_ns_total": 206673700, "members_executed_total": 47764 } }, "callback_ns_total": 4298167931, "cycles_total": 11941, "member_service_ns_total": 4344245200, "members_executed_total": 143292, "thread_cpu_ns_total": 1390625000, "wall_ns_total": 4416552100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 549457594, "member_service_ns_total": 559344700, "members_executed_total": 35841 }, "example_device_plugin.stress_read": { "callback_ns_total": 2858234469, "member_service_ns_total": 2879542100, "members_executed_total": 35841 }, "example_device_plugin.stress_write": { "callback_ns_total": 134911097, "member_service_ns_total": 144523200, "members_executed_total": 35841 } }, "callback_ns_total": 3542603160, "cycles_total": 11947, "member_service_ns_total": 3583410000, "members_executed_total": 107523, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3648842900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 475641795, "member_service_ns_total": 484814900, "members_executed_total": 35859 }, "example_device_plugin.stress_read": { "callback_ns_total": 2740853838, "member_service_ns_total": 2758583500, "members_executed_total": 35859 }, "example_device_plugin.stress_write": { "callback_ns_total": 128521500, "member_service_ns_total": 137762300, "members_executed_total": 35859 } }, "callback_ns_total": 3345017133, "cycles_total": 11953, "member_service_ns_total": 3381160700, "members_executed_total": 107577, "thread_cpu_ns_total": 953125000, "wall_ns_total": 3439677300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 430198417, "member_service_ns_total": 440074600, "members_executed_total": 35850 }, "example_device_plugin.stress_read": { "callback_ns_total": 2799631860, "member_service_ns_total": 2816867300, "members_executed_total": 35850 }, "example_device_plugin.stress_write": { "callback_ns_total": 128220295, "member_service_ns_total": 137843900, "members_executed_total": 35850 } }, "callback_ns_total": 3358050572, "cycles_total": 11950, "member_service_ns_total": 3394785800, "members_executed_total": 107550, "thread_cpu_ns_total": 1187500000, "wall_ns_total": 3456330600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 407754235, "member_service_ns_total": 417921000, "members_executed_total": 35839 }, "example_device_plugin.stress_read": { "callback_ns_total": 2816806956, "member_service_ns_total": 2833437200, "members_executed_total": 35839 }, "example_device_plugin.stress_write": { "callback_ns_total": 127962998, "member_service_ns_total": 137150700, "members_executed_total": 35839 } }, "callback_ns_total": 3352877689, "cycles_total": 11947, "member_service_ns_total": 3388863700, "members_executed_total": 107523, "thread_cpu_ns_total": 453125000, "wall_ns_total": 3446505900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 417744675, "member_service_ns_total": 427240300, "members_executed_total": 35857 }, "example_device_plugin.stress_read": { "callback_ns_total": 2646903730, "member_service_ns_total": 2663751100, "members_executed_total": 35857 }, "example_device_plugin.stress_write": { "callback_ns_total": 128397399, "member_service_ns_total": 137454600, "members_executed_total": 35857 } }, "callback_ns_total": 3193403704, "cycles_total": 11953, "member_service_ns_total": 3228805200, "members_executed_total": 107577, "thread_cpu_ns_total": 515625000, "wall_ns_total": 3284646500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 410157505, "member_service_ns_total": 421540500, "members_executed_total": 35841 }, "example_device_plugin.stress_read": { "callback_ns_total": 2791443633, "member_service_ns_total": 2814773600, "members_executed_total": 35841 }, "example_device_plugin.stress_write": { "callback_ns_total": 131961697, "member_service_ns_total": 143047200, "members_executed_total": 35841 } }, "callback_ns_total": 3333562835, "cycles_total": 11947, "member_service_ns_total": 3379361300, "members_executed_total": 107523, "thread_cpu_ns_total": 921875000, "wall_ns_total": 3438354600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 439853996, "member_service_ns_total": 450284700, "members_executed_total": 35839 }, "example_device_plugin.stress_read": { "callback_ns_total": 2921315607, "member_service_ns_total": 2941064200, "members_executed_total": 35839 }, "example_device_plugin.stress_write": { "callback_ns_total": 136171896, "member_service_ns_total": 145770100, "members_executed_total": 35839 } }, "callback_ns_total": 3497692199, "cycles_total": 11947, "member_service_ns_total": 3537471400, "members_executed_total": 107523, "thread_cpu_ns_total": 781250000, "wall_ns_total": 3598637500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.32678924999999, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 120, "hz": 0.9969435977340159, "failed": 0, "missed": 0, "callback_last_ms": 0.0037, "callback_ewma_ms": 0.020669, "release_lateness_last_ms": 0.4441 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 120, "hz": 0.9969435977340159, "failed": 0, "missed": 0, "callback_last_ms": 12.7389, "callback_ewma_ms": 8.118821, "release_lateness_last_ms": 0.2623 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 602, "hz": 5.00133371529898, "failed": 0, "missed": 0, "callback_last_ms": 0.0014, "callback_ewma_ms": 0.001155, "release_lateness_last_ms": 0.2884 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 602, "hz": 5.00133371529898, "failed": 0, "missed": 0, "callback_last_ms": 0.221, "callback_ewma_ms": 0.276702, "release_lateness_last_ms": 0.443 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 120, "hz": 0.9969435977340159, "failed": 0, "missed": 0, "callback_last_ms": 0.26, "callback_ewma_ms": 0.343673, "release_lateness_last_ms": 0.3536 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 120, "hz": 0.9969435977340159, "failed": 0, "missed": 0, "callback_last_ms": 0.8256, "callback_ewma_ms": 0.47424, "release_lateness_last_ms": 0.5576 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 2.002195058782482, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.001118, "release_lateness_last_ms": 0.102 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.9969435977340159, "failed": 0, "missed": 0, "callback_last_ms": 2.3727, "callback_ewma_ms": 2.103635, "release_lateness_last_ms": 0.2201 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.002195058782482, "failed": 0, "missed": 0, "callback_last_ms": 3.4989, "callback_ewma_ms": 4.126465, "release_lateness_last_ms": 0.4954 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 5.00133371529898, "failed": 0, "missed": 0, "callback_last_ms": 3.6902, "callback_ewma_ms": 5.574205, "release_lateness_last_ms": 0.3574 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1204, "hz": 10.00266743059796, "failed": 0, "missed": 0, "callback_last_ms": 0.0248, "callback_ewma_ms": 0.87674, "release_lateness_last_ms": 0.5195 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 602, "hz": 5.00133371529898, "failed": 0, "missed": 0, "callback_last_ms": 3.9226, "callback_ewma_ms": 5.11824, "release_lateness_last_ms": 0.4679 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.002195058782482, "failed": 0, "missed": 0, "callback_last_ms": 5.0875, "callback_ewma_ms": 8.30167, "release_lateness_last_ms": 0.2421 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 602, "hz": 5.00133371529898, "failed": 0, "missed": 0, "callback_last_ms": 0.0022, "callback_ewma_ms": 0.001814, "release_lateness_last_ms": 0.4181 }, { "name": "general_program_checks_thread", "running": true, "completed": 1204, "hz": 10.00266743059796, "failed": 0, "missed": 0, "callback_last_ms": 0.2812, "callback_ewma_ms": 0.190202, "release_lateness_last_ms": 0.4183 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.9969435977340159, "failed": 0, "missed": 0, "callback_last_ms": 0.1216, "callback_ewma_ms": 0.115714, "release_lateness_last_ms": 0.1634 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1204, "hz": 10.00266743059796, "failed": 0, "missed": 0, "callback_last_ms": 0.0044, "callback_ewma_ms": 0.016203, "release_lateness_last_ms": 0.4509 }, { "name": "engine_diagnostics", "running": true, "completed": 240, "hz": 1.9938871954680317, "failed": 0, "missed": 0, "callback_last_ms": 42.951, "callback_ewma_ms": 39.754135, "release_lateness_last_ms": 0.2246 } ], "peer_counter_seconds": 120.3650999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.999613465824144, "complete_hz": 4.999613465824144, "catalog_hz": 1.9965178565687096, "bytes_per_second": 1550434.3911248192, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.8263999999981, "gap_p95_ms": 207.5502999999735, "gap_p99_ms": 229.82689999998547, "gap_max_ms": 337.03839999999036, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 13, "p95": 22, "p99": 46, "max": 145, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.999613465824144, "complete_hz": 4.999613465824144, "catalog_hz": 1.9965178565687096, "bytes_per_second": 1550434.3911248192, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7346000000034, "gap_p95_ms": 209.93950000000768, "gap_p99_ms": 231.45760000000155, "gap_max_ms": 352.60719999999856, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 17, "p95": 26, "p99": 60, "max": 163, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.999613465824144, "complete_hz": 4.999613465824144, "catalog_hz": 1.9965178565687096, "bytes_per_second": 1550434.3911248192, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9585999999981, "gap_p95_ms": 210.5271999999968, "gap_p99_ms": 228.12780000001658, "gap_max_ms": 369.09189999999944, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 19, "p95": 30, "p99": 45, "max": 184, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.999613465824144, "complete_hz": 4.999613465824144, "catalog_hz": 1.9965178565687096, "bytes_per_second": 1550434.3911248192, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0699999999997, "gap_p95_ms": 208.77619999999297, "gap_p99_ms": 225.38610000000335, "gap_max_ms": 309.18090000000666, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 15, "p95": 24, "p99": 43, "max": 122, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 66.91587297297231, "p95": 79.64050000000861, "max": 186.13299999998708, "p50": 64.49009999999544, "p99": 183.22209999999905, "min": 52.05419999999867 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 11.843606306306233, "p95": 17.008700000005774, "max": 19.510600000001432, "p50": 15.16349999999511, "p99": 17.314900000004855, "min": 4.555899999999383 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2188906, "accepted_rows_total": 13835300, "committed_rows_total": 2189624, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 10.15625, "maximum": 85.9375, "mean": 35.15547157180222 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 1.1715418135082383, "maximum": 22.66662678174068, "mean": 7.409829156880425 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 1000677376, "maximum": 1015341056, "mean": 1007377177.3693694 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 95.86571891493861, "maximum": 166.40670262596166, "mean": 122.18783218719216 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0, "maximum": 0.011095932160703651, "mean": 0.0056090694088443815 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 12448.865041953672, "maximum": 39385.70781423328, "mean": 18193.38853779039 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 8313.0239374304, "maximum": 40814.68496277596, "mean": 18108.214144808662 } ], "graceful_shutdown": true, "shutdown_seconds": 1.912895199999999 }, { "id": "verified-capacity-stable-release-25-web-8", "healthy": true, "failure_reasons": [], "seconds": 120.6821885, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.01864009999996, "wait_seconds": 20.01864009999996, "completed_at": "2026-10-30T00:00:08.002Z", "accepted_cpu": { "count": 20, "mean": 12.969542556625285, "p95": 21.475430127491023, "max": 28.628329599203383 } }, "requested_client_count": 8, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 8, "transport": "web", "actual_connections": { "loadClients": 8, "actualClients": 9, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 2000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 35.750512355556914, "p50": 35.39615029392991, "p95": 47.81729050487128, "p99": 59.823498296087884, "min": 22.793060393292485, "max": 65.48266982395647 }, "engine_cpu_percent": { "count": 109, "mean": 9.379919190122195, "p50": 7.365508342974433, "p95": 19.141561180391307, "p99": 20.47478536983498, "min": 3.8759660204009654, "max": 22.011189037529096 }, "client_generator_cpu_percent": { "count": 109, "mean": 2.556579633859341, "p50": 2.4660202142636187, "p95": 3.8377129816271176, "p99": 5.094592973137113, "min": 1.2302627056020656, "max": 5.17561242152281 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1009575691.7431192, "p50": 1010147328, "p95": 1015267328, "p99": 1016807424, "min": 992014336, "max": 1016811520 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1148785412844033, "p95": 1.1267385, "max": 1.4631074, "p50": 1.110439, "p99": 1.1480515, "min": 1.1005249 }, "achieved_read_hz": { "minimum": 99.59807608454649, "maximum": 99.68918641080742 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48152, "hz": 99.65087281940016 }, { "group": "stress_group_1", "controls": 3, "completed": 36126, "hz": 99.683984922015 }, { "group": "stress_group_2", "controls": 3, "completed": 36108, "hz": 99.63431676809273 }, { "group": "stress_group_3", "controls": 3, "completed": 36102, "hz": 99.61776071678531 }, { "group": "stress_group_4", "controls": 3, "completed": 36093, "hz": 99.59292663982417 }, { "group": "stress_group_5", "controls": 3, "completed": 36105, "hz": 99.62603874243901 }, { "group": "stress_group_6", "controls": 3, "completed": 36117, "hz": 99.65915084505387 }, { "group": "stress_group_7", "controls": 3, "completed": 36126, "hz": 99.683984922015 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 3213, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 549655937, "member_service_ns_total": 562314200, "members_executed_total": 48152 }, "example_device_plugin.stress_read": { "callback_ns_total": 3301837080, "member_service_ns_total": 3325548400, "members_executed_total": 48152 }, "example_device_plugin.stress_write": { "callback_ns_total": 162961899, "member_service_ns_total": 174363300, "members_executed_total": 48152 } }, "callback_ns_total": 4014454916, "cycles_total": 12038, "member_service_ns_total": 4062225900, "members_executed_total": 144456, "thread_cpu_ns_total": 1375000000, "wall_ns_total": 4126391100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 499505100, "member_service_ns_total": 511119800, "members_executed_total": 36126 }, "example_device_plugin.stress_read": { "callback_ns_total": 2617315543, "member_service_ns_total": 2636262300, "members_executed_total": 36126 }, "example_device_plugin.stress_write": { "callback_ns_total": 134110097, "member_service_ns_total": 144659000, "members_executed_total": 36126 } }, "callback_ns_total": 3250930740, "cycles_total": 12042, "member_service_ns_total": 3292041100, "members_executed_total": 108378, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3354583100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 442902410, "member_service_ns_total": 452569000, "members_executed_total": 36108 }, "example_device_plugin.stress_read": { "callback_ns_total": 2680374612, "member_service_ns_total": 2700277900, "members_executed_total": 36108 }, "example_device_plugin.stress_write": { "callback_ns_total": 136412597, "member_service_ns_total": 144884500, "members_executed_total": 36108 } }, "callback_ns_total": 3259689619, "cycles_total": 12036, "member_service_ns_total": 3297731400, "members_executed_total": 108324, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 3357997300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 443650690, "member_service_ns_total": 457525200, "members_executed_total": 36102 }, "example_device_plugin.stress_read": { "callback_ns_total": 2781726109, "member_service_ns_total": 2802563400, "members_executed_total": 36102 }, "example_device_plugin.stress_write": { "callback_ns_total": 140458491, "member_service_ns_total": 150319100, "members_executed_total": 36102 } }, "callback_ns_total": 3365835290, "cycles_total": 12034, "member_service_ns_total": 3410407700, "members_executed_total": 108306, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3471835000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 396438015, "member_service_ns_total": 406870000, "members_executed_total": 36093 }, "example_device_plugin.stress_read": { "callback_ns_total": 2592466384, "member_service_ns_total": 2610409500, "members_executed_total": 36093 }, "example_device_plugin.stress_write": { "callback_ns_total": 130189498, "member_service_ns_total": 140228700, "members_executed_total": 36093 } }, "callback_ns_total": 3119093897, "cycles_total": 12031, "member_service_ns_total": 3157508200, "members_executed_total": 108279, "thread_cpu_ns_total": 640625000, "wall_ns_total": 3226302100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 400565893, "member_service_ns_total": 410626600, "members_executed_total": 36105 }, "example_device_plugin.stress_read": { "callback_ns_total": 2545953001, "member_service_ns_total": 2563983200, "members_executed_total": 36105 }, "example_device_plugin.stress_write": { "callback_ns_total": 144954599, "member_service_ns_total": 153638400, "members_executed_total": 36105 } }, "callback_ns_total": 3091473493, "cycles_total": 12035, "member_service_ns_total": 3128248200, "members_executed_total": 108315, "thread_cpu_ns_total": 609375000, "wall_ns_total": 3183070300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 472777304, "member_service_ns_total": 484526600, "members_executed_total": 36117 }, "example_device_plugin.stress_read": { "callback_ns_total": 2670007306, "member_service_ns_total": 2690063600, "members_executed_total": 36117 }, "example_device_plugin.stress_write": { "callback_ns_total": 142657298, "member_service_ns_total": 152823700, "members_executed_total": 36117 } }, "callback_ns_total": 3285441908, "cycles_total": 12039, "member_service_ns_total": 3327413900, "members_executed_total": 108351, "thread_cpu_ns_total": 1062500000, "wall_ns_total": 3393680600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 458870694, "member_service_ns_total": 471288600, "members_executed_total": 36126 }, "example_device_plugin.stress_read": { "callback_ns_total": 2668886684, "member_service_ns_total": 2688117300, "members_executed_total": 36126 }, "example_device_plugin.stress_write": { "callback_ns_total": 135170695, "member_service_ns_total": 146301000, "members_executed_total": 36126 } }, "callback_ns_total": 3262928073, "cycles_total": 12042, "member_service_ns_total": 3305706900, "members_executed_total": 108378, "thread_cpu_ns_total": 765625000, "wall_ns_total": 3369588800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.80175175, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0011076337596085, "failed": 0, "missed": 0, "callback_last_ms": 0.0048, "callback_ewma_ms": 0.004602, "release_lateness_last_ms": 0.2803 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0011076337596085, "failed": 0, "missed": 0, "callback_last_ms": 4.3292, "callback_ewma_ms": 5.468281, "release_lateness_last_ms": 0.141 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 604, "hz": 4.997264551990112, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001176, "release_lateness_last_ms": 0.436 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 605, "hz": 5.005538168798043, "failed": 0, "missed": 0, "callback_last_ms": 0.4037, "callback_ewma_ms": 0.548834, "release_lateness_last_ms": 0.5597 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0011076337596085, "failed": 0, "missed": 0, "callback_last_ms": 1.2783, "callback_ewma_ms": 0.850556, "release_lateness_last_ms": 0.3126 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0011076337596085, "failed": 0, "missed": 0, "callback_last_ms": 0.7071, "callback_ewma_ms": 0.709121, "release_lateness_last_ms": 0.4531 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.002215267519217, "failed": 0, "missed": 0, "callback_last_ms": 0.0014, "callback_ewma_ms": 0.001094, "release_lateness_last_ms": 0.4014 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0011076337596085, "failed": 0, "missed": 0, "callback_last_ms": 2.9445, "callback_ewma_ms": 2.196299, "release_lateness_last_ms": 0.4081 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.002215267519217, "failed": 0, "missed": 0, "callback_last_ms": 3.7144, "callback_ewma_ms": 4.431795, "release_lateness_last_ms": 0.2517 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 605, "hz": 5.005538168798043, "failed": 0, "missed": 0, "callback_last_ms": 5.1354, "callback_ewma_ms": 5.120919, "release_lateness_last_ms": 0.3385 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1208, "hz": 9.994529103980224, "failed": 0, "missed": 0, "callback_last_ms": 7.4359, "callback_ewma_ms": 1.085969, "release_lateness_last_ms": 0.0753 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 605, "hz": 5.005538168798043, "failed": 0, "missed": 0, "callback_last_ms": 5.7025, "callback_ewma_ms": 5.741104, "release_lateness_last_ms": 0.4375 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.002215267519217, "failed": 0, "missed": 0, "callback_last_ms": 4.9544, "callback_ewma_ms": 7.327028, "release_lateness_last_ms": 0.3626 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 605, "hz": 5.005538168798043, "failed": 0, "missed": 0, "callback_last_ms": 0.0016, "callback_ewma_ms": 0.00179, "release_lateness_last_ms": 0.3915 }, { "name": "general_program_checks_thread", "running": true, "completed": 1208, "hz": 9.994529103980224, "failed": 0, "missed": 0, "callback_last_ms": 0.1361, "callback_ewma_ms": 0.155913, "release_lateness_last_ms": 0.163 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0011076337596085, "failed": 0, "missed": 0, "callback_last_ms": 0.1146, "callback_ewma_ms": 0.117201, "release_lateness_last_ms": 0.323 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1208, "hz": 9.994529103980224, "failed": 0, "missed": 0, "callback_last_ms": 0.0061, "callback_ewma_ms": 0.00593, "release_lateness_last_ms": 0.3587 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 1.9939416507112864, "failed": 0, "missed": 0, "callback_last_ms": 43.0266, "callback_ewma_ms": 38.984673, "release_lateness_last_ms": 0.4564 } ], "peer_counter_seconds": 120.8547324, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.25320000000647, "gap_p95_ms": 211.31939999999304, "gap_p99_ms": 219.00340000000142, "gap_max_ms": 662.7664999999979, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 14, "p95": 25, "p99": 37, "max": 477, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0341999999946, "gap_p95_ms": 211.52019999999902, "gap_p99_ms": 220.69500000000698, "gap_max_ms": 639.7180999999982, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 15, "p95": 27, "p99": 36, "max": 456, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9899999999907, "gap_p95_ms": 213.26339999999618, "gap_p99_ms": 223.7844000000041, "gap_max_ms": 642.3258999999962, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 21, "p95": 33, "p99": 48, "max": 467, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.06590000000142, "gap_p95_ms": 215.5616999999911, "gap_p99_ms": 223.78940000000875, "gap_max_ms": 637.0046000000002, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 11, "p95": 29, "p99": 44, "max": 447, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.87780000000203, "gap_p95_ms": 213.9496999999974, "gap_p99_ms": 221.36670000001322, "gap_max_ms": 687.6140999999989, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 18, "p95": 31, "p99": 49, "max": 510, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7627999999968, "gap_p95_ms": 210.27159999999276, "gap_p99_ms": 219.74950000000536, "gap_max_ms": 704.940800000004, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 23, "p95": 32, "p99": 50, "max": 524, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.63060000000405, "gap_p95_ms": 210.49169999999867, "gap_p99_ms": 224.6533999999956, "gap_max_ms": 665.722299999994, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 22, "p95": 31, "p99": 51, "max": 487, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.996594696420989, "complete_hz": 4.996594696420989, "catalog_hz": 2.005266859923515, "bytes_per_second": 1549615.9628502738, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.06089999999676, "gap_p95_ms": 213.3364999999976, "gap_p99_ms": 219.02619999999297, "gap_max_ms": 687.5813999999955, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 15, "p95": 28, "p99": 36, "max": 499, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 69.77646846846712, "p95": 83.17479999999341, "max": 607.034799999994, "p50": 63.9375999999902, "p99": 132.10429999999178, "min": 50.34909999999218 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 12.47675045045079, "p95": 21.64309999998659, "max": 26.046199999997043, "p50": 15.208299999998417, "p99": 25.775900000007823, "min": 4.652900000000955 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2222227, "accepted_rows_total": 13979254, "committed_rows_total": 2222115, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 21.875, "maximum": 63.671875, "mean": 36.27491812339863 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 1.5619673691271279, "maximum": 23.444430173559304, "mean": 9.099597063887291 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 994451456, "maximum": 1016844288, "mean": 1009464366.1261262 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.048828125, "mean": 0.000879786036036036 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 188.70358060786128, "maximum": 302.2480954935139, "mean": 238.05072277312544 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0, "maximum": 0.021924065087068227, "mean": 0.011636239784812943 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 12489.002184676176, "maximum": 39452.9584715181, "mean": 18556.522593187994 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 6460.577770530442, "maximum": 39743.13113052552, "mean": 18478.56324323166 } ], "graceful_shutdown": true, "shutdown_seconds": 2.1809440999999934 }, { "id": "verified-capacity-stable-release-25-web-16", "healthy": true, "failure_reasons": [], "seconds": 120.204402, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.014762800000025, "wait_seconds": 20.014762800000025, "completed_at": "2026-10-30T00:03:27.098Z", "accepted_cpu": { "count": 20, "mean": 23.9937273250129, "p95": 34.48577134298553, "max": 41.41814131914367 } }, "requested_client_count": 16, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 16, "transport": "web", "actual_connections": { "loadClients": 16, "actualClients": 17, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 4000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 36.45657072079273, "p50": 34.79870410126832, "p95": 54.423407594030536, "p99": 71.09095400996146, "min": 19.729374899797435, "max": 73.35298362349678 }, "engine_cpu_percent": { "count": 108, "mean": 10.649802410801986, "p50": 8.333175227586036, "p95": 21.230926651832128, "p99": 21.947432471973865, "min": 4.861210063742631, "max": 22.808138833139687 }, "client_generator_cpu_percent": { "count": 108, "mean": 3.994213099529747, "p50": 3.7160592559949364, "p95": 5.497888327068233, "p99": 6.765744290827646, "min": 2.3036508110620058, "max": 6.90366537814107 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1021331418.074074, "p50": 1022255104, "p95": 1025298432, "p99": 1025699840, "min": 1001746432, "max": 1028177920 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.117430821296296, "p95": 1.1499292, "max": 1.2145491, "p50": 1.1128686, "p99": 1.1783474, "min": 1.1007099 }, "achieved_read_hz": { "minimum": 99.69932085228452, "maximum": 99.80743277660675 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48052, "hz": 99.8152156383723 }, { "group": "stress_group_1", "controls": 3, "completed": 36039, "hz": 99.8152156383723 }, { "group": "stress_group_2", "controls": 3, "completed": 36033, "hz": 99.79859777178802 }, { "group": "stress_group_3", "controls": 3, "completed": 36012, "hz": 99.7404352387431 }, { "group": "stress_group_4", "controls": 3, "completed": 36015, "hz": 99.74874417203525 }, { "group": "stress_group_5", "controls": 3, "completed": 36009, "hz": 99.73212630545099 }, { "group": "stress_group_6", "controls": 3, "completed": 36000, "hz": 99.70719950557458 }, { "group": "stress_group_7", "controls": 3, "completed": 36039, "hz": 99.8152156383723 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 2103, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 605071110, "member_service_ns_total": 617918500, "members_executed_total": 48052 }, "example_device_plugin.stress_read": { "callback_ns_total": 3748288044, "member_service_ns_total": 3770906600, "members_executed_total": 48052 }, "example_device_plugin.stress_write": { "callback_ns_total": 177773398, "member_service_ns_total": 189814000, "members_executed_total": 48052 } }, "callback_ns_total": 4531132552, "cycles_total": 12013, "member_service_ns_total": 4578639100, "members_executed_total": 144156, "thread_cpu_ns_total": 1250000000, "wall_ns_total": 4647872600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 481021290, "member_service_ns_total": 490868800, "members_executed_total": 36039 }, "example_device_plugin.stress_read": { "callback_ns_total": 3106952678, "member_service_ns_total": 3128613000, "members_executed_total": 36039 }, "example_device_plugin.stress_write": { "callback_ns_total": 137947594, "member_service_ns_total": 147371200, "members_executed_total": 36039 } }, "callback_ns_total": 3725921562, "cycles_total": 12013, "member_service_ns_total": 3766853000, "members_executed_total": 108117, "thread_cpu_ns_total": 1000000000, "wall_ns_total": 3831430400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 439192248, "member_service_ns_total": 450301700, "members_executed_total": 36033 }, "example_device_plugin.stress_read": { "callback_ns_total": 2994290419, "member_service_ns_total": 3013890900, "members_executed_total": 36033 }, "example_device_plugin.stress_write": { "callback_ns_total": 129451398, "member_service_ns_total": 138955100, "members_executed_total": 36033 } }, "callback_ns_total": 3562934065, "cycles_total": 12011, "member_service_ns_total": 3603147700, "members_executed_total": 108099, "thread_cpu_ns_total": 1000000000, "wall_ns_total": 3662822800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 417989480, "member_service_ns_total": 427599900, "members_executed_total": 36012 }, "example_device_plugin.stress_read": { "callback_ns_total": 2954611253, "member_service_ns_total": 2973426500, "members_executed_total": 36012 }, "example_device_plugin.stress_write": { "callback_ns_total": 141955395, "member_service_ns_total": 152654700, "members_executed_total": 36012 } }, "callback_ns_total": 3514556128, "cycles_total": 12004, "member_service_ns_total": 3553681100, "members_executed_total": 108036, "thread_cpu_ns_total": 906250000, "wall_ns_total": 3612527600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 412230710, "member_service_ns_total": 422372200, "members_executed_total": 36015 }, "example_device_plugin.stress_read": { "callback_ns_total": 2952687419, "member_service_ns_total": 2973101700, "members_executed_total": 36015 }, "example_device_plugin.stress_write": { "callback_ns_total": 136725996, "member_service_ns_total": 146189500, "members_executed_total": 36015 } }, "callback_ns_total": 3501644125, "cycles_total": 12005, "member_service_ns_total": 3541663400, "members_executed_total": 108045, "thread_cpu_ns_total": 515625000, "wall_ns_total": 3596743400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 430803478, "member_service_ns_total": 442274900, "members_executed_total": 36009 }, "example_device_plugin.stress_read": { "callback_ns_total": 2876964491, "member_service_ns_total": 2894182700, "members_executed_total": 36009 }, "example_device_plugin.stress_write": { "callback_ns_total": 134274695, "member_service_ns_total": 143642400, "members_executed_total": 36009 } }, "callback_ns_total": 3442042664, "cycles_total": 12003, "member_service_ns_total": 3480100000, "members_executed_total": 108027, "thread_cpu_ns_total": 390625000, "wall_ns_total": 3537192600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 421890109, "member_service_ns_total": 438000000, "members_executed_total": 36000 }, "example_device_plugin.stress_read": { "callback_ns_total": 2960015106, "member_service_ns_total": 2979885000, "members_executed_total": 36000 }, "example_device_plugin.stress_write": { "callback_ns_total": 139236296, "member_service_ns_total": 149059300, "members_executed_total": 36000 } }, "callback_ns_total": 3521141511, "cycles_total": 12000, "member_service_ns_total": 3566944300, "members_executed_total": 108000, "thread_cpu_ns_total": 687500000, "wall_ns_total": 3628320900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 479526787, "member_service_ns_total": 490170000, "members_executed_total": 36039 }, "example_device_plugin.stress_read": { "callback_ns_total": 3073640302, "member_service_ns_total": 3092779300, "members_executed_total": 36039 }, "example_device_plugin.stress_write": { "callback_ns_total": 137438094, "member_service_ns_total": 147097000, "members_executed_total": 36039 } }, "callback_ns_total": 3690605183, "cycles_total": 12013, "member_service_ns_total": 3730046300, "members_executed_total": 108117, "thread_cpu_ns_total": 750000000, "wall_ns_total": 3788884000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.35239240000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 120, "hz": 0.9966084931284733, "failed": 0, "missed": 0, "callback_last_ms": 0.0037, "callback_ewma_ms": 0.004537, "release_lateness_last_ms": 0.4509 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 120, "hz": 0.9966084931284733, "failed": 0, "missed": 0, "callback_last_ms": 7.0746, "callback_ewma_ms": 8.878717, "release_lateness_last_ms": 0.3996 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 602, "hz": 4.999652607194508, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.000937, "release_lateness_last_ms": 0.1868 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 602, "hz": 4.999652607194508, "failed": 0, "missed": 0, "callback_last_ms": 0.9575, "callback_ewma_ms": 1.047336, "release_lateness_last_ms": 0.2919 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 120, "hz": 0.9966084931284733, "failed": 0, "missed": 0, "callback_last_ms": 0.8522, "callback_ewma_ms": 1.188158, "release_lateness_last_ms": 0.4472 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 120, "hz": 0.9966084931284733, "failed": 0, "missed": 0, "callback_last_ms": 0.9526, "callback_ewma_ms": 1.349877, "release_lateness_last_ms": 0.2413 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 2.0015220570330174, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.002358, "release_lateness_last_ms": 0.3617 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.9966084931284733, "failed": 0, "missed": 0, "callback_last_ms": 0.8682, "callback_ewma_ms": 2.080511, "release_lateness_last_ms": 0.2324 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.0015220570330174, "failed": 0, "missed": 0, "callback_last_ms": 3.2807, "callback_ewma_ms": 5.060072, "release_lateness_last_ms": 0.2398 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 4.999652607194508, "failed": 0, "missed": 0, "callback_last_ms": 5.3697, "callback_ewma_ms": 8.701944, "release_lateness_last_ms": 0.477 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1204, "hz": 9.999305214389016, "failed": 0, "missed": 0, "callback_last_ms": 0.0194, "callback_ewma_ms": 0.642668, "release_lateness_last_ms": 0.1889 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 602, "hz": 4.999652607194508, "failed": 0, "missed": 0, "callback_last_ms": 8.2064, "callback_ewma_ms": 15.05321, "release_lateness_last_ms": 0.5456 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.0015220570330174, "failed": 0, "missed": 0, "callback_last_ms": 4.4762, "callback_ewma_ms": 6.559631, "release_lateness_last_ms": 0.2007 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 602, "hz": 4.999652607194508, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.001581, "release_lateness_last_ms": 0.476 }, { "name": "general_program_checks_thread", "running": true, "completed": 1202, "hz": 9.982695072836874, "failed": 0, "missed": 1, "callback_last_ms": 0.17, "callback_ewma_ms": 0.681489, "release_lateness_last_ms": 0.2835 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.9966084931284733, "failed": 0, "missed": 0, "callback_last_ms": 0.1021, "callback_ewma_ms": 0.10091, "release_lateness_last_ms": 0.1512 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1204, "hz": 9.999305214389016, "failed": 0, "missed": 0, "callback_last_ms": 0.0043, "callback_ewma_ms": 0.005086, "release_lateness_last_ms": 0.3411 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 2.0015220570330174, "failed": 0, "missed": 0, "callback_last_ms": 41.9558, "callback_ewma_ms": 58.781184, "release_lateness_last_ms": 0.3474 } ], "peer_counter_seconds": 120.3975269, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.23159999999916, "gap_p95_ms": 225.33280000000377, "gap_p99_ms": 240.10839999999735, "gap_max_ms": 579.6885000000184, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 21, "p95": 48, "p99": 66, "max": 398, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.50220000000263, "gap_p95_ms": 224.97060000000056, "gap_p99_ms": 238.73519999999553, "gap_max_ms": 511.38099999999395, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 22, "p95": 47, "p99": 66, "max": 332, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.36359999999695, "gap_p95_ms": 228.69249999997555, "gap_p99_ms": 244.36110000000917, "gap_max_ms": 481.6128000000026, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 25, "p95": 52, "p99": 68, "max": 306, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.43619999999646, "gap_p95_ms": 230.68909999998868, "gap_p99_ms": 240.08220000000438, "gap_max_ms": 604.0184000000008, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 22, "p95": 51, "p99": 63, "max": 415, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.00419999999576, "gap_p95_ms": 224.6070999999938, "gap_p99_ms": 233.8698000000004, "gap_max_ms": 530.286300000007, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 27, "p95": 51, "p99": 65, "max": 358, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.31180000002496, "gap_p95_ms": 226.32749999999942, "gap_p99_ms": 238.2489999999816, "gap_max_ms": 557.6340999999957, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 26, "p95": 52, "p99": 66, "max": 380, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.67939999999362, "gap_p95_ms": 224.94539999999688, "gap_p99_ms": 239.40780000000086, "gap_max_ms": 514.4555999999866, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 24, "p95": 49, "p99": 67, "max": 341, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.4988000000012, "gap_p95_ms": 224.43989999999758, "gap_p99_ms": 238.2207999999955, "gap_max_ms": 610.012699999992, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 28, "p95": 51, "p99": 69, "max": 440, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.808100000002, "gap_p95_ms": 225.70920000001206, "gap_p99_ms": 235.1365000000078, "gap_max_ms": 353.2190999999875, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 27, "p95": 52, "p99": 68, "max": 282, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.04630000001634, "gap_p95_ms": 231.91790000000037, "gap_p99_ms": 246.4215000000113, "gap_max_ms": 507.9747999999963, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 29, "p95": 56, "p99": 75, "max": 322, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.69569999999658, "gap_p95_ms": 231.09470000000147, "gap_p99_ms": 239.0791999999783, "gap_max_ms": 656.1266000000178, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 28, "p95": 58, "p99": 72, "max": 472, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.50739999999496, "gap_p95_ms": 223.92540000000736, "gap_p99_ms": 241.54040000000532, "gap_max_ms": 535.4823000000033, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 31, "p95": 53, "p99": 72, "max": 369, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.38769999999204, "gap_p95_ms": 225.8582000000024, "gap_p99_ms": 236.21540000000095, "gap_max_ms": 551.1086999999825, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 35, "p95": 57, "p99": 74, "max": 387, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.1826000000001, "gap_p95_ms": 224.78050000000803, "gap_p99_ms": 237.4375, "gap_max_ms": 511.578600000008, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 35, "p95": 56, "p99": 73, "max": 350, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.73540000000503, "gap_p95_ms": 225.82889999999315, "gap_p99_ms": 236.92660000000615, "gap_max_ms": 617.5791000000027, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 36, "p95": 57, "p99": 75, "max": 458, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9998126318136435, "complete_hz": 4.9998126318136435, "catalog_hz": 2.0049165462014775, "bytes_per_second": 1550603.5050458529, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.37209999999322, "gap_p95_ms": 224.97030000000086, "gap_p99_ms": 237.45930000000226, "gap_max_ms": 586.4765999999945, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1202, "p50": 36, "p95": 57, "p99": 73, "max": 429, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 110, "mean": 67.34526727272757, "p95": 90.07240000000456, "max": 116.17559999998775, "p50": 63.85770000000048, "p99": 108.48730000000069, "min": 51.26510000001872 }, { "operation": "rapid/get-persistence-diagnostics", "count": 110, "mean": 13.511830909090415, "p95": 36.4349000000002, "max": 61.26480000000447, "p50": 14.794900000008056, "p99": 47.05789999999979, "min": 4.162700000000768 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2211159, "accepted_rows_total": 13930354, "committed_rows_total": 2211812, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 110, "minimum": 17.1875, "maximum": 100, "mean": 37.200519277731566 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 110, "minimum": 3.125237518051372, "maximum": 25.77758442749441, "mean": 10.353677493372842 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 110, "minimum": 1002745856, "maximum": 1029799936, "mean": 1022147788.8 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0.08544921875, "mean": 0.0007941506125710227 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0.06103515625, "mean": 0.0005548650568181819 }, { "channel": "engine_tempest_tx_mbps", "observations": 110, "minimum": 373.31864817114183, "maximum": 602.0271977029355, "mean": 477.79204744919883 }, { "channel": "engine_tempest_rx_mbps", "observations": 110, "minimum": 0, "maximum": 0.046858962675900964, "mean": 0.02340608628288776 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 110, "minimum": 12253.479988316682, "maximum": 39445.548700488536, "mean": 18289.11393467164 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 110, "minimum": 11134.906210520945, "maximum": 41371.775756746574, "mean": 18433.727571426465 } ], "graceful_shutdown": true, "shutdown_seconds": 1.3427067000000097 }, { "id": "verified-capacity-stable-release-25-web-32", "healthy": true, "failure_reasons": [], "seconds": 120.7439702, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.011153099999994, "wait_seconds": 20.011153099999994, "completed_at": "2026-10-30T00:06:45.245Z", "accepted_cpu": { "count": 20, "mean": 20.11000311033851, "p95": 28.025398406374503, "max": 34.181279328229195 } }, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 31.053540649303645, "p50": 29.250425455287665, "p95": 51.28208478379713, "p99": 58.12245663719977, "min": 16.329779604963335, "max": 74.78184193934521 }, "engine_cpu_percent": { "count": 109, "mean": 10.362229759890925, "p50": 7.377783433922267, "p95": 23.37738742113925, "p99": 24.900524294448967, "min": 3.350758140985049, "max": 25.718025617968962 }, "client_generator_cpu_percent": { "count": 109, "mean": 6.940467066824755, "p50": 6.802105484244499, "p95": 8.943422018410402, "p99": 9.41095518739406, "min": 4.756458501155283, "max": 9.616754210269946 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1025018363.3027523, "p50": 1029419008, "p95": 1035288576, "p99": 1036812288, "min": 993046528, "max": 1037996032 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1177278559633028, "p95": 1.147489, "max": 1.157303, "p50": 1.1149493, "p99": 1.154346, "min": 1.1022737 }, "achieved_read_hz": { "minimum": 99.95057939338285, "maximum": 99.9836976154677 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48360, "hz": 100.01027301553783 }, { "group": "stress_group_1", "controls": 3, "completed": 36261, "hz": 99.98545657062083 }, { "group": "stress_group_2", "controls": 3, "completed": 36261, "hz": 99.98545657062083 }, { "group": "stress_group_3", "controls": 3, "completed": 36270, "hz": 100.01027301553783 }, { "group": "stress_group_4", "controls": 3, "completed": 36273, "hz": 100.0185451638435 }, { "group": "stress_group_5", "controls": 3, "completed": 36270, "hz": 100.01027301553783 }, { "group": "stress_group_6", "controls": 3, "completed": 36267, "hz": 100.00200086723216 }, { "group": "stress_group_7", "controls": 3, "completed": 36270, "hz": 100.01027301553783 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 279, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 607402884, "member_service_ns_total": 621888800, "members_executed_total": 48360 }, "example_device_plugin.stress_read": { "callback_ns_total": 3665414590, "member_service_ns_total": 3689063000, "members_executed_total": 48360 }, "example_device_plugin.stress_write": { "callback_ns_total": 183515294, "member_service_ns_total": 196772400, "members_executed_total": 48360 } }, "callback_ns_total": 4456332768, "cycles_total": 12090, "member_service_ns_total": 4507724200, "members_executed_total": 145080, "thread_cpu_ns_total": 1250000000, "wall_ns_total": 4581331400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 518932619, "member_service_ns_total": 529578800, "members_executed_total": 36261 }, "example_device_plugin.stress_read": { "callback_ns_total": 2988209806, "member_service_ns_total": 3007319600, "members_executed_total": 36261 }, "example_device_plugin.stress_write": { "callback_ns_total": 150453892, "member_service_ns_total": 161288500, "members_executed_total": 36261 } }, "callback_ns_total": 3657596317, "cycles_total": 12087, "member_service_ns_total": 3698186900, "members_executed_total": 108783, "thread_cpu_ns_total": 1046875000, "wall_ns_total": 3766671300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 467875965, "member_service_ns_total": 478342500, "members_executed_total": 36261 }, "example_device_plugin.stress_read": { "callback_ns_total": 2877277024, "member_service_ns_total": 2897748500, "members_executed_total": 36261 }, "example_device_plugin.stress_write": { "callback_ns_total": 145930296, "member_service_ns_total": 155924000, "members_executed_total": 36261 } }, "callback_ns_total": 3491083285, "cycles_total": 12087, "member_service_ns_total": 3532015000, "members_executed_total": 108783, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3594064300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 440177456, "member_service_ns_total": 451236900, "members_executed_total": 36270 }, "example_device_plugin.stress_read": { "callback_ns_total": 2912569891, "member_service_ns_total": 2934740800, "members_executed_total": 36270 }, "example_device_plugin.stress_write": { "callback_ns_total": 141178196, "member_service_ns_total": 151723100, "members_executed_total": 36270 } }, "callback_ns_total": 3493925543, "cycles_total": 12090, "member_service_ns_total": 3537700800, "members_executed_total": 108810, "thread_cpu_ns_total": 1031250000, "wall_ns_total": 3601185000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 422085090, "member_service_ns_total": 431609200, "members_executed_total": 36273 }, "example_device_plugin.stress_read": { "callback_ns_total": 2752506285, "member_service_ns_total": 2773161800, "members_executed_total": 36273 }, "example_device_plugin.stress_write": { "callback_ns_total": 140838195, "member_service_ns_total": 151920900, "members_executed_total": 36273 } }, "callback_ns_total": 3315429570, "cycles_total": 12091, "member_service_ns_total": 3356691900, "members_executed_total": 108819, "thread_cpu_ns_total": 578125000, "wall_ns_total": 3418098800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 426029258, "member_service_ns_total": 436567100, "members_executed_total": 36270 }, "example_device_plugin.stress_read": { "callback_ns_total": 2691833464, "member_service_ns_total": 2709514200, "members_executed_total": 36270 }, "example_device_plugin.stress_write": { "callback_ns_total": 137671695, "member_service_ns_total": 147397400, "members_executed_total": 36270 } }, "callback_ns_total": 3255534417, "cycles_total": 12090, "member_service_ns_total": 3293478700, "members_executed_total": 108810, "thread_cpu_ns_total": 500000000, "wall_ns_total": 3354035900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 443367761, "member_service_ns_total": 453897200, "members_executed_total": 36267 }, "example_device_plugin.stress_read": { "callback_ns_total": 2830151338, "member_service_ns_total": 2850593500, "members_executed_total": 36267 }, "example_device_plugin.stress_write": { "callback_ns_total": 142074896, "member_service_ns_total": 152336000, "members_executed_total": 36267 } }, "callback_ns_total": 3415593995, "cycles_total": 12089, "member_service_ns_total": 3456826700, "members_executed_total": 108801, "thread_cpu_ns_total": 843750000, "wall_ns_total": 3518226100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 494814226, "member_service_ns_total": 506955400, "members_executed_total": 36270 }, "example_device_plugin.stress_read": { "callback_ns_total": 2909769720, "member_service_ns_total": 2930539500, "members_executed_total": 36270 }, "example_device_plugin.stress_write": { "callback_ns_total": 151718390, "member_service_ns_total": 162707000, "members_executed_total": 36270 } }, "callback_ns_total": 3556302336, "cycles_total": 12090, "member_service_ns_total": 3600201900, "members_executed_total": 108810, "thread_cpu_ns_total": 1046875000, "wall_ns_total": 3664912000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.88758120000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.000212412878244, "failed": 0, "missed": 0, "callback_last_ms": 0.0028, "callback_ewma_ms": 0.004425, "release_lateness_last_ms": 0.2191 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.000212412878244, "failed": 0, "missed": 0, "callback_last_ms": 4.9743, "callback_ewma_ms": 5.225523, "release_lateness_last_ms": 0.1419 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 605, "hz": 5.00106206439122, "failed": 0, "missed": 0, "callback_last_ms": 0.0007, "callback_ewma_ms": 0.001507, "release_lateness_last_ms": 0.2416 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 605, "hz": 5.00106206439122, "failed": 0, "missed": 0, "callback_last_ms": 1.9236, "callback_ewma_ms": 2.279372, "release_lateness_last_ms": 0.4332 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.000212412878244, "failed": 0, "missed": 0, "callback_last_ms": 1.99, "callback_ewma_ms": 3.676393, "release_lateness_last_ms": 0.2145 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.000212412878244, "failed": 0, "missed": 0, "callback_last_ms": 3.9801, "callback_ewma_ms": 3.508228, "release_lateness_last_ms": 0.3945 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.000424825756488, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.000996, "release_lateness_last_ms": 0.1717 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.000212412878244, "failed": 0, "missed": 0, "callback_last_ms": 2.5007, "callback_ewma_ms": 2.715173, "release_lateness_last_ms": 0.4037 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.000424825756488, "failed": 0, "missed": 0, "callback_last_ms": 3.5199, "callback_ewma_ms": 6.561681, "release_lateness_last_ms": 0.3649 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 605, "hz": 5.00106206439122, "failed": 0, "missed": 0, "callback_last_ms": 6.9609, "callback_ewma_ms": 6.849939, "release_lateness_last_ms": 0.53 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1210, "hz": 10.00212412878244, "failed": 0, "missed": 0, "callback_last_ms": 0.0285, "callback_ewma_ms": 0.460083, "release_lateness_last_ms": 0.5773 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 605, "hz": 5.00106206439122, "failed": 0, "missed": 0, "callback_last_ms": 13.9947, "callback_ewma_ms": 17.120294, "release_lateness_last_ms": 0.3889 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.000424825756488, "failed": 0, "missed": 0, "callback_last_ms": 4.6762, "callback_ewma_ms": 6.36189, "release_lateness_last_ms": 0.3172 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 605, "hz": 5.00106206439122, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.001906, "release_lateness_last_ms": 0.3477 }, { "name": "general_program_checks_thread", "running": true, "completed": 1210, "hz": 10.00212412878244, "failed": 0, "missed": 0, "callback_last_ms": 0.1276, "callback_ewma_ms": 0.198429, "release_lateness_last_ms": 0.5286 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.000212412878244, "failed": 0, "missed": 0, "callback_last_ms": 0.0989, "callback_ewma_ms": 0.113216, "release_lateness_last_ms": 0.2973 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1210, "hz": 10.00212412878244, "failed": 0, "missed": 0, "callback_last_ms": 0.0054, "callback_ewma_ms": 0.008364, "release_lateness_last_ms": 0.4489 }, { "name": "engine_diagnostics", "running": true, "completed": 242, "hz": 2.000424825756488, "failed": 0, "missed": 0, "callback_last_ms": 37.5817, "callback_ewma_ms": 42.83297, "release_lateness_last_ms": 0.5443 } ], "peer_counter_seconds": 120.98548120000001, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.66589999999997, "gap_p95_ms": 266.2317999999941, "gap_p99_ms": 284.6072999999742, "gap_max_ms": 296.55539999999746, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 40, "p95": 90, "p99": 102, "max": 123, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.79649999999674, "gap_p95_ms": 253.03439999998955, "gap_p99_ms": 281.1909000000014, "gap_max_ms": 324.8029000000024, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 50, "p95": 88, "p99": 104, "max": 169, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.50729999999749, "gap_p95_ms": 246.49399999999878, "gap_p99_ms": 275.9328000000096, "gap_max_ms": 320.6258000000016, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 55, "p95": 89, "p99": 108, "max": 161, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.4192000000039, "gap_p95_ms": 249.7774000000063, "gap_p99_ms": 275.0356000000029, "gap_max_ms": 312.059500000003, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 49, "p95": 90, "p99": 106, "max": 126, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.18359999999666, "gap_p95_ms": 258.1594999999943, "gap_p99_ms": 278.2752999999939, "gap_max_ms": 308.1439000000246, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 42, "p95": 87, "p99": 106, "max": 157, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9817000000039, "gap_p95_ms": 259.0001999999804, "gap_p99_ms": 281.12970000000496, "gap_max_ms": 313.23309999999765, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 52, "p95": 92, "p99": 119, "max": 146, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.73539999999775, "gap_p95_ms": 254.25339999998687, "gap_p99_ms": 282.34249999999884, "gap_max_ms": 292.820000000007, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 57, "p95": 91, "p99": 110, "max": 155, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.5182000000059, "gap_p95_ms": 248.2216999999946, "gap_p99_ms": 263.1432999999961, "gap_max_ms": 310.1735999999946, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 66, "p95": 94, "p99": 108, "max": 180, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.07890000000043, "gap_p95_ms": 253.6133000000118, "gap_p99_ms": 271.3048000000126, "gap_max_ms": 316.0635000000184, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 65, "p95": 98, "p99": 123, "max": 173, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.72550000000047, "gap_p95_ms": 251.19700000001467, "gap_p99_ms": 271.57189999999537, "gap_max_ms": 295.4441000000079, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 57, "p95": 91, "p99": 109, "max": 173, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.69490000000224, "gap_p95_ms": 251.17719999997644, "gap_p99_ms": 271.111699999994, "gap_max_ms": 304.9207999999999, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 58, "p95": 95, "p99": 111, "max": 156, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.25729999999749, "gap_p95_ms": 253.72179999999935, "gap_p99_ms": 269.7768999999971, "gap_max_ms": 300.6701999999932, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 55, "p95": 96, "p99": 109, "max": 165, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.45330000002286, "gap_p95_ms": 246.63799999999173, "gap_p99_ms": 271.156799999997, "gap_max_ms": 286.86269999999786, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 57, "p95": 89, "p99": 110, "max": 131, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.20470000000205, "gap_p95_ms": 248.49669999998878, "gap_p99_ms": 268.0466000000015, "gap_max_ms": 299.4125000000058, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 54, "p95": 91, "p99": 105, "max": 169, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.91790000000037, "gap_p95_ms": 254.17329999999492, "gap_p99_ms": 277.7447000000102, "gap_max_ms": 299.0136000000057, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 44, "p95": 92, "p99": 107, "max": 128, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.113400000002, "gap_p95_ms": 253.54889999999432, "gap_p99_ms": 273.32050000000163, "gap_max_ms": 319.6637000000046, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 66, "p95": 97, "p99": 126, "max": 152, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.33480000001146, "gap_p95_ms": 249.70119999999588, "gap_p99_ms": 275.81840000000375, "gap_max_ms": 291.76290000000154, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 58, "p95": 93, "p99": 110, "max": 179, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.86790000000474, "gap_p95_ms": 247.21710000000166, "gap_p99_ms": 270.03229999999166, "gap_max_ms": 324.47750000000815, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 59, "p95": 90, "p99": 118, "max": 168, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.0208999999959, "gap_p95_ms": 251.7627000000066, "gap_p99_ms": 268.73489999999583, "gap_max_ms": 287.63930000000255, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 55, "p95": 94, "p99": 113, "max": 150, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.80439999999362, "gap_p95_ms": 252.34539999999106, "gap_p99_ms": 273.94280000000435, "gap_max_ms": 302.86069999999745, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 68, "p95": 98, "p99": 114, "max": 158, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.67500000001746, "gap_p95_ms": 253.0847999999969, "gap_p99_ms": 271.74559999999474, "gap_max_ms": 295.2427000000025, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 66, "p95": 97, "p99": 119, "max": 166, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.1006999999954, "gap_p95_ms": 250.52270000000135, "gap_p99_ms": 273.4753999999957, "gap_max_ms": 290.4021000000066, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 52, "p95": 99, "p99": 116, "max": 134, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.58189999999013, "gap_p95_ms": 252.72000000000116, "gap_p99_ms": 270.1883000000089, "gap_max_ms": 300.15200000000186, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 70, "p95": 101, "p99": 123, "max": 165, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.60049999997136, "gap_p95_ms": 249.13810000000376, "gap_p99_ms": 274.30799999999, "gap_max_ms": 292.7601999999897, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 57, "p95": 95, "p99": 120, "max": 183, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7997000000032, "gap_p95_ms": 253.93459999999322, "gap_p99_ms": 269.2786999999953, "gap_max_ms": 291.049199999994, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 68, "p95": 100, "p99": 121, "max": 162, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.78859999999986, "gap_p95_ms": 249.40559999999823, "gap_p99_ms": 272.16339999999036, "gap_max_ms": 295.38790000000154, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 52, "p95": 93, "p99": 108, "max": 158, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.3560999999754, "gap_p95_ms": 250.90290000000095, "gap_p99_ms": 272.2515999999887, "gap_max_ms": 309.16419999999925, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 49, "p95": 87, "p99": 110, "max": 155, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.6324999999997, "gap_p95_ms": 252.36749999999302, "gap_p99_ms": 270.9994999999908, "gap_max_ms": 302.94409999999334, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 58, "p95": 96, "p99": 116, "max": 157, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.77150000000256, "gap_p95_ms": 249.91270000000077, "gap_p99_ms": 271.53280000000086, "gap_max_ms": 305.06750000000466, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 57, "p95": 103, "p99": 127, "max": 140, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.1053000000029, "gap_p95_ms": 254.33329999999842, "gap_p99_ms": 271.32410000001255, "gap_max_ms": 305.35920000000624, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 66, "p95": 102, "p99": 125, "max": 162, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.2945000000036, "gap_p95_ms": 252.5387000000046, "gap_p99_ms": 271.38180000000284, "gap_max_ms": 288.05539999999746, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 52, "p95": 89, "p99": 107, "max": 142, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002317754362827, "complete_hz": 5.002317754362827, "catalog_hz": 2.0042398949599405, "bytes_per_second": 1551323.3453556532, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.96680000000924, "gap_p95_ms": 248.74610000000393, "gap_p99_ms": 276.74929999999586, "gap_max_ms": 326.7390000000014, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 52, "p95": 88, "p99": 121, "max": 174, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 107, "mean": 81.90140747663521, "p95": 146.92639999999665, "max": 158.82510000000184, "p50": 65.37859999999637, "p99": 156.99929999999586, "min": 51.3067999999912 }, { "operation": "rapid/get-persistence-diagnostics", "count": 107, "mean": 22.968890654205214, "p95": 81.63259999999718, "max": 140.4391000000178, "p50": 15.698000000003958, "p99": 100.9941000000108, "min": 4.693699999988894 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2222302, "accepted_rows_total": 14013060, "committed_rows_total": 2222919, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 107, "minimum": 14.84375, "maximum": 90.15748031496064, "mean": 31.290794198173955 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 107, "minimum": 1.9544947098926926, "maximum": 27.719317866534897, "mean": 10.083141809153103 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 107, "minimum": 993591296, "maximum": 1037312000, "mean": 1025271788.8598131 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 107, "minimum": 744.2116050098459, "maximum": 1185.697223683182, "mean": 952.7824784451665 }, { "channel": "engine_tempest_rx_mbps", "observations": 107, "minimum": 0.027344216917994935, "maximum": 0.07107920771303729, "mean": 0.043033845940609126 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 107, "minimum": 12491.87778106675, "maximum": 39501.89541515281, "mean": 18415.253371573923 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 107, "minimum": 11235.395336470681, "maximum": 41505.45423725832, "mean": 18667.020320181728 } ], "graceful_shutdown": true, "shutdown_seconds": 1.5888851000000142 }, { "id": "verified-capacity-stable-release-25-web-32-r2", "healthy": true, "failure_reasons": [], "seconds": 121.1664841, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.011306700000073, "wait_seconds": 20.011306700000073, "completed_at": "2026-10-30T00:10:05.415Z", "accepted_cpu": { "count": 20, "mean": 21.477795936318156, "p95": 28.923153444416815, "max": 32.23561575742513 } }, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 40.517922481994795, "p50": 38.064058017779544, "p95": 55.6943543476617, "p99": 62.74829749785778, "min": 29.59421043134872, "max": 66.87366961567595 }, "engine_cpu_percent": { "count": 108, "mean": 10.643238767645922, "p50": 7.726360909410324, "p95": 22.345947263028155, "p99": 25.621170175180776, "min": 4.726500561835971, "max": 25.882806010689944 }, "client_generator_cpu_percent": { "count": 108, "mean": 7.075231885640826, "p50": 6.960944047463971, "p95": 8.763604928770317, "p99": 9.298349598946814, "min": 5.066832428264275, "max": 9.660283994082707 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1021475081.4814814, "p50": 1022984192, "p95": 1031458816, "p99": 1034072064, "min": 997109760, "max": 1035206656 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1182933888888889, "p95": 1.1350753, "max": 1.1946658, "p50": 1.1162284, "p99": 1.154543, "min": 1.1026373 }, "achieved_read_hz": { "minimum": 99.95590397648917, "maximum": 99.98065731531523 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48488, "hz": 99.96576490047782 }, { "group": "stress_group_1", "controls": 3, "completed": 36369, "hz": 99.97401154005053 }, { "group": "stress_group_2", "controls": 3, "completed": 36369, "hz": 99.97401154005053 }, { "group": "stress_group_3", "controls": 3, "completed": 36372, "hz": 99.98225817962326 }, { "group": "stress_group_4", "controls": 3, "completed": 36372, "hz": 99.98225817962326 }, { "group": "stress_group_5", "controls": 3, "completed": 36372, "hz": 99.98225817962326 }, { "group": "stress_group_6", "controls": 3, "completed": 36366, "hz": 99.96576490047782 }, { "group": "stress_group_7", "controls": 3, "completed": 36372, "hz": 99.98225817962326 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 285, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 597367670, "member_service_ns_total": 610525500, "members_executed_total": 48488 }, "example_device_plugin.stress_read": { "callback_ns_total": 3485981550, "member_service_ns_total": 3509300000, "members_executed_total": 48488 }, "example_device_plugin.stress_write": { "callback_ns_total": 183686697, "member_service_ns_total": 195929600, "members_executed_total": 48488 } }, "callback_ns_total": 4267035917, "cycles_total": 12122, "member_service_ns_total": 4315755100, "members_executed_total": 145464, "thread_cpu_ns_total": 1328125000, "wall_ns_total": 4385590400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 496953947, "member_service_ns_total": 510032400, "members_executed_total": 36369 }, "example_device_plugin.stress_read": { "callback_ns_total": 2864613293, "member_service_ns_total": 2885014100, "members_executed_total": 36369 }, "example_device_plugin.stress_write": { "callback_ns_total": 157049197, "member_service_ns_total": 167526800, "members_executed_total": 36369 } }, "callback_ns_total": 3518616437, "cycles_total": 12123, "member_service_ns_total": 3562573300, "members_executed_total": 109107, "thread_cpu_ns_total": 1250000000, "wall_ns_total": 3628053500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 466891681, "member_service_ns_total": 477959100, "members_executed_total": 36369 }, "example_device_plugin.stress_read": { "callback_ns_total": 2679974984, "member_service_ns_total": 2699376300, "members_executed_total": 36369 }, "example_device_plugin.stress_write": { "callback_ns_total": 140114995, "member_service_ns_total": 149092600, "members_executed_total": 36369 } }, "callback_ns_total": 3286981660, "cycles_total": 12123, "member_service_ns_total": 3326428000, "members_executed_total": 109107, "thread_cpu_ns_total": 890625000, "wall_ns_total": 3389156300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 446717520, "member_service_ns_total": 457214500, "members_executed_total": 36372 }, "example_device_plugin.stress_read": { "callback_ns_total": 2722643134, "member_service_ns_total": 2741292300, "members_executed_total": 36372 }, "example_device_plugin.stress_write": { "callback_ns_total": 139819394, "member_service_ns_total": 148892200, "members_executed_total": 36372 } }, "callback_ns_total": 3309180048, "cycles_total": 12124, "member_service_ns_total": 3347399000, "members_executed_total": 109116, "thread_cpu_ns_total": 1203125000, "wall_ns_total": 3410001800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 417328161, "member_service_ns_total": 427713100, "members_executed_total": 36372 }, "example_device_plugin.stress_read": { "callback_ns_total": 2551813731, "member_service_ns_total": 2569239800, "members_executed_total": 36372 }, "example_device_plugin.stress_write": { "callback_ns_total": 140218499, "member_service_ns_total": 149307500, "members_executed_total": 36372 } }, "callback_ns_total": 3109360391, "cycles_total": 12124, "member_service_ns_total": 3146260400, "members_executed_total": 109116, "thread_cpu_ns_total": 500000000, "wall_ns_total": 3202832700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 409990687, "member_service_ns_total": 419015800, "members_executed_total": 36372 }, "example_device_plugin.stress_read": { "callback_ns_total": 2593979052, "member_service_ns_total": 2612421300, "members_executed_total": 36372 }, "example_device_plugin.stress_write": { "callback_ns_total": 138345595, "member_service_ns_total": 148859200, "members_executed_total": 36372 } }, "callback_ns_total": 3142315334, "cycles_total": 12124, "member_service_ns_total": 3180296300, "members_executed_total": 109116, "thread_cpu_ns_total": 656250000, "wall_ns_total": 3236855000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 433332462, "member_service_ns_total": 445119200, "members_executed_total": 36366 }, "example_device_plugin.stress_read": { "callback_ns_total": 2688313978, "member_service_ns_total": 2707269100, "members_executed_total": 36366 }, "example_device_plugin.stress_write": { "callback_ns_total": 142547796, "member_service_ns_total": 151436500, "members_executed_total": 36366 } }, "callback_ns_total": 3264194236, "cycles_total": 12122, "member_service_ns_total": 3303824800, "members_executed_total": 109098, "thread_cpu_ns_total": 890625000, "wall_ns_total": 3363718500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 489297442, "member_service_ns_total": 499716500, "members_executed_total": 36372 }, "example_device_plugin.stress_read": { "callback_ns_total": 2801326402, "member_service_ns_total": 2821229600, "members_executed_total": 36372 }, "example_device_plugin.stress_write": { "callback_ns_total": 146494895, "member_service_ns_total": 159162600, "members_executed_total": 36372 } }, "callback_ns_total": 3437118739, "cycles_total": 12124, "member_service_ns_total": 3480108700, "members_executed_total": 109116, "thread_cpu_ns_total": 1171875000, "wall_ns_total": 3543556200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.261514, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 122, "hz": 1.005340411942865, "failed": 0, "missed": 0, "callback_last_ms": 0.0047, "callback_ewma_ms": 0.005029, "release_lateness_last_ms": 0.4194 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 122, "hz": 1.005340411942865, "failed": 0, "missed": 0, "callback_last_ms": 4.8225, "callback_ewma_ms": 4.348182, "release_lateness_last_ms": 0.263 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 607, "hz": 5.0019805741747465, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.00109, "release_lateness_last_ms": 0.3969 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 607, "hz": 5.0019805741747465, "failed": 0, "missed": 0, "callback_last_ms": 2.2357, "callback_ewma_ms": 2.086437, "release_lateness_last_ms": 0.6065 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 122, "hz": 1.005340411942865, "failed": 0, "missed": 0, "callback_last_ms": 2.5674, "callback_ewma_ms": 3.384832, "release_lateness_last_ms": 0.3198 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 122, "hz": 1.005340411942865, "failed": 0, "missed": 0, "callback_last_ms": 3.8275, "callback_ewma_ms": 3.593249, "release_lateness_last_ms": 0.5446 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 243, "hz": 2.0024403287058705, "failed": 0, "missed": 0, "callback_last_ms": 0.0015, "callback_ewma_ms": 0.001116, "release_lateness_last_ms": 0.4018 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 122, "hz": 1.005340411942865, "failed": 0, "missed": 0, "callback_last_ms": 0.8398, "callback_ewma_ms": 2.111139, "release_lateness_last_ms": 0.1458 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 243, "hz": 2.0024403287058705, "failed": 0, "missed": 0, "callback_last_ms": 7.5617, "callback_ewma_ms": 6.816462, "release_lateness_last_ms": 0.5628 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 607, "hz": 5.0019805741747465, "failed": 0, "missed": 0, "callback_last_ms": 4.8193, "callback_ewma_ms": 6.707821, "release_lateness_last_ms": 0.0803 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1214, "hz": 10.003961148349493, "failed": 0, "missed": 0, "callback_last_ms": 0.0212, "callback_ewma_ms": 0.596202, "release_lateness_last_ms": 0.1015 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 607, "hz": 5.0019805741747465, "failed": 0, "missed": 0, "callback_last_ms": 17.2811, "callback_ewma_ms": 17.680356, "release_lateness_last_ms": 0.4669 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 243, "hz": 2.0024403287058705, "failed": 0, "missed": 0, "callback_last_ms": 7.3213, "callback_ewma_ms": 6.917959, "release_lateness_last_ms": 0.501 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 607, "hz": 5.0019805741747465, "failed": 0, "missed": 0, "callback_last_ms": 0.0021, "callback_ewma_ms": 0.002429, "release_lateness_last_ms": 0.4054 }, { "name": "general_program_checks_thread", "running": true, "completed": 1214, "hz": 10.003961148349493, "failed": 0, "missed": 0, "callback_last_ms": 0.1649, "callback_ewma_ms": 0.173641, "release_lateness_last_ms": 0.3295 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 122, "hz": 1.005340411942865, "failed": 0, "missed": 0, "callback_last_ms": 0.0641, "callback_ewma_ms": 0.10511, "release_lateness_last_ms": 0.5349 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1214, "hz": 10.003961148349493, "failed": 0, "missed": 0, "callback_last_ms": 0.004, "callback_ewma_ms": 0.005133, "release_lateness_last_ms": 0.1632 }, { "name": "engine_diagnostics", "running": true, "completed": 243, "hz": 2.0024403287058705, "failed": 0, "missed": 0, "callback_last_ms": 45.4265, "callback_ewma_ms": 43.566601, "release_lateness_last_ms": 0.224 } ], "peer_counter_seconds": 121.36135069999997, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.7963000000018, "gap_p95_ms": 247.80599999999686, "gap_p99_ms": 276.7372999999934, "gap_max_ms": 291.43940000000293, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 48, "p95": 86, "p99": 108, "max": 121, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.0063000000082, "gap_p95_ms": 257.078199999989, "gap_p99_ms": 281.40810000000056, "gap_max_ms": 306.52429999999003, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 51, "p95": 99, "p99": 116, "max": 127, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.63869999999588, "gap_p95_ms": 252.29880000000412, "gap_p99_ms": 274.4824999999837, "gap_max_ms": 322.3720999999932, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 41, "p95": 88, "p99": 103, "max": 151, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.65840000000026, "gap_p95_ms": 254.09049999999115, "gap_p99_ms": 274.0679999999993, "gap_max_ms": 305.8960000000079, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 50, "p95": 90, "p99": 106, "max": 138, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.4829999999929, "gap_p95_ms": 256.2195000000065, "gap_p99_ms": 276.95580000001064, "gap_max_ms": 293.3511999999755, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 38, "p95": 91, "p99": 110, "max": 128, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.90269999999146, "gap_p95_ms": 248.39560000001802, "gap_p99_ms": 267.6337000000058, "gap_max_ms": 278.3608000000022, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 60, "p95": 101, "p99": 114, "max": 129, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.8136000000086, "gap_p95_ms": 259.9637999999977, "gap_p99_ms": 286.8255000000063, "gap_max_ms": 308.3396999999968, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 36, "p95": 88, "p99": 111, "max": 123, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.33139999999548, "gap_p95_ms": 250.64259999999922, "gap_p99_ms": 271.6923999999999, "gap_max_ms": 286.98260000001756, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 55, "p95": 91, "p99": 109, "max": 147, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.5089999999982, "gap_p95_ms": 247.41250000000582, "gap_p99_ms": 268.55859999999666, "gap_max_ms": 297.83110000001034, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 62, "p95": 97, "p99": 113, "max": 131, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.9010999999955, "gap_p95_ms": 250.10929999999644, "gap_p99_ms": 272.1003999999957, "gap_max_ms": 277.73349999998754, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 46, "p95": 89, "p99": 102, "max": 124, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.96719999999914, "gap_p95_ms": 246.06010000000242, "gap_p99_ms": 262.2810000000027, "gap_max_ms": 294.7387999999919, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 58, "p95": 93, "p99": 108, "max": 140, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0460000000021, "gap_p95_ms": 254.02010000000882, "gap_p99_ms": 275.1970999999903, "gap_max_ms": 293.4618999999948, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 47, "p95": 92, "p99": 106, "max": 135, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.01789999999892, "gap_p95_ms": 247.09909999999218, "gap_p99_ms": 262.8096000000005, "gap_max_ms": 275.39100000000326, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 65, "p95": 103, "p99": 117, "max": 138, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.30359999999928, "gap_p95_ms": 253.25209999999788, "gap_p99_ms": 280.63300000000163, "gap_max_ms": 300.0652000000118, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 53, "p95": 93, "p99": 116, "max": 128, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.89360000001034, "gap_p95_ms": 249.36470000000554, "gap_p99_ms": 266.6718000000037, "gap_max_ms": 295.2447000000029, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 62, "p95": 93, "p99": 108, "max": 149, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.3698999999906, "gap_p95_ms": 250.8622000000032, "gap_p99_ms": 269.50359999999637, "gap_max_ms": 310.42399999999907, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 59, "p95": 98, "p99": 110, "max": 134, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.19950000000244, "gap_p95_ms": 250.681700000001, "gap_p99_ms": 270.8954000000085, "gap_max_ms": 279.9933999999921, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 51, "p95": 92, "p99": 108, "max": 126, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.11609999999928, "gap_p95_ms": 244.65150000000722, "gap_p99_ms": 262.1644999999844, "gap_max_ms": 297.6782999999996, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 63, "p95": 91, "p99": 108, "max": 120, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.07769999999437, "gap_p95_ms": 252.4930999999924, "gap_p99_ms": 271.9597000000067, "gap_max_ms": 283.5993000000017, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 50, "p95": 92, "p99": 107, "max": 139, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.29829999999492, "gap_p95_ms": 248.65260000000126, "gap_p99_ms": 259.3267000000051, "gap_max_ms": 290.98820000000705, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 68, "p95": 103, "p99": 116, "max": 141, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.38930000000983, "gap_p95_ms": 250.3853999999992, "gap_p99_ms": 274.3948000000091, "gap_max_ms": 299.99890000000596, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 58, "p95": 93, "p99": 113, "max": 132, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.41159999999218, "gap_p95_ms": 249.96110000000044, "gap_p99_ms": 268.6769000000204, "gap_max_ms": 289.11290000000736, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 65, "p95": 94, "p99": 107, "max": 123, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.30600000000413, "gap_p95_ms": 249.78629999999248, "gap_p99_ms": 264.62599999999657, "gap_max_ms": 297.59480000002077, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 73, "p95": 101, "p99": 116, "max": 138, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.1934000000183, "gap_p95_ms": 252.02419999998529, "gap_p99_ms": 281.65230000000156, "gap_max_ms": 292.09049999999115, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 54, "p95": 95, "p99": 115, "max": 138, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.97020000001066, "gap_p95_ms": 248.85430000000633, "gap_p99_ms": 269.36699999999837, "gap_max_ms": 295.3787000000011, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 70, "p95": 97, "p99": 115, "max": 142, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.08729999999923, "gap_p95_ms": 251.78399999999965, "gap_p99_ms": 270.76189999998314, "gap_max_ms": 282.1389999999956, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 53, "p95": 99, "p99": 114, "max": 143, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.96290000001318, "gap_p95_ms": 254.4757999999856, "gap_p99_ms": 278.0408999999927, "gap_max_ms": 303.3619000000035, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 75, "p95": 107, "p99": 118, "max": 143, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.07330000000366, "gap_p95_ms": 256.3133999999991, "gap_p99_ms": 278.72080000001006, "gap_max_ms": 303.93880000000354, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 48, "p95": 94, "p99": 109, "max": 124, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.29910000000382, "gap_p95_ms": 251.56009999997332, "gap_p99_ms": 271.9654000000155, "gap_max_ms": 286.3842999999906, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 57, "p95": 97, "p99": 116, "max": 144, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.18490000000747, "gap_p95_ms": 252.42909999999392, "gap_p99_ms": 268.81059999999707, "gap_max_ms": 282.7268999999942, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 73, "p95": 100, "p99": 115, "max": 132, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.07330000000366, "gap_p95_ms": 247.5740000000078, "gap_p99_ms": 264.6007999999856, "gap_max_ms": 294.7509999999893, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 74, "p95": 102, "p99": 121, "max": 145, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001381776145003, "complete_hz": 5.001381776145003, "catalog_hz": 2.005504573602699, "bytes_per_second": 1550993.366442866, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.8081999999995, "gap_p95_ms": 253.78819999998086, "gap_p99_ms": 280.2161000000051, "gap_max_ms": 299.7073000000091, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 40, "p95": 90, "p99": 110, "max": 146, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 108, "mean": 82.65815462962941, "p95": 141.24259999999776, "max": 164.53239999999641, "p50": 66.26000000000931, "p99": 145.35169999999925, "min": 52.156099999992875 }, { "operation": "rapid/get-persistence-diagnostics", "count": 108, "mean": 15.25453148148093, "p95": 51.87919999999576, "max": 90.34330000000773, "p50": 10.161500000016531, "p99": 76.01310000001104, "min": 4.374299999995856 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2224650, "accepted_rows_total": 14042948, "committed_rows_total": 2225108, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 108, "minimum": 26.612903225806452, "maximum": 77.8225806451613, "mean": 42.19713263770526 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 108, "minimum": 2.735705099819532, "maximum": 30.87879158414811, "mean": 10.397717106575753 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 108, "minimum": 997937152, "maximum": 1034321920, "mean": 1022011316.1481482 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 108, "minimum": 0, "maximum": 0.128173828125, "mean": 0.001186794704861111 }, { "channel": "engine_tempest_tx_mbps", "observations": 108, "minimum": 743.2891125260792, "maximum": 1169.153220288526, "mean": 936.5192986753167 }, { "channel": "engine_tempest_rx_mbps", "observations": 108, "minimum": 0.02457190755598132, "maximum": 0.06563834853339316, "mean": 0.043596406305353545 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 108, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 108, "minimum": 11752.609079215586, "maximum": 39392.78594091402, "mean": 18626.834143124885 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 108, "minimum": 11050.223214508933, "maximum": 41393.04453662009, "mean": 18636.148181832952 } ], "graceful_shutdown": true, "shutdown_seconds": 1.6255308999999833 }, { "id": "verified-capacity-stable-release-25-web-32-r3", "healthy": true, "failure_reasons": [], "seconds": 120.37710829999999, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.03261620000005, "wait_seconds": 20.03261620000005, "completed_at": "2026-10-30T00:13:24.626Z", "accepted_cpu": { "count": 20, "mean": 21.492266440763704, "p95": 29.61946350592639, "max": 32.42939265183704 } }, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 41.13909091093478, "p50": 39.96613141384243, "p95": 55.2332735455692, "p99": 59.678579971395365, "min": 28.06195707227569, "max": 64.08843040679287 }, "engine_cpu_percent": { "count": 108, "mean": 10.668374583687235, "p50": 8.12273331948259, "p95": 22.47259853137793, "p99": 23.593609895593, "min": 4.946594418423411, "max": 24.36684521057098 }, "client_generator_cpu_percent": { "count": 108, "mean": 7.278140551145217, "p50": 7.165409036948445, "p95": 8.97550285351396, "p99": 9.68724549668631, "min": 5.2568401836250445, "max": 10.17889234218259 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1023094366.8148148, "p50": 1027756032, "p95": 1033707520, "p99": 1034121216, "min": 993861632, "max": 1034649600 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1180935916666663, "p95": 1.1352799, "max": 1.194269, "p50": 1.1152386, "p99": 1.1529068, "min": 1.1008485 }, "achieved_read_hz": { "minimum": 99.91535523805271, "maximum": 99.98178541431308 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48212, "hz": 99.99426557402245 }, { "group": "stress_group_1", "controls": 3, "completed": 36162, "hz": 100.00256178787578 }, { "group": "stress_group_2", "controls": 3, "completed": 36156, "hz": 99.98596936016914 }, { "group": "stress_group_3", "controls": 3, "completed": 36162, "hz": 100.00256178787578 }, { "group": "stress_group_4", "controls": 3, "completed": 36147, "hz": 99.9610807186092 }, { "group": "stress_group_5", "controls": 3, "completed": 36138, "hz": 99.93619207704923 }, { "group": "stress_group_6", "controls": 3, "completed": 36159, "hz": 99.99426557402245 }, { "group": "stress_group_7", "controls": 3, "completed": 36156, "hz": 99.98596936016914 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 324, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 600131104, "member_service_ns_total": 614884100, "members_executed_total": 48212 }, "example_device_plugin.stress_read": { "callback_ns_total": 3551132510, "member_service_ns_total": 3576614900, "members_executed_total": 48212 }, "example_device_plugin.stress_write": { "callback_ns_total": 182671689, "member_service_ns_total": 196022000, "members_executed_total": 48212 } }, "callback_ns_total": 4333935303, "cycles_total": 12053, "member_service_ns_total": 4387521000, "members_executed_total": 144636, "thread_cpu_ns_total": 1593750000, "wall_ns_total": 4455833900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 507792328, "member_service_ns_total": 519363100, "members_executed_total": 36162 }, "example_device_plugin.stress_read": { "callback_ns_total": 2912190239, "member_service_ns_total": 2931948000, "members_executed_total": 36162 }, "example_device_plugin.stress_write": { "callback_ns_total": 147773695, "member_service_ns_total": 157345500, "members_executed_total": 36162 } }, "callback_ns_total": 3567756262, "cycles_total": 12054, "member_service_ns_total": 3608656600, "members_executed_total": 108486, "thread_cpu_ns_total": 968750000, "wall_ns_total": 3673382300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 463513565, "member_service_ns_total": 473741900, "members_executed_total": 36156 }, "example_device_plugin.stress_read": { "callback_ns_total": 2756552545, "member_service_ns_total": 2775286900, "members_executed_total": 36156 }, "example_device_plugin.stress_write": { "callback_ns_total": 142991397, "member_service_ns_total": 154261400, "members_executed_total": 36156 } }, "callback_ns_total": 3363057507, "cycles_total": 12052, "member_service_ns_total": 3403290200, "members_executed_total": 108468, "thread_cpu_ns_total": 1203125000, "wall_ns_total": 3469461100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 457515319, "member_service_ns_total": 467137300, "members_executed_total": 36162 }, "example_device_plugin.stress_read": { "callback_ns_total": 2664664122, "member_service_ns_total": 2684310100, "members_executed_total": 36162 }, "example_device_plugin.stress_write": { "callback_ns_total": 140957392, "member_service_ns_total": 151194000, "members_executed_total": 36162 } }, "callback_ns_total": 3263136833, "cycles_total": 12054, "member_service_ns_total": 3302641400, "members_executed_total": 108486, "thread_cpu_ns_total": 1109375000, "wall_ns_total": 3365898600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 421160946, "member_service_ns_total": 431135900, "members_executed_total": 36147 }, "example_device_plugin.stress_read": { "callback_ns_total": 2732038944, "member_service_ns_total": 2751287700, "members_executed_total": 36147 }, "example_device_plugin.stress_write": { "callback_ns_total": 136409297, "member_service_ns_total": 146565700, "members_executed_total": 36147 } }, "callback_ns_total": 3289609187, "cycles_total": 12049, "member_service_ns_total": 3328989300, "members_executed_total": 108441, "thread_cpu_ns_total": 703125000, "wall_ns_total": 3387947400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 412738079, "member_service_ns_total": 424464900, "members_executed_total": 36138 }, "example_device_plugin.stress_read": { "callback_ns_total": 2803295416, "member_service_ns_total": 2821372500, "members_executed_total": 36138 }, "example_device_plugin.stress_write": { "callback_ns_total": 138170095, "member_service_ns_total": 148313600, "members_executed_total": 36138 } }, "callback_ns_total": 3354203590, "cycles_total": 12046, "member_service_ns_total": 3394151000, "members_executed_total": 108414, "thread_cpu_ns_total": 703125000, "wall_ns_total": 3452246900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 427504474, "member_service_ns_total": 438310300, "members_executed_total": 36159 }, "example_device_plugin.stress_read": { "callback_ns_total": 2714789197, "member_service_ns_total": 2733352800, "members_executed_total": 36159 }, "example_device_plugin.stress_write": { "callback_ns_total": 141045197, "member_service_ns_total": 150802900, "members_executed_total": 36159 } }, "callback_ns_total": 3283338868, "cycles_total": 12053, "member_service_ns_total": 3322466000, "members_executed_total": 108477, "thread_cpu_ns_total": 1015625000, "wall_ns_total": 3383091600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 480596642, "member_service_ns_total": 490761100, "members_executed_total": 36156 }, "example_device_plugin.stress_read": { "callback_ns_total": 2861461609, "member_service_ns_total": 2881646500, "members_executed_total": 36156 }, "example_device_plugin.stress_write": { "callback_ns_total": 147171198, "member_service_ns_total": 157120800, "members_executed_total": 36156 } }, "callback_ns_total": 3489229449, "cycles_total": 12052, "member_service_ns_total": 3529528400, "members_executed_total": 108468, "thread_cpu_ns_total": 1250000000, "wall_ns_total": 3590834800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.5369121, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0032018363919695, "failed": 0, "missed": 0, "callback_last_ms": 0.0041, "callback_ewma_ms": 0.004118, "release_lateness_last_ms": 0.0573 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0032018363919695, "failed": 0, "missed": 0, "callback_last_ms": 6.2998, "callback_ewma_ms": 6.855859, "release_lateness_last_ms": 0.5186 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 603, "hz": 4.9994273334244435, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.000953, "release_lateness_last_ms": 0.266 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 603, "hz": 4.9994273334244435, "failed": 0, "missed": 0, "callback_last_ms": 2.2577, "callback_ewma_ms": 2.618399, "release_lateness_last_ms": 0.0827 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0032018363919695, "failed": 0, "missed": 0, "callback_last_ms": 3.2371, "callback_ewma_ms": 2.900804, "release_lateness_last_ms": 0.0868 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0032018363919695, "failed": 0, "missed": 0, "callback_last_ms": 4.3277, "callback_ewma_ms": 3.595505, "release_lateness_last_ms": 0.2441 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.006403672783939, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.001124, "release_lateness_last_ms": 0.482 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0032018363919695, "failed": 0, "missed": 0, "callback_last_ms": 2.5973, "callback_ewma_ms": 2.212385, "release_lateness_last_ms": 0.4755 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.006403672783939, "failed": 0, "missed": 0, "callback_last_ms": 4.2464, "callback_ewma_ms": 5.576422, "release_lateness_last_ms": 0.2744 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 603, "hz": 4.9994273334244435, "failed": 0, "missed": 0, "callback_last_ms": 5.1428, "callback_ewma_ms": 6.239854, "release_lateness_last_ms": 0.4905 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1206, "hz": 9.998854666848887, "failed": 0, "missed": 0, "callback_last_ms": 0.0153, "callback_ewma_ms": 0.332495, "release_lateness_last_ms": 0.5373 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 603, "hz": 4.9994273334244435, "failed": 0, "missed": 0, "callback_last_ms": 13.075, "callback_ewma_ms": 15.463772, "release_lateness_last_ms": 0.4542 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.006403672783939, "failed": 0, "missed": 0, "callback_last_ms": 5.3072, "callback_ewma_ms": 7.551738, "release_lateness_last_ms": 0.2242 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 603, "hz": 4.9994273334244435, "failed": 0, "missed": 0, "callback_last_ms": 0.0016, "callback_ewma_ms": 0.00214, "release_lateness_last_ms": 0.4193 }, { "name": "general_program_checks_thread", "running": true, "completed": 1206, "hz": 9.998854666848887, "failed": 0, "missed": 0, "callback_last_ms": 0.1213, "callback_ewma_ms": 0.211223, "release_lateness_last_ms": 0.5443 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0032018363919695, "failed": 0, "missed": 0, "callback_last_ms": 0.0932, "callback_ewma_ms": 0.117177, "release_lateness_last_ms": 0.3859 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1206, "hz": 9.998854666848887, "failed": 0, "missed": 0, "callback_last_ms": 0.0034, "callback_ewma_ms": 0.0056, "release_lateness_last_ms": 0.2039 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 1.9981127485162369, "failed": 0, "missed": 0, "callback_last_ms": 40.0003, "callback_ewma_ms": 40.064581, "release_lateness_last_ms": 0.3779 } ], "peer_counter_seconds": 120.61097099999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.01309999999648, "gap_p95_ms": 258.2129999999888, "gap_p99_ms": 280.9968999999983, "gap_max_ms": 320.36539999999513, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 37, "p95": 92, "p99": 110, "max": 142, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.6261999999988, "gap_p95_ms": 254.5263999999952, "gap_p99_ms": 276.5329999999958, "gap_max_ms": 297.35599999999977, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 40, "p95": 87, "p99": 109, "max": 130, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.05169999999634, "gap_p95_ms": 257.05199999999604, "gap_p99_ms": 277.8808999999965, "gap_max_ms": 312.0346000000136, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 37, "p95": 92, "p99": 104, "max": 143, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.31120000000374, "gap_p95_ms": 251.20349999999598, "gap_p99_ms": 276.03749999999854, "gap_max_ms": 290.95369999998366, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 31, "p95": 82, "p99": 102, "max": 116, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.3168000000005, "gap_p95_ms": 252.92639999999665, "gap_p99_ms": 275.48829999999725, "gap_max_ms": 308.9667999999947, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 48, "p95": 94, "p99": 115, "max": 151, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.27500000000873, "gap_p95_ms": 254.2826999999961, "gap_p99_ms": 276.0170000000071, "gap_max_ms": 288.6417999999976, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 42, "p95": 90, "p99": 108, "max": 120, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.10269999998854, "gap_p95_ms": 253.58299999999872, "gap_p99_ms": 274.1612000000023, "gap_max_ms": 329.01690000000235, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 36, "p95": 81, "p99": 106, "max": 148, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.63650000002235, "gap_p95_ms": 258.0771999999997, "gap_p99_ms": 279.83150000000023, "gap_max_ms": 299.22279999998864, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 47, "p95": 93, "p99": 117, "max": 139, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.94970000000467, "gap_p95_ms": 251.08789999998407, "gap_p99_ms": 273.54549999997835, "gap_max_ms": 283.4207000000024, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 48, "p95": 98, "p99": 116, "max": 165, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.45140000000538, "gap_p95_ms": 247.10059999999066, "gap_p99_ms": 270.2155999999959, "gap_max_ms": 330.8585000000021, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 50, "p95": 93, "p99": 109, "max": 145, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.00440000000526, "gap_p95_ms": 246.05539999999746, "gap_p99_ms": 275.39970000000176, "gap_max_ms": 285.51290000000154, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 43, "p95": 87, "p99": 107, "max": 126, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.72759999999835, "gap_p95_ms": 249.73530000000028, "gap_p99_ms": 265.96970000000147, "gap_max_ms": 272.3145000000077, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 58, "p95": 101, "p99": 115, "max": 138, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.70420000000013, "gap_p95_ms": 254.76949999999488, "gap_p99_ms": 278.70940000002156, "gap_max_ms": 290.66380000000936, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 41, "p95": 90, "p99": 105, "max": 125, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.83239999999932, "gap_p95_ms": 244.95700000002398, "gap_p99_ms": 263.84539999999106, "gap_max_ms": 290.7954000000027, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 56, "p95": 87, "p99": 110, "max": 134, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.1811000000016, "gap_p95_ms": 251.85599999999977, "gap_p99_ms": 283.7335999999923, "gap_max_ms": 305.06679999998596, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 54, "p95": 93, "p99": 117, "max": 144, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.08000000000175, "gap_p95_ms": 249.3298000000068, "gap_p99_ms": 267.834400000007, "gap_max_ms": 287.2511000000013, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 60, "p95": 100, "p99": 120, "max": 171, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.1246000000101, "gap_p95_ms": 249.31979999999749, "gap_p99_ms": 268.72149999999965, "gap_max_ms": 319.76849999999104, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 59, "p95": 88, "p99": 113, "max": 136, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.83799999998882, "gap_p95_ms": 252.56589999998687, "gap_p99_ms": 272.87140000000363, "gap_max_ms": 304.1914000000106, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 61, "p95": 96, "p99": 115, "max": 154, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.20190000001458, "gap_p95_ms": 260.26130000001285, "gap_p99_ms": 275.9311999999991, "gap_max_ms": 307.69079999999667, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 43, "p95": 96, "p99": 110, "max": 135, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.34940000000643, "gap_p95_ms": 246.8531999999832, "gap_p99_ms": 272.758600000001, "gap_max_ms": 293.4985000000015, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 61, "p95": 97, "p99": 112, "max": 149, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.0619000000006, "gap_p95_ms": 249.25469999999768, "gap_p99_ms": 265.22570000000997, "gap_max_ms": 280.1984000000084, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 63, "p95": 89, "p99": 109, "max": 138, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.28629999997793, "gap_p95_ms": 251.3341000000073, "gap_p99_ms": 271.5987000000023, "gap_max_ms": 284.9186000000045, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 49, "p95": 91, "p99": 115, "max": 141, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.56010000000242, "gap_p95_ms": 247.29739999999583, "gap_p99_ms": 273.6426999999967, "gap_max_ms": 299.30049999999756, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 67, "p95": 102, "p99": 119, "max": 176, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.2511000000086, "gap_p95_ms": 253.33870000000024, "gap_p99_ms": 279.6820000000007, "gap_max_ms": 335.7804999999935, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 63, "p95": 92, "p99": 111, "max": 157, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.50120000001334, "gap_p95_ms": 250.28019999999378, "gap_p99_ms": 273.4985999999917, "gap_max_ms": 315.84550000001036, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 48, "p95": 92, "p99": 114, "max": 140, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.10130000000936, "gap_p95_ms": 248.89549999999872, "gap_p99_ms": 260.7097999999969, "gap_max_ms": 284.2526999999973, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 72, "p95": 106, "p99": 121, "max": 134, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.1932000000088, "gap_p95_ms": 246.43740000000253, "gap_p99_ms": 274.13510000001406, "gap_max_ms": 290.44429999999556, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 50, "p95": 93, "p99": 116, "max": 154, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.41710000000603, "gap_p95_ms": 252.61219999998866, "gap_p99_ms": 266.9181000000099, "gap_max_ms": 302.2914999999921, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 68, "p95": 91, "p99": 110, "max": 143, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.33740000000398, "gap_p95_ms": 252.6490999999951, "gap_p99_ms": 272.5991000000213, "gap_max_ms": 298.18649999999616, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 70, "p95": 99, "p99": 121, "max": 143, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.73059999999532, "gap_p95_ms": 254.51989999999932, "gap_p99_ms": 277.0162999999884, "gap_max_ms": 333.63300000000163, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 70, "p95": 93, "p99": 113, "max": 161, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.4643999999971, "gap_p95_ms": 254.4395000000004, "gap_p99_ms": 274.94749999999476, "gap_max_ms": 317.4288000000088, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 53, "p95": 95, "p99": 117, "max": 145, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000949237485124, "complete_hz": 5.000949237485124, "catalog_hz": 2.002041139923447, "bytes_per_second": 1550954.6630581499, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.48610000000917, "gap_p95_ms": 250.44330000001355, "gap_p99_ms": 274.3350000000064, "gap_max_ms": 296.3429999999935, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1204, "p50": 52, "p95": 98, "p99": 117, "max": 159, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 107, "mean": 81.26479532710202, "p95": 143.1146000000008, "max": 174.33280000000377, "p50": 66.52839999998105, "p99": 158.9591000000073, "min": 51.09429999999702 }, { "operation": "rapid/get-persistence-diagnostics", "count": 107, "mean": 18.742838317757187, "p95": 65.767399999997, "max": 99.61749999999302, "p50": 15.185200000007171, "p99": 77.59509999999864, "min": 4.8320999999996275 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2218294, "accepted_rows_total": 13977513, "committed_rows_total": 2220452, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 107, "minimum": 27.734375, "maximum": 69.921875, "mean": 40.63445732761793 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 107, "minimum": 1.952419395630419, "maximum": 26.179524657104807, "mean": 10.223494458925115 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 107, "minimum": 993808384, "maximum": 1034231808, "mean": 1023650844.7102804 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0.3814697265625, "mean": 0.018909489997079438 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0.01220703125, "mean": 0.00011408440420560748 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 107, "minimum": 0, "maximum": 0.06103515625, "mean": 0.0013690128504672897 }, { "channel": "engine_tempest_tx_mbps", "observations": 107, "minimum": 743.3509720338293, "maximum": 1190.5959112866508, "mean": 955.3270159234299 }, { "channel": "engine_tempest_rx_mbps", "observations": 107, "minimum": 0.030082360657677807, "maximum": 0.06021283364043959, "mean": 0.04367210070932255 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 107, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 107, "minimum": 12488.343380288838, "maximum": 39427.414058682494, "mean": 18374.274160168927 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 107, "minimum": 9872.55483758187, "maximum": 41251.18530418775, "mean": 18174.159639850805 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7713126000000048 }, { "id": "verified-capacity-stable-release-25-web-32-soak", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_3: control callbacks below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz", "client transport warning/error", "client 0: complete snapshots below 4.75 Hz", "client 1: complete snapshots below 4.75 Hz", "client 2: complete snapshots below 4.75 Hz", "client 3: complete snapshots below 4.75 Hz", "client 4: complete snapshots below 4.75 Hz", "client 5: complete snapshots below 4.75 Hz", "client 6: complete snapshots below 4.75 Hz", "client 7: complete snapshots below 4.75 Hz", "client 8: complete snapshots below 4.75 Hz", "client 9: complete snapshots below 4.75 Hz", "client 10: complete snapshots below 4.75 Hz", "client 11: complete snapshots below 4.75 Hz", "client 12: complete snapshots below 4.75 Hz", "client 13: complete snapshots below 4.75 Hz", "client 14: complete snapshots below 4.75 Hz", "client 15: complete snapshots below 4.75 Hz", "client 16: complete snapshots below 4.75 Hz", "client 17: complete snapshots below 4.75 Hz", "client 18: complete snapshots below 4.75 Hz", "client 19: complete snapshots below 4.75 Hz", "client 20: complete snapshots below 4.75 Hz", "client 21: complete snapshots below 4.75 Hz", "client 22: complete snapshots below 4.75 Hz", "client 23: complete snapshots below 4.75 Hz", "client 24: complete snapshots below 4.75 Hz", "client 25: complete snapshots below 4.75 Hz", "client 26: complete snapshots below 4.75 Hz", "client 27: complete snapshots below 4.75 Hz", "client 28: complete snapshots below 4.75 Hz", "client 29: complete snapshots below 4.75 Hz", "client 30: complete snapshots below 4.75 Hz", "client 31: complete snapshots below 4.75 Hz", "engine_tempest_dropped_messages_total increased" ], "seconds": 600.1926337, "measurement_completed": true, "background_preflight": { "enabled": true, "limit_percent": 45, "required_stable_seconds": 20, "stable_seconds": 20.013658599999967, "wait_seconds": 20.013658599999967, "completed_at": "2026-10-30T00:16:43.751Z", "accepted_cpu": { "count": 20, "mean": 21.970427546545505, "p95": 28.254087108448765, "max": 29.57290499315154 } }, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 482, "mean": 49.47362183605289, "p50": 45.49228573724202, "p95": 95.74414052711535, "p99": 100, "min": 17.484345478334586, "max": 100 }, "engine_cpu_percent": { "count": 482, "mean": 11.379426222523184, "p50": 9.127922500380492, "p95": 21.305587880961635, "p99": 25.016816399589473, "min": 2.9726117514009576, "max": 26.69737666044061 }, "client_generator_cpu_percent": { "count": 482, "mean": 6.958524738924697, "p50": 6.955754810911645, "p95": 9.438141586687902, "p99": 10.719197377741043, "min": 0, "max": 12.777721563374882 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 482, "mean": 1059035017.0290457, "p50": 1049604096, "p95": 1121918976, "p99": 1333149696, "min": 996823040, "max": 1370484736 }, "system_samples": 482, "system_sample_interval_seconds": { "count": 482, "mean": 1.243827436721992, "p95": 1.657409, "max": 7.2817945, "p50": 1.1290607, "p99": 3.970608, "min": 1.1014649 }, "achieved_read_hz": { "minimum": 94.29608095469094, "maximum": 98.1945538989634 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 235496, "hz": 98.0223889051834 }, { "group": "stress_group_1", "controls": 3, "completed": 176877, "hz": 98.16390983219601 }, { "group": "stress_group_2", "controls": 3, "completed": 176757, "hz": 98.09731174889596 }, { "group": "stress_group_3", "controls": 3, "completed": 169899, "hz": 94.29123128829792 }, { "group": "stress_group_4", "controls": 3, "completed": 169932, "hz": 94.30954576120543 }, { "group": "stress_group_5", "controls": 3, "completed": 170040, "hz": 94.36948403617549 }, { "group": "stress_group_6", "controls": 3, "completed": 169938, "hz": 94.31287566537044 }, { "group": "stress_group_7", "controls": 3, "completed": 176922, "hz": 98.18888411343353 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 166770, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 4097341328, "member_service_ns_total": 4218610400, "members_executed_total": 235496 }, "example_device_plugin.stress_read": { "callback_ns_total": 23070443490, "member_service_ns_total": 23224556800, "members_executed_total": 235496 }, "example_device_plugin.stress_write": { "callback_ns_total": 1263171343, "member_service_ns_total": 1338796900, "members_executed_total": 235496 } }, "callback_ns_total": 28430956161, "cycles_total": 58874, "member_service_ns_total": 28781964100, "members_executed_total": 706488, "thread_cpu_ns_total": 8750000000, "wall_ns_total": 29228416000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3296555400, "member_service_ns_total": 3428069400, "members_executed_total": 176877 }, "example_device_plugin.stress_read": { "callback_ns_total": 18820556907, "member_service_ns_total": 18965729500, "members_executed_total": 176877 }, "example_device_plugin.stress_write": { "callback_ns_total": 962812651, "member_service_ns_total": 1032865900, "members_executed_total": 176877 } }, "callback_ns_total": 23079924958, "cycles_total": 58959, "member_service_ns_total": 23426664800, "members_executed_total": 530631, "thread_cpu_ns_total": 6921875000, "wall_ns_total": 23861493200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3203832727, "member_service_ns_total": 3350283900, "members_executed_total": 176757 }, "example_device_plugin.stress_read": { "callback_ns_total": 18221555213, "member_service_ns_total": 18330136700, "members_executed_total": 176757 }, "example_device_plugin.stress_write": { "callback_ns_total": 927259148, "member_service_ns_total": 986000800, "members_executed_total": 176757 } }, "callback_ns_total": 22352647088, "cycles_total": 58919, "member_service_ns_total": 22666421400, "members_executed_total": 530271, "thread_cpu_ns_total": 6890625000, "wall_ns_total": 23068728600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3470203780, "member_service_ns_total": 3641122200, "members_executed_total": 169899 }, "example_device_plugin.stress_read": { "callback_ns_total": 22321054271, "member_service_ns_total": 22525442000, "members_executed_total": 169899 }, "example_device_plugin.stress_write": { "callback_ns_total": 1154884259, "member_service_ns_total": 1218215500, "members_executed_total": 169899 } }, "callback_ns_total": 26946142310, "cycles_total": 56633, "member_service_ns_total": 27384779700, "members_executed_total": 509697, "thread_cpu_ns_total": 6828125000, "wall_ns_total": 27839382500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3118154446, "member_service_ns_total": 3430058300, "members_executed_total": 169932 }, "example_device_plugin.stress_read": { "callback_ns_total": 22600410001, "member_service_ns_total": 22798729200, "members_executed_total": 169932 }, "example_device_plugin.stress_write": { "callback_ns_total": 1078533746, "member_service_ns_total": 1182363600, "members_executed_total": 169932 } }, "callback_ns_total": 26797098193, "cycles_total": 56644, "member_service_ns_total": 27411151100, "members_executed_total": 509796, "thread_cpu_ns_total": 4843750000, "wall_ns_total": 27924074700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3198962454, "member_service_ns_total": 3412206500, "members_executed_total": 170040 }, "example_device_plugin.stress_read": { "callback_ns_total": 21870176581, "member_service_ns_total": 22005354200, "members_executed_total": 170040 }, "example_device_plugin.stress_write": { "callback_ns_total": 1064951944, "member_service_ns_total": 1137438700, "members_executed_total": 170040 } }, "callback_ns_total": 26134090979, "cycles_total": 56680, "member_service_ns_total": 26554999400, "members_executed_total": 510120, "thread_cpu_ns_total": 4078125000, "wall_ns_total": 27123814300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3297489670, "member_service_ns_total": 3547670200, "members_executed_total": 169938 }, "example_device_plugin.stress_read": { "callback_ns_total": 23259279715, "member_service_ns_total": 23401547500, "members_executed_total": 169938 }, "example_device_plugin.stress_write": { "callback_ns_total": 1168058550, "member_service_ns_total": 1248837200, "members_executed_total": 169938 } }, "callback_ns_total": 27724827935, "cycles_total": 56646, "member_service_ns_total": 28198054900, "members_executed_total": 509814, "thread_cpu_ns_total": 6281250000, "wall_ns_total": 28691081700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 3348337905, "member_service_ns_total": 3440338300, "members_executed_total": 176922 }, "example_device_plugin.stress_read": { "callback_ns_total": 18888469847, "member_service_ns_total": 19035963800, "members_executed_total": 176922 }, "example_device_plugin.stress_write": { "callback_ns_total": 988067651, "member_service_ns_total": 1061032700, "members_executed_total": 176922 } }, "callback_ns_total": 23224875403, "cycles_total": 58974, "member_service_ns_total": 23537334800, "members_executed_total": 530766, "thread_cpu_ns_total": 7031250000, "wall_ns_total": 23928385000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 600.6178859500001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 601, "hz": 1.0003606083608982, "failed": 0, "missed": 0, "callback_last_ms": 0.0058, "callback_ewma_ms": 0.015624, "release_lateness_last_ms": 13.3562 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 601, "hz": 1.0003606083608982, "failed": 0, "missed": 0, "callback_last_ms": 8.6034, "callback_ewma_ms": 44.322175, "release_lateness_last_ms": 12.7048 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 2999, "hz": 4.991816080656129, "failed": 0, "missed": 4, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.001511, "release_lateness_last_ms": 1.2388 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 2998, "hz": 4.990151587131402, "failed": 0, "missed": 6, "callback_last_ms": 0.0125, "callback_ewma_ms": 0.01402, "release_lateness_last_ms": 3.5445 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 601, "hz": 1.0003606083608982, "failed": 0, "missed": 0, "callback_last_ms": 0.0144, "callback_ewma_ms": 0.042561, "release_lateness_last_ms": 0.8762 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 601, "hz": 1.0003606083608982, "failed": 0, "missed": 0, "callback_last_ms": 0.0084, "callback_ewma_ms": 0.0525, "release_lateness_last_ms": 1.2248 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 1202, "hz": 2.0007212167217965, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.001828, "release_lateness_last_ms": 3.3503 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 601, "hz": 1.0003606083608982, "failed": 0, "missed": 0, "callback_last_ms": 7.1294, "callback_ewma_ms": 24.457606, "release_lateness_last_ms": 8.6401 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 1202, "hz": 2.0007212167217965, "failed": 0, "missed": 0, "callback_last_ms": 7.4402, "callback_ewma_ms": 16.590858, "release_lateness_last_ms": 3.5308 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 2980, "hz": 4.960190703686317, "failed": 0, "missed": 24, "callback_last_ms": 10.5476, "callback_ewma_ms": 14.050655, "release_lateness_last_ms": 2.4773 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 5957, "hz": 9.915387926798452, "failed": 0, "missed": 51, "callback_last_ms": 0.0169, "callback_ewma_ms": 2.083009, "release_lateness_last_ms": 3.5899 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 2978, "hz": 4.956861716636863, "failed": 0, "missed": 26, "callback_last_ms": 0.4707, "callback_ewma_ms": 0.736347, "release_lateness_last_ms": 4.2348 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 1202, "hz": 2.0007212167217965, "failed": 0, "missed": 0, "callback_last_ms": 9.1669, "callback_ewma_ms": 21.185365, "release_lateness_last_ms": 3.1767 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 2999, "hz": 4.991816080656129, "failed": 0, "missed": 4, "callback_last_ms": 0.0172, "callback_ewma_ms": 0.003505, "release_lateness_last_ms": 1.441 }, { "name": "general_program_checks_thread", "running": true, "completed": 5973, "hz": 9.942019823194084, "failed": 0, "missed": 35, "callback_last_ms": 0.2909, "callback_ewma_ms": 0.605459, "release_lateness_last_ms": 11.7931 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 601, "hz": 1.0003606083608982, "failed": 0, "missed": 0, "callback_last_ms": 0.0928, "callback_ewma_ms": 1.105109, "release_lateness_last_ms": 8.474 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 5995, "hz": 9.978638680738078, "failed": 0, "missed": 12, "callback_last_ms": 0.0075, "callback_ewma_ms": 0.009683, "release_lateness_last_ms": 0.2902 }, { "name": "engine_diagnostics", "running": true, "completed": 1187, "hz": 1.9757538138508919, "failed": 0, "missed": 14, "callback_last_ms": 118.4256, "callback_ewma_ms": 141.689319, "release_lateness_last_ms": 4.2495 } ], "peer_counter_seconds": 600.7808838999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.5101493772470045, "complete_hz": 4.5101493772470045, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1398625.9855746862, "stale_episodes": 0, "sequence_gaps": 12, "gap_p50_ms": 198.60839999996824, "gap_p95_ms": 277.45569999999134, "gap_p99_ms": 895.5013000000035, "gap_max_ms": 4730.713499999954, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5414, "p50": 61, "p95": 1169, "p99": 5076, "max": 7433, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.518479908050933, "complete_hz": 4.518479908050933, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1401225.1361771044, "stale_episodes": 0, "sequence_gaps": 7, "gap_p50_ms": 197.8018999999622, "gap_p95_ms": 288.7406000000192, "gap_p99_ms": 888.2942000000039, "gap_max_ms": 7300.953899999964, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5424, "p50": 52, "p95": 938, "p99": 3016, "max": 7318, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.515147695729362, "complete_hz": 4.515147695729362, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1400185.5809008253, "stale_episodes": 0, "sequence_gaps": 9, "gap_p50_ms": 197.7798000000039, "gap_p95_ms": 284.01580000000104, "gap_p99_ms": 704.6638000000385, "gap_max_ms": 12894.403700000024, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5420, "p50": 49, "p95": 1199, "p99": 10529, "max": 12768, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.5101493772470045, "complete_hz": 4.5101493772470045, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1398625.9855746862, "stale_episodes": 0, "sequence_gaps": 12, "gap_p50_ms": 198.07069999998203, "gap_p95_ms": 281.22120000002906, "gap_p99_ms": 936.8089999999502, "gap_max_ms": 5163.737700000056, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5414, "p50": 65, "p95": 1318, "p99": 5052, "max": 7985, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.508483271086219, "complete_hz": 4.508483271086219, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1398106.4853432225, "stale_episodes": 0, "sequence_gaps": 13, "gap_p50_ms": 198.4472999999998, "gap_p95_ms": 298.289700000023, "gap_p99_ms": 870.9977000000654, "gap_max_ms": 2931.6501000000862, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5412, "p50": 55, "p95": 863, "p99": 2363, "max": 3629, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.508483271086219, "complete_hz": 4.508483271086219, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1398106.4853432225, "stale_episodes": 0, "sequence_gaps": 13, "gap_p50_ms": 198.2476999999999, "gap_p95_ms": 280.0634999999893, "gap_p99_ms": 836.7795999999507, "gap_max_ms": 12882.705799999996, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5412, "p50": 58, "p95": 926, "p99": 9693, "max": 12739, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.516813801890148, "complete_hz": 4.516813801890148, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1400705.379365292, "stale_episodes": 0, "sequence_gaps": 8, "gap_p50_ms": 198.49100000000908, "gap_p95_ms": 281.062099999981, "gap_p99_ms": 873.981599999941, "gap_max_ms": 2789.8380999999354, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5422, "p50": 61, "p95": 879, "p99": 2651, "max": 3632, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.513481589568576, "complete_hz": 4.513481589568576, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1399665.774105828, "stale_episodes": 0, "sequence_gaps": 10, "gap_p50_ms": 197.30239999998594, "gap_p95_ms": 290.48100000002887, "gap_p99_ms": 878.1298999999417, "gap_max_ms": 9999.569099999964, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5418, "p50": 49, "p95": 974, "p99": 9688, "max": 11793, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.508483271086219, "complete_hz": 4.508483271086219, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1398106.4853432225, "stale_episodes": 0, "sequence_gaps": 13, "gap_p50_ms": 198.18969999998808, "gap_p95_ms": 289.39850000001024, "gap_p99_ms": 883.3151000000071, "gap_max_ms": 3331.9497000000556, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5412, "p50": 62, "p95": 985, "p99": 2585, "max": 3758, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.511815483407791, "complete_hz": 4.511815483407791, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1399145.8840055226, "stale_episodes": 0, "sequence_gaps": 11, "gap_p50_ms": 197.7858000000124, "gap_p95_ms": 282.2240999999922, "gap_p99_ms": 842.2145000000019, "gap_max_ms": 5176.810600000026, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5416, "p50": 53, "p95": 1065, "p99": 3370, "max": 7432, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.505151058764647, "complete_hz": 4.505151058764647, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1397077.0416652332, "stale_episodes": 0, "sequence_gaps": 15, "gap_p50_ms": 197.71940000000177, "gap_p95_ms": 292.7235999999684, "gap_p99_ms": 840.6393999999855, "gap_max_ms": 10517.630599999917, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5408, "p50": 65, "p95": 1002, "p99": 9740, "max": 13503, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.503484952603862, "complete_hz": 4.503484952603862, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1396557.2598618283, "stale_episodes": 0, "sequence_gaps": 16, "gap_p50_ms": 198.65480000001844, "gap_p95_ms": 291.4848999999813, "gap_p99_ms": 1002.1826000000583, "gap_max_ms": 4285.165800000075, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5406, "p50": 56, "p95": 1096, "p99": 5702, "max": 7667, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.503484952603862, "complete_hz": 4.503484952603862, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1396557.2598618283, "stale_episodes": 0, "sequence_gaps": 16, "gap_p50_ms": 198.19170000000304, "gap_p95_ms": 284.2041999999783, "gap_p99_ms": 958.7632999999914, "gap_max_ms": 4347.73749999993, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5406, "p50": 61, "p95": 979, "p99": 5690, "max": 7541, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.503484952603862, "complete_hz": 4.503484952603862, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1396557.2598618283, "stale_episodes": 0, "sequence_gaps": 16, "gap_p50_ms": 198.2722000000067, "gap_p95_ms": 293.9550000000163, "gap_p99_ms": 896.2256000000052, "gap_max_ms": 2701.4238999999943, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5406, "p50": 67, "p95": 893, "p99": 2570, "max": 3801, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.503484952603862, "complete_hz": 4.503484952603862, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1396557.2598618283, "stale_episodes": 0, "sequence_gaps": 16, "gap_p50_ms": 197.15160000004107, "gap_p95_ms": 287.16659999999683, "gap_p99_ms": 837.07779999997, "gap_max_ms": 3392.470499999996, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5406, "p50": 61, "p95": 805, "p99": 2704, "max": 3869, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.501818846443076, "complete_hz": 4.501818846443076, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1396042.5380228336, "stale_episodes": 0, "sequence_gaps": 17, "gap_p50_ms": 198.6850999999442, "gap_p95_ms": 287.1476000000257, "gap_p99_ms": 961.553099999961, "gap_max_ms": 2790.091299999971, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5404, "p50": 70, "p95": 908, "p99": 3534, "max": 4761, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.501818846443076, "complete_hz": 4.501818846443076, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1396042.5380228336, "stale_episodes": 0, "sequence_gaps": 17, "gap_p50_ms": 198.70609999998123, "gap_p95_ms": 282.2649000000092, "gap_p99_ms": 981.5755999999819, "gap_max_ms": 3532.0495999999694, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5404, "p50": 60, "p95": 967, "p99": 3688, "max": 6390, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.498486634121504, "complete_hz": 4.498486634121504, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1395007.9177530026, "stale_episodes": 0, "sequence_gaps": 19, "gap_p50_ms": 198.3643999999622, "gap_p95_ms": 294.84539999999106, "gap_p99_ms": 915.6601999999839, "gap_max_ms": 3109.7334000000264, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5400, "p50": 70, "p95": 915, "p99": 3068, "max": 4445, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.5101493772470045, "complete_hz": 4.5101493772470045, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1398625.9855746862, "stale_episodes": 0, "sequence_gaps": 12, "gap_p50_ms": 197.54129999998258, "gap_p95_ms": 296.5180999999866, "gap_p99_ms": 946.7969999999623, "gap_max_ms": 4527.178300000029, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5414, "p50": 56, "p95": 925, "p99": 3463, "max": 6561, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.496820527960718, "complete_hz": 4.496820527960718, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1394488.0359832281, "stale_episodes": 0, "sequence_gaps": 20, "gap_p50_ms": 196.96270000000368, "gap_p95_ms": 283.5742999999784, "gap_p99_ms": 933.7333999999973, "gap_max_ms": 3020.56879999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5398, "p50": 65, "p95": 848, "p99": 4153, "max": 6982, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.491822209478362, "complete_hz": 4.491822209478362, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1392933.517282561, "stale_episodes": 0, "sequence_gaps": 23, "gap_p50_ms": 198.37390000000596, "gap_p95_ms": 292.12140000000363, "gap_p99_ms": 1018.795299999998, "gap_max_ms": 3245.5960000000196, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5392, "p50": 71, "p95": 776, "p99": 2388, "max": 3566, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.518479908050933, "complete_hz": 4.518479908050933, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1401225.1361771044, "stale_episodes": 0, "sequence_gaps": 7, "gap_p50_ms": 198.64909999998054, "gap_p95_ms": 288.5564000000013, "gap_p99_ms": 854.2015999999712, "gap_max_ms": 4605.771399999969, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5424, "p50": 45, "p95": 945, "p99": 3459, "max": 7249, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.491822209478362, "complete_hz": 4.491822209478362, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1392933.517282561, "stale_episodes": 0, "sequence_gaps": 23, "gap_p50_ms": 198.25929999997607, "gap_p95_ms": 285.53570000000764, "gap_p99_ms": 905.2530000000261, "gap_max_ms": 3143.570299999963, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5392, "p50": 69, "p95": 790, "p99": 2579, "max": 3709, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.493488315639147, "complete_hz": 4.493488315639147, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1393453.4073828664, "stale_episodes": 0, "sequence_gaps": 22, "gap_p50_ms": 198.14190000000235, "gap_p95_ms": 279.7663000000175, "gap_p99_ms": 970.2527000000118, "gap_max_ms": 2774.2651000000187, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5394, "p50": 64, "p95": 985, "p99": 3671, "max": 5411, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.516813801890148, "complete_hz": 4.516813801890148, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1400705.379365292, "stale_episodes": 0, "sequence_gaps": 8, "gap_p50_ms": 197.93839999998454, "gap_p95_ms": 276.66969999999856, "gap_p99_ms": 912.2366000000038, "gap_max_ms": 4202.334199999925, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5422, "p50": 58, "p95": 1199, "p99": 4613, "max": 7357, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.495154421799933, "complete_hz": 4.495154421799933, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1393968.129221861, "stale_episodes": 0, "sequence_gaps": 21, "gap_p50_ms": 198.7485999999917, "gap_p95_ms": 284.7452999999514, "gap_p99_ms": 993.0578000000678, "gap_max_ms": 7287.227600000042, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5396, "p50": 83, "p95": 904, "p99": 3074, "max": 7338, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.495154421799933, "complete_hz": 4.495154421799933, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1393968.129221861, "stale_episodes": 0, "sequence_gaps": 21, "gap_p50_ms": 198.25779999996303, "gap_p95_ms": 289.3535999999731, "gap_p99_ms": 952.3016000000061, "gap_max_ms": 2672.4377000000095, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5396, "p50": 72, "p95": 1000, "p99": 5444, "max": 6513, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.495154421799933, "complete_hz": 4.495154421799933, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1393968.129221861, "stale_episodes": 0, "sequence_gaps": 21, "gap_p50_ms": 198.51459999999497, "gap_p95_ms": 288.13939999998547, "gap_p99_ms": 990.2447999999858, "gap_max_ms": 3082.6744999999646, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5396, "p50": 74, "p95": 912, "p99": 2729, "max": 3927, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.501818846443076, "complete_hz": 4.501818846443076, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1396042.5380228336, "stale_episodes": 0, "sequence_gaps": 17, "gap_p50_ms": 197.8640000000014, "gap_p95_ms": 281.8765999999887, "gap_p99_ms": 874.8335000000661, "gap_max_ms": 3006.7869000000646, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5404, "p50": 60, "p95": 876, "p99": 2694, "max": 3973, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.493488315639147, "complete_hz": 4.493488315639147, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1393448.264113148, "stale_episodes": 0, "sequence_gaps": 22, "gap_p50_ms": 198.81120000000374, "gap_p95_ms": 284.2961999999825, "gap_p99_ms": 933.7109000000055, "gap_max_ms": 6663.741399999999, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5394, "p50": 77, "p95": 867, "p99": 3695, "max": 6820, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.493488315639147, "complete_hz": 4.493488315639147, "catalog_hz": 1.8210540337388164, "bytes_per_second": 1393448.264113148, "stale_episodes": 0, "sequence_gaps": 22, "gap_p50_ms": 198.2743000000046, "gap_p95_ms": 282.39669999998296, "gap_p99_ms": 815.0279000000446, "gap_max_ms": 12953.15969999996, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5394, "p50": 68, "p95": 991, "p99": 14174, "max": 16823, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.516813801890148, "complete_hz": 4.516813801890148, "catalog_hz": 1.8227201398996022, "bytes_per_second": 1400705.379365292, "stale_episodes": 0, "sequence_gaps": 8, "gap_p50_ms": 198.6767000000109, "gap_p95_ms": 279.3028999999515, "gap_p99_ms": 952.6107000000193, "gap_max_ms": 2569.865300000063, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 5422, "p50": 55, "p95": 911, "p99": 2807, "max": 3782, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 433, "mean": 127.46961501154829, "p95": 299.6146000000881, "max": 2081.2680999999866, "p50": 71.63750000001164, "p99": 1113.767499999958, "min": 50.84169999998994 }, { "operation": "rapid/get-persistence-diagnostics", "count": 433, "mean": 92.7024408775986, "p95": 169.62579999999434, "max": 12241.664500000072, "p50": 15.346599999989849, "p99": 1317.8300999999628, "min": 4.394200000002456 } ], "transport_event_count": 919, "transport_events": [ { "at": "2026-10-30T00:22:39.502Z", "client": 31, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:22:39.502Z", "client": 5, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:22:39.502Z", "client": 4, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:22:39.502Z", "client": 30, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 25, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 18, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 23, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 16, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 15, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 26, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 19, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.774Z", "client": 21, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 20, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 17, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 22, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 24, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 1, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 2, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 29, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 27, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 0, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:23:35.775Z", "client": 28, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.568Z", "client": 15, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 9, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 30, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 27, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 24, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 17, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 10, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 31, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 0, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 25, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 19, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 13, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 7, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 2, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 22, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 11, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.569Z", "client": 5, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 29, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 20, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 16, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 4, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 28, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 18, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 12, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 1, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 21, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 14, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 8, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 3, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 26, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:28.570Z", "client": 23, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-30T00:26:36.912Z", "client": 18, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.912Z", "client": 19, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 20, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 21, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 22, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 23, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 24, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 25, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.913Z", "client": 26, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.914Z", "client": 27, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.914Z", "client": 28, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.914Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.916Z", "client": 29, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.917Z", "client": 1, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.917Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.917Z", "client": 3, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.917Z", "client": 30, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.917Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 31, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 5, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 9, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.918Z", "client": 10, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.919Z", "client": 11, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.920Z", "client": 12, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.921Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.921Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.922Z", "client": 15, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.922Z", "client": 16, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:36.922Z", "client": 17, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.321Z", "client": 18, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.321Z", "client": 19, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.321Z", "client": 20, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.321Z", "client": 21, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.322Z", "client": 22, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.322Z", "client": 23, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.322Z", "client": 24, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.323Z", "client": 25, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.323Z", "client": 26, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.323Z", "client": 27, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.323Z", "client": 28, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.324Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.324Z", "client": 29, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-30T00:26:37.324Z", "client": 1, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." } ], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 10619482, "accepted_rows_total": 67126003, "committed_rows_total": 10619749, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 433, "minimum": 14.453125, "maximum": 100, "mean": 46.68453304594677 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 433, "minimum": 1.9520022083297686, "maximum": 27.348317168967217, "mean": 10.922461926336162 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 433, "minimum": 995938304, "maximum": 1236811776, "mean": 1049180467.4364896 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 433, "minimum": 0, "maximum": 1.141357421875, "mean": 0.3561464798643187 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 433, "minimum": 0, "maximum": 0.08544921875, "mean": 0.0006766021939953811 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 433, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 433, "minimum": 0, "maximum": 20.849609375, "mean": 0.1157271669312933 }, { "channel": "engine_tempest_tx_mbps", "observations": 433, "minimum": 0.1687880909991156, "maximum": 1194.8778546719034, "mean": 920.5911893686629 }, { "channel": "engine_tempest_rx_mbps", "observations": 433, "minimum": 0, "maximum": 0.09186003756689734, "mean": 0.04497717597361596 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 433, "minimum": 0, "maximum": 5084, "mean": 199.6027713625866 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 433, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 433, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 433, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 433, "minimum": 5437.025902032125, "maximum": 39834.57361786009, "mean": 17957.93903916323 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 433, "minimum": 0, "maximum": 41061.64044622568, "mean": 17955.245292873282 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7397676999999676 } ], "context_runs": [ { "id": "verified-background-release-25-alone", "healthy": true, "failure_reasons": [], "seconds": 120.31821280000001, "measurement_completed": true, "background_preflight": { "enabled": false, "accepted_cpu": null }, "requested_client_count": 0, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 0, "transport": "web", "actual_connections": { "loadClients": 0, "actualClients": 1, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 0, "recipient_channel_subscriptions": 0, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 107, "mean": 36.45128581106318, "p50": 37.48060764790825, "p95": 57.29137541973104, "p99": 67.21794363118386, "min": 13.50339720373589, "max": 69.49069981221469 }, "engine_cpu_percent": { "count": 107, "mean": 7.416653794848311, "p50": 5.135394207875998, "p95": 17.534709410194544, "p99": 18.192556883653186, "min": 2.308909479517867, "max": 18.374979067223848 }, "client_generator_cpu_percent": { "count": 107, "mean": 0.44979482088058415, "p50": 0.353360454530323, "p95": 1.05072682725374, "p99": 1.2268525040959768, "min": 0.1684020248336134, "max": 2.467570428013317 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 107, "mean": 1001888988.1121495, "p50": 1001336832, "p95": 1008271360, "p99": 1012727808, "min": 987508736, "max": 1013796864 }, "system_samples": 107, "system_sample_interval_seconds": { "count": 107, "mean": 1.1301900663551403, "p95": 1.1324443, "max": 2.319598, "p50": 1.1097954, "p99": 1.9194613, "min": 1.0985081 }, "achieved_read_hz": { "minimum": 99.98201221150534, "maximum": 99.9986317594772 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48168, "hz": 100.00471252068952 }, { "group": "stress_group_1", "controls": 3, "completed": 36126, "hz": 100.00471252068952 }, { "group": "stress_group_2", "controls": 3, "completed": 36126, "hz": 100.00471252068952 }, { "group": "stress_group_3", "controls": 3, "completed": 36120, "hz": 99.98810320122088 }, { "group": "stress_group_4", "controls": 3, "completed": 36123, "hz": 99.99640786095519 }, { "group": "stress_group_5", "controls": 3, "completed": 36123, "hz": 99.99640786095519 }, { "group": "stress_group_6", "controls": 3, "completed": 36123, "hz": 99.99640786095519 }, { "group": "stress_group_7", "controls": 3, "completed": 36126, "hz": 100.00471252068952 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 45, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 568861489, "member_service_ns_total": 582804700, "members_executed_total": 48168 }, "example_device_plugin.stress_read": { "callback_ns_total": 3405216885, "member_service_ns_total": 3427885200, "members_executed_total": 48168 }, "example_device_plugin.stress_write": { "callback_ns_total": 174890186, "member_service_ns_total": 187847500, "members_executed_total": 48168 } }, "callback_ns_total": 4148968560, "cycles_total": 12042, "member_service_ns_total": 4198537400, "members_executed_total": 144504, "thread_cpu_ns_total": 1515625000, "wall_ns_total": 4269065800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 459442179, "member_service_ns_total": 470505000, "members_executed_total": 36126 }, "example_device_plugin.stress_read": { "callback_ns_total": 2788984224, "member_service_ns_total": 2810365600, "members_executed_total": 36126 }, "example_device_plugin.stress_write": { "callback_ns_total": 136665991, "member_service_ns_total": 146086800, "members_executed_total": 36126 } }, "callback_ns_total": 3385092394, "cycles_total": 12042, "member_service_ns_total": 3426957400, "members_executed_total": 108378, "thread_cpu_ns_total": 1343750000, "wall_ns_total": 3492109000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 441259013, "member_service_ns_total": 451136800, "members_executed_total": 36126 }, "example_device_plugin.stress_read": { "callback_ns_total": 2650367758, "member_service_ns_total": 2668233300, "members_executed_total": 36126 }, "example_device_plugin.stress_write": { "callback_ns_total": 131907495, "member_service_ns_total": 141078000, "members_executed_total": 36126 } }, "callback_ns_total": 3223534266, "cycles_total": 12042, "member_service_ns_total": 3260448100, "members_executed_total": 108378, "thread_cpu_ns_total": 859375000, "wall_ns_total": 3320880200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 421180769, "member_service_ns_total": 431750700, "members_executed_total": 36120 }, "example_device_plugin.stress_read": { "callback_ns_total": 2679620063, "member_service_ns_total": 2698364000, "members_executed_total": 36120 }, "example_device_plugin.stress_write": { "callback_ns_total": 131323393, "member_service_ns_total": 141354800, "members_executed_total": 36120 } }, "callback_ns_total": 3232124225, "cycles_total": 12040, "member_service_ns_total": 3271469500, "members_executed_total": 108360, "thread_cpu_ns_total": 1265625000, "wall_ns_total": 3332581100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 407963570, "member_service_ns_total": 417782200, "members_executed_total": 36123 }, "example_device_plugin.stress_read": { "callback_ns_total": 2526391947, "member_service_ns_total": 2545110000, "members_executed_total": 36123 }, "example_device_plugin.stress_write": { "callback_ns_total": 132514794, "member_service_ns_total": 142548000, "members_executed_total": 36123 } }, "callback_ns_total": 3066870311, "cycles_total": 12041, "member_service_ns_total": 3105440200, "members_executed_total": 108369, "thread_cpu_ns_total": 718750000, "wall_ns_total": 3161073000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 402803456, "member_service_ns_total": 412405700, "members_executed_total": 36123 }, "example_device_plugin.stress_read": { "callback_ns_total": 2545446517, "member_service_ns_total": 2563227900, "members_executed_total": 36123 }, "example_device_plugin.stress_write": { "callback_ns_total": 133228189, "member_service_ns_total": 142696100, "members_executed_total": 36123 } }, "callback_ns_total": 3081478162, "cycles_total": 12041, "member_service_ns_total": 3118329700, "members_executed_total": 108369, "thread_cpu_ns_total": 625000000, "wall_ns_total": 3173661000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 421878278, "member_service_ns_total": 431542000, "members_executed_total": 36123 }, "example_device_plugin.stress_read": { "callback_ns_total": 2673815672, "member_service_ns_total": 2691372300, "members_executed_total": 36123 }, "example_device_plugin.stress_write": { "callback_ns_total": 132745692, "member_service_ns_total": 142209600, "members_executed_total": 36123 } }, "callback_ns_total": 3228439642, "cycles_total": 12041, "member_service_ns_total": 3265123900, "members_executed_total": 108369, "thread_cpu_ns_total": 1187500000, "wall_ns_total": 3321267100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 467412436, "member_service_ns_total": 479318500, "members_executed_total": 36126 }, "example_device_plugin.stress_read": { "callback_ns_total": 2702790168, "member_service_ns_total": 2722881900, "members_executed_total": 36126 }, "example_device_plugin.stress_write": { "callback_ns_total": 140502995, "member_service_ns_total": 151338100, "members_executed_total": 36126 } }, "callback_ns_total": 3310705599, "cycles_total": 12042, "member_service_ns_total": 3353538500, "members_executed_total": 108378, "thread_cpu_ns_total": 1000000000, "wall_ns_total": 3413686900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.41432545000002, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 120, "hz": 0.9959727427159637, "failed": 0, "missed": 0, "callback_last_ms": 0.0047, "callback_ewma_ms": 0.006649, "release_lateness_last_ms": 0.5757 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 120, "hz": 0.9959727427159637, "failed": 0, "missed": 0, "callback_last_ms": 8.0561, "callback_ewma_ms": 6.198018, "release_lateness_last_ms": 0.4197 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 602, "hz": 4.996463259291751, "failed": 0, "missed": 0, "callback_last_ms": 0.0009, "callback_ewma_ms": 0.001254, "release_lateness_last_ms": 0.1966 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 602, "hz": 4.996463259291751, "failed": 0, "missed": 0, "callback_last_ms": 0.0095, "callback_ewma_ms": 0.014411, "release_lateness_last_ms": 0.1838 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 120, "hz": 0.9959727427159637, "failed": 0, "missed": 0, "callback_last_ms": 0.0092, "callback_ewma_ms": 0.008578, "release_lateness_last_ms": 0.5946 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 120, "hz": 0.9959727427159637, "failed": 0, "missed": 0, "callback_last_ms": 0.0124, "callback_ewma_ms": 0.013167, "release_lateness_last_ms": 0.0996 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 240, "hz": 1.9919454854319274, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001581, "release_lateness_last_ms": 0.4236 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.9959727427159637, "failed": 0, "missed": 0, "callback_last_ms": 0.7176, "callback_ewma_ms": 1.837596, "release_lateness_last_ms": 0.094 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.000245258287894, "failed": 0, "missed": 0, "callback_last_ms": 3.6615, "callback_ewma_ms": 3.987184, "release_lateness_last_ms": 0.1712 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 4.996463259291751, "failed": 0, "missed": 0, "callback_last_ms": 3.3392, "callback_ewma_ms": 3.816232, "release_lateness_last_ms": 0.1404 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1204, "hz": 9.992926518583502, "failed": 0, "missed": 0, "callback_last_ms": 0.0233, "callback_ewma_ms": 0.836495, "release_lateness_last_ms": 0.5243 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 602, "hz": 4.996463259291751, "failed": 0, "missed": 0, "callback_last_ms": 0.3034, "callback_ewma_ms": 0.326688, "release_lateness_last_ms": 0.2921 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.000245258287894, "failed": 0, "missed": 0, "callback_last_ms": 4.6703, "callback_ewma_ms": 5.792717, "release_lateness_last_ms": 0.1284 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 602, "hz": 4.996463259291751, "failed": 0, "missed": 0, "callback_last_ms": 0.0021, "callback_ewma_ms": 0.002293, "release_lateness_last_ms": 0.0903 }, { "name": "general_program_checks_thread", "running": true, "completed": 1205, "hz": 10.001226291439469, "failed": 0, "missed": 0, "callback_last_ms": 0.3258, "callback_ewma_ms": 0.280414, "release_lateness_last_ms": 0.4148 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.9959727427159637, "failed": 0, "missed": 0, "callback_last_ms": 0.0695, "callback_ewma_ms": 0.110241, "release_lateness_last_ms": 0.5207 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1205, "hz": 10.001226291439469, "failed": 0, "missed": 0, "callback_last_ms": 0.0157, "callback_ewma_ms": 0.006457, "release_lateness_last_ms": 0.5026 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 2.000245258287894, "failed": 0, "missed": 0, "callback_last_ms": 42.3507, "callback_ewma_ms": 40.221073, "release_lateness_last_ms": 0.1674 } ], "peer_counter_seconds": 120.4747845, "peer_counter_deltas": [], "observations": [], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 110, "mean": 81.06976636363537, "p95": 79.60399999999208, "max": 1207.6731, "p50": 63.909700000003795, "p99": 804.6005999999907, "min": 51.167600000000675 }, { "operation": "rapid/get-persistence-diagnostics", "count": 110, "mean": 11.034894545453822, "p95": 16.92059999999765, "max": 17.6247999999905, "p50": 8.251700000007986, "p99": 17.221799999999348, "min": 4.610000000000582 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2224176, "accepted_rows_total": 13991705, "committed_rows_total": 2224963, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 110, "minimum": 12.5, "maximum": 70.3125, "mean": 37.12713068181818 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 110, "minimum": 0.7804699982837152, "maximum": 20.71505687275871, "mean": 6.953095670724621 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 110, "minimum": 986628096, "maximum": 1012776960, "mean": 1001587321.0181818 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0.0244140625, "mean": 0.00044389204545454544 }, { "channel": "engine_tempest_tx_mbps", "observations": 110, "minimum": 3.363789845271327, "maximum": 6.774388179418175, "mean": 4.335567552516604 }, { "channel": "engine_tempest_rx_mbps", "observations": 110, "minimum": 0, "maximum": 0.0031209194228619752, "mean": 0.0003019821557954982 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 110, "minimum": 12487.519972539445, "maximum": 39511.254266283286, "mean": 18462.437389267045 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 110, "minimum": 0, "maximum": 40874.96331529019, "mean": 18085.817472041574 } ], "graceful_shutdown": true, "shutdown_seconds": 1.6112755999999935 }, { "id": "verified-capacity-release-25-alone", "healthy": true, "failure_reasons": [], "seconds": 120.62522290000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 0, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 0, "transport": "web", "actual_connections": { "loadClients": 0, "actualClients": 1, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 0, "recipient_channel_subscriptions": 0, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 110, "mean": 36.10768560226962, "p50": 33.59896648111612, "p95": 63.23828189197578, "p99": 77.86290681785282, "min": 19.418388524649487, "max": 80.74880759918686 }, "engine_cpu_percent": { "count": 110, "mean": 8.43340554073439, "p50": 5.741226303892202, "p95": 19.676598016011244, "p99": 20.451457087285927, "min": 2.5077156247959436, "max": 20.46212498484956 }, "client_generator_cpu_percent": { "count": 110, "mean": 0.4687940933129271, "p50": 0.35824508925656334, "p95": 1.067525940595683, "p99": 1.2347834042581631, "min": 0, "max": 1.2562240950799062 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 110, "mean": 1003887671.8545455, "p50": 1003417600, "p95": 1012744192, "p99": 1013555200, "min": 994881536, "max": 1013555200 }, "system_samples": 110, "system_sample_interval_seconds": { "count": 110, "mean": 1.1033644090909092, "p95": 1.1447432, "max": 1.2524413, "p50": 1.0979185, "p99": 1.1704469, "min": 1.0847863 }, "achieved_read_hz": { "minimum": 98.8314461839116, "maximum": 99.48612561229929 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48027, "hz": 99.43570931656708 }, { "group": "stress_group_1", "controls": 3, "completed": 36020, "hz": 99.43501917901033 }, { "group": "stress_group_2", "controls": 3, "completed": 36015, "hz": 99.42121642787498 }, { "group": "stress_group_3", "controls": 3, "completed": 35811, "hz": 98.85806418155299 }, { "group": "stress_group_4", "controls": 3, "completed": 35802, "hz": 98.83321922950937 }, { "group": "stress_group_5", "controls": 3, "completed": 35811, "hz": 98.85806418155299 }, { "group": "stress_group_6", "controls": 3, "completed": 35811, "hz": 98.85806418155299 }, { "group": "stress_group_7", "controls": 3, "completed": 36039, "hz": 99.48746963332462 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 7641, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 652333911, "member_service_ns_total": 675408900, "members_executed_total": 48027 }, "example_device_plugin.stress_read": { "callback_ns_total": 4066116092, "member_service_ns_total": 4097532500, "members_executed_total": 48027 }, "example_device_plugin.stress_write": { "callback_ns_total": 209447990, "member_service_ns_total": 222677400, "members_executed_total": 48027 } }, "callback_ns_total": 4928101593, "cycles_total": 12007, "member_service_ns_total": 4995823500, "members_executed_total": 144084, "thread_cpu_ns_total": 1312500000, "wall_ns_total": 5081981500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 558678026, "member_service_ns_total": 571714000, "members_executed_total": 36020 }, "example_device_plugin.stress_read": { "callback_ns_total": 3314936163, "member_service_ns_total": 3338184000, "members_executed_total": 36020 }, "example_device_plugin.stress_write": { "callback_ns_total": 180160491, "member_service_ns_total": 191884300, "members_executed_total": 36020 } }, "callback_ns_total": 4053948780, "cycles_total": 12007, "member_service_ns_total": 4101957500, "members_executed_total": 108063, "thread_cpu_ns_total": 1218750000, "wall_ns_total": 4172157000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 493609961, "member_service_ns_total": 505004100, "members_executed_total": 36015 }, "example_device_plugin.stress_read": { "callback_ns_total": 3182028487, "member_service_ns_total": 3201787900, "members_executed_total": 36014 }, "example_device_plugin.stress_write": { "callback_ns_total": 163736994, "member_service_ns_total": 174745800, "members_executed_total": 36015 } }, "callback_ns_total": 3839585142, "cycles_total": 12005, "member_service_ns_total": 3881748200, "members_executed_total": 108045, "thread_cpu_ns_total": 1093750000, "wall_ns_total": 3948880000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 511113969, "member_service_ns_total": 526968600, "members_executed_total": 35811 }, "example_device_plugin.stress_read": { "callback_ns_total": 3570873839, "member_service_ns_total": 3608112100, "members_executed_total": 35811 }, "example_device_plugin.stress_write": { "callback_ns_total": 176893787, "member_service_ns_total": 188239100, "members_executed_total": 35811 } }, "callback_ns_total": 4258881595, "cycles_total": 11937, "member_service_ns_total": 4323319800, "members_executed_total": 107433, "thread_cpu_ns_total": 1312500000, "wall_ns_total": 4396185000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 556689138, "member_service_ns_total": 572520100, "members_executed_total": 35802 }, "example_device_plugin.stress_read": { "callback_ns_total": 3378982735, "member_service_ns_total": 3405053900, "members_executed_total": 35802 }, "example_device_plugin.stress_write": { "callback_ns_total": 165686390, "member_service_ns_total": 176662400, "members_executed_total": 35802 } }, "callback_ns_total": 4101358263, "cycles_total": 11934, "member_service_ns_total": 4154236400, "members_executed_total": 107406, "thread_cpu_ns_total": 718750000, "wall_ns_total": 4227543300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 502560659, "member_service_ns_total": 518207800, "members_executed_total": 35811 }, "example_device_plugin.stress_read": { "callback_ns_total": 3450854194, "member_service_ns_total": 3475554000, "members_executed_total": 35811 }, "example_device_plugin.stress_write": { "callback_ns_total": 170204190, "member_service_ns_total": 182615500, "members_executed_total": 35811 } }, "callback_ns_total": 4123619043, "cycles_total": 11937, "member_service_ns_total": 4176377300, "members_executed_total": 107433, "thread_cpu_ns_total": 796875000, "wall_ns_total": 4249685900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 505914456, "member_service_ns_total": 522536900, "members_executed_total": 35811 }, "example_device_plugin.stress_read": { "callback_ns_total": 3496449111, "member_service_ns_total": 3519566500, "members_executed_total": 35811 }, "example_device_plugin.stress_write": { "callback_ns_total": 161845396, "member_service_ns_total": 173663200, "members_executed_total": 35811 } }, "callback_ns_total": 4164208963, "cycles_total": 11937, "member_service_ns_total": 4215766600, "members_executed_total": 107433, "thread_cpu_ns_total": 1125000000, "wall_ns_total": 4288369900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 540970734, "member_service_ns_total": 554304600, "members_executed_total": 36039 }, "example_device_plugin.stress_read": { "callback_ns_total": 3357613532, "member_service_ns_total": 3379874500, "members_executed_total": 36039 }, "example_device_plugin.stress_write": { "callback_ns_total": 156619797, "member_service_ns_total": 168292500, "members_executed_total": 36039 } }, "callback_ns_total": 4055204063, "cycles_total": 12013, "member_service_ns_total": 4102471600, "members_executed_total": 108117, "thread_cpu_ns_total": 890625000, "wall_ns_total": 4175366500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.74887465000003, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0013463159146598, "failed": 0, "missed": 0, "callback_last_ms": 0.0036, "callback_ewma_ms": 0.005156, "release_lateness_last_ms": 0.3347 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0013463159146598, "failed": 0, "missed": 0, "callback_last_ms": 8.3267, "callback_ewma_ms": 6.740938, "release_lateness_last_ms": 0.1551 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 604, "hz": 4.998455990185574, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001241, "release_lateness_last_ms": 0.3128 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 604, "hz": 4.998455990185574, "failed": 0, "missed": 0, "callback_last_ms": 0.0315, "callback_ewma_ms": 0.016259, "release_lateness_last_ms": 0.3746 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0013463159146598, "failed": 0, "missed": 0, "callback_last_ms": 0.0198, "callback_ewma_ms": 0.016411, "release_lateness_last_ms": 0.2612 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0013463159146598, "failed": 0, "missed": 0, "callback_last_ms": 0.0281, "callback_ewma_ms": 0.015636, "release_lateness_last_ms": 0.4825 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.0026926318293197, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.001479, "release_lateness_last_ms": 0.212 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0013463159146598, "failed": 0, "missed": 0, "callback_last_ms": 2.5143, "callback_ewma_ms": 2.640536, "release_lateness_last_ms": 0.3126 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.0026926318293197, "failed": 0, "missed": 0, "callback_last_ms": 5.7193, "callback_ewma_ms": 4.578854, "release_lateness_last_ms": 0.2913 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 604, "hz": 4.998455990185574, "failed": 0, "missed": 0, "callback_last_ms": 4.0328, "callback_ewma_ms": 4.783993, "release_lateness_last_ms": 0.1085 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1208, "hz": 9.996911980371149, "failed": 0, "missed": 0, "callback_last_ms": 0.0227, "callback_ewma_ms": 0.629237, "release_lateness_last_ms": 0.2398 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.998455990185574, "failed": 0, "missed": 0, "callback_last_ms": 0.2621, "callback_ewma_ms": 0.342111, "release_lateness_last_ms": 0.4041 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.0026926318293197, "failed": 0, "missed": 0, "callback_last_ms": 10.0826, "callback_ewma_ms": 6.927844, "release_lateness_last_ms": 0.2244 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.998455990185574, "failed": 0, "missed": 0, "callback_last_ms": 0.0015, "callback_ewma_ms": 0.001977, "release_lateness_last_ms": 0.3842 }, { "name": "general_program_checks_thread", "running": true, "completed": 1209, "hz": 10.005187569758874, "failed": 0, "missed": 0, "callback_last_ms": 0.1374, "callback_ewma_ms": 0.199617, "release_lateness_last_ms": 0.3345 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0013463159146598, "failed": 0, "missed": 0, "callback_last_ms": 0.1044, "callback_ewma_ms": 0.127054, "release_lateness_last_ms": 0.2188 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1208, "hz": 9.996911980371149, "failed": 0, "missed": 0, "callback_last_ms": 0.007, "callback_ewma_ms": 0.007653, "release_lateness_last_ms": 0.1947 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 1.9944170424415952, "failed": 0, "missed": 0, "callback_last_ms": 52.2313, "callback_ewma_ms": 45.55883, "release_lateness_last_ms": 0.2964 } ], "peer_counter_seconds": 120.8366196, "peer_counter_deltas": [], "observations": [], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 73.06633153153084, "p95": 94.80870000000141, "max": 165.16569999999774, "p50": 70.57589999999618, "p99": 153.25849999999627, "min": 51.981199999994715 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 11.336140540540745, "p95": 17.165200000003097, "max": 47.872800000004645, "p50": 7.815499999996973, "p99": 46.2614000000176, "min": 4.498999999996158 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2198420, "accepted_rows_total": 13883373, "committed_rows_total": 2197292, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 18.359375, "maximum": 77.34375, "mean": 36.42082675408827 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 1.5621166565724771, "maximum": 22.653182759054424, "mean": 7.968949409502164 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 991154176, "maximum": 1013555200, "mean": 1003361658.2342342 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.9735107421875, "mean": 0.30748521959459457 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.0244140625, "mean": 0.0005498662725225225 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.0732421875, "mean": 0.000879786036036036 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 3.460567027296394, "maximum": 6.7778254786522805, "mean": 4.471662125367461 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0, "maximum": 0.0031209487684256015, "mean": 0.00029917937122761555 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 11261.387290999843, "maximum": 39409.21345212042, "mean": 18100.74124307077 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 10581.906957558858, "maximum": 40754.205631759134, "mean": 18284.593493657754 } ], "graceful_shutdown": true, "shutdown_seconds": 1.8272970999999962 }, { "id": "verified-capacity-release-25-web-1", "healthy": true, "failure_reasons": [], "seconds": 121.0504817, "measurement_completed": true, "background_preflight": null, "requested_client_count": 1, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 1, "transport": "web", "actual_connections": { "loadClients": 1, "actualClients": 2, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 250, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 41.36348980256133, "p50": 37.83243483794756, "p95": 63.45444304894674, "p99": 78.9864678391065, "min": 25.146627690049083, "max": 79.33866738693428 }, "engine_cpu_percent": { "count": 108, "mean": 8.89784347121141, "p50": 6.703854627778958, "p95": 19.84320224550822, "p99": 22.217380698392798, "min": 2.8073472693169585, "max": 24.07763646172544 }, "client_generator_cpu_percent": { "count": 108, "mean": 0.8693047239077375, "p50": 0.7098091951113521, "p95": 1.599345654097059, "p99": 2.2825581966094552, "min": 0.17632841824121268, "max": 2.4952280972698886 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1005354704.5925926, "p50": 1005957120, "p95": 1012269056, "p99": 1013424128, "min": 993017856, "max": 1014005760 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1159934666666669, "p95": 1.1742217, "max": 1.446234, "p50": 1.1057054, "p99": 1.3031685, "min": 1.0902125 }, "achieved_read_hz": { "minimum": 99.08907700518938, "maximum": 99.47717671155151 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48192, "hz": 99.4004417074952 }, { "group": "stress_group_1", "controls": 3, "completed": 36174, "hz": 99.48294539417141 }, { "group": "stress_group_2", "controls": 3, "completed": 36141, "hz": 99.3921913388276 }, { "group": "stress_group_3", "controls": 3, "completed": 36039, "hz": 99.11167880412847 }, { "group": "stress_group_4", "controls": 3, "completed": 36033, "hz": 99.09517806679325 }, { "group": "stress_group_5", "controls": 3, "completed": 36051, "hz": 99.14468027879896 }, { "group": "stress_group_6", "controls": 3, "completed": 36048, "hz": 99.13642991013135 }, { "group": "stress_group_7", "controls": 3, "completed": 36168, "hz": 99.46644465683617 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 6537, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 775609593, "member_service_ns_total": 791736500, "members_executed_total": 48192 }, "example_device_plugin.stress_read": { "callback_ns_total": 4371508315, "member_service_ns_total": 4398512400, "members_executed_total": 48192 }, "example_device_plugin.stress_write": { "callback_ns_total": 224112092, "member_service_ns_total": 241169100, "members_executed_total": 48192 } }, "callback_ns_total": 5371230000, "cycles_total": 12048, "member_service_ns_total": 5431418000, "members_executed_total": 144576, "thread_cpu_ns_total": 1687500000, "wall_ns_total": 5513880800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 651351367, "member_service_ns_total": 665675900, "members_executed_total": 36174 }, "example_device_plugin.stress_read": { "callback_ns_total": 3614596239, "member_service_ns_total": 3637025600, "members_executed_total": 36174 }, "example_device_plugin.stress_write": { "callback_ns_total": 173065293, "member_service_ns_total": 184857300, "members_executed_total": 36174 } }, "callback_ns_total": 4439012899, "cycles_total": 12058, "member_service_ns_total": 4487558800, "members_executed_total": 108522, "thread_cpu_ns_total": 1359375000, "wall_ns_total": 4564195500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 584592348, "member_service_ns_total": 598196300, "members_executed_total": 36141 }, "example_device_plugin.stress_read": { "callback_ns_total": 3523449804, "member_service_ns_total": 3543318400, "members_executed_total": 36141 }, "example_device_plugin.stress_write": { "callback_ns_total": 171771289, "member_service_ns_total": 183280900, "members_executed_total": 36141 } }, "callback_ns_total": 4279813441, "cycles_total": 12047, "member_service_ns_total": 4324795600, "members_executed_total": 108423, "thread_cpu_ns_total": 1359375000, "wall_ns_total": 4393912000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 562961159, "member_service_ns_total": 578800800, "members_executed_total": 36039 }, "example_device_plugin.stress_read": { "callback_ns_total": 3793053384, "member_service_ns_total": 3822141100, "members_executed_total": 36039 }, "example_device_plugin.stress_write": { "callback_ns_total": 175455093, "member_service_ns_total": 186763600, "members_executed_total": 36039 } }, "callback_ns_total": 4531469636, "cycles_total": 12013, "member_service_ns_total": 4587705500, "members_executed_total": 108117, "thread_cpu_ns_total": 1531250000, "wall_ns_total": 4669738500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 561781651, "member_service_ns_total": 574106700, "members_executed_total": 36033 }, "example_device_plugin.stress_read": { "callback_ns_total": 3568603938, "member_service_ns_total": 3591816300, "members_executed_total": 36033 }, "example_device_plugin.stress_write": { "callback_ns_total": 170540787, "member_service_ns_total": 183965100, "members_executed_total": 36033 } }, "callback_ns_total": 4300926376, "cycles_total": 12011, "member_service_ns_total": 4349888100, "members_executed_total": 108099, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 4425418400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 556861751, "member_service_ns_total": 569563300, "members_executed_total": 36051 }, "example_device_plugin.stress_read": { "callback_ns_total": 3758808472, "member_service_ns_total": 3781364100, "members_executed_total": 36051 }, "example_device_plugin.stress_write": { "callback_ns_total": 177801797, "member_service_ns_total": 190441300, "members_executed_total": 36051 } }, "callback_ns_total": 4493472020, "cycles_total": 12017, "member_service_ns_total": 4541368700, "members_executed_total": 108153, "thread_cpu_ns_total": 984375000, "wall_ns_total": 4611683700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 554290156, "member_service_ns_total": 568270100, "members_executed_total": 36048 }, "example_device_plugin.stress_read": { "callback_ns_total": 3794833244, "member_service_ns_total": 3817958600, "members_executed_total": 36048 }, "example_device_plugin.stress_write": { "callback_ns_total": 189220295, "member_service_ns_total": 201256100, "members_executed_total": 36048 } }, "callback_ns_total": 4538343695, "cycles_total": 12016, "member_service_ns_total": 4587484800, "members_executed_total": 108144, "thread_cpu_ns_total": 1031250000, "wall_ns_total": 4680653100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 620255715, "member_service_ns_total": 633214300, "members_executed_total": 36168 }, "example_device_plugin.stress_read": { "callback_ns_total": 3532656141, "member_service_ns_total": 3552163000, "members_executed_total": 36168 }, "example_device_plugin.stress_write": { "callback_ns_total": 177996091, "member_service_ns_total": 189453000, "members_executed_total": 36168 } }, "callback_ns_total": 4330907947, "cycles_total": 12056, "member_service_ns_total": 4374830300, "members_executed_total": 108504, "thread_cpu_ns_total": 1375000000, "wall_ns_total": 4443236400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.20670484999998, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 0.9978667736063167, "failed": 0, "missed": 0, "callback_last_ms": 0.0063, "callback_ewma_ms": 0.004966, "release_lateness_last_ms": 0.2542 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 0.9978667736063167, "failed": 0, "missed": 0, "callback_last_ms": 4.1827, "callback_ewma_ms": 5.713986, "release_lateness_last_ms": 0.5825 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 607, "hz": 5.0058275337110265, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.001194, "release_lateness_last_ms": 0.7475 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 607, "hz": 5.0058275337110265, "failed": 0, "missed": 0, "callback_last_ms": 0.094, "callback_ewma_ms": 0.087587, "release_lateness_last_ms": 0.2851 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 0.9978667736063167, "failed": 0, "missed": 0, "callback_last_ms": 0.0444, "callback_ewma_ms": 0.07067, "release_lateness_last_ms": 0.3302 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 0.9978667736063167, "failed": 0, "missed": 0, "callback_last_ms": 0.0663, "callback_ewma_ms": 0.108989, "release_lateness_last_ms": 0.2991 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 1.9957335472126334, "failed": 0, "missed": 0, "callback_last_ms": 0.0007, "callback_ewma_ms": 0.001161, "release_lateness_last_ms": 0.2705 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 122, "hz": 1.0061136064460383, "failed": 0, "missed": 0, "callback_last_ms": 2.7584, "callback_ewma_ms": 2.46117, "release_lateness_last_ms": 0.7108 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 1.9957335472126334, "failed": 0, "missed": 0, "callback_last_ms": 3.9665, "callback_ewma_ms": 4.334117, "release_lateness_last_ms": 0.1592 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 607, "hz": 5.0058275337110265, "failed": 0, "missed": 0, "callback_last_ms": 6.5088, "callback_ewma_ms": 4.855355, "release_lateness_last_ms": 0.9722 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1213, "hz": 10.00340823458233, "failed": 0, "missed": 0, "callback_last_ms": 0.031, "callback_ewma_ms": 0.957353, "release_lateness_last_ms": 0.4315 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 607, "hz": 5.0058275337110265, "failed": 0, "missed": 0, "callback_last_ms": 4.5898, "callback_ewma_ms": 3.347363, "release_lateness_last_ms": 0.697 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 1.9957335472126334, "failed": 0, "missed": 0, "callback_last_ms": 5.4821, "callback_ewma_ms": 5.597493, "release_lateness_last_ms": 0.1732 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 607, "hz": 5.0058275337110265, "failed": 0, "missed": 0, "callback_last_ms": 0.002, "callback_ewma_ms": 0.001812, "release_lateness_last_ms": 0.5824 }, { "name": "general_program_checks_thread", "running": true, "completed": 1212, "hz": 9.99516140174261, "failed": 0, "missed": 0, "callback_last_ms": 0.3135, "callback_ewma_ms": 0.225931, "release_lateness_last_ms": 0.33 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 122, "hz": 1.0061136064460383, "failed": 0, "missed": 0, "callback_last_ms": 0.0921, "callback_ewma_ms": 0.109771, "release_lateness_last_ms": 0.5527 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1212, "hz": 9.99516140174261, "failed": 0, "missed": 0, "callback_last_ms": 0.0038, "callback_ewma_ms": 0.004763, "release_lateness_last_ms": 1.0819 }, { "name": "engine_diagnostics", "running": true, "completed": 243, "hz": 2.003980380052355, "failed": 0, "missed": 0, "callback_last_ms": 63.0453, "callback_ewma_ms": 45.993877, "release_lateness_last_ms": 0.1357 } ], "peer_counter_seconds": 121.2494889, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.997902260636777, "complete_hz": 4.997902260636777, "catalog_hz": 1.9991609042547107, "bytes_per_second": 1550077.9571142416, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.99160000000848, "gap_p95_ms": 208.25370000000112, "gap_p99_ms": 227.6420999999973, "gap_max_ms": 282.6468000000459, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 13, "p95": 21, "p99": 42, "max": 77, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 78.06599099099004, "p95": 114.31969999999274, "max": 338.0341999999946, "p50": 72.21199999999953, "p99": 188.56999999994878, "min": 54.756700000027195 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 9.806844144145044, "p95": 17.196100000001024, "max": 18.22070000000531, "p50": 6.973800000036135, "p99": 17.32869999995455, "min": 4.648800000024494 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2220568, "accepted_rows_total": 13980078, "committed_rows_total": 2221967, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 19.53125, "maximum": 86.36363636363636, "mean": 41.62629553254554 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 1.9534520078661168, "maximum": 23.835456786507528, "mean": 8.533612569482631 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 992849920, "maximum": 1013362688, "mean": 1005070907.963964 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.20751953125, "mean": 0.0026943447353603603 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.0244140625, "mean": 0.000219946509009009 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 26.781332555323306, "maximum": 43.84521440329301, "mean": 33.802904429436275 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0, "maximum": 0.0058517902221142115, "mean": 0.0017075945962420059 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 11641.278212347715, "maximum": 39528.804497520854, "mean": 18515.10742174454 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 11488.972885824007, "maximum": 41127.632091435815, "mean": 18510.66929667578 } ], "graceful_shutdown": true, "shutdown_seconds": 2.037753299999982 }, { "id": "verified-capacity-release-25-web-2", "healthy": true, "failure_reasons": [], "seconds": 120.55269690000004, "measurement_completed": true, "background_preflight": null, "requested_client_count": 2, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 2, "transport": "web", "actual_connections": { "loadClients": 2, "actualClients": 3, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 500, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 38.175457755110216, "p50": 38.922775239316884, "p95": 66.61045046807241, "p99": 79.97709584193782, "min": 11.349070803885875, "max": 85.51632714254198 }, "engine_cpu_percent": { "count": 108, "mean": 8.748899666593575, "p50": 6.0464599396904175, "p95": 20.628162523559226, "p99": 23.21158911279692, "min": 1.6173227858163879, "max": 23.297121974528643 }, "client_generator_cpu_percent": { "count": 108, "mean": 1.2063953735116077, "p50": 1.2089858927954322, "p95": 2.091782331802885, "p99": 2.338290989881316, "min": 0.5262434906346509, "max": 2.624247877961786 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1004459235.5555556, "p50": 1004937216, "p95": 1009475584, "p99": 1011712000, "min": 996384768, "max": 1013211136 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1175183777777777, "p95": 1.2209851, "max": 1.3133125, "p50": 1.1001607, "p99": 1.2541775, "min": 1.0850867 }, "achieved_read_hz": { "minimum": 98.62499577759665, "maximum": 99.36285980785917 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47944, "hz": 99.29132029594199 }, { "group": "stress_group_1", "controls": 3, "completed": 35979, "hz": 99.34930788496848 }, { "group": "stress_group_2", "controls": 3, "completed": 35979, "hz": 99.34930788496848 }, { "group": "stress_group_3", "controls": 3, "completed": 35728, "hz": 98.65621813041368 }, { "group": "stress_group_4", "controls": 3, "completed": 35725, "hz": 98.64793418912417 }, { "group": "stress_group_5", "controls": 3, "completed": 35739, "hz": 98.68659258180851 }, { "group": "stress_group_6", "controls": 3, "completed": 35715, "hz": 98.62032105149251 }, { "group": "stress_group_7", "controls": 3, "completed": 35983, "hz": 99.36035314002113 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 8958, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 730482894, "member_service_ns_total": 747863600, "members_executed_total": 47944 }, "example_device_plugin.stress_read": { "callback_ns_total": 4143639777, "member_service_ns_total": 4168571200, "members_executed_total": 47944 }, "example_device_plugin.stress_write": { "callback_ns_total": 195060889, "member_service_ns_total": 208828200, "members_executed_total": 47944 } }, "callback_ns_total": 5069183560, "cycles_total": 11986, "member_service_ns_total": 5125263000, "members_executed_total": 143832, "thread_cpu_ns_total": 2125000000, "wall_ns_total": 5203123000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 528319986, "member_service_ns_total": 541394900, "members_executed_total": 35979 }, "example_device_plugin.stress_read": { "callback_ns_total": 3346785255, "member_service_ns_total": 3368577100, "members_executed_total": 35979 }, "example_device_plugin.stress_write": { "callback_ns_total": 158022697, "member_service_ns_total": 169069500, "members_executed_total": 35979 } }, "callback_ns_total": 4033127938, "cycles_total": 11993, "member_service_ns_total": 4079041500, "members_executed_total": 107937, "thread_cpu_ns_total": 1375000000, "wall_ns_total": 4149941900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 529315901, "member_service_ns_total": 543105400, "members_executed_total": 35979 }, "example_device_plugin.stress_read": { "callback_ns_total": 3216909492, "member_service_ns_total": 3235858200, "members_executed_total": 35979 }, "example_device_plugin.stress_write": { "callback_ns_total": 151558191, "member_service_ns_total": 163251700, "members_executed_total": 35979 } }, "callback_ns_total": 3897783584, "cycles_total": 11993, "member_service_ns_total": 3942215300, "members_executed_total": 107937, "thread_cpu_ns_total": 1328125000, "wall_ns_total": 4007957000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 548895465, "member_service_ns_total": 560856500, "members_executed_total": 35728 }, "example_device_plugin.stress_read": { "callback_ns_total": 3807860204, "member_service_ns_total": 3834623100, "members_executed_total": 35728 }, "example_device_plugin.stress_write": { "callback_ns_total": 172867095, "member_service_ns_total": 183732200, "members_executed_total": 35728 } }, "callback_ns_total": 4530031364, "cycles_total": 11910, "member_service_ns_total": 4579622600, "members_executed_total": 107190, "thread_cpu_ns_total": 1500000000, "wall_ns_total": 4666232400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 546824268, "member_service_ns_total": 574559000, "members_executed_total": 35725 }, "example_device_plugin.stress_read": { "callback_ns_total": 3543589410, "member_service_ns_total": 3569157500, "members_executed_total": 35724 }, "example_device_plugin.stress_write": { "callback_ns_total": 183473181, "member_service_ns_total": 196154100, "members_executed_total": 35725 } }, "callback_ns_total": 4274392859, "cycles_total": 11909, "member_service_ns_total": 4340379000, "members_executed_total": 107181, "thread_cpu_ns_total": 1328125000, "wall_ns_total": 4413543800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 515677180, "member_service_ns_total": 530774300, "members_executed_total": 35739 }, "example_device_plugin.stress_read": { "callback_ns_total": 3514634567, "member_service_ns_total": 3546657400, "members_executed_total": 35739 }, "example_device_plugin.stress_write": { "callback_ns_total": 164025683, "member_service_ns_total": 175365500, "members_executed_total": 35739 } }, "callback_ns_total": 4194337430, "cycles_total": 11913, "member_service_ns_total": 4252797200, "members_executed_total": 107217, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 4333571300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 517459704, "member_service_ns_total": 532038400, "members_executed_total": 35715 }, "example_device_plugin.stress_read": { "callback_ns_total": 3702817866, "member_service_ns_total": 3725995800, "members_executed_total": 35715 }, "example_device_plugin.stress_write": { "callback_ns_total": 165107987, "member_service_ns_total": 177306400, "members_executed_total": 35715 } }, "callback_ns_total": 4385385557, "cycles_total": 11905, "member_service_ns_total": 4435340600, "members_executed_total": 107145, "thread_cpu_ns_total": 1562500000, "wall_ns_total": 4513357000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 547751043, "member_service_ns_total": 560847800, "members_executed_total": 35983 }, "example_device_plugin.stress_read": { "callback_ns_total": 3367531927, "member_service_ns_total": 3390218000, "members_executed_total": 35983 }, "example_device_plugin.stress_write": { "callback_ns_total": 166047693, "member_service_ns_total": 176610100, "members_executed_total": 35983 } }, "callback_ns_total": 4081687063, "cycles_total": 11995, "member_service_ns_total": 4128034400, "members_executed_total": 107955, "thread_cpu_ns_total": 1468750000, "wall_ns_total": 4201417100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.71548615000002, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0019470754712088, "failed": 0, "missed": 0, "callback_last_ms": 0.0041, "callback_ewma_ms": 0.006512, "release_lateness_last_ms": 0.2654 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0019470754712088, "failed": 0, "missed": 0, "callback_last_ms": 5.6621, "callback_ewma_ms": 6.639281, "release_lateness_last_ms": 0.3314 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 603, "hz": 4.993174268670569, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.000942, "release_lateness_last_ms": 0.4628 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 603, "hz": 4.993174268670569, "failed": 0, "missed": 0, "callback_last_ms": 0.4968, "callback_ewma_ms": 0.192541, "release_lateness_last_ms": 0.5941 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0019470754712088, "failed": 0, "missed": 0, "callback_last_ms": 0.209, "callback_ewma_ms": 0.218896, "release_lateness_last_ms": 0.3533 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0019470754712088, "failed": 0, "missed": 0, "callback_last_ms": 0.3994, "callback_ewma_ms": 0.27989, "release_lateness_last_ms": 0.5313 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.0038941509424175, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.004047, "release_lateness_last_ms": 0.2153 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.9936665211284715, "failed": 0, "missed": 0, "callback_last_ms": 2.8346, "callback_ewma_ms": 2.292482, "release_lateness_last_ms": 0.3236 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.0038941509424175, "failed": 0, "missed": 0, "callback_last_ms": 4.0732, "callback_ewma_ms": 4.184557, "release_lateness_last_ms": 0.3564 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 603, "hz": 4.993174268670569, "failed": 0, "missed": 0, "callback_last_ms": 7.6617, "callback_ewma_ms": 4.833648, "release_lateness_last_ms": 0.2703 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1207, "hz": 9.994629091683876, "failed": 0, "missed": 0, "callback_last_ms": 0.0178, "callback_ewma_ms": 0.438593, "release_lateness_last_ms": 0.2279 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 603, "hz": 4.993174268670569, "failed": 0, "missed": 0, "callback_last_ms": 3.2182, "callback_ewma_ms": 3.36702, "release_lateness_last_ms": 0.4414 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.0038941509424175, "failed": 0, "missed": 0, "callback_last_ms": 9.5202, "callback_ewma_ms": 6.445686, "release_lateness_last_ms": 0.2743 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 603, "hz": 4.993174268670569, "failed": 0, "missed": 0, "callback_last_ms": 0.0032, "callback_ewma_ms": 0.002312, "release_lateness_last_ms": 0.4142 }, { "name": "general_program_checks_thread", "running": true, "completed": 1208, "hz": 10.002909646026612, "failed": 0, "missed": 0, "callback_last_ms": 0.1414, "callback_ewma_ms": 0.176254, "release_lateness_last_ms": 0.4829 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.9936665211284715, "failed": 0, "missed": 0, "callback_last_ms": 0.1064, "callback_ewma_ms": 0.102463, "release_lateness_last_ms": 0.2701 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1208, "hz": 10.002909646026612, "failed": 0, "missed": 0, "callback_last_ms": 0.0039, "callback_ewma_ms": 0.004455, "release_lateness_last_ms": 0.3837 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 1.9956135965996802, "failed": 0, "missed": 0, "callback_last_ms": 38.8996, "callback_ewma_ms": 43.784293, "release_lateness_last_ms": 0.1609 } ], "peer_counter_seconds": 120.76585060000001, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.00195949067132, "complete_hz": 5.00195949067132, "catalog_hz": 2.0074198950952895, "bytes_per_second": 1551168.6735761354, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9661999999662, "gap_p95_ms": 213.1298999999999, "gap_p99_ms": 232.94680000003427, "gap_max_ms": 353.8654000000097, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 8, "p95": 22, "p99": 40, "max": 163, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.00195949067132, "complete_hz": 5.00195949067132, "catalog_hz": 2.0074198950952895, "bytes_per_second": 1551168.6735761354, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.96000000002095, "gap_p95_ms": 216.08679999999003, "gap_p99_ms": 232.91310000000522, "gap_max_ms": 350.0609000000404, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 9, "p95": 24, "p99": 46, "max": 168, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 72.16716756756462, "p95": 99.18899999995483, "max": 176.86269999999786, "p50": 68.44289999996545, "p99": 131.00839999999152, "min": 51.34429999999702 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 10.309625225225547, "p95": 16.92379999998957, "max": 19.26339999999618, "p50": 7.4874999999883585, "p99": 18.835299999976996, "min": 4.777100000006612 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2190075, "accepted_rows_total": 13848290, "committed_rows_total": 2189755, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 10.15625, "maximum": 97.65625, "mean": 39.509786314611915 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 0.7811479820735411, "maximum": 27.7237966438655, "mean": 8.103532077150318 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 993099776, "maximum": 1012805632, "mean": 1004272077.2612612 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.3875732421875, "mean": 0.008220500774211712 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.0244140625, "mean": 0.001319679054054054 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 49.63836305722806, "maximum": 81.16874500746655, "mean": 62.76074349763118 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0, "maximum": 0.008210298883687434, "mean": 0.0030328247420870424 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 11421.234358920105, "maximum": 39446.634256163336, "mean": 18461.92975152092 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 11089.376777025132, "maximum": 40150.85776650652, "mean": 18465.290409115358 } ], "graceful_shutdown": true, "shutdown_seconds": 1.8535783000000519 }, { "id": "verified-capacity-release-25-web-2-r2", "healthy": true, "failure_reasons": [], "seconds": 120.7679265, "measurement_completed": true, "background_preflight": null, "requested_client_count": 2, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 2, "transport": "web", "actual_connections": { "loadClients": 2, "actualClients": 3, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 500, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 23.352809619785692, "p50": 21.204324374689932, "p95": 46.223717356084784, "p99": 55.28858241346632, "min": 11.544481980861077, "max": 75.76495458645172 }, "engine_cpu_percent": { "count": 109, "mean": 8.104598114024723, "p50": 6.155108070585581, "p95": 17.86636806911151, "p99": 19.57210502172347, "min": 2.6744707771490024, "max": 20.263888372038785 }, "client_generator_cpu_percent": { "count": 109, "mean": 0.852157491446813, "p50": 0.7118184457966996, "p95": 1.600845150615708, "p99": 2.1013878977809615, "min": 0.3517204611763189, "max": 2.1267872839762605 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1003974205.0642202, "p50": 1004056576, "p95": 1009278976, "p99": 1009995776, "min": 993988608, "max": 1010016256 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1069445458715594, "p95": 1.1229587, "max": 1.1861525, "p50": 1.1031723, "p99": 1.1662526, "min": 1.095427 }, "achieved_read_hz": { "minimum": 99.94533879750296, "maximum": 99.97845103914709 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48336, "hz": 99.9637168271938 }, { "group": "stress_group_1", "controls": 3, "completed": 36249, "hz": 99.95544442427861 }, { "group": "stress_group_2", "controls": 3, "completed": 36255, "hz": 99.971989230109 }, { "group": "stress_group_3", "controls": 3, "completed": 36255, "hz": 99.971989230109 }, { "group": "stress_group_4", "controls": 3, "completed": 36261, "hz": 99.98853403593938 }, { "group": "stress_group_5", "controls": 3, "completed": 36258, "hz": 99.9802616330242 }, { "group": "stress_group_6", "controls": 3, "completed": 36258, "hz": 99.9802616330242 }, { "group": "stress_group_7", "controls": 3, "completed": 36255, "hz": 99.971989230109 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 294, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 539145240, "member_service_ns_total": 551267000, "members_executed_total": 48336 }, "example_device_plugin.stress_read": { "callback_ns_total": 3340470386, "member_service_ns_total": 3362781600, "members_executed_total": 48336 }, "example_device_plugin.stress_write": { "callback_ns_total": 167969891, "member_service_ns_total": 179364200, "members_executed_total": 48336 } }, "callback_ns_total": 4047585517, "cycles_total": 12084, "member_service_ns_total": 4093412800, "members_executed_total": 145008, "thread_cpu_ns_total": 1265625000, "wall_ns_total": 4165949700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 430945424, "member_service_ns_total": 441748600, "members_executed_total": 36249 }, "example_device_plugin.stress_read": { "callback_ns_total": 2692240213, "member_service_ns_total": 2713317000, "members_executed_total": 36249 }, "example_device_plugin.stress_write": { "callback_ns_total": 133187095, "member_service_ns_total": 141932500, "members_executed_total": 36249 } }, "callback_ns_total": 3256372732, "cycles_total": 12083, "member_service_ns_total": 3296998100, "members_executed_total": 108747, "thread_cpu_ns_total": 906250000, "wall_ns_total": 3363998100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 424052833, "member_service_ns_total": 434003300, "members_executed_total": 36255 }, "example_device_plugin.stress_read": { "callback_ns_total": 2589342700, "member_service_ns_total": 2606350900, "members_executed_total": 36255 }, "example_device_plugin.stress_write": { "callback_ns_total": 125291297, "member_service_ns_total": 134929800, "members_executed_total": 36255 } }, "callback_ns_total": 3138686830, "cycles_total": 12085, "member_service_ns_total": 3175284000, "members_executed_total": 108765, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3234974800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 385407285, "member_service_ns_total": 395764400, "members_executed_total": 36255 }, "example_device_plugin.stress_read": { "callback_ns_total": 2562556361, "member_service_ns_total": 2579746200, "members_executed_total": 36255 }, "example_device_plugin.stress_write": { "callback_ns_total": 124760199, "member_service_ns_total": 133056800, "members_executed_total": 36255 } }, "callback_ns_total": 3072723845, "cycles_total": 12085, "member_service_ns_total": 3108567400, "members_executed_total": 108765, "thread_cpu_ns_total": 843750000, "wall_ns_total": 3166658700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 381824618, "member_service_ns_total": 390909100, "members_executed_total": 36261 }, "example_device_plugin.stress_read": { "callback_ns_total": 2469076508, "member_service_ns_total": 2486509500, "members_executed_total": 36261 }, "example_device_plugin.stress_write": { "callback_ns_total": 125298798, "member_service_ns_total": 134333400, "members_executed_total": 36261 } }, "callback_ns_total": 2976199924, "cycles_total": 12087, "member_service_ns_total": 3011752000, "members_executed_total": 108783, "thread_cpu_ns_total": 312500000, "wall_ns_total": 3068177000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 388870820, "member_service_ns_total": 398148200, "members_executed_total": 36258 }, "example_device_plugin.stress_read": { "callback_ns_total": 2426811143, "member_service_ns_total": 2444243000, "members_executed_total": 36258 }, "example_device_plugin.stress_write": { "callback_ns_total": 119875797, "member_service_ns_total": 128757700, "members_executed_total": 36258 } }, "callback_ns_total": 2935557760, "cycles_total": 12086, "member_service_ns_total": 2971148900, "members_executed_total": 108774, "thread_cpu_ns_total": 234375000, "wall_ns_total": 3026075600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 384201729, "member_service_ns_total": 394777800, "members_executed_total": 36258 }, "example_device_plugin.stress_read": { "callback_ns_total": 2513614485, "member_service_ns_total": 2532318700, "members_executed_total": 36258 }, "example_device_plugin.stress_write": { "callback_ns_total": 125218495, "member_service_ns_total": 135323400, "members_executed_total": 36258 } }, "callback_ns_total": 3023034709, "cycles_total": 12086, "member_service_ns_total": 3062419900, "members_executed_total": 108774, "thread_cpu_ns_total": 765625000, "wall_ns_total": 3121746600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 440127089, "member_service_ns_total": 450138700, "members_executed_total": 36255 }, "example_device_plugin.stress_read": { "callback_ns_total": 2626919727, "member_service_ns_total": 2647476000, "members_executed_total": 36255 }, "example_device_plugin.stress_write": { "callback_ns_total": 133003696, "member_service_ns_total": 142208200, "members_executed_total": 36255 } }, "callback_ns_total": 3200050512, "cycles_total": 12085, "member_service_ns_total": 3239822900, "members_executed_total": 108765, "thread_cpu_ns_total": 984375000, "wall_ns_total": 3299545600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.88386050000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 0.0046, "callback_ewma_ms": 0.003948, "release_lateness_last_ms": 0.6549 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 3.3147, "callback_ewma_ms": 3.57347, "release_lateness_last_ms": 0.2639 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 605, "hz": 5.002523119691857, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.000819, "release_lateness_last_ms": 0.1984 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 605, "hz": 5.002523119691857, "failed": 0, "missed": 0, "callback_last_ms": 0.1263, "callback_ewma_ms": 0.129983, "release_lateness_last_ms": 0.491 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 0.1186, "callback_ewma_ms": 0.115877, "release_lateness_last_ms": 0.4832 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 0.1473, "callback_ewma_ms": 0.142317, "release_lateness_last_ms": 0.1259 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.001009247876743, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.000997, "release_lateness_last_ms": 0.1313 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 2.288, "callback_ewma_ms": 2.025875, "release_lateness_last_ms": 0.4687 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.001009247876743, "failed": 0, "missed": 0, "callback_last_ms": 3.8824, "callback_ewma_ms": 4.000775, "release_lateness_last_ms": 0.439 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 605, "hz": 5.002523119691857, "failed": 0, "missed": 0, "callback_last_ms": 3.2264, "callback_ewma_ms": 3.412754, "release_lateness_last_ms": 0.2804 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1210, "hz": 10.005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 0.022, "callback_ewma_ms": 0.325801, "release_lateness_last_ms": 0.3968 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 605, "hz": 5.002523119691857, "failed": 0, "missed": 0, "callback_last_ms": 2.8124, "callback_ewma_ms": 2.973896, "release_lateness_last_ms": 0.3962 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.001009247876743, "failed": 0, "missed": 0, "callback_last_ms": 5.0916, "callback_ewma_ms": 5.184048, "release_lateness_last_ms": 0.5135 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 605, "hz": 5.002523119691857, "failed": 0, "missed": 0, "callback_last_ms": 0.0018, "callback_ewma_ms": 0.002695, "release_lateness_last_ms": 0.548 }, { "name": "general_program_checks_thread", "running": true, "completed": 1210, "hz": 10.005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 0.1552, "callback_ewma_ms": 0.171202, "release_lateness_last_ms": 0.5312 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0005046239383715, "failed": 0, "missed": 0, "callback_last_ms": 0.1502, "callback_ewma_ms": 0.109999, "release_lateness_last_ms": 0.3968 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1209, "hz": 9.996777606128026, "failed": 0, "missed": 0, "callback_last_ms": 0.0041, "callback_ewma_ms": 0.006251, "release_lateness_last_ms": 0.5698 }, { "name": "engine_diagnostics", "running": true, "completed": 242, "hz": 2.001009247876743, "failed": 0, "missed": 0, "callback_last_ms": 37.4561, "callback_ewma_ms": 37.801162, "release_lateness_last_ms": 0.1199 } ], "peer_counter_seconds": 120.9456823, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.001326092505053, "complete_hz": 5.001326092505053, "catalog_hz": 2.0038425734871237, "bytes_per_second": 1551199.2529549026, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9985000000015, "gap_p95_ms": 202.47760000000562, "gap_p99_ms": 215.74010000001, "gap_max_ms": 221.59399999998277, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 13, "p95": 16, "p99": 27, "max": 33, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.001326092505053, "complete_hz": 5.001326092505053, "catalog_hz": 2.0038425734871237, "bytes_per_second": 1551199.2529549026, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.00350000000617, "gap_p95_ms": 203.1140000000014, "gap_p99_ms": 215.3798999999999, "gap_max_ms": 222.38250000000698, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1208, "p50": 14, "p95": 17, "p99": 30, "max": 36, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 112, "mean": 63.20133214285742, "p95": 74.57130000001052, "max": 85.50869999999122, "p50": 62.64969999999448, "p99": 76.64869999999792, "min": 51.23209999999381 }, { "operation": "rapid/get-persistence-diagnostics", "count": 112, "mean": 12.910213392858102, "p95": 17.58330000001297, "max": 19.05240000000049, "p50": 15.304100000001199, "p99": 18.403299999976298, "min": 4.7942000000039116 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2239622, "accepted_rows_total": 14059315, "committed_rows_total": 2238316, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 112, "minimum": 8.984375, "maximum": 75, "mean": 23.894279233870968 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 112, "minimum": 1.9529695436243273, "maximum": 21.40649815969503, "mean": 7.7637551670281395 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 112, "minimum": 991203328, "maximum": 1009995776, "mean": 1003405897.1428572 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0.2288818359375, "mean": 0.0020435878208705355 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0.01220703125, "mean": 0.00010899135044642858 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_tx_mbps", "observations": 112, "minimum": 49.5778583140252, "maximum": 80.97145337544683, "mean": 62.186500144819476 }, { "channel": "engine_tempest_rx_mbps", "observations": 112, "minimum": 0, "maximum": 0.00858731647759312, "mean": 0.003155629193238299 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 112, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 112, "minimum": 12474.33828659635, "maximum": 39549.44200321964, "mean": 19037.208682766057 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 112, "minimum": 10723.05362467431, "maximum": 40605.09840913338, "mean": 18956.831585996184 } ], "graceful_shutdown": true, "shutdown_seconds": 2.1211633000000147 }, { "id": "verified-capacity-release-25-web-4", "healthy": true, "failure_reasons": [], "seconds": 121.0045435, "measurement_completed": true, "background_preflight": null, "requested_client_count": 4, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 4, "transport": "web", "actual_connections": { "loadClients": 4, "actualClients": 5, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 1000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 35.811080086932215, "p50": 30.274111685222525, "p95": 76.66650902230911, "p99": 93.22491046664383, "min": 17.117355274654766, "max": 100 }, "engine_cpu_percent": { "count": 109, "mean": 8.837187525915667, "p50": 6.692846640422066, "p95": 19.339480421842712, "p99": 21.086105696274142, "min": 3.3389395651657527, "max": 22.099395059706364 }, "client_generator_cpu_percent": { "count": 109, "mean": 1.783521479284408, "p50": 1.730803927796155, "p95": 2.8480408440955096, "p99": 3.219979373813903, "min": 0.7075080619694651, "max": 5.1580735390742065 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1007763794.2018349, "p50": 1007800320, "p95": 1014087680, "p99": 1014648832, "min": 991793152, "max": 1015754752 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1107854513761466, "p95": 1.1509174, "max": 1.599662, "p50": 1.0971329, "p99": 1.3255198, "min": 1.0880973 }, "achieved_read_hz": { "minimum": 98.36223068740367, "maximum": 99.21324532650279 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48040, "hz": 99.16722608555818 }, { "group": "stress_group_1", "controls": 3, "completed": 36039, "hz": 99.19199724944299 }, { "group": "stress_group_2", "controls": 3, "completed": 36039, "hz": 99.19199724944299 }, { "group": "stress_group_3", "controls": 3, "completed": 35742, "hz": 98.37454884124399 }, { "group": "stress_group_4", "controls": 3, "completed": 35760, "hz": 98.42409116901361 }, { "group": "stress_group_5", "controls": 3, "completed": 35757, "hz": 98.41583411438535 }, { "group": "stress_group_6", "controls": 3, "completed": 35742, "hz": 98.37454884124399 }, { "group": "stress_group_7", "controls": 3, "completed": 36051, "hz": 99.22502546795609 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 10965, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 658298101, "member_service_ns_total": 675119200, "members_executed_total": 48040 }, "example_device_plugin.stress_read": { "callback_ns_total": 4021205064, "member_service_ns_total": 4046811500, "members_executed_total": 48040 }, "example_device_plugin.stress_write": { "callback_ns_total": 189447991, "member_service_ns_total": 204115600, "members_executed_total": 48040 } }, "callback_ns_total": 4868951156, "cycles_total": 12010, "member_service_ns_total": 4926046300, "members_executed_total": 144120, "thread_cpu_ns_total": 1625000000, "wall_ns_total": 5000190500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 540256235, "member_service_ns_total": 556490300, "members_executed_total": 36039 }, "example_device_plugin.stress_read": { "callback_ns_total": 3323610665, "member_service_ns_total": 3346969200, "members_executed_total": 36039 }, "example_device_plugin.stress_write": { "callback_ns_total": 158791394, "member_service_ns_total": 168664200, "members_executed_total": 36039 } }, "callback_ns_total": 4022658294, "cycles_total": 12013, "member_service_ns_total": 4072123700, "members_executed_total": 108117, "thread_cpu_ns_total": 1343750000, "wall_ns_total": 4139128700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 555275787, "member_service_ns_total": 567353400, "members_executed_total": 36039 }, "example_device_plugin.stress_read": { "callback_ns_total": 3282704373, "member_service_ns_total": 3302911700, "members_executed_total": 36039 }, "example_device_plugin.stress_write": { "callback_ns_total": 147123896, "member_service_ns_total": 161333500, "members_executed_total": 36039 } }, "callback_ns_total": 3985104056, "cycles_total": 12013, "member_service_ns_total": 4031598600, "members_executed_total": 108117, "thread_cpu_ns_total": 1093750000, "wall_ns_total": 4093715300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 577049386, "member_service_ns_total": 594466600, "members_executed_total": 35742 }, "example_device_plugin.stress_read": { "callback_ns_total": 3642786133, "member_service_ns_total": 3663920400, "members_executed_total": 35742 }, "example_device_plugin.stress_write": { "callback_ns_total": 159936794, "member_service_ns_total": 200247400, "members_executed_total": 35742 } }, "callback_ns_total": 4379772313, "cycles_total": 11914, "member_service_ns_total": 4458634400, "members_executed_total": 107226, "thread_cpu_ns_total": 1218750000, "wall_ns_total": 4537430400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 485591108, "member_service_ns_total": 499596000, "members_executed_total": 35760 }, "example_device_plugin.stress_read": { "callback_ns_total": 3400562573, "member_service_ns_total": 3421090600, "members_executed_total": 35760 }, "example_device_plugin.stress_write": { "callback_ns_total": 152491792, "member_service_ns_total": 162425200, "members_executed_total": 35760 } }, "callback_ns_total": 4038645473, "cycles_total": 11920, "member_service_ns_total": 4083111800, "members_executed_total": 107280, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 4148564200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 486590388, "member_service_ns_total": 500243600, "members_executed_total": 35757 }, "example_device_plugin.stress_read": { "callback_ns_total": 3353166213, "member_service_ns_total": 3374698500, "members_executed_total": 35757 }, "example_device_plugin.stress_write": { "callback_ns_total": 165784693, "member_service_ns_total": 176587300, "members_executed_total": 35757 } }, "callback_ns_total": 4005541294, "cycles_total": 11919, "member_service_ns_total": 4051529400, "members_executed_total": 107271, "thread_cpu_ns_total": 875000000, "wall_ns_total": 4117423000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 534796482, "member_service_ns_total": 549515000, "members_executed_total": 35742 }, "example_device_plugin.stress_read": { "callback_ns_total": 3428844127, "member_service_ns_total": 3450377400, "members_executed_total": 35742 }, "example_device_plugin.stress_write": { "callback_ns_total": 165971891, "member_service_ns_total": 176663000, "members_executed_total": 35742 } }, "callback_ns_total": 4129612500, "cycles_total": 11914, "member_service_ns_total": 4176555400, "members_executed_total": 107226, "thread_cpu_ns_total": 1484375000, "wall_ns_total": 4248996500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 555001002, "member_service_ns_total": 573117200, "members_executed_total": 36051 }, "example_device_plugin.stress_read": { "callback_ns_total": 3230700280, "member_service_ns_total": 3254694200, "members_executed_total": 36051 }, "example_device_plugin.stress_write": { "callback_ns_total": 154063491, "member_service_ns_total": 168554200, "members_executed_total": 36051 } }, "callback_ns_total": 3939764773, "cycles_total": 12017, "member_service_ns_total": 3996365600, "members_executed_total": 108153, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 4067695000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.10856049999998, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 0.9985879792664515, "failed": 0, "missed": 0, "callback_last_ms": 0.0047, "callback_ewma_ms": 0.004868, "release_lateness_last_ms": 0.3116 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 0.9985879792664515, "failed": 0, "missed": 0, "callback_last_ms": 8.275, "callback_ewma_ms": 6.98922, "release_lateness_last_ms": 0.5394 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 606, "hz": 5.0011926895493355, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.0015, "release_lateness_last_ms": 0.1803 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 606, "hz": 5.0011926895493355, "failed": 0, "missed": 0, "callback_last_ms": 0.2404, "callback_ewma_ms": 0.343521, "release_lateness_last_ms": 0.236 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 0.9985879792664515, "failed": 0, "missed": 0, "callback_last_ms": 0.4204, "callback_ewma_ms": 0.390653, "release_lateness_last_ms": 0.278 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 0.9985879792664515, "failed": 0, "missed": 0, "callback_last_ms": 0.8942, "callback_ewma_ms": 0.525992, "release_lateness_last_ms": 0.3537 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 1.997175958532903, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.000998, "release_lateness_last_ms": 0.0745 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 122, "hz": 1.0068407724835295, "failed": 0, "missed": 0, "callback_last_ms": 2.8042, "callback_ewma_ms": 2.25341, "release_lateness_last_ms": 0.1516 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 1.997175958532903, "failed": 0, "missed": 0, "callback_last_ms": 5.0024, "callback_ewma_ms": 4.271131, "release_lateness_last_ms": 0.4918 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 606, "hz": 5.0011926895493355, "failed": 0, "missed": 0, "callback_last_ms": 6.1147, "callback_ewma_ms": 4.943626, "release_lateness_last_ms": 0.381 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1212, "hz": 10.002385379098671, "failed": 0, "missed": 0, "callback_last_ms": 0.0284, "callback_ewma_ms": 0.741614, "release_lateness_last_ms": 0.6524 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 606, "hz": 5.0011926895493355, "failed": 0, "missed": 0, "callback_last_ms": 5.5153, "callback_ewma_ms": 4.389667, "release_lateness_last_ms": 0.4327 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 1.997175958532903, "failed": 0, "missed": 0, "callback_last_ms": 10.947, "callback_ewma_ms": 6.640505, "release_lateness_last_ms": 0.3902 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 606, "hz": 5.0011926895493355, "failed": 0, "missed": 0, "callback_last_ms": 0.0015, "callback_ewma_ms": 0.002945, "release_lateness_last_ms": 0.2013 }, { "name": "general_program_checks_thread", "running": true, "completed": 1212, "hz": 10.002385379098671, "failed": 0, "missed": 0, "callback_last_ms": 0.4337, "callback_ewma_ms": 0.1877, "release_lateness_last_ms": 0.59 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 122, "hz": 1.0068407724835295, "failed": 0, "missed": 0, "callback_last_ms": 0.1219, "callback_ewma_ms": 0.17441, "release_lateness_last_ms": 0.5988 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1212, "hz": 10.002385379098671, "failed": 0, "missed": 0, "callback_last_ms": 0.0042, "callback_ewma_ms": 0.005325, "release_lateness_last_ms": 0.441 }, { "name": "engine_diagnostics", "running": true, "completed": 243, "hz": 2.005428751749981, "failed": 0, "missed": 0, "callback_last_ms": 42.9508, "callback_ewma_ms": 40.860955, "release_lateness_last_ms": 0.5417 } ], "peer_counter_seconds": 121.17957949999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.99981204425582, "complete_hz": 4.99981204425582, "catalog_hz": 1.999924817702328, "bytes_per_second": 1550584.5938500655, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.85080000000016, "gap_p95_ms": 211.50879999999597, "gap_p99_ms": 232.27690000001166, "gap_max_ms": 265.7902000000031, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 11, "p95": 24, "p99": 47, "max": 111, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.99981204425582, "complete_hz": 4.99981204425582, "catalog_hz": 1.999924817702328, "bytes_per_second": 1550584.5938500655, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.03789999999572, "gap_p95_ms": 211.4774999999936, "gap_p99_ms": 230.56530000000203, "gap_max_ms": 320.0969000000041, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 12, "p95": 26, "p99": 77, "max": 144, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.99981204425582, "complete_hz": 4.99981204425582, "catalog_hz": 1.999924817702328, "bytes_per_second": 1550584.5938500655, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0268999999971, "gap_p95_ms": 212.99599999999919, "gap_p99_ms": 247.6423000000068, "gap_max_ms": 319.46140000000014, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 14, "p95": 28, "p99": 78, "max": 158, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.99981204425582, "complete_hz": 4.99981204425582, "catalog_hz": 1.999924817702328, "bytes_per_second": 1550584.5938500655, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.97150000000693, "gap_p95_ms": 211.9979999999996, "gap_p99_ms": 260.81089999999676, "gap_max_ms": 321.62459999999555, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 16, "p95": 29, "p99": 84, "max": 174, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 72.26886486486438, "p95": 104.94199999999546, "max": 180.67149999999674, "p50": 68.33239999999932, "p99": 143.1981000000087, "min": 50.98800000000483 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 12.31266936936863, "p95": 20.864099999991595, "max": 26.264300000009825, "p50": 15.007599999997183, "p99": 24.66219999999157, "min": 4.501300000003539 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2196349, "accepted_rows_total": 13872608, "committed_rows_total": 2195374, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 13.28125, "maximum": 100, "mean": 35.4372799579657 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 1.9518047992337983, "maximum": 24.572530947097924, "mean": 8.53646362806193 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 991604736, "maximum": 1014599680, "mean": 1007927148.3963964 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 1.068115234375, "mean": 0.21997400232263514 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.06103515625, "mean": 0.0015396255630630632 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.146484375, "mean": 0.002859304617117117 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 95.86945789867735, "maximum": 155.25737648076512, "mean": 123.20634531368799 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0, "maximum": 0.011331875501421486, "mean": 0.006382673477091373 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 10933.155193018776, "maximum": 39862.330487614185, "mean": 17892.089025626068 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 10577.247800122408, "maximum": 40200.26546892378, "mean": 17834.949517308516 } ], "graceful_shutdown": true, "shutdown_seconds": 2.165185599999997 }, { "id": "verified-capacity-release-25-web-4-r2", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_0: control callbacks below 95 Hz", "stress_group_1: control callbacks below 95 Hz", "stress_group_2: control callbacks below 95 Hz", "stress_group_3: control callbacks below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz", "stress_group_7: control callbacks below 95 Hz" ], "seconds": 120.52905750000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 4, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 4, "transport": "web", "actual_connections": { "loadClients": 4, "actualClients": 5, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 1000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 36, "mean": 99.00533920925132, "p50": 99.61284457645422, "p95": 100, "p99": 100, "min": 86.44465529839967, "max": 100 }, "engine_cpu_percent": { "count": 36, "mean": 10.69312586035754, "p50": 9.52610397576964, "p95": 18.274239458368154, "p99": 19.366603272148307, "min": 5.712063605818126, "max": 19.366603272148307 }, "client_generator_cpu_percent": { "count": 36, "mean": 1.5060078440556801, "p50": 1.5554984319381182, "p95": 1.9776905826190883, "p99": 2.2101587545519004, "min": 0.7789177644297445, "max": 2.2101587545519004 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 36, "mean": 1010816682.6666666, "p50": 1012293632, "p95": 1017782272, "p99": 1017999360, "min": 998608896, "max": 1017999360 }, "system_samples": 36, "system_sample_interval_seconds": { "count": 36, "mean": 3.4043766166666662, "p95": 7.156711, "max": 7.7691258, "p50": 2.8588673, "p99": 7.7691258, "min": 1.2151187 }, "achieved_read_hz": { "minimum": 81.6053511450613, "maximum": 93.80802498556352 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 45116, "hz": 93.39650440814836 }, { "group": "stress_group_1", "controls": 3, "completed": 33987, "hz": 93.81053271033893 }, { "group": "stress_group_2", "controls": 3, "completed": 33978, "hz": 93.78569101220751 }, { "group": "stress_group_3", "controls": 3, "completed": 29590, "hz": 81.67398307879274 }, { "group": "stress_group_4", "controls": 3, "completed": 29628, "hz": 81.77887024868103 }, { "group": "stress_group_5", "controls": 3, "completed": 29724, "hz": 82.04384836208298 }, { "group": "stress_group_6", "controls": 3, "completed": 29567, "hz": 81.61049873912353 }, { "group": "stress_group_7", "controls": 3, "completed": 33945, "hz": 93.69460478572557 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 109235, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1251480728, "member_service_ns_total": 1283788100, "members_executed_total": 45116 }, "example_device_plugin.stress_read": { "callback_ns_total": 6432038840, "member_service_ns_total": 6455861200, "members_executed_total": 45116 }, "example_device_plugin.stress_write": { "callback_ns_total": 254681485, "member_service_ns_total": 270837800, "members_executed_total": 45116 } }, "callback_ns_total": 7938201053, "cycles_total": 11279, "member_service_ns_total": 8010487100, "members_executed_total": 135348, "thread_cpu_ns_total": 2265625000, "wall_ns_total": 8129277300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1067766556, "member_service_ns_total": 1098403800, "members_executed_total": 33987 }, "example_device_plugin.stress_read": { "callback_ns_total": 5231808793, "member_service_ns_total": 5259339900, "members_executed_total": 33987 }, "example_device_plugin.stress_write": { "callback_ns_total": 234687294, "member_service_ns_total": 250802000, "members_executed_total": 33987 } }, "callback_ns_total": 6534262643, "cycles_total": 11329, "member_service_ns_total": 6608545700, "members_executed_total": 101961, "thread_cpu_ns_total": 1640625000, "wall_ns_total": 6692181600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1034214643, "member_service_ns_total": 1060364100, "members_executed_total": 33978 }, "example_device_plugin.stress_read": { "callback_ns_total": 4741691290, "member_service_ns_total": 4771864500, "members_executed_total": 33978 }, "example_device_plugin.stress_write": { "callback_ns_total": 212093990, "member_service_ns_total": 221660900, "members_executed_total": 33978 } }, "callback_ns_total": 5987999923, "cycles_total": 11326, "member_service_ns_total": 6053889500, "members_executed_total": 101934, "thread_cpu_ns_total": 1703125000, "wall_ns_total": 6142816100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 956588868, "member_service_ns_total": 1034727000, "members_executed_total": 29590 }, "example_device_plugin.stress_read": { "callback_ns_total": 6956895518, "member_service_ns_total": 6985497300, "members_executed_total": 29590 }, "example_device_plugin.stress_write": { "callback_ns_total": 234654294, "member_service_ns_total": 259352800, "members_executed_total": 29589 } }, "callback_ns_total": 8148091580, "cycles_total": 9863, "member_service_ns_total": 8279528900, "members_executed_total": 88767, "thread_cpu_ns_total": 1328125000, "wall_ns_total": 8418528700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 878543593, "member_service_ns_total": 998223000, "members_executed_total": 29628 }, "example_device_plugin.stress_read": { "callback_ns_total": 6425982309, "member_service_ns_total": 6465881500, "members_executed_total": 29628 }, "example_device_plugin.stress_write": { "callback_ns_total": 303925391, "member_service_ns_total": 339246200, "members_executed_total": 29628 } }, "callback_ns_total": 7608451293, "cycles_total": 9876, "member_service_ns_total": 7803350700, "members_executed_total": 88884, "thread_cpu_ns_total": 1390625000, "wall_ns_total": 7981370000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1041340897, "member_service_ns_total": 1182065500, "members_executed_total": 29724 }, "example_device_plugin.stress_read": { "callback_ns_total": 6737538653, "member_service_ns_total": 6761817600, "members_executed_total": 29724 }, "example_device_plugin.stress_write": { "callback_ns_total": 265090796, "member_service_ns_total": 303540700, "members_executed_total": 29724 } }, "callback_ns_total": 8043970346, "cycles_total": 9908, "member_service_ns_total": 8247423800, "members_executed_total": 89172, "thread_cpu_ns_total": 1718750000, "wall_ns_total": 8401462200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1044360219, "member_service_ns_total": 1143728500, "members_executed_total": 29567 }, "example_device_plugin.stress_read": { "callback_ns_total": 6579958171, "member_service_ns_total": 6617129200, "members_executed_total": 29567 }, "example_device_plugin.stress_write": { "callback_ns_total": 283903584, "member_service_ns_total": 296034500, "members_executed_total": 29567 } }, "callback_ns_total": 7908098774, "cycles_total": 9855, "member_service_ns_total": 8056766200, "members_executed_total": 88695, "thread_cpu_ns_total": 1500000000, "wall_ns_total": 8170454900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1175772196, "member_service_ns_total": 1192799900, "members_executed_total": 33945 }, "example_device_plugin.stress_read": { "callback_ns_total": 5266545823, "member_service_ns_total": 5288491600, "members_executed_total": 33945 }, "example_device_plugin.stress_write": { "callback_ns_total": 195313496, "member_service_ns_total": 210098300, "members_executed_total": 33945 } }, "callback_ns_total": 6637631515, "cycles_total": 11315, "member_service_ns_total": 6691389800, "members_executed_total": 101835, "thread_cpu_ns_total": 1859375000, "wall_ns_total": 6782480200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.76469105, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.001383452619275, "failed": 0, "missed": 0, "callback_last_ms": 0.0044, "callback_ewma_ms": 0.004907, "release_lateness_last_ms": 70.4327 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.001383452619275, "failed": 0, "missed": 0, "callback_last_ms": 14.9556, "callback_ewma_ms": 15.817459, "release_lateness_last_ms": 67.7943 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 604, "hz": 4.998641366793737, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001197, "release_lateness_last_ms": 3.8672 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 604, "hz": 4.998641366793737, "failed": 0, "missed": 0, "callback_last_ms": 0.3969, "callback_ewma_ms": 0.349899, "release_lateness_last_ms": 4.8679 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.001383452619275, "failed": 0, "missed": 0, "callback_last_ms": 0.2797, "callback_ewma_ms": 0.267271, "release_lateness_last_ms": 77.3017 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.001383452619275, "failed": 0, "missed": 0, "callback_last_ms": 0.3559, "callback_ewma_ms": 0.294712, "release_lateness_last_ms": 81.1751 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.00276690523855, "failed": 0, "missed": 0, "callback_last_ms": 0.0016, "callback_ewma_ms": 0.010647, "release_lateness_last_ms": 0.5094 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.001383452619275, "failed": 0, "missed": 0, "callback_last_ms": 8.8194, "callback_ewma_ms": 10.51572, "release_lateness_last_ms": 11.347 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.00276690523855, "failed": 0, "missed": 0, "callback_last_ms": 5.0746, "callback_ewma_ms": 6.12265, "release_lateness_last_ms": 0.3997 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 604, "hz": 4.998641366793737, "failed": 0, "missed": 0, "callback_last_ms": 13.7728, "callback_ewma_ms": 15.680011, "release_lateness_last_ms": 2.5895 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1207, "hz": 9.989006837284835, "failed": 0, "missed": 1, "callback_last_ms": 15.5438, "callback_ewma_ms": 2.132472, "release_lateness_last_ms": 2.0422 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.998641366793737, "failed": 0, "missed": 0, "callback_last_ms": 6.8411, "callback_ewma_ms": 11.692062, "release_lateness_last_ms": 3.8356 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.00276690523855, "failed": 0, "missed": 0, "callback_last_ms": 8.4194, "callback_ewma_ms": 8.411791, "release_lateness_last_ms": 1.1923 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.998641366793737, "failed": 0, "missed": 0, "callback_last_ms": 0.0021, "callback_ewma_ms": 0.002097, "release_lateness_last_ms": 2.8728 }, { "name": "general_program_checks_thread", "running": true, "completed": 1206, "hz": 9.980730940982196, "failed": 0, "missed": 3, "callback_last_ms": 0.1697, "callback_ewma_ms": 0.301101, "release_lateness_last_ms": 2.5513 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.001383452619275, "failed": 0, "missed": 0, "callback_last_ms": 0.0765, "callback_ewma_ms": 0.113848, "release_lateness_last_ms": 11.1789 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1208, "hz": 9.997282733587474, "failed": 0, "missed": 0, "callback_last_ms": 0.0055, "callback_ewma_ms": 0.008533, "release_lateness_last_ms": 0.8945 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 1.9944910089359116, "failed": 0, "missed": 0, "callback_last_ms": 81.0661, "callback_ewma_ms": 80.447398, "release_lateness_last_ms": 0.6117 } ], "peer_counter_seconds": 120.8217517, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.002940716937965, "complete_hz": 5.002940716937965, "catalog_hz": 1.9995169366203143, "bytes_per_second": 1551966.172461193, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7716000000073, "gap_p95_ms": 267.03489999999874, "gap_p99_ms": 319.7274000000034, "gap_max_ms": 481.2214000000022, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 26, "p95": 98, "p99": 141, "max": 329, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002940716937965, "complete_hz": 5.002940716937965, "catalog_hz": 1.9995169366203143, "bytes_per_second": 1551966.172461193, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.5752999999968, "gap_p95_ms": 265.0501000000004, "gap_p99_ms": 340.7864999999874, "gap_max_ms": 463.3663000000088, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 22, "p95": 101, "p99": 169, "max": 287, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002940716937965, "complete_hz": 5.002940716937965, "catalog_hz": 1.9995169366203143, "bytes_per_second": 1551966.172461193, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.80499999999302, "gap_p95_ms": 263.579899999997, "gap_p99_ms": 328.87950000001, "gap_max_ms": 497.5175999999992, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 32, "p95": 109, "p99": 189, "max": 337, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.002940716937965, "complete_hz": 5.002940716937965, "catalog_hz": 1.9995169366203143, "bytes_per_second": 1551966.172461193, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.46319999999832, "gap_p95_ms": 274.58120000000054, "gap_p99_ms": 338.67600000000675, "gap_max_ms": 502.449099999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 38, "p95": 125, "p99": 186, "max": 349, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 99, "mean": 159.0497232323232, "p95": 296.4560999999958, "max": 522.7964999999967, "p50": 140.39040000000386, "p99": 522.7964999999967, "min": 72.23189999999886 }, { "operation": "rapid/get-persistence-diagnostics", "count": 99, "mean": 26.628944444444386, "p95": 89.4149999999936, "max": 439.0391000000018, "p50": 15.122499999997672, "p99": 439.0391000000018, "min": 4.789099999994505 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1957501, "accepted_rows_total": 12377434, "committed_rows_total": 1958233, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 99, "minimum": 71.96969696969697, "maximum": 100, "mean": 99.1911900738703 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 99, "minimum": 2.949997866561543, "maximum": 22.24014668795428, "mean": 10.763798726943882 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 99, "minimum": 999944192, "maximum": 1017782272, "mean": 1011393918.7070707 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 99, "minimum": 0, "maximum": 0.433349609375, "mean": 0.04482076625631313 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 99, "minimum": 0, "maximum": 0.048828125, "mean": 0.0007398200757575758 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 99, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 99, "minimum": 0, "maximum": 0.238037109375, "mean": 0.009864267676767676 }, { "channel": "engine_tempest_tx_mbps", "observations": 99, "minimum": 92.94138584760535, "maximum": 155.22741866925904, "mean": 123.83610736731211 }, { "channel": "engine_tempest_rx_mbps", "observations": 99, "minimum": 0, "maximum": 0.014129640658643798, "mean": 0.005487419625087254 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 99, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 99, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 99, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 99, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 99, "minimum": 8424.690630630308, "maximum": 34501.90999436292, "mean": 16003.354893569454 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 99, "minimum": 4925.888138433295, "maximum": 36494.49977875519, "mean": 15780.255849590587 } ], "graceful_shutdown": true, "shutdown_seconds": 2.2787875000000057 }, { "id": "verified-capacity-release-25-web-4-r3", "healthy": true, "failure_reasons": [], "seconds": 120.98008719999999, "measurement_completed": true, "background_preflight": null, "requested_client_count": 4, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 4, "transport": "web", "actual_connections": { "loadClients": 4, "actualClients": 5, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 1000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 103, "mean": 31.73080189038361, "p50": 26.121250642097493, "p95": 65.42399137370573, "p99": 100, "min": 15.30055794427544, "max": 100 }, "engine_cpu_percent": { "count": 103, "mean": 9.03851407390301, "p50": 9.394694967626775, "p95": 14.280888360535961, "p99": 15.474606300421764, "min": 4.054203759458211, "max": 15.63654222784495 }, "client_generator_cpu_percent": { "count": 103, "mean": 1.5207248688197545, "p50": 1.3909758472289224, "p95": 2.753812995565171, "p99": 2.8367728182516423, "min": 0.7004479998163817, "max": 2.960788669165147 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 103, "mean": 1009967859.5728155, "p50": 1010089984, "p95": 1015422976, "p99": 1017319424, "min": 996896768, "max": 1018081280 }, "system_samples": 103, "system_sample_interval_seconds": { "count": 103, "mean": 1.1932518815533986, "p95": 1.1347563, "max": 4.4621164, "p50": 1.108961, "p99": 4.0230493, "min": 1.0983798 }, "achieved_read_hz": { "minimum": 98.58488556521164, "maximum": 99.56759602390791 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47800, "hz": 98.48893836694765 }, { "group": "stress_group_1", "controls": 3, "completed": 36237, "hz": 99.55212439618082 }, { "group": "stress_group_2", "controls": 3, "completed": 36243, "hz": 99.56860790051 }, { "group": "stress_group_3", "controls": 3, "completed": 36234, "hz": 99.54388264401621 }, { "group": "stress_group_4", "controls": 3, "completed": 36234, "hz": 99.54388264401621 }, { "group": "stress_group_5", "controls": 3, "completed": 35943, "hz": 98.7444326840502 }, { "group": "stress_group_6", "controls": 3, "completed": 35931, "hz": 98.7114656753918 }, { "group": "stress_group_7", "controls": 3, "completed": 35931, "hz": 98.7114656753918 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 8508, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 634755899, "member_service_ns_total": 670830400, "members_executed_total": 47800 }, "example_device_plugin.stress_read": { "callback_ns_total": 4105622337, "member_service_ns_total": 4136572100, "members_executed_total": 47800 }, "example_device_plugin.stress_write": { "callback_ns_total": 195749397, "member_service_ns_total": 210851500, "members_executed_total": 47800 } }, "callback_ns_total": 4936127633, "cycles_total": 11950, "member_service_ns_total": 5018254000, "members_executed_total": 143400, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 5092028000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 511143873, "member_service_ns_total": 522188500, "members_executed_total": 36237 }, "example_device_plugin.stress_read": { "callback_ns_total": 2871284325, "member_service_ns_total": 2895733500, "members_executed_total": 36237 }, "example_device_plugin.stress_write": { "callback_ns_total": 149414293, "member_service_ns_total": 159161900, "members_executed_total": 36237 } }, "callback_ns_total": 3531842491, "cycles_total": 12079, "member_service_ns_total": 3577083900, "members_executed_total": 108711, "thread_cpu_ns_total": 1015625000, "wall_ns_total": 3644499200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 505381094, "member_service_ns_total": 518880900, "members_executed_total": 36243 }, "example_device_plugin.stress_read": { "callback_ns_total": 2858044736, "member_service_ns_total": 2879150700, "members_executed_total": 36243 }, "example_device_plugin.stress_write": { "callback_ns_total": 141682298, "member_service_ns_total": 150818600, "members_executed_total": 36243 } }, "callback_ns_total": 3505108128, "cycles_total": 12081, "member_service_ns_total": 3548850200, "members_executed_total": 108729, "thread_cpu_ns_total": 1203125000, "wall_ns_total": 3610092500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 519751564, "member_service_ns_total": 531090000, "members_executed_total": 36234 }, "example_device_plugin.stress_read": { "callback_ns_total": 2945928259, "member_service_ns_total": 2966711500, "members_executed_total": 36234 }, "example_device_plugin.stress_write": { "callback_ns_total": 143514396, "member_service_ns_total": 153957800, "members_executed_total": 36234 } }, "callback_ns_total": 3609194219, "cycles_total": 12078, "member_service_ns_total": 3651759300, "members_executed_total": 108702, "thread_cpu_ns_total": 953125000, "wall_ns_total": 3716662500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 522474077, "member_service_ns_total": 533486600, "members_executed_total": 36234 }, "example_device_plugin.stress_read": { "callback_ns_total": 2832524617, "member_service_ns_total": 2852376800, "members_executed_total": 36234 }, "example_device_plugin.stress_write": { "callback_ns_total": 142974297, "member_service_ns_total": 153670300, "members_executed_total": 36234 } }, "callback_ns_total": 3497972991, "cycles_total": 12078, "member_service_ns_total": 3539533700, "members_executed_total": 108702, "thread_cpu_ns_total": 562500000, "wall_ns_total": 3599395400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 514244610, "member_service_ns_total": 527529600, "members_executed_total": 35943 }, "example_device_plugin.stress_read": { "callback_ns_total": 3161141625, "member_service_ns_total": 3182215600, "members_executed_total": 35943 }, "example_device_plugin.stress_write": { "callback_ns_total": 169505498, "member_service_ns_total": 178504500, "members_executed_total": 35943 } }, "callback_ns_total": 3844891733, "cycles_total": 11981, "member_service_ns_total": 3888249700, "members_executed_total": 107829, "thread_cpu_ns_total": 546875000, "wall_ns_total": 3997754700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 487349240, "member_service_ns_total": 500434300, "members_executed_total": 35931 }, "example_device_plugin.stress_read": { "callback_ns_total": 3285793550, "member_service_ns_total": 3306478500, "members_executed_total": 35931 }, "example_device_plugin.stress_write": { "callback_ns_total": 151130000, "member_service_ns_total": 163084000, "members_executed_total": 35931 } }, "callback_ns_total": 3924272790, "cycles_total": 11977, "member_service_ns_total": 3969996800, "members_executed_total": 107793, "thread_cpu_ns_total": 687500000, "wall_ns_total": 4038723600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 499056076, "member_service_ns_total": 511008100, "members_executed_total": 35931 }, "example_device_plugin.stress_read": { "callback_ns_total": 3401265183, "member_service_ns_total": 3423661000, "members_executed_total": 35931 }, "example_device_plugin.stress_write": { "callback_ns_total": 161020095, "member_service_ns_total": 171702900, "members_executed_total": 35931 } }, "callback_ns_total": 4061341354, "cycles_total": 11977, "member_service_ns_total": 4106372000, "members_executed_total": 107793, "thread_cpu_ns_total": 718750000, "wall_ns_total": 4189689000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.3334228, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 0.9986342941494843, "failed": 0, "missed": 0, "callback_last_ms": 0.0036, "callback_ewma_ms": 0.00629, "release_lateness_last_ms": 0.4595 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 0.9986342941494843, "failed": 0, "missed": 0, "callback_last_ms": 3.213, "callback_ewma_ms": 3.787269, "release_lateness_last_ms": 0.5072 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 606, "hz": 5.001424646732128, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001343, "release_lateness_last_ms": 0.4079 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 606, "hz": 5.001424646732128, "failed": 0, "missed": 0, "callback_last_ms": 0.3826, "callback_ewma_ms": 0.246872, "release_lateness_last_ms": 0.3929 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 0.9986342941494843, "failed": 0, "missed": 0, "callback_last_ms": 0.2656, "callback_ewma_ms": 0.303914, "release_lateness_last_ms": 0.5097 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 0.9986342941494843, "failed": 0, "missed": 0, "callback_last_ms": 0.2088, "callback_ewma_ms": 0.224383, "release_lateness_last_ms": 0.3791 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 1.9972685882989687, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001109, "release_lateness_last_ms": 0.2557 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 0.9986342941494843, "failed": 0, "missed": 0, "callback_last_ms": 2.7012, "callback_ewma_ms": 2.387689, "release_lateness_last_ms": 0.3478 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 1.9972685882989687, "failed": 0, "missed": 0, "callback_last_ms": 3.0794, "callback_ewma_ms": 3.418834, "release_lateness_last_ms": 0.3267 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 606, "hz": 5.001424646732128, "failed": 0, "missed": 0, "callback_last_ms": 4.7629, "callback_ewma_ms": 4.219151, "release_lateness_last_ms": 0.267 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1212, "hz": 10.002849293464257, "failed": 0, "missed": 0, "callback_last_ms": 0.0241, "callback_ewma_ms": 0.599398, "release_lateness_last_ms": 0.2679 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 606, "hz": 5.001424646732128, "failed": 0, "missed": 0, "callback_last_ms": 2.8531, "callback_ewma_ms": 3.123857, "release_lateness_last_ms": 0.5098 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 1.9972685882989687, "failed": 0, "missed": 0, "callback_last_ms": 4.9205, "callback_ewma_ms": 4.896607, "release_lateness_last_ms": 0.2675 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 606, "hz": 5.001424646732128, "failed": 0, "missed": 0, "callback_last_ms": 0.0015, "callback_ewma_ms": 0.002048, "release_lateness_last_ms": 0.2015 }, { "name": "general_program_checks_thread", "running": true, "completed": 1212, "hz": 10.002849293464257, "failed": 0, "missed": 0, "callback_last_ms": 0.2589, "callback_ewma_ms": 0.181887, "release_lateness_last_ms": 0.6622 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 0.9986342941494843, "failed": 0, "missed": 0, "callback_last_ms": 0.0846, "callback_ewma_ms": 0.091589, "release_lateness_last_ms": 0.5914 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1212, "hz": 10.002849293464257, "failed": 0, "missed": 0, "callback_last_ms": 0.0078, "callback_ewma_ms": 0.006857, "release_lateness_last_ms": 0.4099 }, { "name": "engine_diagnostics", "running": true, "completed": 242, "hz": 1.9972685882989687, "failed": 0, "missed": 0, "callback_last_ms": 38.0037, "callback_ewma_ms": 38.947233, "release_lateness_last_ms": 0.1166 } ], "peer_counter_seconds": 121.16502660000002, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.000820055963905, "complete_hz": 5.000820055963905, "catalog_hz": 2.0003280223855615, "bytes_per_second": 1551059.027010908, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9933999999921, "gap_p95_ms": 207.22630000000208, "gap_p99_ms": 226.87449999999808, "gap_max_ms": 344.6405999999988, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 8, "p95": 21, "p99": 58, "max": 137, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000820055963905, "complete_hz": 5.000820055963905, "catalog_hz": 2.0003280223855615, "bytes_per_second": 1551059.027010908, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.92140000000654, "gap_p95_ms": 207.77799999999843, "gap_p99_ms": 232.3081999999995, "gap_max_ms": 322.0685999999987, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 9, "p95": 23, "p99": 56, "max": 133, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000820055963905, "complete_hz": 5.000820055963905, "catalog_hz": 2.0003280223855615, "bytes_per_second": 1551059.027010908, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.8633000000118, "gap_p95_ms": 208.65090000000055, "gap_p99_ms": 228.70880000000034, "gap_max_ms": 332.56810000000405, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 11, "p95": 29, "p99": 65, "max": 127, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000820055963905, "complete_hz": 5.000820055963905, "catalog_hz": 2.0003280223855615, "bytes_per_second": 1551059.027010908, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0052999999898, "gap_p95_ms": 208.65329999999813, "gap_p99_ms": 228.107600000003, "gap_max_ms": 325.73919999999634, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 12, "p95": 30, "p99": 65, "max": 123, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 111, "mean": 73.257210810811, "p95": 153.6972999999998, "max": 277.1843000000008, "p50": 64.84679999999935, "p99": 175.38989999999467, "min": 51.97640000001411 }, { "operation": "rapid/get-persistence-diagnostics", "count": 111, "mean": 12.540142342342897, "p95": 19.271199999995588, "max": 23.56640000000334, "p50": 15.19550000000163, "p99": 22.647400000001653, "min": 4.950599999996484 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2219805, "accepted_rows_total": 13965192, "committed_rows_total": 2219347, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 111, "minimum": 14.0625, "maximum": 100, "mean": 36.54034005596505 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 111, "minimum": 1.9525724220045726, "maximum": 23.952565143559625, "mean": 9.02837882421747 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 111, "minimum": 997609472, "maximum": 1016520704, "mean": 1009353589.6216216 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.4364013671875, "mean": 0.01248196438626126 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.01220703125, "mean": 0.0001099732545045045 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 111, "minimum": 0, "maximum": 0.048828125, "mean": 0.0014846389358108107 }, { "channel": "engine_tempest_tx_mbps", "observations": 111, "minimum": 89.33740098953199, "maximum": 167.08527731321823, "mean": 121.75418769129152 }, { "channel": "engine_tempest_rx_mbps", "observations": 111, "minimum": 0.0027331061871690254, "maximum": 0.014659264319901954, "mean": 0.005907994201362534 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 111, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 111, "minimum": 11630.152332938376, "maximum": 27412.554950167174, "mean": 18478.020222830597 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 111, "minimum": 4229.885537890732, "maximum": 28515.059897433024, "mean": 18511.980590321735 } ], "graceful_shutdown": true, "shutdown_seconds": 1.6887091000000074 }, { "id": "verified-capacity-release-25-web-8", "healthy": true, "failure_reasons": [], "seconds": 120.13295620000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 8, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 8, "transport": "web", "actual_connections": { "loadClients": 8, "actualClients": 9, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 2000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 109, "mean": 30.32068769815017, "p50": 27.833269327605215, "p95": 50.87344317382518, "p99": 63.692350982543864, "min": 14.889964280338308, "max": 76.31793204946455 }, "engine_cpu_percent": { "count": 109, "mean": 9.507055394220995, "p50": 7.069438399289121, "p95": 20.14389774788785, "p99": 21.80912012654867, "min": 3.5385917342482953, "max": 22.23290832256442 }, "client_generator_cpu_percent": { "count": 109, "mean": 2.450543208148288, "p50": 2.4891528691927465, "p95": 3.695758264059008, "p99": 4.204768858913605, "min": 1.233187830511251, "max": 4.632215435223692 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 109, "mean": 1009988485.8715596, "p50": 1011310592, "p95": 1015922688, "p99": 1018286080, "min": 992931840, "max": 1018957824 }, "system_samples": 109, "system_sample_interval_seconds": { "count": 109, "mean": 1.1092145376146787, "p95": 1.1515435, "max": 1.2617736, "p50": 1.1034056, "p99": 1.2077061, "min": 1.090453 }, "achieved_read_hz": { "minimum": 99.13316776660498, "maximum": 99.44933018063952 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47834, "hz": 99.43193000574826 }, { "group": "stress_group_1", "controls": 3, "completed": 35877, "hz": 99.43608738041922 }, { "group": "stress_group_2", "controls": 3, "completed": 35869, "hz": 99.41391471550736 }, { "group": "stress_group_3", "controls": 3, "completed": 35774, "hz": 99.15061431967884 }, { "group": "stress_group_4", "controls": 3, "completed": 35770, "hz": 99.13952798722289 }, { "group": "stress_group_5", "controls": 3, "completed": 35769, "hz": 99.13675640410891 }, { "group": "stress_group_6", "controls": 3, "completed": 35772, "hz": 99.14507115345087 }, { "group": "stress_group_7", "controls": 3, "completed": 35886, "hz": 99.46103162844508 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 6417, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 587744603, "member_service_ns_total": 602058800, "members_executed_total": 47834 }, "example_device_plugin.stress_read": { "callback_ns_total": 3416037008, "member_service_ns_total": 3437759100, "members_executed_total": 47834 }, "example_device_plugin.stress_write": { "callback_ns_total": 174595497, "member_service_ns_total": 186678300, "members_executed_total": 47834 } }, "callback_ns_total": 4177568709, "cycles_total": 11958, "member_service_ns_total": 4225684900, "members_executed_total": 143496, "thread_cpu_ns_total": 1343750000, "wall_ns_total": 4295642500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 464719498, "member_service_ns_total": 477286800, "members_executed_total": 35877 }, "example_device_plugin.stress_read": { "callback_ns_total": 2766530965, "member_service_ns_total": 2786918200, "members_executed_total": 35877 }, "example_device_plugin.stress_write": { "callback_ns_total": 140138196, "member_service_ns_total": 149913600, "members_executed_total": 35877 } }, "callback_ns_total": 3371388659, "cycles_total": 11959, "member_service_ns_total": 3414118600, "members_executed_total": 107631, "thread_cpu_ns_total": 1125000000, "wall_ns_total": 3482454200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 446779404, "member_service_ns_total": 458106700, "members_executed_total": 35869 }, "example_device_plugin.stress_read": { "callback_ns_total": 2704370915, "member_service_ns_total": 2722660700, "members_executed_total": 35869 }, "example_device_plugin.stress_write": { "callback_ns_total": 129795996, "member_service_ns_total": 139736500, "members_executed_total": 35869 } }, "callback_ns_total": 3280185315, "cycles_total": 11956, "member_service_ns_total": 3319741400, "members_executed_total": 107604, "thread_cpu_ns_total": 968750000, "wall_ns_total": 3378524600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 454159194, "member_service_ns_total": 469124900, "members_executed_total": 35774 }, "example_device_plugin.stress_read": { "callback_ns_total": 2931270522, "member_service_ns_total": 2952040000, "members_executed_total": 35774 }, "example_device_plugin.stress_write": { "callback_ns_total": 150622690, "member_service_ns_total": 162079700, "members_executed_total": 35774 } }, "callback_ns_total": 3535137406, "cycles_total": 11924, "member_service_ns_total": 3582314800, "members_executed_total": 107316, "thread_cpu_ns_total": 1062500000, "wall_ns_total": 3648840800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 444503114, "member_service_ns_total": 457210400, "members_executed_total": 35770 }, "example_device_plugin.stress_read": { "callback_ns_total": 2697964697, "member_service_ns_total": 2717666100, "members_executed_total": 35770 }, "example_device_plugin.stress_write": { "callback_ns_total": 140866797, "member_service_ns_total": 151149700, "members_executed_total": 35770 } }, "callback_ns_total": 3282316608, "cycles_total": 11923, "member_service_ns_total": 3325005900, "members_executed_total": 107307, "thread_cpu_ns_total": 703125000, "wall_ns_total": 3386045400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 431176305, "member_service_ns_total": 444478300, "members_executed_total": 35769 }, "example_device_plugin.stress_read": { "callback_ns_total": 2650460379, "member_service_ns_total": 2669381900, "members_executed_total": 35769 }, "example_device_plugin.stress_write": { "callback_ns_total": 137434598, "member_service_ns_total": 147185200, "members_executed_total": 35769 } }, "callback_ns_total": 3219071282, "cycles_total": 11923, "member_service_ns_total": 3261045400, "members_executed_total": 107307, "thread_cpu_ns_total": 515625000, "wall_ns_total": 3326462200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 453078455, "member_service_ns_total": 463957000, "members_executed_total": 35772 }, "example_device_plugin.stress_read": { "callback_ns_total": 2802890397, "member_service_ns_total": 2822394800, "members_executed_total": 35772 }, "example_device_plugin.stress_write": { "callback_ns_total": 145231297, "member_service_ns_total": 156086700, "members_executed_total": 35772 } }, "callback_ns_total": 3401200149, "cycles_total": 11924, "member_service_ns_total": 3442438500, "members_executed_total": 107316, "thread_cpu_ns_total": 1078125000, "wall_ns_total": 3503981200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 456883592, "member_service_ns_total": 473878100, "members_executed_total": 35886 }, "example_device_plugin.stress_read": { "callback_ns_total": 2808215889, "member_service_ns_total": 2838472300, "members_executed_total": 35886 }, "example_device_plugin.stress_write": { "callback_ns_total": 140668194, "member_service_ns_total": 149189900, "members_executed_total": 35886 } }, "callback_ns_total": 3405767675, "cycles_total": 11962, "member_service_ns_total": 3461540300, "members_executed_total": 107658, "thread_cpu_ns_total": 781250000, "wall_ns_total": 3524646600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.2682076, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.005503959984971, "failed": 0, "missed": 0, "callback_last_ms": 0.0043, "callback_ewma_ms": 0.003849, "release_lateness_last_ms": 0.2974 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.005503959984971, "failed": 0, "missed": 0, "callback_last_ms": 8.1665, "callback_ewma_ms": 5.484207, "release_lateness_last_ms": 0.2334 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 602, "hz": 5.0025899496772945, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.001785, "release_lateness_last_ms": 0.325 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 602, "hz": 5.0025899496772945, "failed": 0, "missed": 0, "callback_last_ms": 0.6604, "callback_ewma_ms": 0.660284, "release_lateness_last_ms": 0.4942 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.005503959984971, "failed": 0, "missed": 0, "callback_last_ms": 0.6386, "callback_ewma_ms": 0.858754, "release_lateness_last_ms": 0.1263 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.005503959984971, "failed": 0, "missed": 0, "callback_last_ms": 0.9458, "callback_ewma_ms": 0.76821, "release_lateness_last_ms": 0.4303 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 2.002697969887422, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.001396, "release_lateness_last_ms": 0.6455 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.005503959984971, "failed": 0, "missed": 0, "callback_last_ms": 3.0236, "callback_ewma_ms": 2.74984, "release_lateness_last_ms": 0.3056 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.002697969887422, "failed": 0, "missed": 0, "callback_last_ms": 5.9403, "callback_ewma_ms": 4.45179, "release_lateness_last_ms": 0.4093 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 5.0025899496772945, "failed": 0, "missed": 0, "callback_last_ms": 5.0863, "callback_ewma_ms": 4.771842, "release_lateness_last_ms": 0.2911 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1201, "hz": 9.980250049107028, "failed": 0, "missed": 2, "callback_last_ms": 0.0374, "callback_ewma_ms": 0.295332, "release_lateness_last_ms": 0.2318 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 601, "hz": 4.994279999594774, "failed": 0, "missed": 1, "callback_last_ms": 6.1905, "callback_ewma_ms": 5.774423, "release_lateness_last_ms": 0.4397 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.002697969887422, "failed": 0, "missed": 0, "callback_last_ms": 6.9554, "callback_ewma_ms": 6.340219, "release_lateness_last_ms": 0.3774 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 602, "hz": 5.0025899496772945, "failed": 0, "missed": 0, "callback_last_ms": 0.0018, "callback_ewma_ms": 0.001953, "release_lateness_last_ms": 0.3642 }, { "name": "general_program_checks_thread", "running": true, "completed": 1202, "hz": 9.988559999189548, "failed": 0, "missed": 1, "callback_last_ms": 0.302, "callback_ewma_ms": 0.306329, "release_lateness_last_ms": 0.1504 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.005503959984971, "failed": 0, "missed": 0, "callback_last_ms": 0.1809, "callback_ewma_ms": 0.206964, "release_lateness_last_ms": 0.1887 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1202, "hz": 9.988559999189548, "failed": 0, "missed": 1, "callback_last_ms": 0.0081, "callback_ewma_ms": 0.007222, "release_lateness_last_ms": 0.7734 }, { "name": "engine_diagnostics", "running": true, "completed": 240, "hz": 1.9943880198049013, "failed": 0, "missed": 0, "callback_last_ms": 38.7496, "callback_ewma_ms": 38.281237, "release_lateness_last_ms": 0.4202 } ], "peer_counter_seconds": 120.34562039999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.95799999999872, "gap_p95_ms": 213.7890999999945, "gap_p99_ms": 251.8391999999949, "gap_max_ms": 595.1361999999936, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 16, "p95": 31, "p99": 54, "max": 380, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.59010000000126, "gap_p95_ms": 217.16930000000866, "gap_p99_ms": 246.99439999999595, "gap_max_ms": 778.9698999999964, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 21, "p95": 37, "p99": 85, "max": 584, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7896999999939, "gap_p95_ms": 214.6463999999978, "gap_p99_ms": 237.14809999999125, "gap_max_ms": 827.4634000000005, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 12, "p95": 27, "p99": 58, "max": 625, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.8454000000056, "gap_p95_ms": 216.6264999999985, "gap_p99_ms": 239.11609999999928, "gap_max_ms": 839.2636999999959, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 14, "p95": 29, "p99": 68, "max": 646, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.0051000000094, "gap_p95_ms": 215.93510000000242, "gap_p99_ms": 257.71830000000773, "gap_max_ms": 582.9421000000002, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 23, "p95": 36, "p99": 68, "max": 396, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.01380000000063, "gap_p95_ms": 214.87739999999758, "gap_p99_ms": 246.80299999999988, "gap_max_ms": 753.3739999999962, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 19, "p95": 32, "p99": 65, "max": 545, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7338000000018, "gap_p95_ms": 215.33699999999953, "gap_p99_ms": 241.87239999999292, "gap_max_ms": 807.6194000000032, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 23, "p95": 36, "p99": 74, "max": 609, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.994465025645171, "complete_hz": 4.994465025645171, "catalog_hz": 2.0061101186341435, "bytes_per_second": 1548753.313022187, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.96809999999823, "gap_p95_ms": 213.93299999999726, "gap_p99_ms": 244.00079999999434, "gap_max_ms": 435.3903999999966, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 20, "p95": 34, "p99": 57, "max": 402, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 110, "mean": 67.5667472727272, "p95": 83.56669999999576, "max": 103.76929999999993, "p50": 66.34789999999339, "p99": 98.03160000000207, "min": 50.79170000000158 }, { "operation": "rapid/get-persistence-diagnostics", "count": 110, "mean": 13.107252727271936, "p95": 28.445599999999104, "max": 41.84549999999581, "p50": 14.973500000000058, "p99": 35.27590000000055, "min": 4.768000000010943 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2185460, "accepted_rows_total": 13833776, "committed_rows_total": 2186589, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 110, "minimum": 15.234375, "maximum": 100, "mean": 31.67518010779919 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 110, "minimum": 1.51573473313174, "maximum": 22.255249602635253, "mean": 9.069018407236166 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 110, "minimum": 990466048, "maximum": 1017835520, "mean": 1009714641.4545455 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 1.13525390625, "mean": 0.37322998046875 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0.0244140625, "mean": 0.0006658380681818181 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 110, "minimum": 0, "maximum": 0.054931640625, "mean": 0.0004993785511363636 }, { "channel": "engine_tempest_tx_mbps", "observations": 110, "minimum": 187.43095045329017, "maximum": 305.1730104775078, "mean": 237.340394077151 }, { "channel": "engine_tempest_rx_mbps", "observations": 110, "minimum": 0, "maximum": 0.019166639479234235, "mean": 0.010963443222401371 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 110, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 110, "minimum": 6158.005814988983, "maximum": 39588.00175333713, "mean": 18542.223553983313 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 110, "minimum": 6206.509326449198, "maximum": 41538.730722123626, "mean": 18397.561705519416 } ], "graceful_shutdown": true, "shutdown_seconds": 1.2441345999999902 }, { "id": "verified-capacity-release-25-web-8-r2", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz", "stress_group_7: control callbacks below 95 Hz" ], "seconds": 120.1725084, "measurement_completed": true, "background_preflight": null, "requested_client_count": 8, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 8, "transport": "web", "actual_connections": { "loadClients": 8, "actualClients": 9, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 2000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 79, "mean": 88.59916056672884, "p50": 92.89781695726727, "p95": 100, "p99": 100, "min": 42.32502077013967, "max": 100 }, "engine_cpu_percent": { "count": 79, "mean": 11.26214023346683, "p50": 9.700757940001036, "p95": 20.35488015579492, "p99": 24.50132806345266, "min": 3.975256140064026, "max": 24.50132806345266 }, "client_generator_cpu_percent": { "count": 79, "mean": 2.8619150035801812, "p50": 2.723746634211809, "p95": 4.2058041982135626, "p99": 4.633165122303918, "min": 1.6034110983098189, "max": 4.633165122303918 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 79, "mean": 1015818421.4683545, "p50": 1017147392, "p95": 1022521344, "p99": 1024217088, "min": 997924864, "max": 1024217088 }, "system_samples": 79, "system_sample_interval_seconds": { "count": 79, "mean": 1.5155441873417719, "p95": 2.5256106, "max": 3.396013, "p50": 1.3466793, "p99": 3.396013, "min": 1.1138437 }, "achieved_read_hz": { "minimum": 92.13684959494825, "maximum": 97.8933220666172 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47055, "hz": 97.74756835116033 }, { "group": "stress_group_1", "controls": 3, "completed": 35336, "hz": 97.87151419279854 }, { "group": "stress_group_2", "controls": 3, "completed": 35310, "hz": 97.79950096637187 }, { "group": "stress_group_3", "controls": 3, "completed": 35350, "hz": 97.91029054548983 }, { "group": "stress_group_4", "controls": 3, "completed": 33351, "hz": 92.37358132907019 }, { "group": "stress_group_5", "controls": 3, "completed": 33405, "hz": 92.52314726087941 }, { "group": "stress_group_6", "controls": 3, "completed": 33276, "hz": 92.16585086822403 }, { "group": "stress_group_7", "controls": 3, "completed": 33297, "hz": 92.22401539726094 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 43695, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1146886606, "member_service_ns_total": 1165932800, "members_executed_total": 47055 }, "example_device_plugin.stress_read": { "callback_ns_total": 5732915547, "member_service_ns_total": 5762667000, "members_executed_total": 47055 }, "example_device_plugin.stress_write": { "callback_ns_total": 306286682, "member_service_ns_total": 325138300, "members_executed_total": 47055 } }, "callback_ns_total": 7185441235, "cycles_total": 11763, "member_service_ns_total": 7253087400, "members_executed_total": 141156, "thread_cpu_ns_total": 2687500000, "wall_ns_total": 7356183100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 978538341, "member_service_ns_total": 1000198200, "members_executed_total": 35336 }, "example_device_plugin.stress_read": { "callback_ns_total": 4878406610, "member_service_ns_total": 4911561800, "members_executed_total": 35336 }, "example_device_plugin.stress_write": { "callback_ns_total": 239284490, "member_service_ns_total": 253493600, "members_executed_total": 35336 } }, "callback_ns_total": 6095804141, "cycles_total": 11778, "member_service_ns_total": 6164825300, "members_executed_total": 106002, "thread_cpu_ns_total": 1750000000, "wall_ns_total": 6273870200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 843102918, "member_service_ns_total": 861113700, "members_executed_total": 35310 }, "example_device_plugin.stress_read": { "callback_ns_total": 4483210553, "member_service_ns_total": 4508288600, "members_executed_total": 35310 }, "example_device_plugin.stress_write": { "callback_ns_total": 217457790, "member_service_ns_total": 236013700, "members_executed_total": 35310 } }, "callback_ns_total": 5543771261, "cycles_total": 11770, "member_service_ns_total": 5605416000, "members_executed_total": 105930, "thread_cpu_ns_total": 1687500000, "wall_ns_total": 5691535300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 890893924, "member_service_ns_total": 904464000, "members_executed_total": 35350 }, "example_device_plugin.stress_read": { "callback_ns_total": 4318928638, "member_service_ns_total": 4343085500, "members_executed_total": 35350 }, "example_device_plugin.stress_write": { "callback_ns_total": 205155978, "member_service_ns_total": 218495000, "members_executed_total": 35350 } }, "callback_ns_total": 5414740640, "cycles_total": 11783, "member_service_ns_total": 5465805400, "members_executed_total": 106047, "thread_cpu_ns_total": 2171875000, "wall_ns_total": 5545040200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 940652180, "member_service_ns_total": 999849000, "members_executed_total": 33351 }, "example_device_plugin.stress_read": { "callback_ns_total": 6341605595, "member_service_ns_total": 6372699100, "members_executed_total": 33351 }, "example_device_plugin.stress_write": { "callback_ns_total": 228345777, "member_service_ns_total": 254555900, "members_executed_total": 33351 } }, "callback_ns_total": 7510603552, "cycles_total": 11117, "member_service_ns_total": 7627104000, "members_executed_total": 100053, "thread_cpu_ns_total": 1609375000, "wall_ns_total": 7782710900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1008005021, "member_service_ns_total": 1062324000, "members_executed_total": 33405 }, "example_device_plugin.stress_read": { "callback_ns_total": 6502175122, "member_service_ns_total": 6554567300, "members_executed_total": 33405 }, "example_device_plugin.stress_write": { "callback_ns_total": 290331178, "member_service_ns_total": 330660700, "members_executed_total": 33405 } }, "callback_ns_total": 7800511321, "cycles_total": 11135, "member_service_ns_total": 7947552000, "members_executed_total": 100215, "thread_cpu_ns_total": 1906250000, "wall_ns_total": 8127262900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 812900495, "member_service_ns_total": 845971100, "members_executed_total": 33276 }, "example_device_plugin.stress_read": { "callback_ns_total": 5973079207, "member_service_ns_total": 6003434700, "members_executed_total": 33276 }, "example_device_plugin.stress_write": { "callback_ns_total": 267614079, "member_service_ns_total": 286048100, "members_executed_total": 33276 } }, "callback_ns_total": 7053593781, "cycles_total": 11092, "member_service_ns_total": 7135453900, "members_executed_total": 99828, "thread_cpu_ns_total": 1578125000, "wall_ns_total": 7307118400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 872049135, "member_service_ns_total": 917884300, "members_executed_total": 33297 }, "example_device_plugin.stress_read": { "callback_ns_total": 5856974245, "member_service_ns_total": 5901175500, "members_executed_total": 33297 }, "example_device_plugin.stress_write": { "callback_ns_total": 345600584, "member_service_ns_total": 362081200, "members_executed_total": 33297 } }, "callback_ns_total": 7074623964, "cycles_total": 11099, "member_service_ns_total": 7181141000, "members_executed_total": 99891, "thread_cpu_ns_total": 1812500000, "wall_ns_total": 7294129800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.34826235, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0050431186505913, "failed": 0, "missed": 0, "callback_last_ms": 0.0915, "callback_ewma_ms": 0.01366, "release_lateness_last_ms": 0.7616 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0050431186505913, "failed": 0, "missed": 0, "callback_last_ms": 6.639, "callback_ewma_ms": 12.295102, "release_lateness_last_ms": 2.66 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 602, "hz": 5.000297168823602, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.0011, "release_lateness_last_ms": 0.2045 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 602, "hz": 5.000297168823602, "failed": 0, "missed": 0, "callback_last_ms": 0.4207, "callback_ewma_ms": 0.688705, "release_lateness_last_ms": 0.1415 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0050431186505913, "failed": 0, "missed": 0, "callback_last_ms": 1.1141, "callback_ewma_ms": 0.869885, "release_lateness_last_ms": 3.0015 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0050431186505913, "failed": 0, "missed": 0, "callback_last_ms": 0.5011, "callback_ewma_ms": 1.291743, "release_lateness_last_ms": 0.6181 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 2.001780095824731, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.001412, "release_lateness_last_ms": 2.8293 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.99673697717414, "failed": 0, "missed": 0, "callback_last_ms": 1.2757, "callback_ewma_ms": 3.221462, "release_lateness_last_ms": 0.447 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 2.001780095824731, "failed": 0, "missed": 0, "callback_last_ms": 5.115, "callback_ewma_ms": 7.806524, "release_lateness_last_ms": 1.9672 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 5.000297168823602, "failed": 0, "missed": 0, "callback_last_ms": 6.2236, "callback_ewma_ms": 11.431839, "release_lateness_last_ms": 0.5569 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1201, "hz": 9.97567591321785, "failed": 0, "missed": 3, "callback_last_ms": 0.0245, "callback_ewma_ms": 1.233036, "release_lateness_last_ms": 0.4811 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 601, "hz": 4.991991027347151, "failed": 0, "missed": 1, "callback_last_ms": 7.4718, "callback_ewma_ms": 13.731686, "release_lateness_last_ms": 0.4333 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 2.001780095824731, "failed": 0, "missed": 0, "callback_last_ms": 6.9028, "callback_ewma_ms": 10.89508, "release_lateness_last_ms": 5.0586 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 602, "hz": 5.000297168823602, "failed": 0, "missed": 0, "callback_last_ms": 0.0024, "callback_ewma_ms": 0.006352, "release_lateness_last_ms": 0.6224 }, { "name": "general_program_checks_thread", "running": true, "completed": 1202, "hz": 9.983982054694302, "failed": 0, "missed": 2, "callback_last_ms": 0.3092, "callback_ewma_ms": 0.286238, "release_lateness_last_ms": 0.2654 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.99673697717414, "failed": 0, "missed": 0, "callback_last_ms": 0.0771, "callback_ewma_ms": 1.41988, "release_lateness_last_ms": 0.5297 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1204, "hz": 10.000594337647204, "failed": 0, "missed": 0, "callback_last_ms": 0.0069, "callback_ewma_ms": 0.00714, "release_lateness_last_ms": 0.2589 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 2.001780095824731, "failed": 0, "missed": 0, "callback_last_ms": 79.0035, "callback_ewma_ms": 78.501269, "release_lateness_last_ms": 2.2794 } ], "peer_counter_seconds": 120.3931616, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.99920000000566, "gap_p95_ms": 253.25869999997667, "gap_p99_ms": 297.0752999999968, "gap_max_ms": 904.6046999999962, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 31, "p95": 92, "p99": 166, "max": 713, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.2969000000012, "gap_p95_ms": 235.3703999999998, "gap_p99_ms": 307.36560000000463, "gap_max_ms": 791.3035000000091, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 28, "p95": 69, "p99": 165, "max": 626, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.59270000000834, "gap_p95_ms": 254.68619999999646, "gap_p99_ms": 341.9935000000114, "gap_max_ms": 901.4104000000079, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 37, "p95": 111, "p99": 190, "max": 728, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.29599999999482, "gap_p95_ms": 249.61019999999553, "gap_p99_ms": 313.8308000000252, "gap_max_ms": 851.5136999999959, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 36, "p95": 99, "p99": 148, "max": 674, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.91250000000582, "gap_p95_ms": 249.23890000000392, "gap_p99_ms": 298.6261999999988, "gap_max_ms": 773.5437999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 34, "p95": 96, "p99": 156, "max": 615, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.23949999999604, "gap_p95_ms": 242.2109999999957, "gap_p99_ms": 291.1883000000089, "gap_max_ms": 814.0694999999978, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 23, "p95": 72, "p99": 135, "max": 644, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.1539000000048, "gap_p95_ms": 243.869200000001, "gap_p99_ms": 301.7859999999928, "gap_max_ms": 766.1071999999986, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 31, "p95": 78, "p99": 193, "max": 604, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.992820021079803, "complete_hz": 4.992820021079803, "catalog_hz": 1.9971280084319212, "bytes_per_second": 1548155.6663057161, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.13469999999506, "gap_p95_ms": 253.22329999999783, "gap_p99_ms": 327.54700000000594, "gap_max_ms": 851.8257000000012, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1200, "p50": 38, "p95": 100, "p99": 187, "max": 697, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 104, "mean": 117.13583365384679, "p95": 225.48029999999562, "max": 609.6229000000021, "p50": 99.46229999999923, "p99": 342.12470000001485, "min": 67.91949999998906 }, { "operation": "rapid/get-persistence-diagnostics", "count": 104, "mean": 16.064030769230996, "p95": 54.67960000000312, "max": 106.79059999999299, "p50": 8.43069999999716, "p99": 75.38539999999921, "min": 4.978900000001886 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2073634, "accepted_rows_total": 13228020, "committed_rows_total": 2074829, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 104, "minimum": 35.9375, "maximum": 100, "mean": 90.97199208239812 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 104, "minimum": 2.7343164856272075, "maximum": 24.238562601070115, "mean": 11.20626680561777 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 104, "minimum": 995995648, "maximum": 1024159744, "mean": 1015499264 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 104, "minimum": 0, "maximum": 0.274658203125, "mean": 0.024384718674879808 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 104, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 104, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 104, "minimum": 0, "maximum": 0.35400390625, "mean": 0.009448711688701924 }, { "channel": "engine_tempest_tx_mbps", "observations": 104, "minimum": 110.90962370296288, "maximum": 306.76063414234795, "mean": 231.76021615553887 }, { "channel": "engine_tempest_rx_mbps", "observations": 104, "minimum": 0, "maximum": 0.0246270783847981, "mean": 0.011249330490303614 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 104, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 104, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 104, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 104, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 104, "minimum": 7350.813173741492, "maximum": 39402.76438765367, "mean": 17287.584974005804 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 104, "minimum": 8331.453109067093, "maximum": 39934.262122294436, "mean": 17380.256894991686 } ], "graceful_shutdown": true, "shutdown_seconds": 2.8038101000000024 }, { "id": "verified-capacity-release-25-web-8-r3", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_0: control callbacks below 95 Hz", "stress_group_1: control callbacks below 95 Hz", "stress_group_2: control callbacks below 95 Hz", "stress_group_3: control callbacks below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz", "stress_group_7: control callbacks below 95 Hz", "client transport warning/error" ], "seconds": 121.34824280000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 8, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 8, "transport": "web", "actual_connections": { "loadClients": 8, "actualClients": 9, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 2000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 29, "mean": 98.04964759616165, "p50": 99.62908870545577, "p95": 100, "p99": 100, "min": 88.70132455071267, "max": 100 }, "engine_cpu_percent": { "count": 29, "mean": 11.589391043954526, "p50": 10.29155254541802, "p95": 22.53984809945569, "p99": 26.57513236837525, "min": 5.065039128296492, "max": 26.57513236837525 }, "client_generator_cpu_percent": { "count": 29, "mean": 2.7660377434686443, "p50": 2.7001471390327794, "p95": 3.881450201708099, "p99": 4.003198877345042, "min": 1.7615377186656933, "max": 4.003198877345042 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 29, "mean": 1023333058.2068965, "p50": 1027715072, "p95": 1041420288, "p99": 1053913088, "min": 1002659840, "max": 1053913088 }, "system_samples": 29, "system_sample_interval_seconds": { "count": 29, "mean": 4.23717434137931, "p95": 12.3341282, "max": 13.7076991, "p50": 2.5097176, "p99": 13.7076991, "min": 1.1953881 }, "achieved_read_hz": { "minimum": 70.99809208463911, "maximum": 89.16155322193477 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 42856, "hz": 88.12187305709348 }, { "group": "stress_group_1", "controls": 3, "completed": 32520, "hz": 89.15821392000124 }, { "group": "stress_group_2", "controls": 3, "completed": 32262, "hz": 88.45087015642929 }, { "group": "stress_group_3", "controls": 3, "completed": 32499, "hz": 89.10063942761748 }, { "group": "stress_group_4", "controls": 3, "completed": 26187, "hz": 71.79539200255451 }, { "group": "stress_group_5", "controls": 3, "completed": 26238, "hz": 71.93521576977223 }, { "group": "stress_group_6", "controls": 3, "completed": 25902, "hz": 71.014023891632 }, { "group": "stress_group_7", "controls": 3, "completed": 26172, "hz": 71.75426736513754 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 177738, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1958607333, "member_service_ns_total": 2034863200, "members_executed_total": 42856 }, "example_device_plugin.stress_read": { "callback_ns_total": 8816965849, "member_service_ns_total": 8872653600, "members_executed_total": 42856 }, "example_device_plugin.stress_write": { "callback_ns_total": 400191980, "member_service_ns_total": 427405600, "members_executed_total": 42856 } }, "callback_ns_total": 11175765162, "cycles_total": 10714, "member_service_ns_total": 11334922400, "members_executed_total": 128568, "thread_cpu_ns_total": 2750000000, "wall_ns_total": 11472405300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1937657699, "member_service_ns_total": 2024827500, "members_executed_total": 32520 }, "example_device_plugin.stress_read": { "callback_ns_total": 7628588752, "member_service_ns_total": 7679159900, "members_executed_total": 32520 }, "example_device_plugin.stress_write": { "callback_ns_total": 276677391, "member_service_ns_total": 293139400, "members_executed_total": 32520 } }, "callback_ns_total": 9842923842, "cycles_total": 10840, "member_service_ns_total": 9997126800, "members_executed_total": 97560, "thread_cpu_ns_total": 2343750000, "wall_ns_total": 10152482500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1375982968, "member_service_ns_total": 1441465300, "members_executed_total": 32262 }, "example_device_plugin.stress_read": { "callback_ns_total": 6974835081, "member_service_ns_total": 6999496400, "members_executed_total": 32262 }, "example_device_plugin.stress_write": { "callback_ns_total": 252621090, "member_service_ns_total": 271374500, "members_executed_total": 32262 } }, "callback_ns_total": 8603439139, "cycles_total": 10754, "member_service_ns_total": 8712336200, "members_executed_total": 96786, "thread_cpu_ns_total": 2062500000, "wall_ns_total": 8826127200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1401682950, "member_service_ns_total": 1447022500, "members_executed_total": 32499 }, "example_device_plugin.stress_read": { "callback_ns_total": 6676836027, "member_service_ns_total": 6715471700, "members_executed_total": 32499 }, "example_device_plugin.stress_write": { "callback_ns_total": 269734491, "member_service_ns_total": 294967100, "members_executed_total": 32499 } }, "callback_ns_total": 8348253468, "cycles_total": 10833, "member_service_ns_total": 8457461300, "members_executed_total": 97497, "thread_cpu_ns_total": 1875000000, "wall_ns_total": 8551016300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 926259084, "member_service_ns_total": 1026862600, "members_executed_total": 26187 }, "example_device_plugin.stress_read": { "callback_ns_total": 7908153492, "member_service_ns_total": 7963220000, "members_executed_total": 26187 }, "example_device_plugin.stress_write": { "callback_ns_total": 333964090, "member_service_ns_total": 348665100, "members_executed_total": 26187 } }, "callback_ns_total": 9168376666, "cycles_total": 8729, "member_service_ns_total": 9338747700, "members_executed_total": 78561, "thread_cpu_ns_total": 1640625000, "wall_ns_total": 9464240400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1133981739, "member_service_ns_total": 1287273200, "members_executed_total": 26238 }, "example_device_plugin.stress_read": { "callback_ns_total": 8448923288, "member_service_ns_total": 8495539100, "members_executed_total": 26238 }, "example_device_plugin.stress_write": { "callback_ns_total": 383541485, "member_service_ns_total": 413389200, "members_executed_total": 26238 } }, "callback_ns_total": 9966446512, "cycles_total": 8746, "member_service_ns_total": 10196201500, "members_executed_total": 78714, "thread_cpu_ns_total": 1796875000, "wall_ns_total": 10413376700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 818049016, "member_service_ns_total": 933792500, "members_executed_total": 25902 }, "example_device_plugin.stress_read": { "callback_ns_total": 7791980280, "member_service_ns_total": 7829576500, "members_executed_total": 25902 }, "example_device_plugin.stress_write": { "callback_ns_total": 268876388, "member_service_ns_total": 289019900, "members_executed_total": 25902 } }, "callback_ns_total": 8878905684, "cycles_total": 8634, "member_service_ns_total": 9052388900, "members_executed_total": 77706, "thread_cpu_ns_total": 1421875000, "wall_ns_total": 9178037100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 984686585, "member_service_ns_total": 1089121600, "members_executed_total": 26172 }, "example_device_plugin.stress_read": { "callback_ns_total": 7102392481, "member_service_ns_total": 7128924800, "members_executed_total": 26172 }, "example_device_plugin.stress_write": { "callback_ns_total": 233665090, "member_service_ns_total": 262691000, "members_executed_total": 26172 } }, "callback_ns_total": 8320744156, "cycles_total": 8724, "member_service_ns_total": 8480737400, "members_executed_total": 78516, "thread_cpu_ns_total": 1453125000, "wall_ns_total": 8655239800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.58161905000001, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 0.9930316995989491, "failed": 0, "missed": 0, "callback_last_ms": 0.0056, "callback_ewma_ms": 0.004896, "release_lateness_last_ms": 4.0731 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 0.9930316995989491, "failed": 0, "missed": 0, "callback_last_ms": 15.279, "callback_ewma_ms": 12.850455, "release_lateness_last_ms": 1.6641 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 609, "hz": 4.997985992196363, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.001185, "release_lateness_last_ms": 1.1314 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 609, "hz": 4.997985992196363, "failed": 0, "missed": 0, "callback_last_ms": 0.8846, "callback_ewma_ms": 0.836163, "release_lateness_last_ms": 0.7234 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 0.9930316995989491, "failed": 0, "missed": 0, "callback_last_ms": 0.6302, "callback_ewma_ms": 0.681454, "release_lateness_last_ms": 1.4811 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 0.9930316995989491, "failed": 0, "missed": 0, "callback_last_ms": 1.0042, "callback_ewma_ms": 1.167263, "release_lateness_last_ms": 5.4926 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 243, "hz": 1.9942702727483028, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001282, "release_lateness_last_ms": 4.8932 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 122, "hz": 1.0012385731493536, "failed": 0, "missed": 0, "callback_last_ms": 0.8893, "callback_ewma_ms": 5.449577, "release_lateness_last_ms": 8.2655 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 243, "hz": 1.9942702727483028, "failed": 0, "missed": 0, "callback_last_ms": 18.8949, "callback_ewma_ms": 9.324335, "release_lateness_last_ms": 3.5726 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 608, "hz": 4.989779118645959, "failed": 0, "missed": 1, "callback_last_ms": 18.9425, "callback_ewma_ms": 13.359008, "release_lateness_last_ms": 4.1616 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1209, "hz": 9.922110122439086, "failed": 0, "missed": 9, "callback_last_ms": 0.08, "callback_ewma_ms": 1.608405, "release_lateness_last_ms": 3.7765 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 605, "hz": 4.965158497994746, "failed": 0, "missed": 4, "callback_last_ms": 17.0345, "callback_ewma_ms": 14.740152, "release_lateness_last_ms": 4.5712 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 243, "hz": 1.9942702727483028, "failed": 0, "missed": 0, "callback_last_ms": 25.9915, "callback_ewma_ms": 14.794771, "release_lateness_last_ms": 3.4964 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 609, "hz": 4.997985992196363, "failed": 0, "missed": 0, "callback_last_ms": 0.0024, "callback_ewma_ms": 0.003083, "release_lateness_last_ms": 4.4952 }, { "name": "general_program_checks_thread", "running": true, "completed": 1212, "hz": 9.946730743090301, "failed": 0, "missed": 6, "callback_last_ms": 0.4446, "callback_ewma_ms": 0.360074, "release_lateness_last_ms": 4.9533 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 122, "hz": 1.0012385731493536, "failed": 0, "missed": 0, "callback_last_ms": 0.0974, "callback_ewma_ms": 0.110348, "release_lateness_last_ms": 8.9259 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1218, "hz": 9.995971984392726, "failed": 0, "missed": 0, "callback_last_ms": 0.0029, "callback_ewma_ms": 0.004802, "release_lateness_last_ms": 3.6249 }, { "name": "engine_diagnostics", "running": true, "completed": 243, "hz": 1.9942702727483028, "failed": 0, "missed": 1, "callback_last_ms": 129.5791, "callback_ewma_ms": 104.090702, "release_lateness_last_ms": 3.6407 } ], "peer_counter_seconds": 121.84967500000002, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 180.5271999999968, "gap_p95_ms": 708.0733000000037, "gap_p99_ms": 1254.0283999999956, "gap_max_ms": 3011.996900000013, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 146, "p95": 1551, "p99": 2151, "max": 2887, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 179.608899999992, "gap_p95_ms": 623.4686999999976, "gap_p99_ms": 1308.2185000000027, "gap_max_ms": 7795.5092, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 135, "p95": 2241, "p99": 6598, "max": 7800, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 174.28530000001774, "gap_p95_ms": 633.8053999999975, "gap_p99_ms": 1365.3842000000004, "gap_max_ms": 3290.7595999999903, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 158, "p95": 1316, "p99": 2193, "max": 3252, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 175.16549999998824, "gap_p95_ms": 721.4204999999929, "gap_p99_ms": 1232.23520000001, "gap_max_ms": 1809.1267999999982, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 164, "p95": 1290, "p99": 1680, "max": 1970, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 177.55629999999655, "gap_p95_ms": 692.1909999999916, "gap_p99_ms": 1296.246399999989, "gap_max_ms": 1731.9475999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 145, "p95": 1473, "p99": 1817, "max": 2019, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 175.08959999999934, "gap_p95_ms": 718.5202999999965, "gap_p99_ms": 1144.6451000000088, "gap_max_ms": 3945.86099999999, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 134, "p95": 3549, "p99": 4265, "max": 4984, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 176.6967999999979, "gap_p95_ms": 674.6122999999934, "gap_p99_ms": 1310.1466999999975, "gap_max_ms": 3275.2978000000003, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 140, "p95": 3297, "p99": 3742, "max": 4044, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969169050211422, "complete_hz": 4.969169050211422, "catalog_hz": 1.9942602158394098, "bytes_per_second": 1541466.3263344592, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 178.87189999999828, "gap_p95_ms": 664.5026999999973, "gap_p99_ms": 1306.8952999999965, "gap_max_ms": 3193.4839999999967, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 151, "p95": 1360, "p99": 2144, "max": 3050, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 55, "mean": 366.7446381818167, "p95": 1408.7286999999924, "max": 1700.4402000000118, "p50": 222.10879999998724, "p99": 1700.4402000000118, "min": 62.374900000002526 }, { "operation": "rapid/get-persistence-diagnostics", "count": 55, "mean": 286.22185636363696, "p95": 852.3776000000071, "max": 7808.456000000006, "p50": 20.960699999995995, "p99": 7808.456000000006, "min": 5.926199999994424 } ], "transport_event_count": 9, "transport_events": [ { "at": "2026-10-29T23:31:10.184Z", "client": 7, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:10.185Z", "client": 1, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:10.185Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:10.186Z", "client": 2, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:10.186Z", "client": 5, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:10.186Z", "client": 3, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:10.186Z", "client": 0, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:24.653Z", "client": 5, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:31:55.978Z", "client": 1, "type": "heartbeat_failed", "message": "fetch failed" } ], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 1814544, "accepted_rows_total": 11441760, "committed_rows_total": 1815994, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 55, "minimum": 84.49612403100775, "maximum": 100, "mean": 98.84947465241233 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 55, "minimum": 3.5230671269990728, "maximum": 27.71239906753944, "mean": 10.680016903559977 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 55, "minimum": 1006166016, "maximum": 1040244736, "mean": 1024401333.5272727 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0.5096435546875, "mean": 0.06153453480113636 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0.10986328125, "mean": 0.0019975142045454545 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 55, "minimum": 0, "maximum": 2.838134765625, "mean": 0.14448686079545456 }, { "channel": "engine_tempest_tx_mbps", "observations": 55, "minimum": 190.72579519706306, "maximum": 309.9949085399928, "mean": 237.48155496437406 }, { "channel": "engine_tempest_rx_mbps", "observations": 55, "minimum": 0, "maximum": 0.027921103559880322, "mean": 0.009926480947340657 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 55, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 55, "minimum": 8607.902284271384, "maximum": 37641.15181564028, "mean": 16585.65172115338 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 55, "minimum": 8966.244725738397, "maximum": 48169.579847586385, "mean": 17237.269997323463 } ], "graceful_shutdown": true, "shutdown_seconds": 2.4663967999999876 }, { "id": "verified-capacity-release-25-web-16", "healthy": true, "failure_reasons": [], "seconds": 120.93154000000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 16, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 16, "transport": "web", "actual_connections": { "loadClients": 16, "actualClients": 17, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 4000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 104, "mean": 53.21872154416045, "p50": 48.30054392627229, "p95": 84.4497153964119, "p99": 89.1313388391942, "min": 26.9864313821092, "max": 92.58192522139971 }, "engine_cpu_percent": { "count": 104, "mean": 10.792102980758285, "p50": 8.403264177382285, "p95": 22.20377510889815, "p99": 24.971912227333988, "min": 4.741963562453395, "max": 24.983851470001078 }, "client_generator_cpu_percent": { "count": 104, "mean": 4.599957583279809, "p50": 4.5484384228431685, "p95": 6.388626377864864, "p99": 6.557290363951648, "min": 2.092492311681049, "max": 6.676332936458079 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 104, "mean": 1021137880.6153846, "p50": 1022144512, "p95": 1030107136, "p99": 1031061504, "min": 990613504, "max": 1031757824 }, "system_samples": 104, "system_sample_interval_seconds": { "count": 104, "mean": 1.1690915951923073, "p95": 1.4297064, "max": 1.8158797, "p50": 1.1327394, "p99": 1.569585, "min": 1.0989882 }, "achieved_read_hz": { "minimum": 97.35801754203159, "maximum": 98.97831512837492 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47845, "hz": 98.81647990472443 }, { "group": "stress_group_1", "controls": 3, "completed": 35937, "hz": 98.96311947151794 }, { "group": "stress_group_2", "controls": 3, "completed": 35919, "hz": 98.91355116724971 }, { "group": "stress_group_3", "controls": 3, "completed": 35358, "hz": 97.36867235088994 }, { "group": "stress_group_4", "controls": 3, "completed": 35351, "hz": 97.34939578811895 }, { "group": "stress_group_5", "controls": 3, "completed": 35381, "hz": 97.43200962856601 }, { "group": "stress_group_6", "controls": 3, "completed": 35397, "hz": 97.47607034347111 }, { "group": "stress_group_7", "controls": 3, "completed": 35940, "hz": 98.97138085556264 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 16383, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 913627317, "member_service_ns_total": 935972200, "members_executed_total": 47845 }, "example_device_plugin.stress_read": { "callback_ns_total": 5150907873, "member_service_ns_total": 5194723100, "members_executed_total": 47846 }, "example_device_plugin.stress_write": { "callback_ns_total": 269195389, "member_service_ns_total": 286099500, "members_executed_total": 47845 } }, "callback_ns_total": 6333281179, "cycles_total": 11961, "member_service_ns_total": 6416343700, "members_executed_total": 143532, "thread_cpu_ns_total": 1968750000, "wall_ns_total": 6508651400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 815523657, "member_service_ns_total": 836188000, "members_executed_total": 35937 }, "example_device_plugin.stress_read": { "callback_ns_total": 4337971129, "member_service_ns_total": 4366763300, "members_executed_total": 35937 }, "example_device_plugin.stress_write": { "callback_ns_total": 199572388, "member_service_ns_total": 212897200, "members_executed_total": 35937 } }, "callback_ns_total": 5353067174, "cycles_total": 11979, "member_service_ns_total": 5415848500, "members_executed_total": 107811, "thread_cpu_ns_total": 1406250000, "wall_ns_total": 5509675700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 723455628, "member_service_ns_total": 739369400, "members_executed_total": 35919 }, "example_device_plugin.stress_read": { "callback_ns_total": 4007010400, "member_service_ns_total": 4029141800, "members_executed_total": 35919 }, "example_device_plugin.stress_write": { "callback_ns_total": 204709585, "member_service_ns_total": 215958000, "members_executed_total": 35919 } }, "callback_ns_total": 4935175613, "cycles_total": 11973, "member_service_ns_total": 4984469200, "members_executed_total": 107757, "thread_cpu_ns_total": 1546875000, "wall_ns_total": 5066367600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 772900516, "member_service_ns_total": 791329900, "members_executed_total": 35358 }, "example_device_plugin.stress_read": { "callback_ns_total": 4519636151, "member_service_ns_total": 4552468100, "members_executed_total": 35358 }, "example_device_plugin.stress_write": { "callback_ns_total": 216959788, "member_service_ns_total": 231124500, "members_executed_total": 35358 } }, "callback_ns_total": 5509496455, "cycles_total": 11786, "member_service_ns_total": 5574922500, "members_executed_total": 106074, "thread_cpu_ns_total": 1875000000, "wall_ns_total": 5664420000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 647735455, "member_service_ns_total": 680786500, "members_executed_total": 35351 }, "example_device_plugin.stress_read": { "callback_ns_total": 4238989810, "member_service_ns_total": 4263006100, "members_executed_total": 35351 }, "example_device_plugin.stress_write": { "callback_ns_total": 197881489, "member_service_ns_total": 212964100, "members_executed_total": 35351 } }, "callback_ns_total": 5084127754, "cycles_total": 11783, "member_service_ns_total": 5156274600, "members_executed_total": 106047, "thread_cpu_ns_total": 1156250000, "wall_ns_total": 5254346400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 642648046, "member_service_ns_total": 670355700, "members_executed_total": 35381 }, "example_device_plugin.stress_read": { "callback_ns_total": 4133170793, "member_service_ns_total": 4155897500, "members_executed_total": 35381 }, "example_device_plugin.stress_write": { "callback_ns_total": 210295796, "member_service_ns_total": 221345600, "members_executed_total": 35381 } }, "callback_ns_total": 4985539035, "cycles_total": 11793, "member_service_ns_total": 5047019600, "members_executed_total": 106137, "thread_cpu_ns_total": 1062500000, "wall_ns_total": 5133333000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 632943722, "member_service_ns_total": 654294800, "members_executed_total": 35397 }, "example_device_plugin.stress_read": { "callback_ns_total": 4421757355, "member_service_ns_total": 4455664800, "members_executed_total": 35397 }, "example_device_plugin.stress_write": { "callback_ns_total": 220076385, "member_service_ns_total": 231686900, "members_executed_total": 35397 } }, "callback_ns_total": 5273817162, "cycles_total": 11798, "member_service_ns_total": 5340683000, "members_executed_total": 106182, "thread_cpu_ns_total": 1578125000, "wall_ns_total": 5456500200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 776255106, "member_service_ns_total": 790362400, "members_executed_total": 35940 }, "example_device_plugin.stress_read": { "callback_ns_total": 4332536677, "member_service_ns_total": 4361714100, "members_executed_total": 35940 }, "example_device_plugin.stress_write": { "callback_ns_total": 196041788, "member_service_ns_total": 206720900, "members_executed_total": 35940 } }, "callback_ns_total": 5304833571, "cycles_total": 11980, "member_service_ns_total": 5358797400, "members_executed_total": 107820, "thread_cpu_ns_total": 1750000000, "wall_ns_total": 5448471000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.045093, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 0.9982255427255342, "failed": 0, "missed": 0, "callback_last_ms": 0.0057, "callback_ewma_ms": 0.022128, "release_lateness_last_ms": 1.8989 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 0.9982255427255342, "failed": 0, "missed": 0, "callback_last_ms": 24.5031, "callback_ewma_ms": 16.262763, "release_lateness_last_ms": 2.087 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 606, "hz": 4.999377511501436, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.003212, "release_lateness_last_ms": 0.5962 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 606, "hz": 4.999377511501436, "failed": 0, "missed": 0, "callback_last_ms": 2.8343, "callback_ewma_ms": 2.707785, "release_lateness_last_ms": 0.6183 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 0.9982255427255342, "failed": 0, "missed": 0, "callback_last_ms": 3.3659, "callback_ewma_ms": 3.223078, "release_lateness_last_ms": 2.8044 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 0.9982255427255342, "failed": 0, "missed": 0, "callback_last_ms": 2.1041, "callback_ewma_ms": 3.135779, "release_lateness_last_ms": 2.8077 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 1.9964510854510684, "failed": 0, "missed": 0, "callback_last_ms": 0.0013, "callback_ewma_ms": 0.001931, "release_lateness_last_ms": 0.2966 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 0.9982255427255342, "failed": 0, "missed": 0, "callback_last_ms": 2.7183, "callback_ewma_ms": 2.949379, "release_lateness_last_ms": 0.8605 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 1.9964510854510684, "failed": 0, "missed": 0, "callback_last_ms": 8.3626, "callback_ewma_ms": 7.923649, "release_lateness_last_ms": 0.5671 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 605, "hz": 4.991127713627671, "failed": 0, "missed": 1, "callback_last_ms": 15.546, "callback_ewma_ms": 14.44695, "release_lateness_last_ms": 2.9055 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1212, "hz": 9.998755023002872, "failed": 0, "missed": 0, "callback_last_ms": 0.0264, "callback_ewma_ms": 1.459885, "release_lateness_last_ms": 0.8385 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.982877915753906, "failed": 0, "missed": 2, "callback_last_ms": 17.6764, "callback_ewma_ms": 20.018238, "release_lateness_last_ms": 0.8495 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 1.9964510854510684, "failed": 0, "missed": 0, "callback_last_ms": 16.6271, "callback_ewma_ms": 14.307379, "release_lateness_last_ms": 0.4586 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 606, "hz": 4.999377511501436, "failed": 0, "missed": 0, "callback_last_ms": 0.0026, "callback_ewma_ms": 0.005432, "release_lateness_last_ms": 0.7619 }, { "name": "general_program_checks_thread", "running": true, "completed": 1211, "hz": 9.990505225129107, "failed": 0, "missed": 1, "callback_last_ms": 2.0214, "callback_ewma_ms": 0.538112, "release_lateness_last_ms": 0.9125 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 0.9982255427255342, "failed": 0, "missed": 0, "callback_last_ms": 0.2697, "callback_ewma_ms": 0.172072, "release_lateness_last_ms": 0.7828 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1212, "hz": 9.998755023002872, "failed": 0, "missed": 0, "callback_last_ms": 0.0061, "callback_ewma_ms": 0.012004, "release_lateness_last_ms": 0.6075 }, { "name": "engine_diagnostics", "running": true, "completed": 243, "hz": 2.004700883324833, "failed": 0, "missed": 0, "callback_last_ms": 84.358, "callback_ewma_ms": 64.737305, "release_lateness_last_ms": 0.5803 } ], "peer_counter_seconds": 121.21289240000002, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.40959999999905, "gap_p95_ms": 254.7980000000025, "gap_p99_ms": 292.41700000001583, "gap_max_ms": 651.3681999999972, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 28, "p95": 87, "p99": 149, "max": 417, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.61420000000362, "gap_p95_ms": 252.2791000000143, "gap_p99_ms": 322.164300000004, "gap_max_ms": 552.4160999999949, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 31, "p95": 94, "p99": 179, "max": 568, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.13409999999567, "gap_p95_ms": 253.28669999999693, "gap_p99_ms": 340.69029999999475, "gap_max_ms": 571.270500000006, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 35, "p95": 94, "p99": 199, "max": 406, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.3353000000061, "gap_p95_ms": 247.1235000000015, "gap_p99_ms": 322.68030000000726, "gap_max_ms": 525.5817000000025, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 35, "p95": 90, "p99": 189, "max": 523, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.20410000000265, "gap_p95_ms": 249.35409999999683, "gap_p99_ms": 316.5417000000016, "gap_max_ms": 667.9628000000012, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 32, "p95": 89, "p99": 152, "max": 484, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.56439999999566, "gap_p95_ms": 248.32519999999204, "gap_p99_ms": 325.4302000000025, "gap_max_ms": 615.5506999999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 42, "p95": 95, "p99": 218, "max": 465, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.10410000001139, "gap_p95_ms": 248.0783999999985, "gap_p99_ms": 323.2262000000046, "gap_max_ms": 555.6879000000044, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 36, "p95": 100, "p99": 199, "max": 579, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.77529999999388, "gap_p95_ms": 251.51959999999963, "gap_p99_ms": 311.16189999999915, "gap_max_ms": 653.5727000000043, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 32, "p95": 98, "p99": 168, "max": 439, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.81489999999758, "gap_p95_ms": 247.60749999999825, "gap_p99_ms": 303.16590000000724, "gap_max_ms": 597.3194999999978, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 46, "p95": 96, "p99": 194, "max": 452, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.7448999999906, "gap_p95_ms": 252.80530000000726, "gap_p99_ms": 340.60959999999614, "gap_max_ms": 564.7445999999982, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 40, "p95": 106, "p99": 223, "max": 434, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.23869999998715, "gap_p95_ms": 250.517399999997, "gap_p99_ms": 322.9910999999847, "gap_max_ms": 530.2223999999987, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 42, "p95": 103, "p99": 224, "max": 559, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.25199999999313, "gap_p95_ms": 250.18709999999555, "gap_p99_ms": 334.00049999999464, "gap_max_ms": 669.0744999999952, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 39, "p95": 103, "p99": 175, "max": 504, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.16999999999825, "gap_p95_ms": 248.35190000000875, "gap_p99_ms": 317.7528999999922, "gap_max_ms": 609.6416000000027, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 45, "p95": 98, "p99": 182, "max": 426, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.4426999999996, "gap_p95_ms": 248.13390000000072, "gap_p99_ms": 321.94079999999667, "gap_max_ms": 559.6463000000003, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 44, "p95": 105, "p99": 189, "max": 443, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.07690000000002, "gap_p95_ms": 250.86270000000513, "gap_p99_ms": 331.71510000000126, "gap_max_ms": 652.0409999999974, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 44, "p95": 107, "p99": 200, "max": 468, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9862849215001965, "complete_hz": 4.9862849215001965, "catalog_hz": 2.0011292719785203, "bytes_per_second": 1546253.6686901401, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.88289999999688, "gap_p95_ms": 248.56729999999516, "gap_p99_ms": 342.7427000000025, "gap_max_ms": 594.9461999999985, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1206, "p50": 49, "p95": 98, "p99": 261, "max": 473, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 106, "mean": 104.3341915094332, "p95": 193.32620000000315, "max": 622.1252000000095, "p50": 87.03689999999915, "p99": 453.065300000002, "min": 53.295500000000175 }, { "operation": "rapid/get-persistence-diagnostics", "count": 106, "mean": 12.029411320755244, "p95": 42.829399999995076, "max": 62.213199999998324, "p50": 7.652100000006612, "p99": 58.62949999999546, "min": 5.0046000000002095 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2165191, "accepted_rows_total": 13743928, "committed_rows_total": 2165731, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 106, "minimum": 24.609375, "maximum": 97.65625, "mean": 53.26611013181522 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 106, "minimum": 3.12269857115306, "maximum": 28.10139482834419, "mean": 10.730499493703185 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 106, "minimum": 996605952, "maximum": 1031307264, "mean": 1020625011.9245284 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 106, "minimum": 0, "maximum": 0.7232666015625, "mean": 0.04499903265035377 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 106, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 106, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 106, "minimum": 0, "maximum": 0.1220703125, "mean": 0.005182230247641509 }, { "channel": "engine_tempest_tx_mbps", "observations": 106, "minimum": 372.8520480931354, "maximum": 604.617035710111, "mean": 484.6266365695545 }, { "channel": "engine_tempest_rx_mbps", "observations": 106, "minimum": 0, "maximum": 0.04687074604203255, "mean": 0.02255724933674385 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 106, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 106, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 106, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 106, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 106, "minimum": 11147.127092234236, "maximum": 39485.57964474977, "mean": 17822.841398991957 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 106, "minimum": 10834.917166758953, "maximum": 40394.572308426585, "mean": 17947.664347969996 } ], "graceful_shutdown": true, "shutdown_seconds": 1.9400971999999892 }, { "id": "verified-capacity-release-25-web-16-r2", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_0: control callbacks below 95 Hz", "stress_group_3: control callbacks below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz" ], "seconds": 120.1329068, "measurement_completed": true, "background_preflight": null, "requested_client_count": 16, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 16, "transport": "web", "actual_connections": { "loadClients": 16, "actualClients": 17, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 4000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 90, "mean": 65.40038253789521, "p50": 62.78116595987273, "p95": 99.22726690957604, "p99": 100, "min": 28.231473829399988, "max": 100 }, "engine_cpu_percent": { "count": 90, "mean": 11.82379326205749, "p50": 9.833991435279028, "p95": 22.773266188980987, "p99": 26.38878296689874, "min": 5.648305567201321, "max": 26.38878296689874 }, "client_generator_cpu_percent": { "count": 90, "mean": 4.860114255604185, "p50": 4.7990181597720385, "p95": 6.688769562764396, "p99": 7.441773679787537, "min": 2.807433004541753, "max": 7.441773679787537 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 90, "mean": 1019077108.6222222, "p50": 1019043840, "p95": 1037975552, "p99": 1063546880, "min": 996802560, "max": 1063546880 }, "system_samples": 90, "system_sample_interval_seconds": { "count": 90, "mean": 1.3307929433333336, "p95": 1.6766055, "max": 5.563166, "p50": 1.1515194, "p99": 5.563166, "min": 1.0997377 }, "achieved_read_hz": { "minimum": 93.87807954290012, "maximum": 95.70040991087558 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 45676, "hz": 94.95389277536508 }, { "group": "stress_group_1", "controls": 3, "completed": 34431, "hz": 95.43618770320211 }, { "group": "stress_group_2", "controls": 3, "completed": 34461, "hz": 95.51934200110504 }, { "group": "stress_group_3", "controls": 3, "completed": 33864, "hz": 93.86457147283659 }, { "group": "stress_group_4", "controls": 3, "completed": 33891, "hz": 93.93941034094922 }, { "group": "stress_group_5", "controls": 3, "completed": 33942, "hz": 94.08077264738422 }, { "group": "stress_group_6", "controls": 3, "completed": 33867, "hz": 93.87288690262687 }, { "group": "stress_group_7", "controls": 3, "completed": 34521, "hz": 95.68565059691093 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 47796, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1341269642, "member_service_ns_total": 1394942400, "members_executed_total": 45676 }, "example_device_plugin.stress_read": { "callback_ns_total": 6588632000, "member_service_ns_total": 6617399000, "members_executed_total": 45676 }, "example_device_plugin.stress_write": { "callback_ns_total": 279951186, "member_service_ns_total": 294975300, "members_executed_total": 45676 } }, "callback_ns_total": 8209852828, "cycles_total": 11419, "member_service_ns_total": 8307316700, "members_executed_total": 137028, "thread_cpu_ns_total": 2375000000, "wall_ns_total": 8418182500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1054930389, "member_service_ns_total": 1084380400, "members_executed_total": 34431 }, "example_device_plugin.stress_read": { "callback_ns_total": 5637935592, "member_service_ns_total": 5663522500, "members_executed_total": 34431 }, "example_device_plugin.stress_write": { "callback_ns_total": 201343391, "member_service_ns_total": 219733600, "members_executed_total": 34431 } }, "callback_ns_total": 6894209372, "cycles_total": 11477, "member_service_ns_total": 6967636500, "members_executed_total": 103293, "thread_cpu_ns_total": 1859375000, "wall_ns_total": 7050334100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 935287386, "member_service_ns_total": 958936300, "members_executed_total": 34461 }, "example_device_plugin.stress_read": { "callback_ns_total": 5535505486, "member_service_ns_total": 5559315800, "members_executed_total": 34461 }, "example_device_plugin.stress_write": { "callback_ns_total": 188338693, "member_service_ns_total": 200592800, "members_executed_total": 34461 } }, "callback_ns_total": 6659131565, "cycles_total": 11487, "member_service_ns_total": 6718844900, "members_executed_total": 103383, "thread_cpu_ns_total": 1937500000, "wall_ns_total": 6807783900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 700767725, "member_service_ns_total": 733028700, "members_executed_total": 33864 }, "example_device_plugin.stress_read": { "callback_ns_total": 6142995086, "member_service_ns_total": 6184521200, "members_executed_total": 33864 }, "example_device_plugin.stress_write": { "callback_ns_total": 244248780, "member_service_ns_total": 265689700, "members_executed_total": 33864 } }, "callback_ns_total": 7088011591, "cycles_total": 11288, "member_service_ns_total": 7183239600, "members_executed_total": 101592, "thread_cpu_ns_total": 1890625000, "wall_ns_total": 7319073700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 785812765, "member_service_ns_total": 813215900, "members_executed_total": 33891 }, "example_device_plugin.stress_read": { "callback_ns_total": 5587424682, "member_service_ns_total": 5612714300, "members_executed_total": 33891 }, "example_device_plugin.stress_write": { "callback_ns_total": 251893483, "member_service_ns_total": 282161400, "members_executed_total": 33891 } }, "callback_ns_total": 6625130930, "cycles_total": 11297, "member_service_ns_total": 6708091600, "members_executed_total": 101673, "thread_cpu_ns_total": 1359375000, "wall_ns_total": 6810855000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 982541974, "member_service_ns_total": 1025217400, "members_executed_total": 33942 }, "example_device_plugin.stress_read": { "callback_ns_total": 5479098930, "member_service_ns_total": 5519495200, "members_executed_total": 33942 }, "example_device_plugin.stress_write": { "callback_ns_total": 276992382, "member_service_ns_total": 294098100, "members_executed_total": 33942 } }, "callback_ns_total": 6738633286, "cycles_total": 11314, "member_service_ns_total": 6838810700, "members_executed_total": 101826, "thread_cpu_ns_total": 1421875000, "wall_ns_total": 6976737900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 784050589, "member_service_ns_total": 807009000, "members_executed_total": 33867 }, "example_device_plugin.stress_read": { "callback_ns_total": 5508013380, "member_service_ns_total": 5537444500, "members_executed_total": 33867 }, "example_device_plugin.stress_write": { "callback_ns_total": 292858182, "member_service_ns_total": 303957400, "members_executed_total": 33867 } }, "callback_ns_total": 6584922151, "cycles_total": 11289, "member_service_ns_total": 6648410900, "members_executed_total": 101601, "thread_cpu_ns_total": 1515625000, "wall_ns_total": 6784636700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1077660638, "member_service_ns_total": 1100100000, "members_executed_total": 34521 }, "example_device_plugin.stress_read": { "callback_ns_total": 5347448048, "member_service_ns_total": 5371913600, "members_executed_total": 34521 }, "example_device_plugin.stress_write": { "callback_ns_total": 222630293, "member_service_ns_total": 236978200, "members_executed_total": 34521 } }, "callback_ns_total": 6647738979, "cycles_total": 11507, "member_service_ns_total": 6708991800, "members_executed_total": 103563, "thread_cpu_ns_total": 1718750000, "wall_ns_total": 6789128400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.25836610000002, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 120, "hz": 0.9955395332390663, "failed": 0, "missed": 0, "callback_last_ms": 0.0034, "callback_ewma_ms": 0.007742, "release_lateness_last_ms": 1.0184 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 120, "hz": 0.9955395332390663, "failed": 0, "missed": 0, "callback_last_ms": 7.1177, "callback_ewma_ms": 23.966417, "release_lateness_last_ms": 0.0969 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 603, "hz": 5.002586154526308, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001135, "release_lateness_last_ms": 4.7567 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 603, "hz": 5.002586154526308, "failed": 0, "missed": 0, "callback_last_ms": 2.6831, "callback_ewma_ms": 1.881987, "release_lateness_last_ms": 21.2603 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0038356960160586, "failed": 0, "missed": 0, "callback_last_ms": 1.617, "callback_ewma_ms": 1.626134, "release_lateness_last_ms": 3.086 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0038356960160586, "failed": 0, "missed": 0, "callback_last_ms": 1.8711, "callback_ewma_ms": 1.907946, "release_lateness_last_ms": 0.7146 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 241, "hz": 1.9993752292551248, "failed": 0, "missed": 0, "callback_last_ms": 0.0014, "callback_ewma_ms": 0.001352, "release_lateness_last_ms": 0.6571 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 120, "hz": 0.9955395332390663, "failed": 0, "missed": 0, "callback_last_ms": 0.9028, "callback_ewma_ms": 4.213304, "release_lateness_last_ms": 0.0629 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 241, "hz": 1.9993752292551248, "failed": 0, "missed": 0, "callback_last_ms": 23.9264, "callback_ewma_ms": 9.155124, "release_lateness_last_ms": 2.2432 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 601, "hz": 4.985993828972323, "failed": 0, "missed": 1, "callback_last_ms": 10.1493, "callback_ewma_ms": 8.898703, "release_lateness_last_ms": 21.1835 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1200, "hz": 9.955395332390664, "failed": 0, "missed": 4, "callback_last_ms": 26.6698, "callback_ewma_ms": 3.132682, "release_lateness_last_ms": 0.5979 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 598, "hz": 4.961105340641347, "failed": 0, "missed": 4, "callback_last_ms": 14.3279, "callback_ewma_ms": 13.313417, "release_lateness_last_ms": 6.7513 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 241, "hz": 1.9993752292551248, "failed": 0, "missed": 0, "callback_last_ms": 24.4054, "callback_ewma_ms": 15.251089, "release_lateness_last_ms": 1.9744 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 603, "hz": 5.002586154526308, "failed": 0, "missed": 0, "callback_last_ms": 0.0022, "callback_ewma_ms": 0.00348, "release_lateness_last_ms": 2.2019 }, { "name": "general_program_checks_thread", "running": true, "completed": 1202, "hz": 9.971987657944647, "failed": 0, "missed": 3, "callback_last_ms": 0.2025, "callback_ewma_ms": 0.486297, "release_lateness_last_ms": 32.8069 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 120, "hz": 0.9955395332390663, "failed": 0, "missed": 0, "callback_last_ms": 0.0862, "callback_ewma_ms": 0.195864, "release_lateness_last_ms": 0.1175 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1204, "hz": 9.988579983498632, "failed": 0, "missed": 1, "callback_last_ms": 0.0072, "callback_ewma_ms": 0.007506, "release_lateness_last_ms": 1.3356 }, { "name": "engine_diagnostics", "running": true, "completed": 240, "hz": 1.9910790664781326, "failed": 0, "missed": 1, "callback_last_ms": 93.7415, "callback_ewma_ms": 89.894568, "release_lateness_last_ms": 0.9095 } ], "peer_counter_seconds": 120.56933609999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.2725999999966, "gap_p95_ms": 279.79449999999633, "gap_p99_ms": 707.9488000000129, "gap_max_ms": 1675.1345000000001, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 33, "p95": 458, "p99": 922, "max": 1574, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.1664000000019, "gap_p95_ms": 287.22000000000116, "gap_p99_ms": 844.7002999999968, "gap_max_ms": 1399.9660999999905, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 34, "p95": 569, "p99": 1087, "max": 1523, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.44439999999304, "gap_p95_ms": 292.0584999999992, "gap_p99_ms": 794.681099999987, "gap_max_ms": 1575.017399999997, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 41, "p95": 495, "p99": 1100, "max": 1503, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.1707000000024, "gap_p95_ms": 282.9005000000179, "gap_p99_ms": 713.785200000013, "gap_max_ms": 1539.1828999999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 38, "p95": 472, "p99": 1014, "max": 1433, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.8475000000035, "gap_p95_ms": 298.7633999999962, "gap_p99_ms": 906.6333000000013, "gap_max_ms": 1732.9941000000108, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 36, "p95": 655, "p99": 1228, "max": 1669, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.92510000000038, "gap_p95_ms": 280.495600000002, "gap_p99_ms": 670.3073999999906, "gap_max_ms": 1854.5531999999948, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 35, "p95": 529, "p99": 2080, "max": 2735, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.34990000000107, "gap_p95_ms": 283.6246000000101, "gap_p99_ms": 725.6625000000058, "gap_max_ms": 1581.2722000000067, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 36, "p95": 793, "p99": 1892, "max": 2456, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.40619999999763, "gap_p95_ms": 271.449099999998, "gap_p99_ms": 715.2930000000051, "gap_max_ms": 3447.3237999999837, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 41, "p95": 770, "p99": 2800, "max": 3406, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.4464999999982, "gap_p95_ms": 283.96519999999146, "gap_p99_ms": 746.7008999999962, "gap_max_ms": 1493.7396000000008, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 47, "p95": 490, "p99": 1106, "max": 1506, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.31320000000414, "gap_p95_ms": 290.5149999999994, "gap_p99_ms": 800.5914000000048, "gap_max_ms": 1599.9375, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 47, "p95": 496, "p99": 1069, "max": 1530, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.1987000000081, "gap_p95_ms": 285.659399999975, "gap_p99_ms": 793.1970000000147, "gap_max_ms": 1560.4744000000064, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 49, "p95": 454, "p99": 1023, "max": 1466, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.4953000000096, "gap_p95_ms": 289.2138999999879, "gap_p99_ms": 793.491399999999, "gap_max_ms": 1614.61109999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 45, "p95": 858, "p99": 2914, "max": 3388, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.8706999999995, "gap_p95_ms": 290.0034999999916, "gap_p99_ms": 759.7056999999913, "gap_max_ms": 1676.6702000000223, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 50, "p95": 475, "p99": 1031, "max": 1584, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.36680000001797, "gap_p95_ms": 296.86470000000554, "gap_p99_ms": 864.6722000000009, "gap_max_ms": 3253.2819000000018, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 48, "p95": 918, "p99": 4456, "max": 5037, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.20700000000215, "gap_p95_ms": 292.8064999999915, "gap_p99_ms": 750.511799999993, "gap_max_ms": 1574.3838999999862, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 50, "p95": 542, "p99": 1334, "max": 1934, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969480633910246, "complete_hz": 4.969480633910246, "catalog_hz": 1.9977811593609027, "bytes_per_second": 1540937.4456403225, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.7036999999982, "gap_p95_ms": 280.111699999994, "gap_p99_ms": 758.0779000000039, "gap_max_ms": 3138.456999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1194, "p50": 47, "p95": 734, "p99": 3317, "max": 3900, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 94, "mean": 125.74893297872342, "p95": 319.26100000001315, "max": 1209.0536999999895, "p50": 91.67979999999807, "p99": 1209.0536999999895, "min": 58.089899999999034 }, { "operation": "rapid/get-persistence-diagnostics", "count": 94, "mean": 58.463561702127215, "p95": 108.68970000000263, "max": 3375.9930999999924, "p50": 7.060400000002119, "p99": 3375.9930999999924, "min": 4.3092000000033295 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2035231, "accepted_rows_total": 13081000, "committed_rows_total": 2035709, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 94, "minimum": 28.515625, "maximum": 100, "mean": 65.91728960143959 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 94, "minimum": 4.294717334011392, "maximum": 28.138894986344255, "mean": 12.005248237955943 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 94, "minimum": 1000677376, "maximum": 1038041088, "mean": 1019917900.2553191 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 0.9521484375, "mean": 0.07778735871010638 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 2.972412109375, "mean": 0.06350253490691489 }, { "channel": "engine_tempest_tx_mbps", "observations": 94, "minimum": 375.0278696119333, "maximum": 808.2053993367281, "mean": 490.89776284779344 }, { "channel": "engine_tempest_rx_mbps", "observations": 94, "minimum": 0, "maximum": 0.046390123866577725, "mean": 0.022022307026057043 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 94, "minimum": 11040.161073323346, "maximum": 37496.52032291403, "mean": 17486.863999391328 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 94, "minimum": 9720.833965966933, "maximum": 38483.0029068354, "mean": 17479.286606583788 } ], "graceful_shutdown": true, "shutdown_seconds": 1.268532999999996 }, { "id": "verified-capacity-release-25-web-16-r3", "healthy": false, "failure_reasons": [ "control below 95 Hz", "stress_group_3: control callbacks below 95 Hz", "stress_group_4: control callbacks below 95 Hz", "stress_group_5: control callbacks below 95 Hz", "stress_group_6: control callbacks below 95 Hz", "client transport warning/error", "load client disconnected", "client 0: complete snapshots below 4.75 Hz", "client 2: complete snapshots below 4.75 Hz", "client 4: complete snapshots below 4.75 Hz", "client 6: complete snapshots below 4.75 Hz", "client 7: complete snapshots below 4.75 Hz", "client 8: complete snapshots below 4.75 Hz", "client 13: complete snapshots below 4.75 Hz", "client 14: complete snapshots below 4.75 Hz" ], "seconds": 120.81280930000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 16, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 16, "transport": "web", "actual_connections": { "loadClients": 16, "actualClients": 9, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 4000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 55, "mean": 95.53636643448982, "p50": 97.48544210359833, "p95": 100, "p99": 100, "min": 67.04307798163043, "max": 100 }, "engine_cpu_percent": { "count": 55, "mean": 12.1604031458444, "p50": 11.539208569071672, "p95": 18.78757061010023, "p99": 19.226099753413735, "min": 6.035732916973121, "max": 19.226099753413735 }, "client_generator_cpu_percent": { "count": 55, "mean": 2.595813136703075, "p50": 2.600721144603591, "p95": 3.7786919945811475, "p99": 4.844142149983596, "min": 0, "max": 4.844142149983596 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 55, "mean": 1068573938.0363636, "p50": 1059999744, "p95": 1139298304, "p99": 1141436416, "min": 1046036480, "max": 1141436416 }, "system_samples": 55, "system_sample_interval_seconds": { "count": 55, "mean": 2.2587357418181813, "p95": 5.6553347, "max": 7.7953629, "p50": 1.7008472, "p99": 7.7953629, "min": 1.1143834 }, "achieved_read_hz": { "minimum": 85.99280089170979, "maximum": 97.29254328437023 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 55680, "hz": 96.14856548106825 }, { "group": "stress_group_1", "controls": 3, "completed": 41934, "hz": 96.54918450390602 }, { "group": "stress_group_2", "controls": 3, "completed": 41856, "hz": 96.3695966660822 }, { "group": "stress_group_3", "controls": 3, "completed": 36836, "hz": 84.8115076163944 }, { "group": "stress_group_4", "controls": 3, "completed": 36810, "hz": 84.75164500378645 }, { "group": "stress_group_5", "controls": 3, "completed": 36882, "hz": 84.9174183925469 }, { "group": "stress_group_6", "controls": 3, "completed": 36813, "hz": 84.75855222831814 }, { "group": "stress_group_7", "controls": 3, "completed": 41926, "hz": 96.5307652384882 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 91605, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1564581685, "member_service_ns_total": 1610584800, "members_executed_total": 55680 }, "example_device_plugin.stress_read": { "callback_ns_total": 7471888264, "member_service_ns_total": 7520913100, "members_executed_total": 55680 }, "example_device_plugin.stress_write": { "callback_ns_total": 386503286, "member_service_ns_total": 408056600, "members_executed_total": 55680 } }, "callback_ns_total": 9422973235, "cycles_total": 13920, "member_service_ns_total": 9539554500, "members_executed_total": 167040, "thread_cpu_ns_total": 2921875000, "wall_ns_total": 9683107000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1308319924, "member_service_ns_total": 1329120800, "members_executed_total": 41934 }, "example_device_plugin.stress_read": { "callback_ns_total": 6057350281, "member_service_ns_total": 6097351100, "members_executed_total": 41934 }, "example_device_plugin.stress_write": { "callback_ns_total": 264704183, "member_service_ns_total": 277162900, "members_executed_total": 41934 } }, "callback_ns_total": 7630374388, "cycles_total": 13978, "member_service_ns_total": 7703634800, "members_executed_total": 125802, "thread_cpu_ns_total": 2468750000, "wall_ns_total": 7823770500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1279783060, "member_service_ns_total": 1304434000, "members_executed_total": 41856 }, "example_device_plugin.stress_read": { "callback_ns_total": 6169988422, "member_service_ns_total": 6199030200, "members_executed_total": 41856 }, "example_device_plugin.stress_write": { "callback_ns_total": 268671185, "member_service_ns_total": 283442100, "members_executed_total": 41856 } }, "callback_ns_total": 7718442667, "cycles_total": 13952, "member_service_ns_total": 7786906300, "members_executed_total": 125568, "thread_cpu_ns_total": 2437500000, "wall_ns_total": 7893863500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1631738125, "member_service_ns_total": 1760479500, "members_executed_total": 36836 }, "example_device_plugin.stress_read": { "callback_ns_total": 11648980747, "member_service_ns_total": 11714988800, "members_executed_total": 36836 }, "example_device_plugin.stress_write": { "callback_ns_total": 469147679, "member_service_ns_total": 489534400, "members_executed_total": 36836 } }, "callback_ns_total": 13749456551, "cycles_total": 12278, "member_service_ns_total": 13964590000, "members_executed_total": 110502, "thread_cpu_ns_total": 2500000000, "wall_ns_total": 14158929900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1293984224, "member_service_ns_total": 1445817000, "members_executed_total": 36810 }, "example_device_plugin.stress_read": { "callback_ns_total": 10410772999, "member_service_ns_total": 10455604300, "members_executed_total": 36810 }, "example_device_plugin.stress_write": { "callback_ns_total": 407084677, "member_service_ns_total": 430518900, "members_executed_total": 36810 } }, "callback_ns_total": 12111841900, "cycles_total": 12270, "member_service_ns_total": 12331940200, "members_executed_total": 110430, "thread_cpu_ns_total": 2625000000, "wall_ns_total": 12512715100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1479161961, "member_service_ns_total": 1675611400, "members_executed_total": 36882 }, "example_device_plugin.stress_read": { "callback_ns_total": 10784627041, "member_service_ns_total": 10845836800, "members_executed_total": 36882 }, "example_device_plugin.stress_write": { "callback_ns_total": 448884978, "member_service_ns_total": 473354500, "members_executed_total": 36882 } }, "callback_ns_total": 12712673980, "cycles_total": 12294, "member_service_ns_total": 12994802700, "members_executed_total": 110646, "thread_cpu_ns_total": 2359375000, "wall_ns_total": 13178671500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1361970077, "member_service_ns_total": 1467423700, "members_executed_total": 36813 }, "example_device_plugin.stress_read": { "callback_ns_total": 10431631974, "member_service_ns_total": 10496865300, "members_executed_total": 36813 }, "example_device_plugin.stress_write": { "callback_ns_total": 386940378, "member_service_ns_total": 409882200, "members_executed_total": 36813 } }, "callback_ns_total": 12180542429, "cycles_total": 12271, "member_service_ns_total": 12374171200, "members_executed_total": 110439, "thread_cpu_ns_total": 2625000000, "wall_ns_total": 12599264400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 1283317990, "member_service_ns_total": 1312860800, "members_executed_total": 41926 }, "example_device_plugin.stress_read": { "callback_ns_total": 6181288482, "member_service_ns_total": 6214364100, "members_executed_total": 41926 }, "example_device_plugin.stress_write": { "callback_ns_total": 272446287, "member_service_ns_total": 288422700, "members_executed_total": 41926 } }, "callback_ns_total": 7737005159, "cycles_total": 13975, "member_service_ns_total": 7815598800, "members_executed_total": 125775, "thread_cpu_ns_total": 2562500000, "wall_ns_total": 7976492500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 144.77595095, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 129, "hz": 0.9781240590365989, "failed": 0, "missed": 0, "callback_last_ms": 0.0028, "callback_ewma_ms": 0.005544, "release_lateness_last_ms": 0.213 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 129, "hz": 0.9781240590365989, "failed": 0, "missed": 0, "callback_last_ms": 7.2594, "callback_ewma_ms": 8.985699, "release_lateness_last_ms": 0.5947 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 645, "hz": 4.890620295182995, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.001182, "release_lateness_last_ms": 0.9466 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 645, "hz": 4.890620295182995, "failed": 0, "missed": 0, "callback_last_ms": 0.5292, "callback_ewma_ms": 1.302175, "release_lateness_last_ms": 0.1614 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 129, "hz": 0.9781240590365989, "failed": 0, "missed": 0, "callback_last_ms": 0.9634, "callback_ewma_ms": 1.236534, "release_lateness_last_ms": 1.0887 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 129, "hz": 0.9781240590365989, "failed": 0, "missed": 0, "callback_last_ms": 0.6938, "callback_ewma_ms": 0.916538, "release_lateness_last_ms": 2.0311 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 258, "hz": 1.9562481180731979, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.001561, "release_lateness_last_ms": 0.1046 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 129, "hz": 0.9781240590365989, "failed": 0, "missed": 0, "callback_last_ms": 3.1742, "callback_ewma_ms": 6.592504, "release_lateness_last_ms": 1.4568 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 258, "hz": 1.9562481180731979, "failed": 0, "missed": 0, "callback_last_ms": 6.271, "callback_ewma_ms": 7.948363, "release_lateness_last_ms": 2.134 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 645, "hz": 4.890620295182995, "failed": 0, "missed": 0, "callback_last_ms": 6.8001, "callback_ewma_ms": 13.824834, "release_lateness_last_ms": 0.175 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1288, "hz": 9.7660758762724, "failed": 0, "missed": 2, "callback_last_ms": 0.0281, "callback_ewma_ms": 0.737765, "release_lateness_last_ms": 0.788 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 645, "hz": 4.890620295182995, "failed": 0, "missed": 0, "callback_last_ms": 4.8941, "callback_ewma_ms": 12.099264, "release_lateness_last_ms": 0.2739 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 258, "hz": 1.9562481180731979, "failed": 0, "missed": 0, "callback_last_ms": 9.5019, "callback_ewma_ms": 11.977162, "release_lateness_last_ms": 1.0307 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 645, "hz": 4.890620295182995, "failed": 0, "missed": 0, "callback_last_ms": 0.0016, "callback_ewma_ms": 0.002231, "release_lateness_last_ms": 0.8127 }, { "name": "general_program_checks_thread", "running": true, "completed": 1282, "hz": 9.720581733991628, "failed": 0, "missed": 8, "callback_last_ms": 0.1862, "callback_ewma_ms": 0.523454, "release_lateness_last_ms": 0.239 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 129, "hz": 0.9781240590365989, "failed": 0, "missed": 0, "callback_last_ms": 0.0857, "callback_ewma_ms": 0.145097, "release_lateness_last_ms": 0.3062 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1290, "hz": 9.78124059036599, "failed": 0, "missed": 0, "callback_last_ms": 0.0057, "callback_ewma_ms": 0.007434, "release_lateness_last_ms": 0.4687 }, { "name": "engine_diagnostics", "running": true, "completed": 252, "hz": 1.910753975792426, "failed": 0, "missed": 6, "callback_last_ms": 56.9278, "callback_ewma_ms": 88.185107, "release_lateness_last_ms": 0.4169 } ], "peer_counter_seconds": 126.5657973, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.0160226081773684, "complete_hz": 5.0160226081773684, "catalog_hz": 2.011375402288945, "bytes_per_second": 1556522.405961301, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.36000000000058, "gap_p95_ms": 269.20509999999194, "gap_p99_ms": 496.90399999998044, "gap_max_ms": 5491.253799999991, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 25, "p95": 908, "p99": 4285, "max": 5413, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.131904318597308, "complete_hz": 5.131904318597308, "catalog_hz": 2.052761727438923, "bytes_per_second": 1592317.0826611207, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.12200000000303, "gap_p95_ms": 275.2484000000113, "gap_p99_ms": 451.38400000000547, "gap_max_ms": 5796.263399999996, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1240, "p50": 38, "p95": 2353, "p99": 4788, "max": 5659, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.0160226081773684, "complete_hz": 5.0160226081773684, "catalog_hz": 2.011375402288945, "bytes_per_second": 1556522.405961301, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.96889999999257, "gap_p95_ms": 265.1131000000023, "gap_p99_ms": 415.3454999999958, "gap_max_ms": 5851.7660999999935, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1212, "p50": 31, "p95": 1214, "p99": 4891, "max": 5779, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.512658509977109, "complete_hz": 5.512658509977109, "catalog_hz": 2.201752497978845, "bytes_per_second": 1710345.0134079526, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 192.11410000000615, "gap_p95_ms": 271.4930000000022, "gap_p99_ms": 476.7856000000029, "gap_max_ms": 5819.894499999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1332, "p50": 49, "p95": 7046, "p99": 11718, "max": 12294, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.330558679317204, "complete_hz": 5.330558679317204, "catalog_hz": 2.13553437773888, "bytes_per_second": 1653685.420883771, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 195.92480000000796, "gap_p95_ms": 275.2624000000069, "gap_p99_ms": 423.5688000000082, "gap_max_ms": 5774.55829999999, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1288, "p50": 45, "p95": 4869, "p99": 7275, "max": 7865, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.0077453431473735, "complete_hz": 5.0077453431473735, "catalog_hz": 2.003098137258949, "bytes_per_second": 1553965.6332889204, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 196.84579999998095, "gap_p95_ms": 278.07279999999446, "gap_p99_ms": 474.3561000000045, "gap_max_ms": 5941.418399999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 38, "p95": 1966, "p99": 5339, "max": 5881, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.0077453431473735, "complete_hz": 5.0077453431473735, "catalog_hz": 2.003098137258949, "bytes_per_second": 1553965.6332889204, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.0624000000098, "gap_p95_ms": 274.6869000000006, "gap_p99_ms": 455.1207999999897, "gap_max_ms": 5985.318899999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 39, "p95": 1278, "p99": 4915, "max": 5824, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 0, "complete_hz": 0, "catalog_hz": 0, "bytes_per_second": 0, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": null, "gap_p95_ms": null, "gap_p99_ms": null, "gap_max_ms": null, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 0, "p50": null, "p95": null, "p99": null, "max": null, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.0077453431473735, "complete_hz": 5.0077453431473735, "catalog_hz": 2.003098137258949, "bytes_per_second": 1553965.6332889204, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.36829999998736, "gap_p95_ms": 279.28489999999874, "gap_p99_ms": 462.62859999999637, "gap_max_ms": 5983.8908999999985, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1210, "p50": 40, "p95": 1317, "p99": 4990, "max": 5909, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 90, "mean": 144.97348333333323, "p95": 283.8545999999915, "max": 617.5884000000078, "p50": 119.47660000000906, "p99": 617.5884000000078, "min": 67.15279999998165 }, { "operation": "rapid/get-persistence-diagnostics", "count": 90, "mean": 102.60777000000138, "p95": 208.4423999999999, "max": 5927.8802, "p50": 15.983699999982491, "p99": 5927.8802, "min": 5.107400000008056 } ], "transport_event_count": 977, "transport_events": [ { "at": "2026-10-29T23:23:37.522Z", "client": 11, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.522Z", "client": 9, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.522Z", "client": 10, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.522Z", "client": 1, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.522Z", "client": 4, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.523Z", "client": 7, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.523Z", "client": 15, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.523Z", "client": 13, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.523Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:37.523Z", "client": 14, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:49.174Z", "client": 0, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:49.175Z", "client": 14, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:49.175Z", "client": 7, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:49.175Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:49.175Z", "client": 8, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:49.175Z", "client": 13, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:57.833Z", "client": 4, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:57.833Z", "client": 2, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:57.833Z", "client": 15, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:23:57.833Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:02.553Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:02.553Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:02.554Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:02.554Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:02.554Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:04.844Z", "client": 5, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 12, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 4, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 2, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 7, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 14, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 13, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 3, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 0, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 9, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.844Z", "client": 11, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.845Z", "client": 8, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.845Z", "client": 10, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:04.845Z", "client": 1, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:10.349Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.425Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.427Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.429Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.432Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.433Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.436Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:10.437Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:11.686Z", "client": 6, "type": "heartbeat_failed", "message": "fetch failed" }, { "at": "2026-10-29T23:24:14.063Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.151Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.152Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.152Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.152Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.153Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.175Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.352Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.353Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.353Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.354Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.354Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.356Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.358Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:14.358Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.023Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.024Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.028Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.029Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.029Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.030Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.053Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:16.054Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.249Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.250Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.250Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.278Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.279Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.280Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.282Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:17.505Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.297Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.311Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.311Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.312Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.312Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.312Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.324Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:18.334Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.782Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.783Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.812Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.814Z", "client": 0, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.815Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.815Z", "client": 8, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.816Z", "client": 2, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:19.817Z", "client": 4, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:20.558Z", "client": 13, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:20.599Z", "client": 14, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:20.599Z", "client": 6, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." }, { "at": "2026-10-29T23:24:20.599Z", "client": 7, "type": "heartbeat_failed", "message": "heartbeat: Client session is invalid or expired." } ], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2430845, "accepted_rows_total": 15238562, "committed_rows_total": 2430214, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 90, "minimum": 74.21875, "maximum": 100, "mean": 98.04161429144301 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 90, "minimum": 3.7064219661186777, "maximum": 27.714714181759458, "mean": 11.89148495625708 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 90, "minimum": 1044283392, "maximum": 1141436416, "mean": 1073836942.2222223 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 90, "minimum": 0, "maximum": 0.2288818359375, "mean": 0.024312337239583332 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 90, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 90, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 90, "minimum": 0, "maximum": 1.422119140625, "mean": 0.04150390625 }, { "channel": "engine_tempest_tx_mbps", "observations": 90, "minimum": 192.35262918821397, "maximum": 297.41682903479386, "mean": 239.45949533586753 }, { "channel": "engine_tempest_rx_mbps", "observations": 90, "minimum": 0, "maximum": 0.045039780200604805, "mean": 0.02161881603303778 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 90, "minimum": 1893, "maximum": 1893, "mean": 1893 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 90, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 90, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 90, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 90, "minimum": 9667.666326558798, "maximum": 36709.95795582613, "mean": 17015.874588575454 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 90, "minimum": 9952.233684615738, "maximum": 37116.87305111472, "mean": 17226.269387053148 } ], "graceful_shutdown": true, "shutdown_seconds": 1.612990300000005 }, { "id": "verified-capacity-release-25-web-32", "healthy": false, "failure_reasons": [ "client transport warning/error" ], "seconds": 120.62850100000001, "measurement_completed": true, "background_preflight": null, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 104, "mean": 45.897684613000834, "p50": 41.962777866834614, "p95": 82.73154895129558, "p99": 97.09335360197888, "min": 23.53054167694747, "max": 99.42223479981739 }, "engine_cpu_percent": { "count": 104, "mean": 11.808427915442033, "p50": 9.762944548706493, "p95": 21.2836668810981, "p99": 23.19170696587868, "min": 4.94737933415308, "max": 24.2381045542178 }, "client_generator_cpu_percent": { "count": 104, "mean": 7.385615968473423, "p50": 7.039356797995178, "p95": 9.56937171114365, "p99": 11.706657896024591, "min": 4.6245201139882015, "max": 13.585042802666058 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 104, "mean": 1058154496, "p50": 1053130752, "p95": 1144197120, "p99": 1262395392, "min": 996925440, "max": 1262530560 }, "system_samples": 104, "system_sample_interval_seconds": { "count": 104, "mean": 1.16485014423077, "p95": 1.2215667, "max": 2.6528369, "p50": 1.1178201, "p99": 2.1071404, "min": 1.0952077 }, "achieved_read_hz": { "minimum": 96.86206948313549, "maximum": 99.37162759609433 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47904, "hz": 99.12227871918519 }, { "group": "stress_group_1", "controls": 3, "completed": 35991, "hz": 99.29609033016573 }, { "group": "stress_group_2", "controls": 3, "completed": 35967, "hz": 99.22987638312553 }, { "group": "stress_group_3", "controls": 3, "completed": 35127, "hz": 96.9123882367184 }, { "group": "stress_group_4", "controls": 3, "completed": 35106, "hz": 96.85445103305823 }, { "group": "stress_group_5", "controls": 3, "completed": 35109, "hz": 96.86272777643823 }, { "group": "stress_group_6", "controls": 3, "completed": 35118, "hz": 96.88755800657832 }, { "group": "stress_group_7", "controls": 3, "completed": 36015, "hz": 99.36230427720592 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 17064, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 795715962, "member_service_ns_total": 817232300, "members_executed_total": 47904 }, "example_device_plugin.stress_read": { "callback_ns_total": 4759638486, "member_service_ns_total": 4788509700, "members_executed_total": 47904 }, "example_device_plugin.stress_write": { "callback_ns_total": 226164183, "member_service_ns_total": 240081900, "members_executed_total": 47904 } }, "callback_ns_total": 5781518631, "cycles_total": 11976, "member_service_ns_total": 5845823900, "members_executed_total": 143712, "thread_cpu_ns_total": 1750000000, "wall_ns_total": 5925585400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 658190563, "member_service_ns_total": 670655700, "members_executed_total": 35991 }, "example_device_plugin.stress_read": { "callback_ns_total": 3835490075, "member_service_ns_total": 3861758000, "members_executed_total": 35991 }, "example_device_plugin.stress_write": { "callback_ns_total": 170657093, "member_service_ns_total": 182396600, "members_executed_total": 35991 } }, "callback_ns_total": 4664337731, "cycles_total": 11997, "member_service_ns_total": 4714810300, "members_executed_total": 107973, "thread_cpu_ns_total": 1656250000, "wall_ns_total": 4804657000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 593768964, "member_service_ns_total": 606532200, "members_executed_total": 35967 }, "example_device_plugin.stress_read": { "callback_ns_total": 3720992763, "member_service_ns_total": 3741272700, "members_executed_total": 35967 }, "example_device_plugin.stress_write": { "callback_ns_total": 178731689, "member_service_ns_total": 189935400, "members_executed_total": 35967 } }, "callback_ns_total": 4493493416, "cycles_total": 11989, "member_service_ns_total": 4537740300, "members_executed_total": 107901, "thread_cpu_ns_total": 1406250000, "wall_ns_total": 4608126700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 678758215, "member_service_ns_total": 700905500, "members_executed_total": 35127 }, "example_device_plugin.stress_read": { "callback_ns_total": 5209108663, "member_service_ns_total": 5238836100, "members_executed_total": 35127 }, "example_device_plugin.stress_write": { "callback_ns_total": 207388886, "member_service_ns_total": 220614700, "members_executed_total": 35127 } }, "callback_ns_total": 6095255764, "cycles_total": 11709, "member_service_ns_total": 6160356300, "members_executed_total": 105381, "thread_cpu_ns_total": 1828125000, "wall_ns_total": 6276305000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 586396031, "member_service_ns_total": 608204100, "members_executed_total": 35106 }, "example_device_plugin.stress_read": { "callback_ns_total": 5031719571, "member_service_ns_total": 5056850600, "members_executed_total": 35106 }, "example_device_plugin.stress_write": { "callback_ns_total": 225072090, "member_service_ns_total": 246909800, "members_executed_total": 35106 } }, "callback_ns_total": 5843187692, "cycles_total": 11702, "member_service_ns_total": 5911964500, "members_executed_total": 105318, "thread_cpu_ns_total": 1171875000, "wall_ns_total": 5992780700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 657095948, "member_service_ns_total": 677810100, "members_executed_total": 35109 }, "example_device_plugin.stress_read": { "callback_ns_total": 5005167675, "member_service_ns_total": 5027241500, "members_executed_total": 35109 }, "example_device_plugin.stress_write": { "callback_ns_total": 173164989, "member_service_ns_total": 187130700, "members_executed_total": 35109 } }, "callback_ns_total": 5835428612, "cycles_total": 11703, "member_service_ns_total": 5892182300, "members_executed_total": 105327, "thread_cpu_ns_total": 953125000, "wall_ns_total": 6014732400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 614117810, "member_service_ns_total": 628324400, "members_executed_total": 35118 }, "example_device_plugin.stress_read": { "callback_ns_total": 5351409674, "member_service_ns_total": 5378516300, "members_executed_total": 35118 }, "example_device_plugin.stress_write": { "callback_ns_total": 200610786, "member_service_ns_total": 215208100, "members_executed_total": 35118 } }, "callback_ns_total": 6166138270, "cycles_total": 11706, "member_service_ns_total": 6222048800, "members_executed_total": 105354, "thread_cpu_ns_total": 1593750000, "wall_ns_total": 6314324700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 619716535, "member_service_ns_total": 635339500, "members_executed_total": 36015 }, "example_device_plugin.stress_read": { "callback_ns_total": 3757758365, "member_service_ns_total": 3782697200, "members_executed_total": 36015 }, "example_device_plugin.stress_write": { "callback_ns_total": 201382492, "member_service_ns_total": 216538800, "members_executed_total": 36015 } }, "callback_ns_total": 4578857392, "cycles_total": 12005, "member_service_ns_total": 4634575500, "members_executed_total": 108045, "thread_cpu_ns_total": 1328125000, "wall_ns_total": 4706534300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 120.82046694999997, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0007062430553415, "failed": 0, "missed": 0, "callback_last_ms": 0.0036, "callback_ewma_ms": 0.004248, "release_lateness_last_ms": 0.1887 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0007062430553415, "failed": 0, "missed": 0, "callback_last_ms": 5.4651, "callback_ewma_ms": 4.926604, "release_lateness_last_ms": 0.3987 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 604, "hz": 4.995260915747325, "failed": 0, "missed": 0, "callback_last_ms": 0.0011, "callback_ewma_ms": 0.000827, "release_lateness_last_ms": 0.1146 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 604, "hz": 4.995260915747325, "failed": 0, "missed": 0, "callback_last_ms": 2.0531, "callback_ewma_ms": 2.109188, "release_lateness_last_ms": 0.2503 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0007062430553415, "failed": 0, "missed": 0, "callback_last_ms": 4.4724, "callback_ewma_ms": 3.426879, "release_lateness_last_ms": 0.5289 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0007062430553415, "failed": 0, "missed": 0, "callback_last_ms": 3.4832, "callback_ewma_ms": 3.641383, "release_lateness_last_ms": 0.1106 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.001412486110683, "failed": 0, "missed": 0, "callback_last_ms": 0.0008, "callback_ewma_ms": 0.00097, "release_lateness_last_ms": 0.4152 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0007062430553415, "failed": 0, "missed": 0, "callback_last_ms": 2.8803, "callback_ewma_ms": 2.391256, "release_lateness_last_ms": 0.4962 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.001412486110683, "failed": 0, "missed": 0, "callback_last_ms": 8.2913, "callback_ewma_ms": 6.838648, "release_lateness_last_ms": 0.5426 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 602, "hz": 4.978720316688559, "failed": 0, "missed": 2, "callback_last_ms": 6.2674, "callback_ewma_ms": 6.438877, "release_lateness_last_ms": 0.2079 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1207, "hz": 9.982251531965266, "failed": 0, "missed": 2, "callback_last_ms": 0.0233, "callback_ewma_ms": 0.928747, "release_lateness_last_ms": 0.5511 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 600, "hz": 4.962179717629793, "failed": 0, "missed": 5, "callback_last_ms": 17.9039, "callback_ewma_ms": 15.368566, "release_lateness_last_ms": 0.124 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.001412486110683, "failed": 0, "missed": 0, "callback_last_ms": 6.3018, "callback_ewma_ms": 6.967824, "release_lateness_last_ms": 0.5035 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.995260915747325, "failed": 0, "missed": 0, "callback_last_ms": 0.0023, "callback_ewma_ms": 0.002024, "release_lateness_last_ms": 0.4104 }, { "name": "general_program_checks_thread", "running": true, "completed": 1205, "hz": 9.965710932906502, "failed": 0, "missed": 4, "callback_last_ms": 0.1885, "callback_ewma_ms": 0.206808, "release_lateness_last_ms": 0.4368 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0007062430553415, "failed": 0, "missed": 0, "callback_last_ms": 0.1378, "callback_ewma_ms": 0.123919, "release_lateness_last_ms": 0.4353 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1209, "hz": 9.998792131024032, "failed": 0, "missed": 0, "callback_last_ms": 0.0048, "callback_ewma_ms": 0.007748, "release_lateness_last_ms": 0.2639 }, { "name": "engine_diagnostics", "running": true, "completed": 242, "hz": 2.001412486110683, "failed": 0, "missed": 0, "callback_last_ms": 44.6438, "callback_ewma_ms": 45.619549, "release_lateness_last_ms": 0.2267 } ], "peer_counter_seconds": 120.88870949999999, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.62080000000424, "gap_p95_ms": 277.5948999999964, "gap_p99_ms": 800.4311999999991, "gap_max_ms": 2459.7270999999964, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 52, "p95": 885, "p99": 2464, "max": 3163, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.31239999999525, "gap_p95_ms": 266.2690999999977, "gap_p99_ms": 881.9256999999998, "gap_max_ms": 1541.2363000000041, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 51, "p95": 881, "p99": 1468, "max": 1745, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.53660000002128, "gap_p95_ms": 288.16949999998906, "gap_p99_ms": 832.1652000000031, "gap_max_ms": 2653.2516000000032, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 52, "p95": 2392, "p99": 3421, "max": 3950, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.12910000000556, "gap_p95_ms": 275.38030000000435, "gap_p99_ms": 942.6338000000032, "gap_max_ms": 2287.1293000000005, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 55, "p95": 1292, "p99": 3047, "max": 3721, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.28819999998086, "gap_p95_ms": 278.3064999999915, "gap_p99_ms": 1018.6163000000015, "gap_max_ms": 1504.509299999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 44, "p95": 1901, "p99": 3155, "max": 3653, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.76690000000235, "gap_p95_ms": 279.6646000000037, "gap_p99_ms": 944.8623999999982, "gap_max_ms": 2438.8197, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 53, "p95": 665, "p99": 1750, "max": 2347, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.09960000000137, "gap_p95_ms": 277.20569999999134, "gap_p99_ms": 1170.2053000000014, "gap_max_ms": 1524.0377000000008, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 48, "p95": 942, "p99": 2683, "max": 3117, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.45669999999518, "gap_p95_ms": 268.4259999999922, "gap_p99_ms": 788.7433000000019, "gap_max_ms": 2499.1419000000024, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 57, "p95": 970, "p99": 2730, "max": 3510, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.6408999999985, "gap_p95_ms": 267.82799999999406, "gap_p99_ms": 990.2989000000016, "gap_max_ms": 2495.2167000000045, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 59, "p95": 1244, "p99": 2863, "max": 3276, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.4586999999883, "gap_p95_ms": 271.8402999999962, "gap_p99_ms": 981.7849999999962, "gap_max_ms": 2706.6731999999975, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 54, "p95": 3688, "p99": 5269, "max": 5659, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.8984000000055, "gap_p95_ms": 274.65690000000177, "gap_p99_ms": 636.4276000000027, "gap_max_ms": 2817.0812000000005, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 52, "p95": 1965, "p99": 5149, "max": 5978, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.55409999999392, "gap_p95_ms": 274.04939999998896, "gap_p99_ms": 914.2021999999997, "gap_max_ms": 2455.9765000000043, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 51, "p95": 1005, "p99": 2626, "max": 3214, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.73329999999987, "gap_p95_ms": 266.4444000000003, "gap_p99_ms": 846.2704000000012, "gap_max_ms": 1565.3764999999985, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 63, "p95": 884, "p99": 2056, "max": 2405, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.30260000001, "gap_p95_ms": 263.6460999999981, "gap_p99_ms": 950.2716, "gap_max_ms": 2746.877099999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 58, "p95": 3486, "p99": 6014, "max": 6405, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.86799999999494, "gap_p95_ms": 261.7525000000023, "gap_p99_ms": 915.7665000000052, "gap_max_ms": 2417.5959999999977, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 65, "p95": 965, "p99": 2250, "max": 2984, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.91909999999916, "gap_p95_ms": 275.5249999999942, "gap_p99_ms": 715.0021999999954, "gap_max_ms": 2830.8289000000004, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 66, "p95": 1989, "p99": 5185, "max": 5641, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.09479999999166, "gap_p95_ms": 267.3738000000012, "gap_p99_ms": 773.8919000000024, "gap_max_ms": 2745.6331000000064, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 69, "p95": 1102, "p99": 3634, "max": 4409, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.721000000005, "gap_p95_ms": 267.06890000001295, "gap_p99_ms": 932.4608999999982, "gap_max_ms": 1625.8485, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 70, "p95": 1471, "p99": 3566, "max": 4141, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.61040000000503, "gap_p95_ms": 269.9968999999983, "gap_p99_ms": 882.2776999999987, "gap_max_ms": 1966.9651999999987, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 70, "p95": 1772, "p99": 2933, "max": 3352, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.89560000000347, "gap_p95_ms": 280.468200000003, "gap_p99_ms": 1015.8079999999973, "gap_max_ms": 1656.5982999999978, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 45, "p95": 1501, "p99": 3723, "max": 4460, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.8336999999883, "gap_p95_ms": 260.12919999999576, "gap_p99_ms": 1142.2998000000007, "gap_max_ms": 2486.2490000000034, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 74, "p95": 2865, "p99": 5282, "max": 6111, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.40059999999357, "gap_p95_ms": 265.4183000000048, "gap_p99_ms": 1095.7184000000052, "gap_max_ms": 1436.1301000000021, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 74, "p95": 1630, "p99": 2428, "max": 2754, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.35250000000815, "gap_p95_ms": 264.7813999999926, "gap_p99_ms": 742.9006999999983, "gap_max_ms": 2395.5074999999997, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 79, "p95": 876, "p99": 2273, "max": 3032, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.94629999999597, "gap_p95_ms": 276.8422000000137, "gap_p99_ms": 796.640800000001, "gap_max_ms": 2846.3882999999987, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 62, "p95": 1047, "p99": 2982, "max": 3573, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.2942999999941, "gap_p95_ms": 261.3274999999994, "gap_p99_ms": 693.0291999999972, "gap_max_ms": 2872.9153000000006, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 77, "p95": 3408, "p99": 6284, "max": 6562, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.94720000000962, "gap_p95_ms": 283.2927999999956, "gap_p99_ms": 1075.1837999999989, "gap_max_ms": 1378.9789999999994, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 58, "p95": 1018, "p99": 2473, "max": 2877, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.5158999999985, "gap_p95_ms": 278.71140000000014, "gap_p99_ms": 944.6763000000064, "gap_max_ms": 1286.8366000000024, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 58, "p95": 698, "p99": 1374, "max": 1878, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.30510000001232, "gap_p95_ms": 274.36269999999786, "gap_p99_ms": 928.2782000000007, "gap_max_ms": 1821.591800000002, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 77, "p95": 825, "p99": 2967, "max": 3439, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.94510000001173, "gap_p95_ms": 262.0439999999944, "gap_p99_ms": 900.5766000000003, "gap_max_ms": 1819.5180000000037, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 83, "p95": 853, "p99": 3171, "max": 3687, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.70960000000196, "gap_p95_ms": 278.62080000000424, "gap_p99_ms": 977.0251000000062, "gap_max_ms": 2280.9115999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 60, "p95": 848, "p99": 2609, "max": 3384, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.32580000000598, "gap_p95_ms": 283.3657999999996, "gap_p99_ms": 901.5038999999961, "gap_max_ms": 2877.140900000006, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 81, "p95": 747, "p99": 2892, "max": 3378, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.9573694537042785, "complete_hz": 4.9573694537042785, "catalog_hz": 2.0061595448100924, "bytes_per_second": 1537634.6283205664, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.68189999999595, "gap_p95_ms": 259.9519, "gap_p99_ms": 1174.1944000000003, "gap_max_ms": 1715.2018000000025, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1196, "p50": 87, "p95": 2367, "p99": 5786, "max": 6616, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 94, "mean": 122.59968936170287, "p95": 200.02300000000105, "max": 1571.4475999999995, "p50": 79.84249999999884, "p99": 1571.4475999999995, "min": 53.052800000004936 }, { "operation": "rapid/get-persistence-diagnostics", "count": 94, "mean": 61.77860744680862, "p95": 119.74490000000515, "max": 2834.443500000001, "p50": 14.913000000000466, "p99": 2834.443500000001, "min": 4.915899999992689 } ], "transport_event_count": 1, "transport_events": [ { "at": "2026-10-29T23:11:56.221Z", "client": 22, "type": "heartbeat_failed", "message": "fetch failed" } ], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2189181, "accepted_rows_total": 13774375, "committed_rows_total": 2188733, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 94, "minimum": 22.265625, "maximum": 86.88524590163935, "mean": 44.804368799588936 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 94, "minimum": 3.5141328991710115, "maximum": 30.718873018587587, "mean": 11.11945786213739 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 94, "minimum": 1003229184, "maximum": 1228128256, "mean": 1051108025.1914893 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 0.0946044921875, "mean": 0.0011687583111702129 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 94, "minimum": 0, "maximum": 25.897216796875, "mean": 0.2793981673869681 }, { "channel": "engine_tempest_tx_mbps", "observations": 94, "minimum": 748.89482081787, "maximum": 1186.9949704043647, "mean": 960.365637022314 }, { "channel": "engine_tempest_rx_mbps", "observations": 94, "minimum": 0, "maximum": 0.09025760123990241, "mean": 0.042446384807543626 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 94, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 94, "minimum": 11856.162833440821, "maximum": 37839.65668038483, "mean": 17702.402714569667 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 94, "minimum": 10947.53899323432, "maximum": 38873.92062454338, "mean": 17619.195689407014 } ], "graceful_shutdown": true, "shutdown_seconds": 1.3906290000000154 }, { "id": "verified-capacity-release-25-web-32-r2", "healthy": true, "failure_reasons": [], "seconds": 122.1767121, "measurement_completed": true, "background_preflight": null, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 108, "mean": 41.47776062368801, "p50": 37.44413519930795, "p95": 69.20272898973927, "p99": 89.41185271952472, "min": 25.09095277371307, "max": 92.26420428243749 }, "engine_cpu_percent": { "count": 108, "mean": 11.552930401611732, "p50": 9.380314920544215, "p95": 22.930039465683056, "p99": 24.63733251522762, "min": 4.564702343878303, "max": 27.279275924351754 }, "client_generator_cpu_percent": { "count": 108, "mean": 7.3016145352765625, "p50": 7.2487764461389625, "p95": 9.671901635361987, "p99": 9.989524963805277, "min": 5.321243015123568, "max": 11.010310162238358 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 108, "mean": 1027107612.4444444, "p50": 1027842048, "p95": 1036558336, "p99": 1100333056, "min": 996511744, "max": 1105821696 }, "system_samples": 108, "system_sample_interval_seconds": { "count": 108, "mean": 1.1299641407407404, "p95": 1.1885167, "max": 1.8609285, "p50": 1.1124767, "p99": 1.4093427, "min": 1.0943217 }, "achieved_read_hz": { "minimum": 98.93269159081325, "maximum": 99.43177949323284 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 48612, "hz": 99.32456472271194 }, { "group": "stress_group_1", "controls": 3, "completed": 36498, "hz": 99.43081168571656 }, { "group": "stress_group_2", "controls": 3, "completed": 36480, "hz": 99.38177462586827 }, { "group": "stress_group_3", "controls": 3, "completed": 36318, "hz": 98.94044108723365 }, { "group": "stress_group_4", "controls": 3, "completed": 36323, "hz": 98.95406249274708 }, { "group": "stress_group_5", "controls": 3, "completed": 36315, "hz": 98.9322682439256 }, { "group": "stress_group_6", "controls": 3, "completed": 36324, "hz": 98.95678677384977 }, { "group": "stress_group_7", "controls": 3, "completed": 36489, "hz": 99.40629315579241 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 7629, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 728067815, "member_service_ns_total": 745852800, "members_executed_total": 48612 }, "example_device_plugin.stress_read": { "callback_ns_total": 4219773792, "member_service_ns_total": 4245782200, "members_executed_total": 48612 }, "example_device_plugin.stress_write": { "callback_ns_total": 220353497, "member_service_ns_total": 235287000, "members_executed_total": 48612 } }, "callback_ns_total": 5168195104, "cycles_total": 12153, "member_service_ns_total": 5226922000, "members_executed_total": 145836, "thread_cpu_ns_total": 2031250000, "wall_ns_total": 5308393300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 600616824, "member_service_ns_total": 617900900, "members_executed_total": 36498 }, "example_device_plugin.stress_read": { "callback_ns_total": 3484860692, "member_service_ns_total": 3506524200, "members_executed_total": 36498 }, "example_device_plugin.stress_write": { "callback_ns_total": 166511596, "member_service_ns_total": 176794800, "members_executed_total": 36498 } }, "callback_ns_total": 4251989112, "cycles_total": 12166, "member_service_ns_total": 4301219900, "members_executed_total": 109494, "thread_cpu_ns_total": 1375000000, "wall_ns_total": 4372287700 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 611336042, "member_service_ns_total": 625641300, "members_executed_total": 36480 }, "example_device_plugin.stress_read": { "callback_ns_total": 3265834515, "member_service_ns_total": 3287889100, "members_executed_total": 36480 }, "example_device_plugin.stress_write": { "callback_ns_total": 195860089, "member_service_ns_total": 207037600, "members_executed_total": 36480 } }, "callback_ns_total": 4073030646, "cycles_total": 12160, "member_service_ns_total": 4120568000, "members_executed_total": 109440, "thread_cpu_ns_total": 1828125000, "wall_ns_total": 4190765200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 540274945, "member_service_ns_total": 554881800, "members_executed_total": 36318 }, "example_device_plugin.stress_read": { "callback_ns_total": 3610402384, "member_service_ns_total": 3634116600, "members_executed_total": 36318 }, "example_device_plugin.stress_write": { "callback_ns_total": 170749491, "member_service_ns_total": 188189700, "members_executed_total": 36318 } }, "callback_ns_total": 4321426820, "cycles_total": 12106, "member_service_ns_total": 4377188100, "members_executed_total": 108954, "thread_cpu_ns_total": 1546875000, "wall_ns_total": 4449976600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 510403770, "member_service_ns_total": 535775000, "members_executed_total": 36323 }, "example_device_plugin.stress_read": { "callback_ns_total": 3443080529, "member_service_ns_total": 3472856400, "members_executed_total": 36323 }, "example_device_plugin.stress_write": { "callback_ns_total": 164502789, "member_service_ns_total": 176050900, "members_executed_total": 36323 } }, "callback_ns_total": 4118363488, "cycles_total": 12108, "member_service_ns_total": 4185060900, "members_executed_total": 108972, "thread_cpu_ns_total": 953125000, "wall_ns_total": 4254994500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 513017456, "member_service_ns_total": 526329400, "members_executed_total": 36315 }, "example_device_plugin.stress_read": { "callback_ns_total": 3407767291, "member_service_ns_total": 3437367400, "members_executed_total": 36315 }, "example_device_plugin.stress_write": { "callback_ns_total": 168210594, "member_service_ns_total": 178584900, "members_executed_total": 36315 } }, "callback_ns_total": 4088995341, "cycles_total": 12105, "member_service_ns_total": 4142281700, "members_executed_total": 108945, "thread_cpu_ns_total": 750000000, "wall_ns_total": 4215908500 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 536124248, "member_service_ns_total": 549868900, "members_executed_total": 36324 }, "example_device_plugin.stress_read": { "callback_ns_total": 3503088352, "member_service_ns_total": 3525016200, "members_executed_total": 36324 }, "example_device_plugin.stress_write": { "callback_ns_total": 164754989, "member_service_ns_total": 175039100, "members_executed_total": 36324 } }, "callback_ns_total": 4203967589, "cycles_total": 12108, "member_service_ns_total": 4249924200, "members_executed_total": 108972, "thread_cpu_ns_total": 1203125000, "wall_ns_total": 4322201200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 607532005, "member_service_ns_total": 621855000, "members_executed_total": 36489 }, "example_device_plugin.stress_read": { "callback_ns_total": 3428153704, "member_service_ns_total": 3450324800, "members_executed_total": 36489 }, "example_device_plugin.stress_write": { "callback_ns_total": 172616494, "member_service_ns_total": 183458900, "members_executed_total": 36489 } }, "callback_ns_total": 4208302203, "cycles_total": 12163, "member_service_ns_total": 4255638700, "members_executed_total": 109467, "thread_cpu_ns_total": 1437500000, "wall_ns_total": 4320960400 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 122.35643855, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 122, "hz": 0.9966241407663176, "failed": 0, "missed": 0, "callback_last_ms": 0.0062, "callback_ewma_ms": 0.005273, "release_lateness_last_ms": 0.5997 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 122, "hz": 0.9966241407663176, "failed": 0, "missed": 0, "callback_last_ms": 5.9462, "callback_ewma_ms": 19.632846, "release_lateness_last_ms": 0.4492 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 612, "hz": 4.999458804499888, "failed": 0, "missed": 0, "callback_last_ms": 0.001, "callback_ewma_ms": 0.001208, "release_lateness_last_ms": 0.2705 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 612, "hz": 4.999458804499888, "failed": 0, "missed": 0, "callback_last_ms": 2.1089, "callback_ewma_ms": 3.621975, "release_lateness_last_ms": 0.4412 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 122, "hz": 0.9966241407663176, "failed": 0, "missed": 0, "callback_last_ms": 7.5911, "callback_ewma_ms": 11.036807, "release_lateness_last_ms": 0.6125 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 122, "hz": 0.9966241407663176, "failed": 0, "missed": 0, "callback_last_ms": 2.1503, "callback_ewma_ms": 10.771454, "release_lateness_last_ms": 0.1974 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 245, "hz": 2.001417331866785, "failed": 0, "missed": 0, "callback_last_ms": 0.0074, "callback_ewma_ms": 0.003502, "release_lateness_last_ms": 0.4515 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 122, "hz": 0.9966241407663176, "failed": 0, "missed": 0, "callback_last_ms": 2.8932, "callback_ewma_ms": 3.783808, "release_lateness_last_ms": 0.4487 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 245, "hz": 2.001417331866785, "failed": 0, "missed": 0, "callback_last_ms": 4.4343, "callback_ewma_ms": 12.640302, "release_lateness_last_ms": 0.5213 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 612, "hz": 4.999458804499888, "failed": 0, "missed": 0, "callback_last_ms": 8.9116, "callback_ewma_ms": 11.496177, "release_lateness_last_ms": 0.5227 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1224, "hz": 9.998917608999776, "failed": 0, "missed": 0, "callback_last_ms": 0.0548, "callback_ewma_ms": 0.473443, "release_lateness_last_ms": 0.3545 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 612, "hz": 4.999458804499888, "failed": 0, "missed": 0, "callback_last_ms": 20.0734, "callback_ewma_ms": 28.625853, "release_lateness_last_ms": 0.1298 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 245, "hz": 2.001417331866785, "failed": 0, "missed": 0, "callback_last_ms": 6.2597, "callback_ewma_ms": 14.89683, "release_lateness_last_ms": 0.3491 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 612, "hz": 4.999458804499888, "failed": 0, "missed": 0, "callback_last_ms": 0.0021, "callback_ewma_ms": 0.048227, "release_lateness_last_ms": 0.3825 }, { "name": "general_program_checks_thread", "running": true, "completed": 1224, "hz": 9.998917608999776, "failed": 0, "missed": 0, "callback_last_ms": 0.1373, "callback_ewma_ms": 0.19821, "release_lateness_last_ms": 0.4987 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 122, "hz": 0.9966241407663176, "failed": 0, "missed": 0, "callback_last_ms": 0.1179, "callback_ewma_ms": 0.151248, "release_lateness_last_ms": 0.3486 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1224, "hz": 9.998917608999776, "failed": 0, "missed": 0, "callback_last_ms": 0.0043, "callback_ewma_ms": 0.004068, "release_lateness_last_ms": 0.5804 }, { "name": "engine_diagnostics", "running": true, "completed": 245, "hz": 2.001417331866785, "failed": 0, "missed": 0, "callback_last_ms": 39.932, "callback_ewma_ms": 74.248481, "release_lateness_last_ms": 0.3442 } ], "peer_counter_seconds": 122.4134144, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.50139999999374, "gap_p95_ms": 267.31140000000596, "gap_p99_ms": 470.23809999998775, "gap_max_ms": 1056.0907999999763, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 53, "p95": 169, "p99": 744, "max": 1004, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.63229999999749, "gap_p95_ms": 272.9439999999886, "gap_p99_ms": 465.7606000000087, "gap_max_ms": 1033.0153999999748, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 46, "p95": 146, "p99": 777, "max": 1019, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.2597999999998, "gap_p95_ms": 259.2573999999877, "gap_p99_ms": 496.92559999998775, "gap_max_ms": 1057.9911000000138, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 50, "p95": 176, "p99": 698, "max": 1034, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.23079999999754, "gap_p95_ms": 266.89110000000073, "gap_p99_ms": 510.79999999998836, "gap_max_ms": 1017.5200999999943, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 58, "p95": 192, "p99": 671, "max": 1008, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.48089999999502, "gap_p95_ms": 264.45980000001146, "gap_p99_ms": 400.3899000000092, "gap_max_ms": 1822.5246000000043, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 42, "p95": 251, "p99": 2357, "max": 3100, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.40949999997974, "gap_p95_ms": 265.078600000008, "gap_p99_ms": 523.8953000000038, "gap_max_ms": 1067.7772000000114, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 56, "p95": 182, "p99": 930, "max": 1428, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.44250000000466, "gap_p95_ms": 264.9311000000016, "gap_p99_ms": 399.55799999999, "gap_max_ms": 1841.1499000000185, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 49, "p95": 140, "p99": 1697, "max": 2335, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.12820000000647, "gap_p95_ms": 263.5755999999965, "gap_p99_ms": 482.446100000001, "gap_max_ms": 1030.0254000000132, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 51, "p95": 185, "p99": 782, "max": 1057, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.3335999999981, "gap_p95_ms": 258.4765999999945, "gap_p99_ms": 393.76459999999497, "gap_max_ms": 989.5024000000267, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 61, "p95": 171, "p99": 750, "max": 945, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.1941999999981, "gap_p95_ms": 260.37470000001485, "gap_p99_ms": 496.0280999999959, "gap_max_ms": 1025.1659999999974, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 59, "p95": 205, "p99": 715, "max": 986, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.60979999999836, "gap_p95_ms": 264.2234000000026, "gap_p99_ms": 486.66109999999753, "gap_max_ms": 2190.589399999997, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 57, "p95": 311, "p99": 2912, "max": 3308, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.6947000000073, "gap_p95_ms": 251.72030000000086, "gap_p99_ms": 490.26579999999376, "gap_max_ms": 953.2032000000181, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 72, "p95": 181, "p99": 682, "max": 1001, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.8573000000033, "gap_p95_ms": 267.6912000000011, "gap_p99_ms": 505.84029999998165, "gap_max_ms": 1055.6150999999954, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 66, "p95": 187, "p99": 613, "max": 980, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.23549999999523, "gap_p95_ms": 268.7424000000028, "gap_p99_ms": 373.1854999999923, "gap_max_ms": 1027.8475999999791, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 54, "p95": 165, "p99": 791, "max": 1136, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.7289000000019, "gap_p95_ms": 257.80659999999625, "gap_p99_ms": 472.1281000000017, "gap_max_ms": 1013.5430999999808, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 67, "p95": 197, "p99": 749, "max": 1036, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.13810000001104, "gap_p95_ms": 262.34760000000824, "gap_p99_ms": 477.82339999999385, "gap_max_ms": 1097.0695000000123, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 67, "p95": 183, "p99": 766, "max": 1048, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.2387000000017, "gap_p95_ms": 261.58159999999043, "gap_p99_ms": 462.8904999999795, "gap_max_ms": 1036.385799999989, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 59, "p95": 198, "p99": 918, "max": 1199, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.36359999999695, "gap_p95_ms": 257.82330000000366, "gap_p99_ms": 502.22119999999995, "gap_max_ms": 2185.8471999999892, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 64, "p95": 307, "p99": 2420, "max": 2705, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.77289999998175, "gap_p95_ms": 259.3246999999974, "gap_p99_ms": 510.18580000000657, "gap_max_ms": 1066.7962000000116, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 61, "p95": 173, "p99": 703, "max": 1107, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.22990000000573, "gap_p95_ms": 267.74379999999655, "gap_p99_ms": 510.9363000000012, "gap_max_ms": 1054.1307000000088, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 69, "p95": 197, "p99": 756, "max": 1041, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.44630000001052, "gap_p95_ms": 264.0691000000079, "gap_p99_ms": 405.98199999998906, "gap_max_ms": 1606.231999999989, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 66, "p95": 213, "p99": 971, "max": 1588, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.4881000000023, "gap_p95_ms": 261.54349999999977, "gap_p99_ms": 442.50659999999334, "gap_max_ms": 1194.3797999999952, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 78, "p95": 251, "p99": 973, "max": 1385, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.52630000001227, "gap_p95_ms": 260.99290000000474, "gap_p99_ms": 478.65260000000126, "gap_max_ms": 1069.1510999999882, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 67, "p95": 192, "p99": 885, "max": 1162, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.42309999998542, "gap_p95_ms": 264.2887000000046, "gap_p99_ms": 495.8196000000171, "gap_max_ms": 1067.3481999999785, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 63, "p95": 212, "p99": 859, "max": 1125, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.18469999999797, "gap_p95_ms": 259.5941000000021, "gap_p99_ms": 403.9222000000009, "gap_max_ms": 2174.8886000000057, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 57, "p95": 202, "p99": 1451, "max": 2103, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.328600000008, "gap_p95_ms": 264.08759999999893, "gap_p99_ms": 494.3249000000069, "gap_max_ms": 1030.17849999998, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 74, "p95": 185, "p99": 760, "max": 1080, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.9331000000093, "gap_p95_ms": 261.5942000000068, "gap_p99_ms": 515.453600000008, "gap_max_ms": 1002.671399999992, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 73, "p95": 203, "p99": 745, "max": 998, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.44380000000092, "gap_p95_ms": 266.11149999999907, "gap_p99_ms": 417.07319999998435, "gap_max_ms": 1099.3368999999948, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 66, "p95": 248, "p99": 1087, "max": 1569, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.12320000000182, "gap_p95_ms": 265.6247000000003, "gap_p99_ms": 384.10719999999856, "gap_max_ms": 1567.2217999999993, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 73, "p95": 209, "p99": 1113, "max": 1570, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.42849999999453, "gap_p95_ms": 258.8127999999997, "gap_p99_ms": 371.2070999999996, "gap_max_ms": 2215.003800000006, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 74, "p95": 236, "p99": 2280, "max": 2867, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.57080000000133, "gap_p95_ms": 267.137999999999, "gap_p99_ms": 534.3984000000055, "gap_max_ms": 1121.5462999999872, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 78, "p95": 241, "p99": 703, "max": 1100, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 5.000950102763345, "complete_hz": 5.000950102763345, "catalog_hz": 2.005290957736529, "bytes_per_second": 1550884.1283225166, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.75010000000475, "gap_p95_ms": 257.56849999999395, "gap_p99_ms": 510.72019999998156, "gap_max_ms": 968.6429000000062, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1222, "p50": 59, "p95": 179, "p99": 729, "max": 1026, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 102, "mean": 104.35008039215673, "p95": 156.2442999999912, "max": 1049.9343999999983, "p50": 78.56689999999799, "p99": 405.6634999999951, "min": 52.590299999996205 }, { "operation": "rapid/get-persistence-diagnostics", "count": 102, "mean": 26.02488039215649, "p95": 80.43439999999828, "max": 539.8055999999924, "p50": 15.55340000000433, "p99": 481.74239999998827, "min": 4.831999999994878 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2225269, "accepted_rows_total": 14059169, "committed_rows_total": 2226313, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 102, "minimum": 19.921875, "maximum": 85.08064516129032, "mean": 40.728924248824406 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 102, "minimum": 3.1252031382039833, "maximum": 29.012499575947032, "mean": 11.284720312501456 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 102, "minimum": 996511744, "maximum": 1097998336, "mean": 1025555395.7647059 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 102, "minimum": 0, "maximum": 0.0396728515625, "mean": 0.000388949525122549 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 102, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 102, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 102, "minimum": 0, "maximum": 21.86279296875, "mean": 0.21625593596813725 }, { "channel": "engine_tempest_tx_mbps", "observations": 102, "minimum": 748.4152443557507, "maximum": 1185.5431773487217, "mean": 954.0331671125717 }, { "channel": "engine_tempest_rx_mbps", "observations": 102, "minimum": 0.01915696548545383, "maximum": 0.0738085836649151, "mean": 0.043977051222024326 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 102, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 102, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 102, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 102, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 102, "minimum": 10764.025745628865, "maximum": 39539.922360298144, "mean": 17849.9269048415 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 102, "minimum": 10894.074383819549, "maximum": 40792.04657101984, "mean": 17993.575047371025 } ], "graceful_shutdown": true, "shutdown_seconds": 1.748886599999998 }, { "id": "verified-capacity-release-25-web-32-r3", "healthy": true, "failure_reasons": [], "seconds": 120.5449527, "measurement_completed": true, "background_preflight": null, "requested_client_count": 32, "requested_peer_count": 0, "requested_channels_per_client": 250, "engine_sha256": "aa468632a1f17da532f266f538790ec95a0fd6102765e2ba23584c137e3c0a2e", "plugin_sha256": "aa3eac37bd7ddb0c711ab36004cd9feed436f0694d61522213df92a2fd738f56", "peer_simulator_sha256": null, "clients": 32, "transport": "web", "actual_connections": { "loadClients": 32, "actualClients": 33, "actualPeers": 0, "note": "Actual clients include the operator measurement connection in addition to the independent load clients." }, "local_channels": 934, "imported_channels": 0, "total_channels": 934, "unique_subscribed_channels_per_client": 250, "recipient_channel_subscriptions": 8000, "imported_channels_per_peer": 0, "peers": [], "peer_simulator_offered": null, "host_cpu_percent": { "count": 101, "mean": 61.855719928232816, "p50": 57.078899183283106, "p95": 92.75103960094326, "p99": 97.8379858800991, "min": 36.526643137325834, "max": 98.83980065496594 }, "engine_cpu_percent": { "count": 101, "mean": 12.334233233764927, "p50": 10.858419817556264, "p95": 22.07787710564161, "p99": 25.59432909427899, "min": 5.654730992232481, "max": 26.14491444947486 }, "client_generator_cpu_percent": { "count": 101, "mean": 7.982240011481917, "p50": 7.720232984204679, "p95": 10.450547976573262, "p99": 12.146035378583178, "min": 5.18426638082008, "max": 14.398697238785141 }, "peer_generator_cpu_percent": { "count": 0, "mean": null, "p50": null, "p95": null, "p99": null, "min": null, "max": null }, "engine_resident_bytes": { "count": 101, "mean": 1056100230.3366337, "p50": 1054195712, "p95": 1085968384, "p99": 1090404352, "min": 1009442816, "max": 1097986048 }, "system_samples": 101, "system_sample_interval_seconds": { "count": 101, "mean": 1.1906568138613864, "p95": 1.3966493, "max": 2.099799, "p50": 1.1349635, "p99": 1.674335, "min": 1.104617 }, "achieved_read_hz": { "minimum": 95.95881739555578, "maximum": 98.67354172568464 }, "control_task_hz": [ { "group": "stress_group_0", "controls": 4, "completed": 47816, "hz": 98.64214002883371 }, { "group": "stress_group_1", "controls": 3, "completed": 35893, "hz": 98.72740873500999 }, { "group": "stress_group_2", "controls": 3, "completed": 35862, "hz": 98.64214002883371 }, { "group": "stress_group_3", "controls": 3, "completed": 34923, "hz": 96.05932341271985 }, { "group": "stress_group_4", "controls": 3, "completed": 34935, "hz": 96.09233065382035 }, { "group": "stress_group_5", "controls": 3, "completed": 34953, "hz": 96.14184151547111 }, { "group": "stress_group_6", "controls": 3, "completed": 34932, "hz": 96.08407884354523 }, { "group": "stress_group_7", "controls": 3, "completed": 35907, "hz": 98.76591718296056 } ], "task_summary": { "count": 75, "failedCycles": 0, "missedCycles": 23997, "activeOverdue": 0 }, "task_groups": [ { "name": "stress_group_0", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 929629278, "member_service_ns_total": 946499300, "members_executed_total": 47816 }, "example_device_plugin.stress_read": { "callback_ns_total": 5480395047, "member_service_ns_total": 5508312000, "members_executed_total": 47816 }, "example_device_plugin.stress_write": { "callback_ns_total": 248779590, "member_service_ns_total": 266158400, "members_executed_total": 47816 } }, "callback_ns_total": 6658803915, "cycles_total": 11954, "member_service_ns_total": 6720969700, "members_executed_total": 143448, "thread_cpu_ns_total": 2343750000, "wall_ns_total": 6809715800 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_1", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 722762375, "member_service_ns_total": 737344500, "members_executed_total": 35893 }, "example_device_plugin.stress_read": { "callback_ns_total": 4541050014, "member_service_ns_total": 4567106300, "members_executed_total": 35893 }, "example_device_plugin.stress_write": { "callback_ns_total": 199174993, "member_service_ns_total": 210483000, "members_executed_total": 35894 } }, "callback_ns_total": 5463095482, "cycles_total": 11965, "member_service_ns_total": 5515043600, "members_executed_total": 107685, "thread_cpu_ns_total": 1406250000, "wall_ns_total": 5588768100 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_2", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 741746584, "member_service_ns_total": 763506300, "members_executed_total": 35862 }, "example_device_plugin.stress_read": { "callback_ns_total": 4539827260, "member_service_ns_total": 4562914400, "members_executed_total": 35862 }, "example_device_plugin.stress_write": { "callback_ns_total": 185253687, "member_service_ns_total": 197854000, "members_executed_total": 35862 } }, "callback_ns_total": 5466827531, "cycles_total": 11954, "member_service_ns_total": 5524274700, "members_executed_total": 107586, "thread_cpu_ns_total": 1812500000, "wall_ns_total": 5607328200 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_3", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 725801227, "member_service_ns_total": 745044500, "members_executed_total": 34923 }, "example_device_plugin.stress_read": { "callback_ns_total": 4725354620, "member_service_ns_total": 4757413700, "members_executed_total": 34923 }, "example_device_plugin.stress_write": { "callback_ns_total": 228862584, "member_service_ns_total": 243660800, "members_executed_total": 34923 } }, "callback_ns_total": 5680018431, "cycles_total": 11641, "member_service_ns_total": 5746119000, "members_executed_total": 104769, "thread_cpu_ns_total": 1687500000, "wall_ns_total": 5835591000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_4", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 688972042, "member_service_ns_total": 725656300, "members_executed_total": 34935 }, "example_device_plugin.stress_read": { "callback_ns_total": 4689061450, "member_service_ns_total": 4716805900, "members_executed_total": 34935 }, "example_device_plugin.stress_write": { "callback_ns_total": 273861884, "member_service_ns_total": 283941100, "members_executed_total": 34935 } }, "callback_ns_total": 5651895376, "cycles_total": 11645, "member_service_ns_total": 5726403300, "members_executed_total": 104805, "thread_cpu_ns_total": 1281250000, "wall_ns_total": 5826601300 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_5", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 764001564, "member_service_ns_total": 787923400, "members_executed_total": 34953 }, "example_device_plugin.stress_read": { "callback_ns_total": 4587314636, "member_service_ns_total": 4618153200, "members_executed_total": 34953 }, "example_device_plugin.stress_write": { "callback_ns_total": 291874390, "member_service_ns_total": 305961000, "members_executed_total": 34953 } }, "callback_ns_total": 5643190590, "cycles_total": 11651, "member_service_ns_total": 5712037600, "members_executed_total": 104859, "thread_cpu_ns_total": 1140625000, "wall_ns_total": 5834769600 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_6", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 671988650, "member_service_ns_total": 700529900, "members_executed_total": 34932 }, "example_device_plugin.stress_read": { "callback_ns_total": 4480390215, "member_service_ns_total": 4512194500, "members_executed_total": 34932 }, "example_device_plugin.stress_write": { "callback_ns_total": 203896384, "member_service_ns_total": 221449200, "members_executed_total": 34932 } }, "callback_ns_total": 5356275249, "cycles_total": 11644, "member_service_ns_total": 5434173600, "members_executed_total": 104796, "thread_cpu_ns_total": 1515625000, "wall_ns_total": 5522918000 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." }, { "name": "stress_group_7", "counters": { "by_task_type": { "DCode": { "callback_ns_total": 735955530, "member_service_ns_total": 753860300, "members_executed_total": 35907 }, "example_device_plugin.stress_read": { "callback_ns_total": 4599195941, "member_service_ns_total": 4624277900, "members_executed_total": 35907 }, "example_device_plugin.stress_write": { "callback_ns_total": 183466390, "member_service_ns_total": 194873200, "members_executed_total": 35907 } }, "callback_ns_total": 5518617861, "cycles_total": 11969, "member_service_ns_total": 5573011400, "members_executed_total": 107721, "thread_cpu_ns_total": 1421875000, "wall_ns_total": 5679393900 }, "note": "Cumulative counter delta between resource captures; midpoint timing is approximate within each capture duration." } ], "task_counter_seconds": 121.1855298, "loops": [ { "name": "engine_tempest_peer", "running": true, "completed": 121, "hz": 1.0006266867051339, "failed": 0, "missed": 0, "callback_last_ms": 0.006, "callback_ewma_ms": 0.00704, "release_lateness_last_ms": 0.6743 }, { "name": "argus/firing_state_updates_loop", "running": true, "completed": 121, "hz": 1.0006266867051339, "failed": 0, "missed": 0, "callback_last_ms": 6.1904, "callback_ewma_ms": 13.4693, "release_lateness_last_ms": 0.2548 }, { "name": "dartwic/project_action_telemetry_loop", "running": true, "completed": 604, "hz": 4.994863791486784, "failed": 0, "missed": 0, "callback_last_ms": 0.0012, "callback_ewma_ms": 0.001148, "release_lateness_last_ms": 0.5869 }, { "name": "dartwic/dcode_execution_lines_telemetry_loop", "running": true, "completed": 603, "hz": 4.9865941494479, "failed": 0, "missed": 1, "callback_last_ms": 2.1299, "callback_ewma_ms": 3.105049, "release_lateness_last_ms": 0.7421 }, { "name": "dartwic/editor_file_change_telemetry_loop", "running": true, "completed": 121, "hz": 1.0006266867051339, "failed": 0, "missed": 0, "callback_last_ms": 4.352, "callback_ewma_ms": 4.913793, "release_lateness_last_ms": 0.634 }, { "name": "dartwic/runtime_views_telemetry_loop", "running": true, "completed": 121, "hz": 1.0006266867051339, "failed": 0, "missed": 0, "callback_last_ms": 2.9808, "callback_ewma_ms": 4.913163, "release_lateness_last_ms": 0.1735 }, { "name": "dartwic/builder_updates_loop", "running": true, "completed": 242, "hz": 2.0012533734102678, "failed": 0, "missed": 0, "callback_last_ms": 0.0015, "callback_ewma_ms": 0.001374, "release_lateness_last_ms": 0.1026 }, { "name": "example_device_plugin.example_device_1_discovery", "running": true, "completed": 121, "hz": 1.0006266867051339, "failed": 0, "missed": 0, "callback_last_ms": 2.8226, "callback_ewma_ms": 4.708712, "release_lateness_last_ms": 0.484 }, { "name": "rapid/channel_catalog_telemetry_loop", "running": true, "completed": 242, "hz": 2.0012533734102678, "failed": 0, "missed": 0, "callback_last_ms": 4.7448, "callback_ewma_ms": 7.426758, "release_lateness_last_ms": 0.6222 }, { "name": "argus/board_snapshot_telemetry_loop", "running": true, "completed": 603, "hz": 4.9865941494479, "failed": 0, "missed": 1, "callback_last_ms": 5.7778, "callback_ewma_ms": 8.653604, "release_lateness_last_ms": 0.7114 }, { "name": "rapid/autorecord_loop", "running": true, "completed": 1207, "hz": 9.981457940934684, "failed": 0, "missed": 2, "callback_last_ms": 0.0247, "callback_ewma_ms": 0.83826, "release_lateness_last_ms": 0.4792 }, { "name": "rapid/channels_telemetry_loop", "running": true, "completed": 600, "hz": 4.961785223331243, "failed": 0, "missed": 4, "callback_last_ms": 17.8883, "callback_ewma_ms": 23.060041, "release_lateness_last_ms": 0.5693 }, { "name": "dartwic/console_telemetry_loop", "running": true, "completed": 242, "hz": 2.0012533734102678, "failed": 0, "missed": 0, "callback_last_ms": 5.9075, "callback_ewma_ms": 10.997602, "release_lateness_last_ms": 0.3765 }, { "name": "rapid/peer_channels_telemetry_loop", "running": true, "completed": 604, "hz": 4.994863791486784, "failed": 0, "missed": 0, "callback_last_ms": 0.0017, "callback_ewma_ms": 0.001785, "release_lateness_last_ms": 0.4442 }, { "name": "general_program_checks_thread", "running": true, "completed": 1207, "hz": 9.981457940934684, "failed": 0, "missed": 2, "callback_last_ms": 0.1657, "callback_ewma_ms": 0.273044, "release_lateness_last_ms": 0.5182 }, { "name": "example_device_plugin.connection_monitor", "running": true, "completed": 121, "hz": 1.0006266867051339, "failed": 0, "missed": 0, "callback_last_ms": 0.1832, "callback_ewma_ms": 0.215349, "release_lateness_last_ms": 0.4064 }, { "name": "example_device_plugin.heartbeat", "running": true, "completed": 1209, "hz": 9.997997225012455, "failed": 0, "missed": 0, "callback_last_ms": 0.0055, "callback_ewma_ms": 0.007129, "release_lateness_last_ms": 0.1365 }, { "name": "engine_diagnostics", "running": true, "completed": 241, "hz": 1.9929837313713825, "failed": 0, "missed": 0, "callback_last_ms": 44.1154, "callback_ewma_ms": 65.836972, "release_lateness_last_ms": 0.1281 } ], "peer_counter_seconds": 120.8150272, "peer_counter_deltas": [], "observations": [ { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.92320000000473, "gap_p95_ms": 397.40639999999985, "gap_p99_ms": 802.0368000000017, "gap_max_ms": 1256.6735999999946, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 44, "p95": 475, "p99": 944, "max": 1497, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.65549999999348, "gap_p95_ms": 389.835699999996, "gap_p99_ms": 742.0746999999974, "gap_max_ms": 1718.4002999999939, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 50, "p95": 525, "p99": 1078, "max": 1616, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.98869999998715, "gap_p95_ms": 382.1193000000203, "gap_p99_ms": 880.0019000000029, "gap_max_ms": 1259.755600000004, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 43, "p95": 637, "p99": 1063, "max": 1280, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.34369999999763, "gap_p95_ms": 358.6938999999984, "gap_p99_ms": 900.5458999999973, "gap_max_ms": 1263.7235999999975, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 53, "p95": 624, "p99": 1420, "max": 1913, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.51980000000913, "gap_p95_ms": 387.4258999999947, "gap_p99_ms": 804.0458999999973, "gap_max_ms": 1396.3450000000012, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 47, "p95": 564, "p99": 1050, "max": 1382, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.66439999997965, "gap_p95_ms": 379.7118999999948, "gap_p99_ms": 793.9306999999972, "gap_max_ms": 1186.8803999999946, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 49, "p95": 501, "p99": 817, "max": 1111, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.1921000000002, "gap_p95_ms": 402.69349999999395, "gap_p99_ms": 848.3187000000034, "gap_max_ms": 1287.8885999999911, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 55, "p95": 486, "p99": 939, "max": 1234, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.00229999999283, "gap_p95_ms": 387.86119999999937, "gap_p99_ms": 812.7932000000146, "gap_max_ms": 1471.47530000002, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 57, "p95": 493, "p99": 1063, "max": 1404, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.3929000000062, "gap_p95_ms": 397.14740000000165, "gap_p99_ms": 825.3280999999843, "gap_max_ms": 1217.6763999999966, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 62, "p95": 505, "p99": 971, "max": 1275, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.69620000000577, "gap_p95_ms": 351.2599999999802, "gap_p99_ms": 889.6466999999975, "gap_max_ms": 1822.7305999999953, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 67, "p95": 626, "p99": 1294, "max": 1637, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.78640000001178, "gap_p95_ms": 377.5630000000019, "gap_p99_ms": 858.5253999999986, "gap_max_ms": 1807.4496999999974, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 63, "p95": 605, "p99": 1112, "max": 1662, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.80379999999423, "gap_p95_ms": 396.96960000001127, "gap_p99_ms": 866.9438000000009, "gap_max_ms": 1247.7848999999987, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 63, "p95": 494, "p99": 899, "max": 1357, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.61159999998927, "gap_p95_ms": 382.2815999999875, "gap_p99_ms": 850.2126999999964, "gap_max_ms": 1450.2776000000013, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 67, "p95": 574, "p99": 1477, "max": 2255, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.68069999999716, "gap_p95_ms": 391.14959999998973, "gap_p99_ms": 843.1010000000097, "gap_max_ms": 1213.666200000007, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 69, "p95": 436, "p99": 805, "max": 1130, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 201.37970000000496, "gap_p95_ms": 378.291000000012, "gap_p99_ms": 810.8083000000042, "gap_max_ms": 1308.1154999999999, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 56, "p95": 488, "p99": 957, "max": 1169, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.15690000000177, "gap_p95_ms": 411.54030000000057, "gap_p99_ms": 758.5256999999983, "gap_max_ms": 1075.7654000000039, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 75, "p95": 477, "p99": 750, "max": 1133, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.80309999999008, "gap_p95_ms": 363.07609999999113, "gap_p99_ms": 906.669900000008, "gap_max_ms": 1880.7220000000088, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 52, "p95": 585, "p99": 1172, "max": 1715, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.51529999999912, "gap_p95_ms": 383.9082000000053, "gap_p99_ms": 849.3755999999994, "gap_max_ms": 1831.9714999999997, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 66, "p95": 567, "p99": 1149, "max": 1697, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.10659999999916, "gap_p95_ms": 394.30489999999554, "gap_p99_ms": 820.0325999999768, "gap_max_ms": 1278.8475000000035, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 74, "p95": 529, "p99": 930, "max": 1440, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.0862999999954, "gap_p95_ms": 356.54709999999614, "gap_p99_ms": 810.3616000000038, "gap_max_ms": 1441.8411999999953, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 74, "p95": 613, "p99": 1458, "max": 2232, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.37140000000363, "gap_p95_ms": 409.4824999999837, "gap_p99_ms": 839.9766000000018, "gap_max_ms": 1332.9786000000022, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 79, "p95": 474, "p99": 900, "max": 1209, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.75219999998808, "gap_p95_ms": 405.70679999999993, "gap_p99_ms": 842.8328000000038, "gap_max_ms": 1219.0587999999989, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 52, "p95": 464, "p99": 969, "max": 1540, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.64680000000226, "gap_p95_ms": 376.6021999999939, "gap_p99_ms": 887.9522999999899, "gap_max_ms": 1882.3081999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 79, "p95": 601, "p99": 1217, "max": 1758, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.25010000000475, "gap_p95_ms": 414.7847000000038, "gap_p99_ms": 847.2654000000039, "gap_max_ms": 1243.8489999999874, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 82, "p95": 475, "p99": 860, "max": 1244, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 197.9377999999997, "gap_p95_ms": 388.9216999999917, "gap_p99_ms": 906.9372000000149, "gap_max_ms": 1848.3499999999913, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 82, "p95": 594, "p99": 1095, "max": 1733, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.93219999999565, "gap_p95_ms": 397.0821999999898, "gap_p99_ms": 770.6815999999963, "gap_max_ms": 1246.2532999999967, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 85, "p95": 522, "p99": 892, "max": 1200, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.60629999999946, "gap_p95_ms": 378.0504000000001, "gap_p99_ms": 807.9949999999953, "gap_max_ms": 1163.6033999999927, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 76, "p95": 495, "p99": 896, "max": 1148, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.066599999991, "gap_p95_ms": 409.06870000000345, "gap_p99_ms": 775.6972999999998, "gap_max_ms": 1347.4037000000098, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 77, "p95": 512, "p99": 925, "max": 1238, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 199.86039999999048, "gap_p95_ms": 391.93330000000424, "gap_p99_ms": 777.1676999999909, "gap_max_ms": 1747.6748999999982, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 48, "p95": 515, "p99": 1046, "max": 1592, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 198.34470000000147, "gap_p95_ms": 366.2896000000037, "gap_p99_ms": 905.8775000000023, "gap_max_ms": 1884.1062999999995, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 58, "p95": 546, "p99": 1119, "max": 1681, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.01269999999204, "gap_p95_ms": 417.03779999999824, "gap_p99_ms": 755.9357000000018, "gap_max_ms": 1121.5290000000095, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 67, "p95": 481, "p99": 745, "max": 1012, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } }, { "snapshot_hz": 4.969132070960145, "complete_hz": 4.969132070960145, "catalog_hz": 1.9992668265465692, "bytes_per_second": 1541183.5041914368, "stale_episodes": 0, "sequence_gaps": 0, "gap_p50_ms": 200.20560000000114, "gap_p95_ms": 396.0448999999935, "gap_p99_ms": 830.3897999999754, "gap_max_ms": 1295.198299999989, "stale_durations_ms": [], "stale_ongoing_at_end": 0, "source_age_ms": { "count": 1198, "p50": 86, "p95": 487, "p99": 831, "max": 1192, "note": "First and last subscribed channel; source age includes producer and delivery delay on this host. UI freshness tracks timestamp advancement and can remain fresh despite delayed delivery." } } ], "operation_round_trip_ms": [ { "operation": "dartwic/diagnostics/get", "count": 86, "mean": 146.52541162790735, "p95": 466.33410000000003, "max": 1171.8025000000198, "p50": 90.10409999999683, "p99": 1171.8025000000198, "min": 57.73189999999886 }, { "operation": "rapid/get-persistence-diagnostics", "count": 86, "mean": 85.84667093023296, "p95": 413.12270000000717, "max": 1744.8617999999988, "p50": 13.285900000002584, "p99": 1744.8617999999988, "min": 4.731599999999162 } ], "transport_event_count": 0, "transport_events": [], "recording": { "rapid": { "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusEvents": { "rejected_recording_jobs_total": 0, "failed_jobs_total": 0, "write_errors_total": 0, "available": true, "lossFree": true }, "argusLogs": { "dropped_total": 0, "rejected_total": 0, "available": true, "lossFree": true, "write_errors_total": 0 }, "allLossFree": true, "allErrorFree": true }, "recording_rows": { "enqueued_rows_total": 2175391, "accepted_rows_total": 13703690, "committed_rows_total": 2175933, "recording_dropped_rows_total": 0, "failed_rows_total": 0, "write_errors_total": 0 }, "metrics": [ { "channel": "engine_system_cpu_usage_percent", "observations": 86, "minimum": 31.25, "maximum": 100, "mean": 58.77673764183553 }, { "channel": "engine_dartwic_cpu_usage_percent", "observations": 86, "minimum": 5.077086228157719, "maximum": 23.83507053955523, "mean": 11.517035193543487 }, { "channel": "engine_dartwic_memory_resident_bytes", "observations": 86, "minimum": 1021788160, "maximum": 1090260992, "mean": 1053376702.5116279 }, { "channel": "engine_rapid_recording_queue_used_percent", "observations": 86, "minimum": 0, "maximum": 0.3662109375, "mean": 0.007416481195494186 }, { "channel": "engine_rapid_publication_queue_used_percent", "observations": 86, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_argus_log_queue_used_percent", "observations": 86, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_queue_used_percent", "observations": 86, "minimum": 0, "maximum": 1.904296875, "mean": 0.06976460301598837 }, { "channel": "engine_tempest_tx_mbps", "observations": 86, "minimum": 751.2535865093253, "maximum": 1167.789863581606, "mean": 952.2130604343813 }, { "channel": "engine_tempest_rx_mbps", "observations": 86, "minimum": 0, "maximum": 0.0902815900071095, "mean": 0.04155094179155758 }, { "channel": "engine_tempest_dropped_messages_total", "observations": 86, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_operations_total", "observations": 86, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_rejected_snapshots_total", "observations": 86, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_tempest_operation_timeouts_total", "observations": 86, "minimum": 0, "maximum": 0, "mean": 0 }, { "channel": "engine_rapid_recording_rows_per_sec", "observations": 86, "minimum": 8761.333617723258, "maximum": 37854.5794208741, "mean": 17424.36883552509 }, { "channel": "engine_rapid_recording_committed_rows_per_sec", "observations": 86, "minimum": 8635.137312957939, "maximum": 38142.35576110247, "mean": 17261.319723036653 } ], "graceful_shutdown": true, "shutdown_seconds": 1.7224973999999929 } ] }