From fe9c82d13bbedfee27efcaef383b5c157a54734b Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 9 Jul 2026 12:38:20 -0500 Subject: [PATCH 01/21] trimming matches rows from logs to avoid confustion on consecutive CE errors --- .../plugins/inband/dmesg/dmesg_analyzer.py | 16 +++++--- nodescraper/plugins/inband/dmesg/mce_utils.py | 41 ++++++++++++++++++- test/unit/plugin/test_dmesg_analyzer.py | 38 +++++++++++++++++ test/unit/plugin/test_mce_utils.py | 15 +++++++ 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 9c7b11b2..04c8737b 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -36,10 +36,13 @@ from .analyzer_args import DmesgAnalyzerArgs from .dmesgdata import DmesgData from .mce_utils import ( + compile_mce_ce_status_regex, + compile_mce_uc_status_regex, ignored_mce_block_line_indices, mce_block_all_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, + trim_mce_status_match_content, ) @@ -343,17 +346,13 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): event_category=EventCategory.RAS, ), ErrorRegex( - regex=re.compile( - r"\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\].*(?:\n.*){0,5}" - ), + regex=compile_mce_ce_status_regex(), message="MCE Corrected Error", event_category=EventCategory.RAS, event_priority=EventPriority.WARNING, ), ErrorRegex( - regex=re.compile( - r"\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\].*(?:\n.*){0,5}" - ), + regex=compile_mce_uc_status_regex(), message="MCE Uncorrected Error", event_category=EventCategory.RAS, ), @@ -731,6 +730,11 @@ def analyze_data( ignore_match_rules=ignore_match_rules, skip_line_indices=ignored_mce_block_lines, ) + for event in known_err_events: + if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"): + event.data["match_content"] = trim_mce_status_match_content( + event.data["match_content"] + ) if args.exclude_category: known_err_events = [ event for event in known_err_events if event.category not in args.exclude_category diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index b08e3a83..a8eeccf3 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -24,7 +24,7 @@ # ############################################################################### import re -from typing import FrozenSet, Optional, Sequence +from typing import FrozenSet, Optional, Sequence, Union from nodescraper.base.match_ignore import extract_mce_banks_from_text @@ -68,6 +68,45 @@ re.IGNORECASE, ) +_MCE_CE_STATUS_LINE_RE = re.compile( + r"\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\][^\n]*", + re.IGNORECASE, +) + +_MCE_UC_STATUS_LINE_RE = re.compile( + r"\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\][^\n]*", + re.IGNORECASE, +) + + +def compile_mce_ce_status_regex() -> re.Pattern[str]: + """Return a single-line regex for corrected MCn_STATUS hardware error rows.""" + return _MCE_CE_STATUS_LINE_RE + + +def compile_mce_uc_status_regex() -> re.Pattern[str]: + """Return a single-line regex for uncorrected MCn_STATUS hardware error rows.""" + return _MCE_UC_STATUS_LINE_RE + + +def trim_mce_status_match_content(match: Union[str, list[str]]) -> str: + """Keep only the MCn_STATUS [Hardware Error] row in match_content.""" + if isinstance(match, list): + for item in match: + trimmed = trim_mce_status_match_content(item) + if _MCE_STATUS_START_RE.search(trimmed): + return trimmed + return match[0] if match else "" + + for line in str(match).splitlines(): + ce_match = _MCE_CE_STATUS_LINE_RE.search(line) + if ce_match: + return ce_match.group(0) + uc_match = _MCE_UC_STATUS_LINE_RE.search(line) + if uc_match: + return uc_match.group(0) + return str(match) + def _normalize_cpu_label(cpu: str) -> str: return cpu.replace(":", "") diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 0de2ae21..87477acf 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1326,3 +1326,41 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] assert len(unknown_events) == 1 assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" + + +def test_mce_match_content_is_single_status_line(system_info): + """Adjacent incidents: events.json match_content must be one MCn_STATUS row.""" + dmesg_content = ( + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xccc\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=False, + mce_threshold=1, + ignore_match_rules=[{"mce_banks": ["60-63"]}], + ), + ) + + mce_events = [event for event in res.events if event.description == "MCE Corrected Error"] + assert len(mce_events) == 1 + match_content = str(mce_events[0].data["match_content"]) + assert match_content.startswith("[Hardware Error]:") + assert "MC49_STATUS" in match_content + assert "MC60_STATUS" not in match_content + assert "CPU:29" in match_content + assert "CPU:8" not in match_content + assert "\n" not in match_content diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index 53a58f00..2e60c73a 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -29,6 +29,7 @@ iter_hardware_error_block_ranges, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, + trim_mce_status_match_content, ) @@ -191,6 +192,20 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) +def test_trim_mce_status_match_content_keeps_status_row_only(): + multiline = ( + "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" + "[Hardware Error]: Corrected error, no action required.\n" + "[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" + ) + + trimmed = trim_mce_status_match_content(multiline) + + assert trimmed == ("[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") + assert "MC60_STATUS" not in trimmed + assert "\n" not in trimmed + + def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " From e5449b245f05c71458dfde6b0ec989b48b122da3 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 9 Jul 2026 14:07:32 -0500 Subject: [PATCH 02/21] doc updates --- docs/PLUGIN_DOC.md | 27 ++++++++++++++----- docs/generate_plugin_doc_bundle.py | 17 +++++++++++- .../plugins/inband/dmesg/dmesg_plugin.py | 2 ++ .../regex_search/regex_search_plugin.py | 5 ++++ .../plugins/inband/storage/storage_plugin.py | 2 ++ .../ooband/redfish_endpoint/analyzer_args.py | 4 +-- .../redfish_endpoint/endpoint_analyzer.py | 2 +- .../serviceability/mi3xx/mi3xx_analyzer.py | 7 +++++ .../serviceability/mi3xx/mi3xx_collector.py | 8 ++++++ test/unit/plugin/test_mi3xx_collector.py | 2 ++ 10 files changed, 66 insertions(+), 10 deletions(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index fe6612ae..360c3c08 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -25,9 +25,9 @@ | PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | - | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | | ProcessPlugin | top -b -n 1
rocm-smi --showpids
top -b -n 1 -o %CPU | **Analyzer Args:**
- `max_kfd_processes`: int — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.
- `max_cpu_usage`: float — Maximum allowed CPU usage (percent) for process checks. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `top_n_process`: int — Number of top processes by CPU usage to collect (e.g. for top -b -n 1 -o %%CPU). | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | | RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | - | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | -| RegexSearchPlugin | - | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | +| RegexSearchPlugin | No collector step: reads local text from the CLI --data path (file or directory).
Directory scans load each file's contents into RegexSearchData for analysis. | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | | RocmPlugin | {rocm_path}/opencl/bin/*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}*
ls -v -d {rocm_path}-[3-7]* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | -| StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | +| StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | **Analyzer Args:**
- `min_required_free_space_abs`: Optional[str] — Minimum required free space per mount (e.g. '10G', '1T').
- `min_required_free_space_prct`: Optional[int] — Minimum required free space as percentage of total (0–100).
- `ignore_devices`: Optional[list[str]] — Mount points or devices to exclude from free-space checks.
- `check_devices`: Optional[list[str]] — If non-empty, only these mount points or devices are checked.
- `regex_match`: bool — If True, match device/mount names with regex; otherwise exact match. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | | ScaleOutAristaPlugin | show interfaces counters bins | json | no-more
show interfaces counters queue | no-more
show interfaces counters queue drop-precedence | no-more
show qos interfaces ecn counters queue | json | no-more
show interfaces counters errors | json | no-more
show interfaces phy | no-more
show interfaces phy detail | no-more
show interfaces counters ip | json | no-more
show ip interface | no-more
show lldp | no-more
show lldp neighbors | json | no-more
show interfaces counters | json | no-more
show interfaces flow-control | json | no-more
show interfaces counters queue detail | no-more
show priority-flow-control counters | json | no-more
show priority-flow-control status | no-more
show interfaces status | json | no-more
show qos interfaces | no-more
show qos interfaces ecn | no-more
show qos interfaces trust | no-more
show qos maps | no-more
show qos profile | no-more
show qos profile summary | no-more
show interfaces counters rates | json | no-more
show running-config | no-more
show startup-config | no-more
show system environment cooling | json | no-more
show platform trident mmu queue status | no-more
show version | json | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Ports are S/P/[SP] where subport is optional (e.g. ['1/1', '1/31', '1/...
- `expected_port_bandwidth`: int — Expected interface bandwidth (bps) from show interfaces status (AristaPortStatus.bandwidth). Ports with a different b... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Re-runs successful... | [ScaleOutAristaDataModel](#ScaleOutAristaDataModel-Model) | [ScaleOutAristaCollector](#Collector-Class-ScaleOutAristaCollector) | [ScaleOutAristaAnalyzer](#Data-Analyzer-Class-ScaleOutAristaAnalyzer) | | ScaleOutDellPlugin | show alarm | no-more
show buffer pool | no-more
show buffer profile | no-more
show clock | no-more
show interface counters {port} | no-more
show event details | no-more
show interface fec status | no-more
show interface counters | no-more
show interface counters rate | no-more
show interface Eth | no-more
show interface phy counters | no-more
show interface status | no-more
show interface transceiver | no-more
show interface transceiver dom | no-more
show interface transceiver summary | no-more
show ip arp | no-more
show ip interfaces | no-more
show ip route | no-more
show lldp neighbor | no-more
show lldp table | no-more
show qos interface Ethall priority-flow-control statistics | no-more
show priority-flow-control watchdog | no-more
show qos interface Ethall queue all priority-flow-control watchdog-statistics | no-more
show platform environment | no-more
show platform firmware detail | no-more
show platform syseeprom | no-more
show qos interface Eth all | no-more
show qos interface Eth all queue all | no-more
show qos map dot1p-tc | no-more
show qos map dscp-tc | no-more
show qos map pfc-priority-pg | no-more
show qos map pfc-priority-queue | no-more
show qos map tc-dot1p | no-more
show qos map tc-dscp | no-more
show qos map tc-pg | no-more
show qos map tc-queue | no-more
show qos scheduler-policy | no-more
show qos wred-policy | no-more
show queue counters | no-more
show queue persistent-watermark multicast | no-more
show queue persistent-watermark unicast | no-more
show queue watermark multicast | no-more
show queue watermark unicast | no-more
show running-configuration | no-more
show version | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Accepts optional Eth prefix (e.g. ['1/1', '1/31', '1/1/1'] or ['Eth1/1...
- `expected_port_speed`: int — Expected interface speed (Mbps) from show interface status (DellInterfaceStatus.speed). Ports with a different speed... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `collection_ports`: Optional[List[str]] — Restrict detail counter collection to these ports. Accepts the same tokens as analysis_ports (e.g. ['1/1', '1/1/2'] o... | [ScaleOutDellDataModel](#ScaleOutDellDataModel-Model) | [ScaleOutDellCollector](#Collector-Class-ScaleOutDellCollector) | [ScaleOutDellAnalyzer](#Data-Analyzer-Class-ScaleOutDellAnalyzer) | | SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '*' are collected with ls -l (e.g. class/net/*/device).
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | @@ -41,9 +41,9 @@ | --- | --- | --- | --- | --- | --- | --- | | OobGenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | | OobBmcArchivePlugin | SSH (BMC) shell: tar+gzip archives for each path in collection_args (see PathSpec entries).
Uses sudo on the BMC when collection_args paths require elevated access. | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[nodescraper.plugins.ooband.bmc_archive.collector_args.PathSpec] — Named BMC paths to archive with tar czf -. Configure in plugin config under plugins.OobBmcArchivePlugin.collection_ar...
- `sudo`: bool — Default sudo setting for paths that do not specify sudo.
- `timeout`: int — Default per-path tar timeout in seconds.
- `skip_if_missing`: bool — Skip paths that do not exist on the BMC instead of failing collection.
- `ignore_failed_read`: bool — When true, pass GNU tar's --ignore-failed-read when the remote tar supports it. | [BmcArchiveDataModel](#BmcArchiveDataModel-Model) | [BmcArchiveCollector](#Collector-Class-BmcArchiveCollector) | - | -| RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key "*" runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or '*' -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | +| RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key `*` runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | | RedfishOemDiagPlugin | Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types (collection_args.log_service_path selects the LogService).
Optional binary archives under the plugin log path when log_path is set. | Summarizes success/failure per OEM diagnostic type from collected results.
When analysis_args.require_all_success is true, fails the run if any type failed collection.
**Analyzer Args:**
- `require_all_success`: bool — If True, analysis fails when any OEM type collection failed. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `log_service_path`: str — Redfish path to the LogService (e.g. DiagLogs).
- `oem_diagnostic_types_allowable`: Optional[list[str]] — Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagno...
- `oem_diagnostic_types`: list[str] — OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.
- `task_timeout_s`: int — Max seconds to wait for each BMC task. | [RedfishOemDiagDataModel](#RedfishOemDiagDataModel-Model) | [RedfishOemDiagCollector](#Collector-Class-RedfishOemDiagCollector) | [RedfishOemDiagAnalyzer](#Data-Analyzer-Class-RedfishOemDiagAnalyzer) | -| ServiceabilityPluginMI3XX | - | **Analyzer Args:**
- `hub_python_module`: Optional[str] — Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.
- `hub_display_name`: Optional[str] — Optional label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to hub config (e.g. AFID_SAG.json); passed as hub_init_path_kwarg.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uri`: Optional[str] — Optional alias for ``rf_event_log_uri``. When both ``uri`` and ``rf_event_log_uri`` are explicitly set to non-empty v...
- `rf_event_log_uri`: str — Redfish URI for the event log ``Entries`` collection.
- `rf_chassis_devices`: Optional[List[str]] — Chassis designations for Assembly GETs; required with ``rf_assembly_uri_template``.
- `rf_assembly_uri_template`: Optional[str] — Redfish URI template containing ``{device}`` for each chassis Assembly resource.
- `rf_firmware_bundle_uri`: Optional[str] — Redfish URI for firmware bundle inventory when subclasses extract component details.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI3XXCollector](#Collector-Class-MI3XXCollector) | [MI3XXAnalyzer](#Data-Analyzer-Class-MI3XXAnalyzer) | +| ServiceabilityPluginMI3XX | Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias).
Paginated Members collection and optional top, reference_time/time_operator filters.
Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model).
Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).
Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details. | Builds AFID events from collected Redfish event log members (and optional assembly metadata).
Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis.
Runs the configured Python service hub (hub_python_module) to produce service recommendations.
When analysis_args.skip_hub is true, only builds AFID events without running the hub.
**Analyzer Args:**
- `hub_python_module`: Optional[str] — Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.
- `hub_display_name`: Optional[str] — Optional label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to hub config (e.g. AFID_SAG.json); passed as hub_init_path_kwarg.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uri`: Optional[str] — Optional alias for ``rf_event_log_uri``. When both ``uri`` and ``rf_event_log_uri`` are explicitly set to non-empty v...
- `rf_event_log_uri`: str — Redfish URI for the event log ``Entries`` collection.
- `rf_chassis_devices`: Optional[List[str]] — Chassis designations for Assembly GETs; required with ``rf_assembly_uri_template``.
- `rf_assembly_uri_template`: Optional[str] — Redfish URI template containing ``{device}`` for each chassis Assembly resource.
- `rf_firmware_bundle_uri`: Optional[str] — Redfish URI for firmware bundle inventory when subclasses extract component details.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI3XXCollector](#Collector-Class-MI3XXCollector) | [MI3XXAnalyzer](#Data-Analyzer-Class-MI3XXAnalyzer) | # Collectors @@ -1313,6 +1313,14 @@ Collect MI3XX BMC Redfish data: event log members (with pagination), firmware in ServiceabilityDataModel +### Documented collection + +- Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias). +- Paginated Members collection and optional top, reference_time/time_operator filters. +- Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model). +- Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices). +- Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details. + # Data Models ## GenericCollectionDataModel Model @@ -2318,7 +2326,7 @@ Checks Redfish endpoint responses against configured thresholds and key/value ru ### Documented analysis - For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.). -- URI key "*" runs checks against every collected response body. +- URI key `*` runs checks against every collected response body. ## Data Analyzer Class RedfishOemDiagAnalyzer @@ -2345,6 +2353,13 @@ Build AFID events from collected data and run the configured service hub. **Link to code**: [mi3xx_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py) +### Documented analysis + +- Builds AFID events from collected Redfish event log members (and optional assembly metadata). +- Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis. +- Runs the configured Python service hub (hub_python_module) to produce service recommendations. +- When analysis_args.skip_hub is true, only builds AFID events without running the hub. + # Analyzer Args ## Analyzer Args Class GenericAnalyzerArgs @@ -2696,7 +2711,7 @@ Analyzer args for config-driven Redfish checks. ### Annotations / fields -- **checks**: `dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]]` — Map: URI or '*' -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match). Use '*' as the key to apply the inner constraints to every collected response body. Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. Constraints: 'eq' — value must equal the given literal (int, float, str, bool). 'min' — value must be numeric and >= the given number. 'max' — value must be numeric and <= the given number. 'anyOf' — value must be in the given list (OR; any match passes). Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "*": { "Status/Health": { "anyOf": ["OK"] } } }. +- **checks**: `dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]]` — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match). Use `*` as the key to apply the inner constraints to every collected response body. Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. Constraints: 'eq' — value must equal the given literal (int, float, str, bool). 'min' — value must be numeric and >= the given number. 'max' — value must be numeric and <= the given number. 'anyOf' — value must be in the given list (OR; any match passes). Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "\*": { "Status/Health": { "anyOf": ["OK"] } } }. ## Analyzer Args Class RedfishOemDiagAnalyzerArgs diff --git a/docs/generate_plugin_doc_bundle.py b/docs/generate_plugin_doc_bundle.py index 75c4d412..4e148d96 100644 --- a/docs/generate_plugin_doc_bundle.py +++ b/docs/generate_plugin_doc_bundle.py @@ -505,12 +505,25 @@ def extract_collection_args_from_collector_args(args_cls: Optional[type]) -> Lis return output +def escape_lone_markdown_asterisks(s: str) -> str: + """Escape single asterisks so GFM does not start emphasis spans; preserve **bold** and `code`.""" + parts = re.split(r"(`[^`]*`)", s) + out: list[str] = [] + for index, part in enumerate(parts): + if index % 2 == 1: + out.append(part) + else: + out.append(re.sub(r"(? str: """Escape content for a markdown table cell so pipes and newlines don't break columns. Use HTML entity for pipe so all markdown parsers treat it as content, not column separator. """ if not s: return s + s = escape_lone_markdown_asterisks(s) # Avoid @ in cells (e.g. OData property names) being turned into mail/mention links in Outlook/HTML viewers. return s.replace("|", "|").replace("@", "@").replace("\n", " ").replace("\r", " ") @@ -524,7 +537,9 @@ def md_kv(key: str, value: str) -> str: def md_list(items: List[str]) -> str: - return "".join(f"- {i}\n" for i in items) + ("\n" if items else "") + return "".join(f"- {escape_lone_markdown_asterisks(i)}\n" for i in items) + ( + "\n" if items else "" + ) def bases_list(cls: type) -> List[str]: diff --git a/nodescraper/plugins/inband/dmesg/dmesg_plugin.py b/nodescraper/plugins/inband/dmesg/dmesg_plugin.py index b1b42900..d4095ab9 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_plugin.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_plugin.py @@ -41,4 +41,6 @@ class DmesgPlugin(InBandDataPlugin[DmesgData, DmesgCollectorArgs, DmesgAnalyzerA ANALYZER = DmesgAnalyzer + ANALYZER_ARGS = DmesgAnalyzerArgs + COLLECTOR_ARGS = DmesgCollectorArgs diff --git a/nodescraper/plugins/inband/regex_search/regex_search_plugin.py b/nodescraper/plugins/inband/regex_search/regex_search_plugin.py index 2a101ff8..31a068ca 100644 --- a/nodescraper/plugins/inband/regex_search/regex_search_plugin.py +++ b/nodescraper/plugins/inband/regex_search/regex_search_plugin.py @@ -41,6 +41,11 @@ class RegexSearchPlugin(InBandDataPlugin[RegexSearchData, CollectorArgs, RegexSe ANALYZER = RegexSearchAnalyzer ANALYZER_ARGS = RegexSearchAnalyzerArgs + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "No collector step: reads local text from the CLI --data path (file or directory).", + "Directory scans load each file's contents into RegexSearchData for analysis.", + ) + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( "Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).", "Emits regex match events with optional per-file source in the description when scanning directories.", diff --git a/nodescraper/plugins/inband/storage/storage_plugin.py b/nodescraper/plugins/inband/storage/storage_plugin.py index 1687e20e..f0ca62fc 100644 --- a/nodescraper/plugins/inband/storage/storage_plugin.py +++ b/nodescraper/plugins/inband/storage/storage_plugin.py @@ -41,4 +41,6 @@ class StoragePlugin(InBandDataPlugin[StorageDataModel, StorageCollectorArgs, Sto ANALYZER = StorageAnalyzer + ANALYZER_ARGS = StorageAnalyzerArgs + COLLECTOR_ARGS = StorageCollectorArgs diff --git a/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py b/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py index 9162980e..5bb5acf9 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py @@ -53,9 +53,9 @@ class RedfishEndpointAnalyzerArgs(AnalyzerArgs): checks: dict[str, dict[str, RedfishConstraint]] = Field( default_factory=dict, description=( - "Map: URI or '*' -> { property_path: constraint }. " + "Map: URI or `*` -> { property_path: constraint }. " "URI keys must match a key in the collected responses (exact match). " - "Use '*' as the key to apply the inner constraints to every collected response body. " + "Use `*` as the key to apply the inner constraints to every collected response body. " "Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. " "Constraints: " "'eq' — value must equal the given literal (int, float, str, bool). " diff --git a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py index 1e43a71a..26412f94 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py @@ -92,7 +92,7 @@ class RedfishEndpointAnalyzer(DataAnalyzer[RedfishEndpointDataModel, RedfishEndp DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( "For each entry in analysis_args.checks, reads JSON paths in collected responses and " "compares values to constraints (eq, min/max, anyOf, regex, etc.).", - 'URI key "*" runs checks against every collected response body.', + "URI key `*` runs checks against every collected response body.", ) def analyze_data( diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py index b3e2644d..87bbaa3f 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py @@ -63,6 +63,13 @@ class MI3XXAnalyzer(DataAnalyzer[ServiceabilityDataModel, ServiceabilityAnalyzer DATA_MODEL = ServiceabilityDataModel + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "Builds AFID events from collected Redfish event log members (and optional assembly metadata).", + "Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis.", + "Runs the configured Python service hub (hub_python_module) to produce service recommendations.", + "When analysis_args.skip_hub is true, only builds AFID events without running the hub.", + ) + def analyze_data( self, data: ServiceabilityDataModel, diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py index d155f14a..910b3c66 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_collector.py @@ -44,6 +44,14 @@ class MI3XXCollector(ServiceabilityCollectorBase[MI3XXCollectorArgs]): """Collect MI3XX BMC Redfish data: event log members (with pagination), firmware inventory, CPER attachment bytes for qualifying events, and optional assembly/chassis metadata.""" + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias).", + "Paginated Members collection and optional top, reference_time/time_operator filters.", + "Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model).", + "Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).", + "Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details.", + ) + def satisfies_reference_time( self, candidate: str, diff --git a/test/unit/plugin/test_mi3xx_collector.py b/test/unit/plugin/test_mi3xx_collector.py index 96a9d556..d27b80c1 100644 --- a/test/unit/plugin/test_mi3xx_collector.py +++ b/test/unit/plugin/test_mi3xx_collector.py @@ -140,6 +140,8 @@ def test_serviceability_plugin_mi3xx_wiring(): assert ServiceabilityPluginMI3XX.COLLECTOR is MI3XXCollector assert ServiceabilityPluginMI3XX.COLLECTOR_ARGS is MI3XXCollectorArgs assert ServiceabilityPluginMI3XX.ANALYZER is MI3XXAnalyzer + assert MI3XXCollector.DOCUMENTATION_COLLECTION_ITEMS + assert MI3XXAnalyzer.DOCUMENTATION_ANALYSIS_ITEMS def test_mi3xx_collector_no_args(mi3xx_collector): From aae7b93138df5143af1676e5a1f65f33f2967e62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 10 Jul 2026 00:25:52 +0000 Subject: [PATCH 03/21] docs: Update plugin documentation [automated] --- docs/PLUGIN_DOC.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index fe6612ae..bf0e1314 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -11,7 +11,7 @@ | DeviceEnumerationPlugin | powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count"
lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l
powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count"
lscpu
lshw
lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l
powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count" | **Analyzer Args:**
- `cpu_count`: Optional[list[int]] — Expected CPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `gpu_count`: Optional[list[int]] — Expected GPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `vf_count`: Optional[list[int]] — Expected virtual function count(s); pass as int or list of ints. Analysis passes if actual is in list. | - | [DeviceEnumerationDataModel](#DeviceEnumerationDataModel-Model) | [DeviceEnumerationCollector](#Collector-Class-DeviceEnumerationCollector) | [DeviceEnumerationAnalyzer](#Data-Analyzer-Class-DeviceEnumerationAnalyzer) | | DimmPlugin | sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\|None\|Module" | grep Size' 2>/dev/null
dmidecode
wmic memorychip get Capacity | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running dmidecode or wmic for memory info. | [DimmDataModel](#DimmDataModel-Model) | [DimmCollector](#Collector-Class-DimmCollector) | - | | DkmsPlugin | dkms status
dkms --version | **Analyzer Args:**
- `dkms_status`: Union[str, list] — Expected dkms status string(s) to match (e.g. 'amd/1.0.0'). At least one of dkms_status or dkms_version required.
- `dkms_version`: Union[str, list] — Expected dkms version string(s) to match. At least one of dkms_status or dkms_version required.
- `regex_match`: bool — If True, match dkms_status and dkms_version as regex; otherwise exact match. | - | [DkmsDataModel](#DkmsDataModel-Model) | [DkmsCollector](#Collector-Class-DkmsCollector) | [DkmsAnalyzer](#Data-Analyzer-Class-DkmsAnalyzer) | -| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...` | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | +| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...` | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | | FabricsPlugin | lspci | grep -i cassini
lsmod | grep cxi
cxi_stat
ibstat
ibv_devinfo
ls -l /sys/class/infiniband/*/device/net
fi_info -p cxi
mst start
mst status -v
ip link show
ofed_info -s | - | - | [FabricsDataModel](#FabricsDataModel-Model) | [FabricsCollector](#Collector-Class-FabricsCollector) | - | | JournalPlugin | journalctl --no-pager --system --output=short-iso
journalctl --no-pager --system --output=json | **Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_priority`: Optional[int] — Check against journal log priority (0=emergency..7=debug). If an entry has priority <= check_priority, an ERROR event...
- `group`: bool — If True, group entries that have the same priority and message. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `boot`: Optional[int] — Optional boot ID to limit journal collection to a specific boot. | [JournalData](#JournalData-Model) | [JournalCollector](#Collector-Class-JournalCollector) | [JournalAnalyzer](#Data-Analyzer-Class-JournalAnalyzer) | | KernelPlugin | sh -c 'uname -a'
sh -c 'cat /proc/sys/kernel/numa_balancing'
wmic os get Version /Value | **Analyzer Args:**
- `exp_kernel`: Union[str, list] — Expected kernel version string(s) to match (e.g. from uname -a).
- `exp_numa`: Optional[int] — Expected value for kernel.numa_balancing (e.g. 0 or 1).
- `regex_match`: bool — If True, match exp_kernel as regex; otherwise exact match. | - | [KernelDataModel](#KernelDataModel-Model) | [KernelCollector](#Collector-Class-KernelCollector) | [KernelAnalyzer](#Data-Analyzer-Class-KernelAnalyzer) | @@ -1993,8 +1993,8 @@ Check dmesg for errors regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.*GPU reset(?:\\(\\d+\\))? failed.*)') message='GPU reset failed' event_category= event_priority=, regex=re.compile('(Accelerator Check Architecture[^\\n]*)(?:\\n[^\\n]*){0,10}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*entry\\[\\d+\\]\\.STATUS=0x[0-9a-fA-F]+)(?:\\n[^\\n]*){0,5}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*entry\\[\\d+\\], re.MULTILINE) message='ACA Error' event_category= event_priority=, regex=re.compile('(Accelerator Check Architecture[^\\n]*)(?:\\n[^\\n]*){0,10}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*CONTROL=0x[0-9a-fA-F]+)(?:\\n[^\\n]*){0,5}?(amdgpu[ 0-9a-fA-F:.]+:? [^\\n]*STATUS=0x[0-9a-fA-F]+)(?:\\n[^\\, re.MULTILINE) message='ACA Error' event_category= event_priority=, - regex=re.compile('\\[Hardware Error\\]:.+MC\\d+_STATUS\\[[^\\]]*\\|CE\\|[^\\]]*\\].*(?:\\n.*){0,5}') message='MCE Corrected Error' event_category= event_priority=, - regex=re.compile('\\[Hardware Error\\]:.+MC\\d+_STATUS\\[[^\\]]*\\|UC\\|[^\\]]*\\].*(?:\\n.*){0,5}') message='MCE Uncorrected Error' event_category= event_priority=, + regex=re.compile('\\[Hardware Error\\]:[^\\n]*MC\\d+_STATUS\\[[^\\]]*\\|CE\\|[^\\]]*\\][^\\n]*', re.IGNORECASE) message='MCE Corrected Error' event_category= event_priority=, + regex=re.compile('\\[Hardware Error\\]:[^\\n]*MC\\d+_STATUS\\[[^\\]]*\\|UC\\|[^\\]]*\\][^\\n]*', re.IGNORECASE) message='MCE Uncorrected Error' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)? (.*Mode2 reset failed.*)') message='Mode 2 Reset Failed' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.*\\[Hardware Error\\]: Corrected error.*)') message='RAS Corrected Error' event_category= event_priority=, regex=re.compile('x86/cpu: SGX disabled by BIOS') message='SGX Error' event_category= event_priority=, @@ -2057,8 +2057,8 @@ Check dmesg for errors - - GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...` - - ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...` -- - MCE Corrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|...` -- - MCE Uncorrected Error: `\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|...` +- - MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...` +- - MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...` - - Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...` - - RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - SGX Error: `x86/cpu: SGX disabled by BIOS` From 9a604320d51cc70efce5426c2708b28516e0d0a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 11 Jul 2026 00:21:30 +0000 Subject: [PATCH 04/21] docs: Update plugin documentation [automated] --- docs/PLUGIN_DOC.md | 81 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index 57aa3e81..8e0992c8 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -4,42 +4,42 @@ | Plugin | Collection | Analyzer Args | Collection Args | DataModel | Collector | Analyzer | | --- | --- | --- | --- | --- | --- | --- | -| GenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | +| GenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_\* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | | AmdSmiPlugin | bad-pages
firmware --json
list --json
metric -g all
partition --json
process --json
ras --cper --folder={folder}
ras --afid --cper-file {cper_file}
static -g all --json
static -g {gpu_id} --json
topology
version --json
xgmi -l
xgmi -m | **Analyzer Args:**
- `check_static_data`: bool — If True, run static data checks (e.g. driver version, partition mode).
- `expected_gpu_processes`: Optional[int] — Expected number of GPU processes.
- `expected_max_power`: Optional[int] — Expected maximum power value (e.g. watts).
- `expected_power_management`: Optional[str] — Expected amd-smi metric power_management value per GPU (e.g. DISABLED for active/full power, ENABLED for power-manage...
- `expected_driver_version`: Optional[str] — Expected AMD driver version string.
- `expected_memory_partition_mode`: Optional[str] — Expected memory partition mode (e.g. sp3, dp).
- `expected_compute_partition_mode`: Optional[str] — Expected compute partition mode.
- `expected_firmware_versions`: Optional[dict[str, str]] — Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).
- `l0_to_recovery_count_error_threshold`: Optional[int] — L0-to-recovery count above which an error is raised.
- `l0_to_recovery_count_warning_threshold`: Optional[int] — L0-to-recovery count above which a warning is raised.
- `vendorid_ep`: Optional[str] — Expected endpoint vendor ID (e.g. for PCIe).
- `vendorid_ep_vf`: Optional[str] — Expected endpoint VF vendor ID.
- `devid_ep`: Optional[str] — Expected endpoint device ID.
- `devid_ep_vf`: Optional[str] — Expected endpoint VF device ID.
- `sku_name`: Optional[str] — Expected SKU name string for GPU.
- `expected_xgmi_speed`: Optional[list[float]] — Expected xGMI speed value(s) (e.g. link rate).
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for time-windowed analysis.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for time-windowed analysis. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `analysis_firmware_ids`: Optional[list[str]] — amd-smi fw_id values to record in analysis_ref.firmware_versions
- `cper_file_path`: Optional[str] — Path to CPER folder or file for RAS AFID collection (ras --afid --cper-file). | [AmdSmiDataModel](#AmdSmiDataModel-Model) | [AmdSmiCollector](#Collector-Class-AmdSmiCollector) | [AmdSmiAnalyzer](#Data-Analyzer-Class-AmdSmiAnalyzer) | | BiosPlugin | sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'
wmic bios get SMBIOSBIOSVersion /Value | **Analyzer Args:**
- `exp_bios_version`: list[str] — Expected BIOS version(s) to match against collected value (str or list).
- `regex_match`: bool — If True, match exp_bios_version as regex; otherwise exact match. | - | [BiosDataModel](#BiosDataModel-Model) | [BiosCollector](#Collector-Class-BiosCollector) | [BiosAnalyzer](#Data-Analyzer-Class-BiosAnalyzer) | | CmdlinePlugin | cat /proc/cmdline | **Analyzer Args:**
- `required_cmdline`: Union[str, List] — Command-line parameters that must be present (e.g. 'pci=bfsort').
- `banned_cmdline`: Union[str, List] — Command-line parameters that must not be present.
- `os_overrides`: Dict[str, nodescraper.plugins.inband.cmdline.cmdlineconfig.OverrideConfig] — Per-OS overrides for required_cmdline and banned_cmdline (keyed by OS identifier).
- `platform_overrides`: Dict[str, nodescraper.plugins.inband.cmdline.cmdlineconfig.OverrideConfig] — Per-platform overrides for required_cmdline and banned_cmdline (keyed by platform). | - | [CmdlineDataModel](#CmdlineDataModel-Model) | [CmdlineCollector](#Collector-Class-CmdlineCollector) | [CmdlineAnalyzer](#Data-Analyzer-Class-CmdlineAnalyzer) | | DeviceEnumerationPlugin | powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count"
lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l
powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count"
lscpu
lshw
lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l
powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count" | **Analyzer Args:**
- `cpu_count`: Optional[list[int]] — Expected CPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `gpu_count`: Optional[list[int]] — Expected GPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `vf_count`: Optional[list[int]] — Expected virtual function count(s); pass as int or list of ints. Analysis passes if actual is in list. | - | [DeviceEnumerationDataModel](#DeviceEnumerationDataModel-Model) | [DeviceEnumerationCollector](#Collector-Class-DeviceEnumerationCollector) | [DeviceEnumerationAnalyzer](#Data-Analyzer-Class-DeviceEnumerationAnalyzer) | | DimmPlugin | sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\|None\|Module" | grep Size' 2>/dev/null
dmidecode
wmic memorychip get Capacity | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running dmidecode or wmic for memory info. | [DimmDataModel](#DimmDataModel-Model) | [DimmCollector](#Collector-Class-DimmCollector) | - | | DkmsPlugin | dkms status
dkms --version | **Analyzer Args:**
- `dkms_status`: Union[str, list] — Expected dkms status string(s) to match (e.g. 'amd/1.0.0'). At least one of dkms_status or dkms_version required.
- `dkms_version`: Union[str, list] — Expected dkms version string(s) to match. At least one of dkms_status or dkms_version required.
- `regex_match`: bool — If True, match dkms_status and dkms_version as regex; otherwise exact match. | - | [DkmsDataModel](#DkmsDataModel-Model) | [DkmsCollector](#Collector-Class-DkmsCollector) | [DkmsAnalyzer](#Data-Analyzer-Class-DkmsAnalyzer) | -| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...` | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | -| FabricsPlugin | lspci | grep -i cassini
lsmod | grep cxi
cxi_stat
ibstat
ibv_devinfo
ls -l /sys/class/infiniband/*/device/net
fi_info -p cxi
mst start
mst status -v
ip link show
ofed_info -s | - | - | [FabricsDataModel](#FabricsDataModel-Model) | [FabricsCollector](#Collector-Class-FabricsCollector) | - | +| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...`
**Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_unknown_dmesg_errors`: Optional[bool] — If True, treat unknown/unmatched dmesg error lines as failures.
- `exclude_category`: Optional[set[str]] — Set of error categories to exclude from analysis.
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one (for rate limiting).
- `num_timestamps`: int — Number of timestamps to include per event in output.
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.
- `priority_override_rules`: Optional[list[dict]] — Rules to override the priority of matched ErrorRegex objects. Each rule is a dict where all keys except 'new_priority...
- `mce_threshold`: Optional[int] — When set, raise ERROR if correctable MCE/RAS error count for any component (CPU, GPU BDF/block, etc.) reaches or exce...
- `ignore_match_rules`: Optional[list[dict[str, object]]] — Rules that skip regex matches during analysis. Each rule may use line_regex, match_regex, message, and/or mce_banks.... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg\*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | +| FabricsPlugin | lspci | grep -i cassini
lsmod | grep cxi
cxi_stat
ibstat
ibv_devinfo
ls -l /sys/class/infiniband/\*/device/net
fi_info -p cxi
mst start
mst status -v
ip link show
ofed_info -s | - | - | [FabricsDataModel](#FabricsDataModel-Model) | [FabricsCollector](#Collector-Class-FabricsCollector) | - | | JournalPlugin | journalctl --no-pager --system --output=short-iso
journalctl --no-pager --system --output=json | **Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_priority`: Optional[int] — Check against journal log priority (0=emergency..7=debug). If an entry has priority <= check_priority, an ERROR event...
- `group`: bool — If True, group entries that have the same priority and message. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `boot`: Optional[int] — Optional boot ID to limit journal collection to a specific boot. | [JournalData](#JournalData-Model) | [JournalCollector](#Collector-Class-JournalCollector) | [JournalAnalyzer](#Data-Analyzer-Class-JournalAnalyzer) | | KernelPlugin | sh -c 'uname -a'
sh -c 'cat /proc/sys/kernel/numa_balancing'
wmic os get Version /Value | **Analyzer Args:**
- `exp_kernel`: Union[str, list] — Expected kernel version string(s) to match (e.g. from uname -a).
- `exp_numa`: Optional[int] — Expected value for kernel.numa_balancing (e.g. 0 or 1).
- `regex_match`: bool — If True, match exp_kernel as regex; otherwise exact match. | - | [KernelDataModel](#KernelDataModel-Model) | [KernelCollector](#Collector-Class-KernelCollector) | [KernelAnalyzer](#Data-Analyzer-Class-KernelAnalyzer) | | KernelModulePlugin | cat /proc/modules
modinfo amdgpu
wmic os get Version /Value | **Analyzer Args:**
- `kernel_modules`: dict[str, dict] — Expected kernel module name -> {version, etc.}. Analyzer checks collected modules match.
- `regex_filter`: list[str] — List of regex patterns to filter which collected modules are checked (default: amd). | - | [KernelModuleDataModel](#KernelModuleDataModel-Model) | [KernelModuleCollector](#Collector-Class-KernelModuleCollector) | [KernelModuleAnalyzer](#Data-Analyzer-Class-KernelModuleAnalyzer) | | MemoryPlugin | free -b
lsmem
numactl -H
wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value | **Analyzer Args:**
- `ratio`: float — Required free-memory ratio (0-1). Analysis fails if free/total < ratio.
- `memory_threshold`: str — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. | - | [MemoryDataModel](#MemoryDataModel-Model) | [MemoryCollector](#Collector-Class-MemoryCollector) | [MemoryAnalyzer](#Data-Analyzer-Class-MemoryAnalyzer) | | NetworkPlugin | ip addr show
curl
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
rdma link -j
ip route show
ip rule show
wget | **Analyzer Args:**
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | -| NicPlugin | niccli --listdev
niccli --list
niccli --list_devices
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | +| NicPlugin | niccli --listdev
niccli --list
niccli --list_devices
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | | NvmePlugin | nvme smart-log {dev}
nvme error-log {dev} --log-entries=256
nvme id-ctrl {dev}
nvme id-ns {dev}{ns}
nvme fw-log {dev}
nvme self-test-log {dev}
nvme get-log {dev} --log-id=6 --log-len=512
nvme telemetry-log {dev} --output-file={dev}_{f_name}
nvme list -o json | - | - | [NvmeDataModel](#NvmeDataModel-Model) | [NvmeCollector](#Collector-Class-NvmeCollector) | - | -| OsPlugin | sh -c '( lsb_release -ds || (cat /etc/*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | -| PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | +| OsPlugin | sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/\*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | +| PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/\*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | | PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | - | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | | ProcessPlugin | top -b -n 1
rocm-smi --showpids
top -b -n 1 -o %CPU | **Analyzer Args:**
- `max_kfd_processes`: int — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.
- `max_cpu_usage`: float — Maximum allowed CPU usage (percent) for process checks. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `top_n_process`: int — Number of top processes by CPU usage to collect (e.g. for top -b -n 1 -o %%CPU). | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | | RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | - | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | | RegexSearchPlugin | No collector step: reads local text from the CLI --data path (file or directory).
Directory scans load each file's contents into RegexSearchData for analysis. | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | -| RocmPlugin | {rocm_path}/opencl/bin/*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}*
ls -v -d {rocm_path}-[3-7]* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | +| RocmPlugin | {rocm_path}/opencl/bin/\*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/\*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}\*
ls -v -d {rocm_path}-[3-7]\* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/\* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | | StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | **Analyzer Args:**
- `min_required_free_space_abs`: Optional[str] — Minimum required free space per mount (e.g. '10G', '1T').
- `min_required_free_space_prct`: Optional[int] — Minimum required free space as percentage of total (0–100).
- `ignore_devices`: Optional[list[str]] — Mount points or devices to exclude from free-space checks.
- `check_devices`: Optional[list[str]] — If non-empty, only these mount points or devices are checked.
- `regex_match`: bool — If True, match device/mount names with regex; otherwise exact match. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | | ScaleOutAristaPlugin | show interfaces counters bins | json | no-more
show interfaces counters queue | no-more
show interfaces counters queue drop-precedence | no-more
show qos interfaces ecn counters queue | json | no-more
show interfaces counters errors | json | no-more
show interfaces phy | no-more
show interfaces phy detail | no-more
show interfaces counters ip | json | no-more
show ip interface | no-more
show lldp | no-more
show lldp neighbors | json | no-more
show interfaces counters | json | no-more
show interfaces flow-control | json | no-more
show interfaces counters queue detail | no-more
show priority-flow-control counters | json | no-more
show priority-flow-control status | no-more
show interfaces status | json | no-more
show qos interfaces | no-more
show qos interfaces ecn | no-more
show qos interfaces trust | no-more
show qos maps | no-more
show qos profile | no-more
show qos profile summary | no-more
show interfaces counters rates | json | no-more
show running-config | no-more
show startup-config | no-more
show system environment cooling | json | no-more
show platform trident mmu queue status | no-more
show version | json | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Ports are S/P/[SP] where subport is optional (e.g. ['1/1', '1/31', '1/...
- `expected_port_bandwidth`: int — Expected interface bandwidth (bps) from show interfaces status (AristaPortStatus.bandwidth). Ports with a different b... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Re-runs successful... | [ScaleOutAristaDataModel](#ScaleOutAristaDataModel-Model) | [ScaleOutAristaCollector](#Collector-Class-ScaleOutAristaCollector) | [ScaleOutAristaAnalyzer](#Data-Analyzer-Class-ScaleOutAristaAnalyzer) | | ScaleOutDellPlugin | show alarm | no-more
show buffer pool | no-more
show buffer profile | no-more
show clock | no-more
show interface counters {port} | no-more
show event details | no-more
show interface fec status | no-more
show interface counters | no-more
show interface counters rate | no-more
show interface Eth | no-more
show interface phy counters | no-more
show interface status | no-more
show interface transceiver | no-more
show interface transceiver dom | no-more
show interface transceiver summary | no-more
show ip arp | no-more
show ip interfaces | no-more
show ip route | no-more
show lldp neighbor | no-more
show lldp table | no-more
show qos interface Ethall priority-flow-control statistics | no-more
show priority-flow-control watchdog | no-more
show qos interface Ethall queue all priority-flow-control watchdog-statistics | no-more
show platform environment | no-more
show platform firmware detail | no-more
show platform syseeprom | no-more
show qos interface Eth all | no-more
show qos interface Eth all queue all | no-more
show qos map dot1p-tc | no-more
show qos map dscp-tc | no-more
show qos map pfc-priority-pg | no-more
show qos map pfc-priority-queue | no-more
show qos map tc-dot1p | no-more
show qos map tc-dscp | no-more
show qos map tc-pg | no-more
show qos map tc-queue | no-more
show qos scheduler-policy | no-more
show qos wred-policy | no-more
show queue counters | no-more
show queue persistent-watermark multicast | no-more
show queue persistent-watermark unicast | no-more
show queue watermark multicast | no-more
show queue watermark unicast | no-more
show running-configuration | no-more
show version | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Accepts optional Eth prefix (e.g. ['1/1', '1/31', '1/1/1'] or ['Eth1/1...
- `expected_port_speed`: int — Expected interface speed (Mbps) from show interface status (DellInterfaceStatus.speed). Ports with a different speed... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `collection_ports`: Optional[List[str]] — Restrict detail counter collection to these ports. Accepts the same tokens as analysis_ports (e.g. ['1/1', '1/1/2'] o... | [ScaleOutDellDataModel](#ScaleOutDellDataModel-Model) | [ScaleOutDellCollector](#Collector-Class-ScaleOutDellCollector) | [ScaleOutDellAnalyzer](#Data-Analyzer-Class-ScaleOutDellAnalyzer) | -| SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '*' are collected with ls -l (e.g. class/net/*/device).
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | +| SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '\*' are collected with ls -l (e.g. class/net/\*/device).
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | | SysctlPlugin | sysctl -n | **Analyzer Args:**
- `exp_vm_swappiness`: Optional[int] — Expected vm.swappiness value.
- `exp_vm_numa_balancing`: Optional[int] — Expected vm.numa_balancing value.
- `exp_vm_oom_kill_allocating_task`: Optional[int] — Expected vm.oom_kill_allocating_task value.
- `exp_vm_compaction_proactiveness`: Optional[int] — Expected vm.compaction_proactiveness value.
- `exp_vm_compact_unevictable_allowed`: Optional[int] — Expected vm.compact_unevictable_allowed value.
- `exp_vm_extfrag_threshold`: Optional[int] — Expected vm.extfrag_threshold value.
- `exp_vm_zone_reclaim_mode`: Optional[int] — Expected vm.zone_reclaim_mode value.
- `exp_vm_dirty_background_ratio`: Optional[int] — Expected vm.dirty_background_ratio value.
- `exp_vm_dirty_ratio`: Optional[int] — Expected vm.dirty_ratio value.
- `exp_vm_dirty_writeback_centisecs`: Optional[int] — Expected vm.dirty_writeback_centisecs value.
- `exp_kernel_numa_balancing`: Optional[int] — Expected kernel.numa_balancing value. | - | [SysctlDataModel](#SysctlDataModel-Model) | [SysctlCollector](#Collector-Class-SysctlCollector) | [SysctlAnalyzer](#Data-Analyzer-Class-SysctlAnalyzer) | -| SyslogPlugin | ls -1 /var/log/syslog* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true
ls -1 /var/log/messages* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true | - | - | [SyslogData](#SyslogData-Model) | [SyslogCollector](#Collector-Class-SyslogCollector) | - | +| SyslogPlugin | ls -1 /var/log/syslog\* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true
ls -1 /var/log/messages\* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true | - | - | [SyslogData](#SyslogData-Model) | [SyslogCollector](#Collector-Class-SyslogCollector) | - | | UptimePlugin | uptime | - | - | [UptimeDataModel](#UptimeDataModel-Model) | [UptimeCollector](#Collector-Class-UptimeCollector) | - | # OOB plugins | Plugin | Collection | Analyzer Args | Collection Args | DataModel | Collector | Analyzer | | --- | --- | --- | --- | --- | --- | --- | -| OobGenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | +| OobGenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_\* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | | OobBmcArchivePlugin | SSH (BMC) shell: tar+gzip archives for each path in collection_args (see PathSpec entries).
Uses sudo on the BMC when collection_args paths require elevated access. | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `paths`: list[nodescraper.plugins.ooband.bmc_archive.collector_args.PathSpec] — Named BMC paths to archive with tar czf -. Configure in plugin config under plugins.OobBmcArchivePlugin.collection_ar...
- `sudo`: bool — Default sudo setting for paths that do not specify sudo.
- `timeout`: int — Default per-path tar timeout in seconds.
- `skip_if_missing`: bool — Skip paths that do not exist on the BMC instead of failing collection.
- `ignore_failed_read`: bool — When true, pass GNU tar's --ignore-failed-read when the remote tar supports it. | [BmcArchiveDataModel](#BmcArchiveDataModel-Model) | [BmcArchiveCollector](#Collector-Class-BmcArchiveCollector) | - | | RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key `*` runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | | RedfishOemDiagPlugin | Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types (collection_args.log_service_path selects the LogService).
Optional binary archives under the plugin log path when log_path is set. | Summarizes success/failure per OEM diagnostic type from collected results.
When analysis_args.require_all_success is true, fails the run if any type failed collection.
**Analyzer Args:**
- `require_all_success`: bool — If True, analysis fails when any OEM type collection failed. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `log_service_path`: str — Redfish path to the LogService (e.g. DiagLogs).
- `oem_diagnostic_types_allowable`: Optional[list[str]] — Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagno...
- `oem_diagnostic_types`: list[str] — OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.
- `task_timeout_s`: int — Max seconds to wait for each BMC task. | [RedfishOemDiagDataModel](#RedfishOemDiagDataModel-Model) | [RedfishOemDiagCollector](#Collector-Class-RedfishOemDiagCollector) | [RedfishOemDiagAnalyzer](#Data-Analyzer-Class-RedfishOemDiagAnalyzer) | @@ -68,7 +68,7 @@ GenericCollectionDataModel ### Documented collection - Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH). -- Commands are user-configured; there are no fixed CMD_* class fields. +- Commands are user-configured; there are no fixed CMD_\* class fields. ## Collector Class AmdSmiCollector @@ -275,7 +275,7 @@ DmesgData ### Commands - dmesg --time-format iso -x -- ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true ## Collector Class FabricsCollector @@ -312,7 +312,7 @@ FabricsDataModel - cxi_stat - ibstat - ibv_devinfo -- ls -l /sys/class/infiniband/*/device/net +- ls -l /sys/class/infiniband/\*/device/net - fi_info -p cxi - mst start - mst status -v @@ -673,8 +673,8 @@ OsDataModel ### Commands -- sh -c '( lsb_release -ds || (cat /etc/*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1' -- cat /etc/*release | grep VERSION_ID +- sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1' +- cat /etc/\*release | grep VERSION_ID - wmic os get Version /value - wmic os get Caption /Value @@ -705,7 +705,7 @@ PackageDataModel - dnf list --installed - dpkg-query -W - pacman -Q -- cat /etc/*release +- cat /etc/\*release - wmic product get name,version ## Collector Class PcieCollector @@ -853,15 +853,15 @@ RocmDataModel ### Commands -- {rocm_path}/opencl/bin/*/clinfo +- {rocm_path}/opencl/bin/\*/clinfo - env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen' - ls /sys/class/kfd/kfd/proc/ -- grep -i -E 'rocm' /etc/ld.so.conf.d/* +- grep -i -E 'rocm' /etc/ld.so.conf.d/\* - {rocm_path}/bin/rocminfo -- ls -v -d {rocm_path}* -- ls -v -d {rocm_path}-[3-7]* | tail -1 +- ls -v -d {rocm_path}\* +- ls -v -d {rocm_path}-[3-7]\* | tail -1 - ldconfig -p | grep -i -E 'rocm' -- grep . -H -r -i {rocm_path}/.info/* +- grep . -H -r -i {rocm_path}/.info/\* ## Collector Class StorageCollector @@ -1208,8 +1208,8 @@ SyslogData ### Commands -- ls -1 /var/log/syslog* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true -- ls -1 /var/log/messages* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/syslog\* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/messages\* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true ## Collector Class UptimeCollector @@ -2447,6 +2447,25 @@ Build AFID events from collected data and run the configured service hub. - **dkms_version**: `Union[str, list]` — Expected dkms version string(s) to match. At least one of dkms_status or dkms_version required. - **regex_match**: `bool` — If True, match dkms_status and dkms_version as regex; otherwise exact match. +## Analyzer Args Class DmesgAnalyzerArgs + +**Bases**: ['TimeRangeAnalysisArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/dmesg/analyzer_args.py) + +### Annotations / fields + +- **analysis_range_start**: `Optional[datetime.datetime]` — Start of time range for analysis (ISO format). Only events on or after this time are analyzed. +- **analysis_range_end**: `Optional[datetime.datetime]` — End of time range for analysis (ISO format). Only events before this time are analyzed. +- **check_unknown_dmesg_errors**: `Optional[bool]` — If True, treat unknown/unmatched dmesg error lines as failures. +- **exclude_category**: `Optional[set[str]]` — Set of error categories to exclude from analysis. +- **interval_to_collapse_event**: `int` — Seconds within which repeated events are collapsed into one (for rate limiting). +- **num_timestamps**: `int` — Number of timestamps to include per event in output. +- **error_regex**: `Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType]` — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. +- **priority_override_rules**: `Optional[list[dict]]` — Rules to override the priority of matched ErrorRegex objects. Each rule is a dict where all keys except 'new_priority' and 'match_all' are filter fields matched against ErrorRegex attributes. 'new_priority' must be an EventPriority name (e.g. 'WARNING', 'ERROR') or 'NO_CHANGE' to leave the priority unchanged. +- **mce_threshold**: `Optional[int]` — When set, raise ERROR if correctable MCE/RAS error count for any component (CPU, GPU BDF/block, etc.) reaches or exceeds this value. +- **ignore_match_rules**: `Optional[list[dict[str, object]]]` — Rules that skip regex matches during analysis. Each rule may use line_regex, match_regex, message, and/or mce_banks. Within a rule all specified fields must match; any matching rule suppresses the hit. mce_banks accepts bank ids and inclusive ranges such as "60-63". When a rule matches an MCA bank, only [Hardware Error]: lines in that incident block are suppressed; interleaved non-MCE dmesg lines in the same block are not. + ## Analyzer Args Class JournalAnalyzerArgs ### Description @@ -2532,7 +2551,7 @@ Analyzer args for niccli/nicctl data, with expected_values keyed by canonical co - **expected_qos_pfc_enabled**: `Optional[int]` — Expected PFC enabled value (0/1 or bitmask). Checked per device when set. - **expected_qos_tsa_map**: `Optional[Dict[Any, Any]]` — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set. - **expected_qos_tc_bandwidth**: `Optional[List[int]]` — Expected TC bandwidth percentages. Checked per device when set. -- **require_qos_consistent_across_adapters**: `bool` — When True and no expected_qos_* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map. +- **require_qos_consistent_across_adapters**: `bool` — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map. - **nicctl_log_error_regex**: `Optional[List[Dict[str, Any]]]` — Optional list of error patterns for nicctl show card logs. ## Analyzer Args Class OsAnalyzerArgs @@ -2632,6 +2651,20 @@ Arguments for RegexSearchAnalyzer (dict items match Dmesg-style error_regex). - **exp_rocm_latest**: `str` — Expected 'latest' ROCm path or version string for versioned installs. - **exp_rocm_sub_versions**: `dict[str, Union[str, list]]` — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. +## Analyzer Args Class StorageAnalyzerArgs + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/storage/analyzer_args.py) + +### Annotations / fields + +- **min_required_free_space_abs**: `Optional[str]` — Minimum required free space per mount (e.g. '10G', '1T'). +- **min_required_free_space_prct**: `Optional[int]` — Minimum required free space as percentage of total (0–100). +- **ignore_devices**: `Optional[list[str]]` — Mount points or devices to exclude from free-space checks. +- **check_devices**: `Optional[list[str]]` — If non-empty, only these mount points or devices are checked. +- **regex_match**: `bool` — If True, match device/mount names with regex; otherwise exact match. + ## Analyzer Args Class ScaleOutAristaAnalyzerArgs ### Description From 94e869113229455541ec95739e4c1415d70564c1 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Mon, 13 Jul 2026 11:40:14 -0700 Subject: [PATCH 05/21] added queue fields --- .../plugins/inband/network/ethtool_vendor.py | 93 +++++++- .../inband/network/network_analyzer.py | 81 +++++-- .../inband/network/network_collector.py | 179 +++++++------- .../plugins/inband/network/networkdata.py | 6 +- .../plugins/inband/rdma/rdma_collector.py | 8 +- .../plugins/inband/rdma/rdma_plugin.py | 4 +- test/unit/plugin/test_network_analyzer.py | 219 +++++++++++++++++- test/unit/plugin/test_network_collector.py | 114 +++++++-- 8 files changed, 553 insertions(+), 151 deletions(-) diff --git a/nodescraper/plugins/inband/network/ethtool_vendor.py b/nodescraper/plugins/inband/network/ethtool_vendor.py index d43791b4..60aaa4ca 100644 --- a/nodescraper/plugins/inband/network/ethtool_vendor.py +++ b/nodescraper/plugins/inband/network/ethtool_vendor.py @@ -25,6 +25,7 @@ ############################################################################### """Vendor-specific ethtool -S statistics models (Pollara / Thor2 / ConnectX-7).""" +import re from typing import ClassVar, Optional, Union from pydantic import BaseModel, Field, model_validator @@ -32,7 +33,18 @@ class PollaraEthtoolStatistics(BaseModel): - """ifname ionic. Keeping only fields of interest. Skip queue-specific stats for now""" + """ionic (Pollara) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^(rx|tx)_\d+" + queue_error_regex: ClassVar[list[str]] = [ + r"dma_map_err", + r"hwstamp_invalid", + r"alloc_err", + r"csum_error", + r"dropped", + ] + queue_warning_regex: ClassVar[list[str]] = [] rx_csum_error: Optional[int] = None hw_tx_dropped: Optional[int] = None @@ -157,7 +169,27 @@ class PollaraEthtoolStatistics(BaseModel): class Thor2EthtoolStatistics(BaseModel): - """ifname bnxt. Keeping only fields of interest. Skip queue-specific stats for now""" + """bnxt (Thor2) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^\[\d+\]" + queue_error_regex: ClassVar[list[str]] = [ + r"rx_discards", + r"rx_errors", + r"tx_errors", + r"tx_discards", + r"rx_l4_csum_errors", + r"rx_buf_errors", + r"so_txtime_cmpl_errors", + r"missed_irqs", + r"xsk_rx_redirect_fail", + r"xsk_rx_alloc_fail", + r"xsk_rx_no_room", + r"xsk_tx_ring_full", + ] + queue_warning_regex: ClassVar[list[str]] = [ + r"rx_resets", + ] rx_total_l4_csum_errors: Optional[int] = None rx_total_resets: Optional[int] = None @@ -356,7 +388,22 @@ class Thor2EthtoolStatistics(BaseModel): class Cx7EthtoolStatistics(BaseModel): - """ifname mlx. Keeping only fields of interest. Skip queue-specific stats for now""" + """mlx (ConnectX-7) ethtool -S counters.""" + + #: Regex matched against raw ``ethtool -S`` field names to identify per-queue + queue_counter_regex: ClassVar[str] = r"^(rx|tx|ch)\d+" + queue_error_regex: ClassVar[list[str]] = [ + r"xdp_drop", + r"wqe_err", + r"oversize_pkts_sw_drop", + r"buff_alloc_err", + r"arfs_err", + r"tls_err", + r"xdp_tx_err", + r"dropped", + r"cqe_err", + ] + queue_warning_regex: ClassVar[list[str]] = [] rx_xdp_drop: Optional[int] = None rx_xdp_tx_err: Optional[int] = None @@ -645,15 +692,51 @@ class Cx7EthtoolStatistics(BaseModel): } +def extract_queue_counters(stats: dict[str, str], queue_counter_regex: str) -> dict[str, int]: + """Extract per-queue counters from a raw ``ethtool -S`` stats dict. + + Devices can report hundreds of per-queue counters. These are matched by the + vendor's ``queue_counter_regex`` and retained (rather than dropped like other + non-enumerated fields) so they remain available in the data model for error + detection. + + Args: + stats: Mapping of raw ``ethtool -S`` field name -> string value. + queue_counter_regex: Vendor regex matched against field names to identify + per-queue counters. + + Returns: + Mapping of matching field name -> integer value. The ``netdev`` marker key + and non-integer values are skipped. + """ + pattern = re.compile(queue_counter_regex) + queue_counters: dict[str, int] = {} + for name, value in stats.items(): + if name == "netdev" or not pattern.match(name): + continue + try: + queue_counters[name] = int(value) + except (TypeError, ValueError): + continue + return queue_counters + + class EthtoolStatistics(BaseModel): """Per-netdev ethtool -S row with optional vendor-parsed counters.""" netdev: Optional[str] = None - rdma_ifname: Optional[str] = Field( + driver: Optional[str] = Field( default=None, - description="RDMA interface name from 'rdma link -j' used for vendor prefix selection", + description="Kernel driver (from 'ethtool -i') used for vendor model selection", ) vendor_statistics: Optional[VendorEthtoolStatisticsModel] = None + queue_statistics: dict[str, int] = Field( + default_factory=dict, + description=( + "High-cardinality per-queue ethtool -S counters (raw field name -> value) " + "captured via the vendor queue_counter_regex for error detection" + ), + ) @model_validator(mode="after") def validate_atleast_one_field(self) -> Self: diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 2967df59..eb590651 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer @@ -49,13 +50,13 @@ def analyze_data( """Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex. Args: - data: Network data model with ethtool_info and/or rdma_ethtool_statistics. + data: Network data model with ethtool_info and/or ethtool_statistics. args: Optional analyzer arguments with custom error regex support. Returns: TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ - if not data.ethtool_info and not data.rdma_ethtool_statistics: + if not data.ethtool_info and not data.ethtool_statistics: self.result.message = "No network devices found" self.result.status = ExecutionStatus.WARNING return self.result @@ -110,7 +111,9 @@ def analyze_data( vendor_warning = False vendor_error_fields: set[str] = set() vendor_warning_fields: set[str] = set() - for stat in data.rdma_ethtool_statistics: + vendor_queue_error_fields: set[str] = set() + vendor_queue_warning_fields: set[str] = set() + for stat in data.ethtool_statistics: if stat.vendor_statistics is None: continue @@ -129,24 +132,18 @@ def analyze_data( else: vendor_error = True vendor_error_fields.add(field_name) - # Use a single grouped description per severity so the run summary - # collapses every occurrence into one "Ethtool warning detected" / - # "Ethtool error detected" entry instead of one line per field. The - # specific field is still preserved in the event data below and in - # the consolidated console line emitted after the loop. + # Use a single grouped description per severity desc = ( "Ethtool warning detected" if is_warning_tier else "Ethtool error detected" ) # Per-field events are still recorded for the run summary and event - # log, but console logging is suppressed here to avoid repeating the - # same message once per device. A single consolidated line is emitted - # after the loop instead. + # log, but console logging is suppressed here self._log_event( category=EventCategory.NETWORK, description=desc, data={ "netdev": stat.netdev, - "rdma_ifname": stat.rdma_ifname, + "driver": stat.driver, "error_field": field_name, "error_count": error_value, }, @@ -154,12 +151,68 @@ def analyze_data( console_log=False, ) - if vendor_error: + # Per-queue counters matched against the vendor's adjustable regex + queue_error_patterns = [ + re.compile(pattern) for pattern in getattr(type(vs), "queue_error_regex", []) + ] + queue_warning_patterns = [ + re.compile(pattern) for pattern in getattr(type(vs), "queue_warning_regex", []) + ] + if stat.queue_statistics and (queue_error_patterns or queue_warning_patterns): + for counter_name, counter_value in stat.queue_statistics.items(): + if counter_value <= 0: + continue + if queue_error_patterns and any( + pattern.search(counter_name) for pattern in queue_error_patterns + ): + vendor_error = True + vendor_queue_error_fields.add(f"{stat.netdev} {counter_name}") + self._log_event( + category=EventCategory.NETWORK, + description="Ethtool queue error detected", + data={ + "netdev": stat.netdev, + "driver": stat.driver, + "error_field": counter_name, + "error_count": counter_value, + }, + priority=EventPriority.ERROR, + console_log=False, + ) + elif queue_warning_patterns and any( + pattern.search(counter_name) for pattern in queue_warning_patterns + ): + vendor_warning = True + vendor_queue_warning_fields.add(f"{stat.netdev} {counter_name}") + self._log_event( + category=EventCategory.NETWORK, + description="Ethtool queue warning detected", + data={ + "netdev": stat.netdev, + "driver": stat.driver, + "error_field": counter_name, + "error_count": counter_value, + }, + priority=EventPriority.WARNING, + console_log=False, + ) + + if vendor_error_fields: self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields))) - if vendor_warning: + if vendor_queue_error_fields: + self.logger.error( + "Ethtool queue error detected: %s", + ", ".join(sorted(vendor_queue_error_fields)), + ) + if vendor_warning_fields: self.logger.warning( "Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields)) ) + if vendor_queue_warning_fields: + self.logger.warning( + "Ethtool queue warning detected: %s", + ", ".join(sorted(vendor_queue_warning_fields)), + ) if regex_error or vendor_error: self.result.message = "Network errors detected in statistics" diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index b960e574..942ae93e 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -23,7 +23,6 @@ # SOFTWARE. # ############################################################################### -import json import re from typing import Dict, List, Optional, Tuple @@ -39,6 +38,7 @@ VENDOR_PREFIX_MAP, EthtoolStatistics, VendorEthtoolStatisticsModel, + extract_queue_counters, ) from .networkdata import ( EthtoolInfo, @@ -61,7 +61,7 @@ class NetworkCollector(InBandDataCollector[NetworkDataModel, NetworkCollectorArg CMD_NEIGHBOR = "ip neighbor show" CMD_ETHTOOL_TEMPLATE = "ethtool {interface}" CMD_ETHTOOL_S_TEMPLATE = "ethtool -S {interface}" - CMD_RDMA_LINK_JSON = "rdma link -j" + CMD_ETHTOOL_I_TEMPLATE = "ethtool -i {interface}" CMD_PING = "ping" CMD_WGET = "wget" CMD_CURL = "curl" @@ -453,6 +453,10 @@ def _parse_ethtool(self, interface: str, output: str) -> EthtoolInfo: def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, str]: """Parse 'ethtool -S ' output into a key-value dictionary. + Per-queue counters keep their full raw field name (e.g. "[0]: rx_ucast_packets") + so vendor ``queue_counter_regex`` patterns can match them; scalar counters are + stored under their plain name (e.g. "rx_total_l4_csum_errors"). + Args: output: Raw output from 'ethtool -S ' command interface: Name of the network interface (for netdev key) @@ -462,24 +466,18 @@ def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, st """ stats_dict: Dict[str, str] = {} for line in output.splitlines(): - if ":" not in line: + line = line.strip() + if not line: continue if "NIC statistics" in line: stats_dict["netdev"] = interface - elif "]: " in line and line.strip().startswith("["): - # Format: " [0]: rx_ucast_packets: 162" - bracket_part, rest = line.split("]: ", 1) - index = bracket_part.strip().lstrip("[") - if ": " in rest: - stat_key, stat_value = rest.split(": ", 1) - key = f"{index}_{stat_key.strip()}" - stats_dict[key] = stat_value.strip() - else: - key, value = line.split(":", 1) - stats_dict[key.strip()] = value.strip() - else: - key, value = line.split(":", 1) - stats_dict[key.strip()] = value.strip() + continue + if ": " not in line: + continue + # Split on the last ": " so per-queue lines like "[0]: rx_ucast_packets: 5" + # preserve their full "[0]: rx_ucast_packets" name for vendor queue matching. + name, value = line.rsplit(": ", 1) + stats_dict[name.strip()] = value.strip() return stats_dict def _collect_ethtool_info( @@ -526,47 +524,12 @@ def _collect_ethtool_info( return ethtool_data, skipped - def _collect_rdma_link_json(self) -> Optional[list[dict]]: - """Parse JSON from `rdma link -j`. Returns None on failure, [] when no links.""" - res = self._run_sut_cmd(self.CMD_RDMA_LINK_JSON) - if res.exit_code != 0: - self._log_event( - category=EventCategory.NETWORK, - description="rdma link -j failed (RDMA-scoped ethtool collection skipped)", - data={ - "command": self.CMD_RDMA_LINK_JSON, - "exit_code": res.exit_code, - "stderr": res.stderr, - }, - priority=EventPriority.WARNING, - ) - return None - if not res.stdout.strip(): - return [] - try: - parsed = json.loads(res.stdout) - except json.JSONDecodeError as e: - self._log_event( - category=EventCategory.NETWORK, - description="Failed to parse rdma link -j JSON", - data={"exception": get_exception_traceback(e)}, - priority=EventPriority.WARNING, - ) - return None - if not isinstance(parsed, list): - self._log_event( - category=EventCategory.NETWORK, - description="Unexpected rdma link -j JSON type", - data={"data_type": type(parsed).__name__}, - priority=EventPriority.WARNING, - ) - return None - return parsed + def _collect_ethtool_statistic(self, netdev: str, driver: str) -> Optional[EthtoolStatistics]: + """Run `ethtool -S` for netdev and attach vendor-parsed stats. - def _collect_rdma_scoped_ethtool_statistic( - self, netdev: str, ifname: str - ) -> Optional[EthtoolStatistics]: - """Run `ethtool -S` for netdev and attach vendor-parsed stats (prefix from RDMA ifname).""" + The vendor model is selected by matching ``driver`` (from ``ethtool -i``) + against the known vendor prefixes in ``VENDOR_PREFIX_MAP``. + """ cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=netdev) res = self._run_sut_cmd(cmd_s, sudo=True) if res.exit_code != 0: @@ -586,11 +549,16 @@ def _collect_rdma_scoped_ethtool_statistic( stats_dict = self._parse_ethtool_statistics(res.stdout, netdev) vendor_stats: Optional[VendorEthtoolStatisticsModel] = None + queue_statistics: Dict[str, int] = {} for prefix, vendor_cls in VENDOR_PREFIX_MAP.items(): - if ifname.startswith(prefix): + if driver.startswith(prefix): vendor_fields = set(vendor_cls.model_fields.keys()) stat_fields = set(stats_dict.keys()) - {"netdev"} + queue_statistics = extract_queue_counters( + stats_dict, vendor_cls.queue_counter_regex + ) + missing_fields = vendor_fields - stat_fields if missing_fields: sorted_missing = sorted(missing_fields) @@ -599,7 +567,7 @@ def _collect_rdma_scoped_ethtool_statistic( description=f"Missing fields in ethtool statistic for {netdev}", data={ "netdev": netdev, - "ifname": ifname, + "driver": driver, "missing_fields_count": len(sorted_missing), "missing_fields": sorted_missing[:50], }, @@ -620,18 +588,40 @@ def _collect_rdma_scoped_ethtool_statistic( return EthtoolStatistics( netdev=netdev, - rdma_ifname=ifname or None, + driver=driver or None, vendor_statistics=vendor_stats, + queue_statistics=queue_statistics, ) - def _collect_rdma_scoped_ethtool( - self, exclusions: Optional[List[re.Pattern]] = None + def _get_netdev_driver(self, netdev: str) -> str: + """Return the kernel driver for a netdev via `ethtool -i` ("" on failure).""" + cmd = self.CMD_ETHTOOL_I_TEMPLATE.format(interface=netdev) + res = self._run_sut_cmd(cmd, sudo=True) + if res.exit_code != 0: + return "" + for line in res.stdout.splitlines(): + line = line.strip() + if line.startswith("driver:"): + return line.split(":", 1)[1].strip() + return "" + + def _collect_ethtool_statistics( + self, + interfaces: List[NetworkInterface], + exclusions: Optional[List[re.Pattern]] = None, ) -> tuple[List[str], List[EthtoolStatistics], set[str]]: - """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity). + """Collect ethtool -S for vendor netdevs discovered from `ip addr`. + + Each interface's driver is resolved via `ethtool -i`; only interfaces whose + driver matches a known vendor prefix (see VENDOR_PREFIX_MAP) are collected. The + matched driver selects the respective vendor data model used to parse the + `ethtool -S` counters. Interfaces without a known vendor driver (lo, docker, + veth, ...) are skipped, which avoids noisy `ethtool -S` failures. This does not + depend on RDMA. Args: - exclusions: Compiled regex patterns; netdevs whose name matches any pattern - are skipped (ethtool -S is not run against them). + interfaces: Interfaces discovered from `ip addr` to consider. + exclusions: Compiled regex patterns; matching netdevs are skipped. Returns: Tuple of (netdev list, statistics list, set of netdev names skipped due to @@ -641,38 +631,27 @@ def _collect_rdma_scoped_ethtool( statistics_list: List[EthtoolStatistics] = [] skipped: set[str] = set() - link_data = self._collect_rdma_link_json() - if link_data is None: - return netdev_list, statistics_list, skipped - - for link in link_data: - if not isinstance(link, dict): - self._log_event( - category=EventCategory.NETWORK, - description="Invalid data type for RDMA link entry", - data={"data_type": type(link).__name__}, - priority=EventPriority.WARNING, - ) + for iface in interfaces: + netdev = iface.name + if not netdev: continue - - netdev = link.get("netdev") or "" - ifname = link.get("ifname") or "" - - if netdev: - if exclusions and any(pattern.search(netdev) for pattern in exclusions): - skipped.add(netdev) - continue - netdev_list.append(netdev) - stat = self._collect_rdma_scoped_ethtool_statistic(netdev, ifname) - if stat is not None: - statistics_list.append(stat) + if exclusions and any(pattern.search(netdev) for pattern in exclusions): + skipped.add(netdev) + continue + driver = self._get_netdev_driver(netdev) + if not any(driver.startswith(prefix) for prefix in VENDOR_PREFIX_MAP): + continue + netdev_list.append(netdev) + stat = self._collect_ethtool_statistic(netdev, driver) + if stat is not None: + statistics_list.append(stat) if netdev_list: self._log_event( category=EventCategory.NETWORK, description=( - f"Collected RDMA-scoped ethtool -S for {len(statistics_list)}/" - f"{len(netdev_list)} netdev(s) from rdma link" + f"Collected ethtool -S for {len(statistics_list)}/" + f"{len(netdev_list)} vendor netdev(s)" ), priority=EventPriority.INFO, ) @@ -778,8 +757,8 @@ def collect_data( neighbors = [] ethtool_data: Dict[str, EthtoolInfo] = {} network_accessible: Optional[bool] = None - rdma_ethtool_netdevs: List[str] = [] - rdma_ethtool_statistics: List[EthtoolStatistics] = [] + ethtool_netdevs: List[str] = [] + ethtool_statistics: List[EthtoolStatistics] = [] skipped_devices: set[str] = set() # Compile device-exclusion regex patterns (netdevs matching any pattern are skipped) @@ -835,11 +814,11 @@ def collect_data( if self.system_info.os_family == OSFamily.LINUX: ( - rdma_ethtool_netdevs, - rdma_ethtool_statistics, - rdma_skipped, - ) = self._collect_rdma_scoped_ethtool(compiled_exclusions) - skipped_devices |= rdma_skipped + ethtool_netdevs, + ethtool_statistics, + ethtool_stat_skipped, + ) = self._collect_ethtool_statistics(interfaces, compiled_exclusions) + skipped_devices |= ethtool_stat_skipped # Collect routing table res_route = self._run_sut_cmd(self.CMD_ROUTE) @@ -905,8 +884,8 @@ def collect_data( rules=rules, neighbors=neighbors, ethtool_info=ethtool_data, - rdma_ethtool_netdevs=rdma_ethtool_netdevs, - rdma_ethtool_statistics=rdma_ethtool_statistics, + ethtool_netdevs=ethtool_netdevs, + ethtool_statistics=ethtool_statistics, accessible=network_accessible, ) self.result.status = ExecutionStatus.OK diff --git a/nodescraper/plugins/inband/network/networkdata.py b/nodescraper/plugins/inband/network/networkdata.py index c90a1fc1..25b5f226 100644 --- a/nodescraper/plugins/inband/network/networkdata.py +++ b/nodescraper/plugins/inband/network/networkdata.py @@ -119,7 +119,7 @@ class NetworkDataModel(DataModel): ethtool_info: Dict[str, EthtoolInfo] = Field( default_factory=dict ) # Interface name -> EthtoolInfo mapping - # RDMA-scoped ethtool -S: netdevs from `rdma link -j` with vendor-parsed counters - rdma_ethtool_netdevs: List[str] = Field(default_factory=list) - rdma_ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) + # ethtool -S vendor-parsed counters for netdevs whose driver matches a known vendor + ethtool_netdevs: List[str] = Field(default_factory=list) + ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) accessible: Optional[bool] = None # Network accessibility check via ping diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 740f5998..90d68aa4 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -31,7 +31,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily -from nodescraper.models import TaskResult +from nodescraper.models import CollectorArgs, TaskResult from nodescraper.utils import get_exception_traceback from .rdmadata import ( @@ -45,7 +45,7 @@ ) -class RdmaCollector(InBandDataCollector[RdmaDataModel, None]): +class RdmaCollector(InBandDataCollector[RdmaDataModel, CollectorArgs]): """Collect RDMA status and statistics via rdma link and rdma statistic commands.""" DATA_MODEL = RdmaDataModel @@ -291,7 +291,9 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: ) return None - def collect_data(self, args=None) -> tuple[TaskResult, Optional[RdmaDataModel]]: + def collect_data( + self, args: Optional[CollectorArgs] = None + ) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. Returns: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index 4bf5e8cc..f435ff7a 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -24,6 +24,7 @@ # ############################################################################### from nodescraper.base import InBandDataPlugin +from nodescraper.models import CollectorArgs from .analyzer_args import RdmaAnalyzerArgs from .rdma_analyzer import RdmaAnalyzer @@ -31,10 +32,11 @@ from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, CollectorArgs, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector + COLLECTOR_ARGS = CollectorArgs ANALYZER = RdmaAnalyzer ANALYZER_ARGS = RdmaAnalyzerArgs diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 832125e5..2eb74490 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -192,13 +192,13 @@ def test_empty_ethtool_info(network_analyzer): def test_rdma_ethtool_vendor_error_only(network_analyzer): - """RDMA-scoped vendor ethtool: error-tier counter raises ERROR.""" + """Vendor ethtool: error-tier counter raises ERROR.""" stat = EthtoolStatistics( netdev="eth0", - rdma_ifname="bnxt0", + driver="bnxt_en", vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "Network errors detected" in result.message @@ -209,13 +209,13 @@ def test_rdma_ethtool_vendor_error_only(network_analyzer): def test_rdma_ethtool_vendor_warning_only(network_analyzer): - """RDMA-scoped vendor ethtool: only warning-tier counters -> WARNING status.""" + """Vendor ethtool: only warning-tier counters -> WARNING status.""" stat = EthtoolStatistics( netdev="eth0", - rdma_ifname="bnxt0", + driver="bnxt_en", vendor_statistics=Thor2EthtoolStatistics(rx_pause_frames=2), ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.WARNING assert "warning counters" in result.message @@ -234,9 +234,9 @@ def test_thor2_tx_pause_counters_are_warning_tier(): def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): - """RDMA ethtool row without parsed vendor statistics is ignored by vendor path.""" - stat = EthtoolStatistics(netdev="eth0", rdma_ifname="unknown0", vendor_statistics=None) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + """Ethtool row without parsed vendor statistics is ignored by vendor path.""" + stat = EthtoolStatistics(netdev="eth0", driver="unknown", vendor_statistics=None) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 @@ -399,6 +399,207 @@ def test_custom_error_regex_detected(network_analyzer): assert result.events[0].data["errors"] == {"custom_tx_drops": 9} +def test_queue_counter_error_detected(network_analyzer): + """A non-zero per-queue error counter (e.g. rx_discards) is flagged as ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 100, # benign, ignored + "[0]: rx_discards": 5, # error + "[3]: tx_errors": 2, # error + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + assert "Network errors detected" in result.message + + queue_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert len(queue_events) == 2 + flagged = {e.data["error_field"] for e in queue_events} + assert flagged == {"[0]: rx_discards", "[3]: tx_errors"} + assert all(e.priority == EventPriority.ERROR for e in queue_events) + + +def test_queue_counter_thor2_specific_error_fields(network_analyzer): + """Each curated Thor2 per-queue error pattern flags its non-zero counter as ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 1, + "[1]: rx_errors": 2, + "[2]: tx_errors": 3, + "[3]: rx_l4_csum_errors": 4, + "[4]: rx_buf_errors": 5, + "[5]: so_txtime_cmpl_errors": 6, + "[6]: missed_irqs": 7, + "[7]: xsk_rx_redirect_fail": 8, + "[8]: xsk_rx_alloc_fail": 9, + "[9]: xsk_rx_no_room": 10, + "[10]: xsk_tx_ring_full": 11, + # benign counters must NOT be flagged + "[0]: rx_ucast_packets": 12345, + "[0]: tx_ucast_bytes": 67890, + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + + error_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + flagged = {e.data["error_field"] for e in error_events} + assert flagged == { + "[0]: rx_discards", + "[1]: rx_errors", + "[2]: tx_errors", + "[3]: rx_l4_csum_errors", + "[4]: rx_buf_errors", + "[5]: so_txtime_cmpl_errors", + "[6]: missed_irqs", + "[7]: xsk_rx_redirect_fail", + "[8]: xsk_rx_alloc_fail", + "[9]: xsk_rx_no_room", + "[10]: xsk_tx_ring_full", + } + assert "[0]: rx_ucast_packets" not in flagged + assert "[0]: tx_ucast_bytes" not in flagged + assert all(e.priority == EventPriority.ERROR for e in error_events) + + +def test_queue_counter_benign_not_reported(network_analyzer): + """Non-zero benign per-queue counters (packets/bytes) do not raise errors.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 12345, + "[0]: tx_ucast_bytes": 67890, + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + + +def test_queue_counter_zero_not_reported(network_analyzer): + """A per-queue error counter that is zero does not raise an error.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={"[0]: rx_discards": 0, "[1]: tx_errors": 0}, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + + +def test_queue_counter_warning_detected(network_analyzer): + """A non-zero warning-tier per-queue counter (e.g. rx_resets) -> WARNING.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_ucast_packets": 100, # benign, ignored + "[0]: rx_resets": 3, # warning-tier + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.WARNING + assert "warning counters" in result.message + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert len(warn_events) == 1 + assert warn_events[0].data["error_field"] == "[0]: rx_resets" + assert warn_events[0].priority == EventPriority.WARNING + + +def test_queue_counter_error_precedence_over_warning(network_analyzer): + """When both error and warning per-queue counters are non-zero, status is ERROR.""" + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 5, # error-tier + "[0]: rx_resets": 3, # warning-tier + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert {e.data["error_field"] for e in err_events} == {"[0]: rx_discards"} + assert {e.data["error_field"] for e in warn_events} == {"[0]: rx_resets"} + + +def test_queue_counter_empty_warning_list_skips_warning_check(network_analyzer, monkeypatch): + """An empty queue_warning_regex skips warning checking; error checking still runs.""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_warning_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_resets": 3, # would be a warning, but warning list is empty + "[0]: rx_discards": 5, # still flagged as an error + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert warn_events == [] + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert {e.data["error_field"] for e in err_events} == {"[0]: rx_discards"} + + +def test_queue_counter_empty_error_list_skips_error_check(network_analyzer, monkeypatch): + """An empty queue_error_regex skips error checking; warning checking still runs.""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_error_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={ + "[0]: rx_discards": 5, # would be an error, but error list is empty + "[0]: rx_resets": 3, # still flagged as a warning + }, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.WARNING + err_events = [e for e in result.events if e.description == "Ethtool queue error detected"] + assert err_events == [] + warn_events = [e for e in result.events if e.description == "Ethtool queue warning detected"] + assert {e.data["error_field"] for e in warn_events} == {"[0]: rx_resets"} + + +def test_queue_counter_both_lists_empty_skips_all_checks(network_analyzer, monkeypatch): + """Empty error and warning lists skip per-queue checking entirely (status OK).""" + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_error_regex", []) + monkeypatch.setattr(Thor2EthtoolStatistics, "queue_warning_regex", []) + stat = EthtoolStatistics( + netdev="benic1p1", + driver="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(), + queue_statistics={"[0]: rx_discards": 5, "[0]: rx_resets": 3}, + ) + model = NetworkDataModel(ethtool_info={}, ethtool_statistics=[stat]) + result = network_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + + def test_custom_warning_priority_regex(network_analyzer): """A user-supplied WARNING-priority custom pattern yields WARNING status via the regex path.""" ethtool = EthtoolInfo( diff --git a/test/unit/plugin/test_network_collector.py b/test/unit/plugin/test_network_collector.py index a7b1faae..909e1319 100644 --- a/test/unit/plugin/test_network_collector.py +++ b/test/unit/plugin/test_network_collector.py @@ -30,6 +30,12 @@ from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.network.ethtool_vendor import ( + Cx7EthtoolStatistics, + PollaraEthtoolStatistics, + Thor2EthtoolStatistics, + extract_queue_counters, +) from nodescraper.plugins.inband.network.network_collector import NetworkCollector from nodescraper.plugins.inband.network.networkdata import ( EthtoolInfo, @@ -510,7 +516,10 @@ def test_parse_ethtool_empty_output(collector): def test_parse_ethtool_statistics(collector): - """Test parsing ethtool -S output (statistics) for error/health analysis.""" + """Test parsing ethtool -S output (statistics) for error/health analysis. + + Per-queue bracket lines keep their full raw name so vendor queue patterns match. + """ output = """NIC statistics: [0]: rx_ucast_packets: 162692536538787551 [0]: rx_errors: 0 @@ -519,13 +528,51 @@ def test_parse_ethtool_statistics(collector): rx_total_buf_errors: 0""" stats = collector._parse_ethtool_statistics(output, "abc1p1") assert stats.get("netdev") == "abc1p1" - assert stats.get("0_rx_ucast_packets") == "162692536538787551" - assert stats.get("0_rx_errors") == "0" - assert stats.get("1_rx_ucast_packets") == "79657418409137764" + assert stats.get("[0]: rx_ucast_packets") == "162692536538787551" + assert stats.get("[0]: rx_errors") == "0" + assert stats.get("[1]: rx_ucast_packets") == "79657418409137764" assert stats.get("rx_total_l4_csum_errors") == "0" assert stats.get("rx_total_buf_errors") == "0" +def test_extract_queue_counters_thor2(): + """Thor2 per-queue counters are the bracket-indexed "[]" lines.""" + stats = { + "netdev": "benic1p1", + "[0]: rx_ucast_packets": "5", + "[15]: tx_errors": "0", + "rx_total_l4_csum_errors": "0", + } + queue = extract_queue_counters(stats, Thor2EthtoolStatistics.queue_counter_regex) + assert queue == {"[0]: rx_ucast_packets": 5, "[15]: tx_errors": 0} + assert "rx_total_l4_csum_errors" not in queue + assert "netdev" not in queue + + +def test_extract_queue_counters_pollara(): + """Pollara per-queue counters start with "rx_" / "tx_".""" + stats = { + "rx_0_pkts": "3", + "tx_12_bytes": "9", + "rx_csum_error": "0", + "frames_tx_pri_0": "1", + } + queue = extract_queue_counters(stats, PollaraEthtoolStatistics.queue_counter_regex) + assert queue == {"rx_0_pkts": 3, "tx_12_bytes": 9} + + +def test_extract_queue_counters_cx7(): + """CX7 per-queue counters start with "rx" / "tx".""" + stats = { + "rx0_packets": "7", + "tx3_bytes": "8", + "rx_out_of_buffer": "0", + "rx_prio0_pause": "2", + } + queue = extract_queue_counters(stats, Cx7EthtoolStatistics.queue_counter_regex) + assert queue == {"rx0_packets": 7, "tx3_bytes": 8} + + def test_network_data_model_creation(collector): """Test creating NetworkDataModel with all components""" interface = NetworkInterface( @@ -650,13 +697,10 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert accessible is False -def test_collect_data_includes_rdma_ethtool(collector, conn_mock): - """RDMA-scoped ethtool -S is stored on NetworkDataModel when rdma link succeeds.""" - import json - +def test_collect_data_includes_ethtool_statistics(collector, conn_mock): + """ethtool -S is collected for vendor NICs via driver detection (no RDMA).""" collector.system_info.os_family = OSFamily.LINUX - rdma_link = [{"netdev": "eth0", "ifname": "bnxt0"}] ethtool_s_bnxt = "NIC statistics:\n tx_pfc_frames: 0\n rx_pause_frames: 0\n" def run_sut_cmd_side_effect(cmd, **kwargs): @@ -668,8 +712,8 @@ def run_sut_cmd_side_effect(cmd, **kwargs): return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) elif "neighbor show" in cmd: return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) - elif "rdma link -j" in cmd: - return MagicMock(exit_code=0, stdout=json.dumps(rdma_link), command=cmd) + elif "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock(exit_code=0, stdout="driver: bnxt_en\nversion: 1.0\n", command=cmd) elif "ethtool -S" in cmd and "eth0" in cmd: return MagicMock(exit_code=0, stdout=ethtool_s_bnxt, command=cmd) elif "ethtool" in cmd: @@ -684,8 +728,46 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert result.status == ExecutionStatus.OK assert data is not None - assert "eth0" in data.rdma_ethtool_netdevs - assert len(data.rdma_ethtool_statistics) == 1 - assert data.rdma_ethtool_statistics[0].netdev == "eth0" - assert data.rdma_ethtool_statistics[0].rdma_ifname == "bnxt0" - assert data.rdma_ethtool_statistics[0].vendor_statistics is not None + # lo has no vendor driver and is skipped; eth0 (bnxt_en) is collected + assert data.ethtool_netdevs == ["eth0"] + assert len(data.ethtool_statistics) == 1 + assert data.ethtool_statistics[0].netdev == "eth0" + assert data.ethtool_statistics[0].driver == "bnxt_en" + assert data.ethtool_statistics[0].vendor_statistics is not None + + +def test_collect_data_skips_non_vendor_netdev(collector, conn_mock): + """ethtool -S is not run for netdevs whose driver is not a known vendor.""" + collector.system_info.os_family = OSFamily.LINUX + + ethtool_s_called = {"value": False} + + def run_sut_cmd_side_effect(cmd, **kwargs): + if "addr show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ADDR_OUTPUT, command=cmd) + elif "route show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ROUTE_OUTPUT, command=cmd) + elif "rule show" in cmd: + return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) + elif "neighbor show" in cmd: + return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) + elif "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock(exit_code=0, stdout="driver: e1000e\n", command=cmd) + elif "ethtool -S" in cmd: + ethtool_s_called["value"] = True + return MagicMock(exit_code=0, stdout="NIC statistics:\n", command=cmd) + elif "ethtool" in cmd: + return MagicMock(exit_code=1, stdout="", command=cmd) + elif "lldpcli" in cmd or "lldpctl" in cmd: + return MagicMock(exit_code=1, stdout="", command=cmd) + return MagicMock(exit_code=1, stdout="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.ethtool_netdevs == [] + assert data.ethtool_statistics == [] + assert ethtool_s_called["value"] is False From e85272b70466dac7a1856efbd953326fcb7c6b89 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 16 Jul 2026 10:04:47 -0500 Subject: [PATCH 06/21] Create label.yml --- .github/workflows/label.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/label.yml diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml new file mode 100644 index 00000000..46135690 --- /dev/null +++ b/.github/workflows/label.yml @@ -0,0 +1,22 @@ +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. +# +# To use this workflow, you will need to set up a .github/labeler.yml +# file with configuration. For more information, see: +# https://github.com/actions/labeler + +name: Labeler +on: [pull_request_target] + +jobs: + label: + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + + steps: + - uses: actions/labeler@v4 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" From fb96f08c4e50c5856490348bf5b2f78f8a8f8688 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 16 Jul 2026 11:34:04 -0500 Subject: [PATCH 07/21] fixed label.yml and added labeler --- .github/labeler.yml | 31 +++++++++++++++++++++++++++++++ .github/workflows/label.yml | 3 ++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..a756fe0f --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,31 @@ +# Path-based PR labels for actions/labeler. +# Labels are created automatically when sync-labels is enabled in label.yml. + +documentation: + - docs/** + - README.md + - .github/**/*.md + +ci: + - .github/** + +tests: + - test/** + +plugins-inband: + - nodescraper/plugins/inband/** + +plugins-ooband: + - nodescraper/plugins/ooband/** + +plugins-serviceability: + - nodescraper/plugins/serviceability/** + +plugins-generic: + - nodescraper/plugins/generic_collection/** + +framework: + - nodescraper/base/** + - nodescraper/interfaces/** + - nodescraper/cli/** + - nodescraper/models/** diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml index 46135690..6535a06c 100644 --- a/.github/workflows/label.yml +++ b/.github/workflows/label.yml @@ -17,6 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" + sync-labels: true From 87688cef8d05d3e33e7bdf4360927f727a24ef9e Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 16 Jul 2026 11:46:37 -0500 Subject: [PATCH 08/21] fixed for v5 schema --- .github/labeler.yml | 46 ++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index a756fe0f..6b948b10 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,31 +1,47 @@ -# Path-based PR labels for actions/labeler. -# Labels are created automatically when sync-labels is enabled in label.yml. +# Path-based PR labels for actions/labeler v5+. +# See: https://github.com/actions/labeler#pull-request-labeler documentation: - - docs/** - - README.md - - .github/**/*.md + - changed-files: + - any-glob-to-any-file: + - docs/** + - README.md + - .github/**/*.md ci: - - .github/** + - changed-files: + - any-glob-to-any-file: + - .github/** tests: - - test/** + - changed-files: + - any-glob-to-any-file: + - test/** plugins-inband: - - nodescraper/plugins/inband/** + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/inband/** plugins-ooband: - - nodescraper/plugins/ooband/** + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/ooband/** plugins-serviceability: - - nodescraper/plugins/serviceability/** + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/serviceability/** plugins-generic: - - nodescraper/plugins/generic_collection/** + - changed-files: + - any-glob-to-any-file: + - nodescraper/plugins/generic_collection/** framework: - - nodescraper/base/** - - nodescraper/interfaces/** - - nodescraper/cli/** - - nodescraper/models/** + - changed-files: + - any-glob-to-any-file: + - nodescraper/base/** + - nodescraper/interfaces/** + - nodescraper/cli/** + - nodescraper/models/** From f84ade094c43ca964cdca66ac9e123e947455bbc Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Mon, 20 Jul 2026 10:43:22 -0500 Subject: [PATCH 09/21] fixed for shell q --- .github/scripts/plugin_convention_warnings.py | 186 ++++++++++++++++++ .../plugins/inband/amdsmi/amdsmi_collector.py | 4 +- .../inband/journal/journal_collector.py | 8 +- .../inband/network/network_collector.py | 9 +- .../inband/process/process_collector.py | 3 +- .../plugins/inband/rocm/rocm_collector.py | 17 +- test/unit/plugin/test_rocm_collector.py | 2 +- 7 files changed, 209 insertions(+), 20 deletions(-) diff --git a/.github/scripts/plugin_convention_warnings.py b/.github/scripts/plugin_convention_warnings.py index caaf03f6..893e5797 100755 --- a/.github/scripts/plugin_convention_warnings.py +++ b/.github/scripts/plugin_convention_warnings.py @@ -14,6 +14,14 @@ ``pydantic.Field(...)`` with a non-empty ``description=`` (for help/CLI text). ``ClassVar`` fields, ``_``-prefixed names, and ``model_config`` are skipped. + +3. **Shell quoting** — In ``Collector`` / ``Analyzer`` methods, values from + ``args.*``, ``cmd_spec.*``, or sensitive function parameters (names ending + in ``_path`` / ``_file``, or ``url``, ``boot``, ``folder``, ``host``) that + are interpolated into shell commands (``_run_sut_cmd``, ``_run_amd_smi``, + ``_run_dell_command``) must be wrapped in ``shell_quote(...)`` or assigned + from a prior ``shell_quote(...)`` call in the same function. + ``GenericCollectionCollector`` is excluded (user commands are intentional). """ from __future__ import annotations @@ -40,6 +48,9 @@ } ) +_SHELL_RUN_METHODS = frozenset({"_run_sut_cmd", "_run_amd_smi", "_run_dell_command"}) +_SKIP_SHELL_QUOTE_CLASSES = frozenset({"GenericCollectionCollector"}) + def _is_stringish(expr: ast.expr) -> bool: if isinstance(expr, ast.Constant) and isinstance(expr.value, str): @@ -224,6 +235,179 @@ def _check_args_fields(path: Path, tree: ast.Module) -> list[str]: return msgs +def _is_shell_quote_call(expr: ast.expr) -> bool: + if not isinstance(expr, ast.Call): + return False + func = expr.func + if isinstance(func, ast.Name) and func.id == "shell_quote": + return True + return isinstance(func, ast.Attribute) and func.attr == "shell_quote" + + +def _func_param_names(func: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + names: set[str] = set() + for arg in func.args.args + func.args.kwonlyargs: + if arg.arg != "self": + names.add(arg.arg) + if func.args.vararg: + names.add(func.args.vararg.arg) + return names + + +def _collect_shell_quote_safe_names(func: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + safe: set[str] = set() + for node in ast.walk(func): + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if isinstance(target, ast.Name) and _is_shell_quote_call(node.value): + safe.add(target.id) + return safe + + +def _expr_preview(expr: ast.expr) -> str: + try: + return ast.unparse(expr) + except Exception: + return "" + + +def _is_sensitive_param_name(name: str) -> bool: + if name in {"url", "boot", "folder", "hostname", "host"}: + return True + return name.endswith("_path") or name.endswith("_file") + + +def _is_user_controlled_leaf(expr: ast.expr, param_names: set[str]) -> bool: + if isinstance(expr, ast.Attribute) and isinstance(expr.value, ast.Name): + base = expr.value.id + if base in ("args", "cmd_spec"): + return True + if isinstance(expr, ast.Name) and expr.id in param_names: + return _is_sensitive_param_name(expr.id) + return False + + +def _contains_user_controlled(expr: ast.expr, param_names: set[str]) -> bool: + if _is_user_controlled_leaf(expr, param_names): + return True + for child in ast.iter_child_nodes(expr): + if isinstance(child, ast.expr) and _contains_user_controlled(child, param_names): + return True + return False + + +def _is_trivially_safe_expr(expr: ast.expr, safe_names: set[str]) -> bool: + if _is_shell_quote_call(expr): + return True + if isinstance(expr, ast.Constant): + return True + if isinstance(expr, ast.Name) and expr.id in safe_names: + return True + if isinstance(expr, ast.Attribute) and isinstance(expr.value, ast.Name): + base = expr.value.id + if base == "self" and ( + expr.attr == "CMD" or expr.attr.startswith("CMD_") or expr.attr in ("AMD_SMI_EXE",) + ): + return True + return False + + +def _command_insert_issues( + expr: ast.expr, + param_names: set[str], + safe_names: set[str], +) -> list[tuple[int, str]]: + issues: list[tuple[int, str]] = [] + + def note(value: ast.expr) -> None: + if _is_trivially_safe_expr(value, safe_names): + return + if _contains_user_controlled(value, param_names): + issues.append((getattr(value, "lineno", 0), _expr_preview(value))) + + if isinstance(expr, ast.JoinedStr): + for part in expr.values: + if isinstance(part, ast.FormattedValue): + note(part.value) + elif isinstance(expr, ast.Call) and isinstance(expr.func, ast.Attribute): + if expr.func.attr == "format": + for kw in expr.keywords: + if kw.arg is not None and kw.value is not None: + note(kw.value) + elif isinstance(expr, ast.BinOp): + note(expr.left) + note(expr.right) + + return issues + + +def _resolve_command_exprs( + func: ast.FunctionDef | ast.AsyncFunctionDef, expr: ast.expr +) -> list[ast.expr]: + if not isinstance(expr, ast.Name): + return [expr] + + matches: list[ast.expr] = [] + for node in ast.walk(func): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == expr.id: + matches.append(node.value) + elif ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == expr.id + and node.value is not None + ): + matches.append(node.value) + return matches or [expr] + + +def _check_function_shell_quoting( + path: Path, + class_name: str, + func: ast.FunctionDef | ast.AsyncFunctionDef, +) -> list[str]: + msgs: list[str] = [] + param_names = _func_param_names(func) + safe_names = _collect_shell_quote_safe_names(func) + + for node in ast.walk(func): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Attribute): + continue + if node.func.attr not in _SHELL_RUN_METHODS: + continue + if not node.args: + continue + + command_expr = node.args[0] + for resolved in _resolve_command_exprs(func, command_expr): + for lineno, preview in _command_insert_issues(resolved, param_names, safe_names): + line = lineno or node.lineno + msgs.append( + f"{path}:{line}: [{class_name}.{func.name}] user-controlled value " + f"{preview} in shell command must use shell_quote(...)." + ) + return msgs + + +def _check_shell_quoting(path: Path, tree: ast.Module) -> list[str]: + """Rule #3: warn when config/param values reach shell commands without shell_quote.""" + msgs: list[str] = [] + for node in tree.body: + if not isinstance(node, ast.ClassDef) or not _is_collector_or_analyzer_class(node): + continue + if node.name in _SKIP_SHELL_QUOTE_CLASSES: + continue + for stmt in node.body: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)): + msgs.extend(_check_function_shell_quoting(path, node.name, stmt)) + return msgs + + def main() -> None: if not PLUGIN_ROOT.is_dir(): sys.stderr.write(f"warning: plugins directory not found: {PLUGIN_ROOT}\n") @@ -242,8 +426,10 @@ def main() -> None: if "collector" in name and name.endswith(".py"): all_msgs.extend(_check_cmd_prefixes(rel, tree)) + all_msgs.extend(_check_shell_quoting(rel, tree)) if "analyzer" in name and name.endswith(".py"): all_msgs.extend(_check_cmd_prefixes(rel, tree)) + all_msgs.extend(_check_shell_quoting(rel, tree)) if name == "collector_args.py" or name == "analyzer_args.py": all_msgs.extend(_check_args_fields(rel, tree)) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py index ef60995a..92d42252 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py @@ -76,7 +76,7 @@ normalize_static_limit_dict, ) from nodescraper.plugins.inband.amdsmi.collector_args import AmdSmiCollectorArgs -from nodescraper.utils import get_exception_traceback +from nodescraper.utils import get_exception_traceback, shell_quote class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs]): @@ -1489,7 +1489,7 @@ def _get_cper_afid(self, cper_file_path: str) -> Optional[int]: Returns: Optional[int]: AFID value or None if command fails or no value found """ - cmd = self.CMD_RAS_AFID.format(cper_file=cper_file_path) + cmd = self.CMD_RAS_AFID.format(cper_file=shell_quote(cper_file_path)) result = self._run_amd_smi(cmd) if result is None: diff --git a/nodescraper/plugins/inband/journal/journal_collector.py b/nodescraper/plugins/inband/journal/journal_collector.py index d244eda3..e7a76c81 100644 --- a/nodescraper/plugins/inband/journal/journal_collector.py +++ b/nodescraper/plugins/inband/journal/journal_collector.py @@ -31,7 +31,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import get_exception_details +from nodescraper.utils import get_exception_details, shell_quote from .collector_args import JournalCollectorArgs from .journaldata import JournalData, JournalJsonEntry @@ -56,7 +56,8 @@ def _read_with_journalctl(self, args: Optional[JournalCollectorArgs] = None): try: # safe check for args.boot if args is not None and getattr(args, "boot", None): - cmd = f"{self.CMD} -b {args.boot}" + boot_q = shell_quote(str(args.boot)) + cmd = f"{self.CMD} -b {boot_q}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False, strip=False) @@ -99,7 +100,8 @@ def _read_with_journalctl_json( try: # safe check for args.boot if args is not None and getattr(args, "boot", None): - cmd = f"{self.CMD_JSON} -b {args.boot}" + boot_q = shell_quote(str(args.boot)) + cmd = f"{self.CMD_JSON} -b {boot_q}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False, strip=False) diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index b960e574..9163e6c5 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -32,7 +32,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import get_exception_traceback +from nodescraper.utils import get_exception_traceback, shell_quote from .collector_args import NetworkCollectorArgs from .ethtool_vendor import ( @@ -733,12 +733,13 @@ def _check_network_connectivity(self, cmd: str, url: str) -> bool: ping_option = "-c 1" if self.system_info.os_family == OSFamily.LINUX else "-n 1" # Build command based on cmd parameter using class constants + qu = shell_quote(url) if cmd == "ping": - result = self._run_sut_cmd(f"{self.CMD_PING} {url} {ping_option}") + result = self._run_sut_cmd(f"{self.CMD_PING} {qu} {ping_option}") elif cmd == "wget": - result = self._run_sut_cmd(f"{self.CMD_WGET} {url}") + result = self._run_sut_cmd(f"{self.CMD_WGET} {qu}") else: # curl - result = self._run_sut_cmd(f"{self.CMD_CURL} {url}") + result = self._run_sut_cmd(f"{self.CMD_CURL} {qu}") if result.exit_code == 0: self._log_event( diff --git a/nodescraper/plugins/inband/process/process_collector.py b/nodescraper/plugins/inband/process/process_collector.py index 9c92080d..72feb8c5 100644 --- a/nodescraper/plugins/inband/process/process_collector.py +++ b/nodescraper/plugins/inband/process/process_collector.py @@ -83,8 +83,9 @@ def collect_data( ) process_data.cpu_usage = 100 - float(cpu_idle) + last_line = args.top_n_process + 7 processes = self._run_sut_cmd( - f"self.CMD_PROCESS | sed -n '8,{args.top_n_process + 7}p'" + f"{self.CMD_PROCESS} | sed -n '8,{last_line}p'" ) # Remove system header if processes.exit_code == 0: for line in processes.stdout.splitlines(): diff --git a/nodescraper/plugins/inband/rocm/rocm_collector.py b/nodescraper/plugins/inband/rocm/rocm_collector.py index 27a6c4f5..c586de9b 100644 --- a/nodescraper/plugins/inband/rocm/rocm_collector.py +++ b/nodescraper/plugins/inband/rocm/rocm_collector.py @@ -31,7 +31,7 @@ from nodescraper.connection.inband import TextFileArtifact from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import strip_ansi_codes +from nodescraper.utils import shell_quote, strip_ansi_codes from .collector_args import RocmCollectorArgs from .rocmdata import RocmDataModel @@ -63,6 +63,7 @@ def collect_data( """ if args is None: args = RocmCollectorArgs() + rocm_path_q = shell_quote(args.rocm_path) version_paths = [ f"{args.rocm_path}/.info/version-rocm", f"{args.rocm_path}/.info/version", @@ -73,7 +74,7 @@ def collect_data( # First, try to collect all sub-versions sub_versions_res = self._run_sut_cmd( - self.CMD_ROCM_SUB_VERSIONS_TMPL.format(rocm_path=args.rocm_path) + self.CMD_ROCM_SUB_VERSIONS_TMPL.format(rocm_path=rocm_path_q) ) if sub_versions_res.exit_code == 0: for line in sub_versions_res.stdout.splitlines(): @@ -86,7 +87,7 @@ def collect_data( # Determine the main ROCm version for path in version_paths: - res = self._run_sut_cmd(f"grep . {path}") + res = self._run_sut_cmd(f"grep . {shell_quote(path)}") if res.exit_code == 0: try: rocm_data = RocmDataModel( @@ -124,15 +125,13 @@ def collect_data( if rocm_data: # Collect latest versioned ROCm path (rocm-[3-7]*) versioned_path_res = self._run_sut_cmd( - self.CMD_ROCM_LATEST_TMPL.format(rocm_path=args.rocm_path) + self.CMD_ROCM_LATEST_TMPL.format(rocm_path=rocm_path_q) ) if versioned_path_res.exit_code == 0: rocm_data.rocm_latest_versioned_path = versioned_path_res.stdout.strip() # Collect all ROCm paths as list - all_paths_res = self._run_sut_cmd( - self.CMD_ROCM_DIRS_TMPL.format(rocm_path=args.rocm_path) - ) + all_paths_res = self._run_sut_cmd(self.CMD_ROCM_DIRS_TMPL.format(rocm_path=rocm_path_q)) if all_paths_res.exit_code == 0: rocm_data.rocm_all_paths = [ path.strip() @@ -141,7 +140,7 @@ def collect_data( ] # Collect rocminfo output as list of lines with ANSI codes stripped - rocminfo_cmd = self.CMD_ROCMINFO_TMPL.format(rocm_path=args.rocm_path) + rocminfo_cmd = self.CMD_ROCMINFO_TMPL.format(rocm_path=rocm_path_q) rocminfo_res = self._run_sut_cmd(rocminfo_cmd) rocminfo_artifact_content = "" if rocminfo_res.exit_code == 0: @@ -178,7 +177,7 @@ def collect_data( ] # Collect clinfo output - clinfo_cmd = self.CMD_CLINFO_TMPL.format(rocm_path=args.rocm_path) + clinfo_cmd = self.CMD_CLINFO_TMPL.format(rocm_path=rocm_path_q) clinfo_res = self._run_sut_cmd(clinfo_cmd) # Always append clinfo section to artifact, even if empty or failed diff --git a/test/unit/plugin/test_rocm_collector.py b/test/unit/plugin/test_rocm_collector.py index 0cb8523c..63cef21d 100644 --- a/test/unit/plugin/test_rocm_collector.py +++ b/test/unit/plugin/test_rocm_collector.py @@ -32,7 +32,7 @@ from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.plugins.inband.rocm.rocm_collector import RocmCollector -ROCM_SUB_VERSIONS_GREP_CMD = "grep . -H -r -i /opt/rocm/.info/*" +ROCM_SUB_VERSIONS_GREP_CMD = "grep . -H -r -i '/opt/rocm'/.info/*" # gfx942 (CDNA3 / MI300) and gfx950 (CDNA4 / MI350) — released ROCm 7.13 LLVM targets ROCM_7_13_GFX_VERSION = "7.13.0-123-gfx942;gfx950" From e74101ba2c529a7913893fdb88246aaf5adaf57e Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Mon, 20 Jul 2026 11:23:10 -0500 Subject: [PATCH 10/21] fixed for shell q --- nodescraper/base/inbandcollectortask.py | 38 +++- nodescraper/connection/inband/inband.py | 14 +- nodescraper/connection/inband/inbandlocal.py | 42 ++++- nodescraper/connection/inband/inbandremote.py | 35 +++- nodescraper/connection/inband/shellcommand.py | 70 ++++++++ .../inband/network/network_collector.py | 16 +- .../connection/inband/test_shellcommand.py | 89 +++++++++ test/unit/test_plugin_convention_warnings.py | 169 ++++++++++++++++++ 8 files changed, 449 insertions(+), 24 deletions(-) create mode 100644 nodescraper/connection/inband/shellcommand.py create mode 100644 test/unit/connection/inband/test_shellcommand.py create mode 100644 test/unit/test_plugin_convention_warnings.py diff --git a/nodescraper/base/inbandcollectortask.py b/nodescraper/base/inbandcollectortask.py index 25a3c55b..21f6c239 100644 --- a/nodescraper/base/inbandcollectortask.py +++ b/nodescraper/base/inbandcollectortask.py @@ -24,7 +24,7 @@ # ############################################################################### import logging -from typing import Generic, Optional, Union +from typing import Generic, Optional, Sequence, Union from nodescraper.connection.inband import InBandConnection from nodescraper.connection.inband.inband import BaseFileArtifact, CommandArtifact @@ -107,6 +107,42 @@ def _run_sut_cmd( return command_res + def _run_sut_argv( + self, + argv: Sequence[str], + sudo: bool = False, + timeout: int = 300, + strip: bool = True, + log_artifact: bool = True, + html_view: Optional[bool] = None, + ) -> CommandArtifact: + """Run a command via argv list (shell=False locally; quoted on remote SSH). + + Prefer this over _run_sut_cmd for commands that include config/user values. + Legacy shell strings (pipes, globs, redirection) still use _run_sut_cmd. + + Args: + argv (Sequence[str]): executable and arguments, e.g. ["ping", host, "-c", "1"]. + sudo (bool, optional): whether to run the command with sudo. Defaults to False. + timeout (int, optional): command timeout in seconds. Defaults to 300. + strip (bool, optional): whether output should be stripped. Defaults to True. + log_artifact (bool, optional): whether we should log the command result. Defaults to True. + html_view (Optional[bool], optional): whether to include this command in HTML + artifacts. When omitted, uses collection_args.html_view. + + Returns: + CommandArtifact: The result of the command execution. + """ + command_res = self.connection.run_command( + command=list(argv), sudo=sudo, timeout=timeout, strip=strip + ) + effective_html_view = self._effective_html_view(html_view) + if log_artifact or effective_html_view: + command_res.log_html = effective_html_view + self.result.artifacts.append(command_res) + + return command_res + def _read_sut_file( self, filename: str, encoding="utf-8", strip: bool = True, log_artifact=True ) -> BaseFileArtifact: diff --git a/nodescraper/connection/inband/inband.py b/nodescraper/connection/inband/inband.py index 9a52cb88..db082bcd 100644 --- a/nodescraper/connection/inband/inband.py +++ b/nodescraper/connection/inband/inband.py @@ -29,6 +29,8 @@ from pydantic import BaseModel +from .shellcommand import ShellCommand + class CommandArtifact(BaseModel): """Artifact for the result of shell command execution""" @@ -151,12 +153,18 @@ class InBandConnection(abc.ABC): @abc.abstractmethod def run_command( - self, command: str, sudo: bool = False, timeout: int = 300, strip: bool = True + self, + command: ShellCommand, + sudo: bool = False, + timeout: int = 300, + strip: bool = True, ) -> CommandArtifact: - """Run an in band shell command + """Run an in band shell command. Args: - command (str): command to run + command (ShellCommand): full shell string (legacy) or argv list (preferred). + When a sequence is passed, collectors should use shell=False locally and + safely quoted execution remotely. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/connection/inband/inbandlocal.py b/nodescraper/connection/inband/inbandlocal.py index e36e460b..f3590d6d 100644 --- a/nodescraper/connection/inband/inbandlocal.py +++ b/nodescraper/connection/inband/inbandlocal.py @@ -25,23 +25,29 @@ ############################################################################### import os import subprocess +from typing import Sequence from .inband import ( BaseFileArtifact, CommandArtifact, InBandConnection, ) +from .shellcommand import ShellCommand, build_exec_argv, format_argv_display class LocalShell(InBandConnection): def run_command( - self, command: str, sudo: bool = False, timeout: int = 300, strip: bool = True + self, + command: ShellCommand, + sudo: bool = False, + timeout: int = 300, + strip: bool = True, ) -> CommandArtifact: - """Run a local in band shell command + """Run a local in band shell command. Args: - command (str): command to run + command (ShellCommand): shell string (legacy) or argv list (preferred). sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. @@ -49,6 +55,13 @@ def run_command( Returns: CommandArtifact: command result object """ + if isinstance(command, str): + return self._run_shell_string(command, sudo=sudo, timeout=timeout, strip=strip) + return self._run_argv(command, sudo=sudo, timeout=timeout, strip=strip) + + def _run_shell_string( + self, command: str, *, sudo: bool, timeout: int, strip: bool + ) -> CommandArtifact: if sudo: command = f"sudo {command}" @@ -69,6 +82,29 @@ def run_command( exit_code=res.returncode, ) + def _run_argv( + self, argv: Sequence[str], *, sudo: bool, timeout: int, strip: bool + ) -> CommandArtifact: + exec_argv = build_exec_argv(argv, sudo=sudo) + display = format_argv_display(exec_argv) + + res = subprocess.run( + exec_argv, + encoding="utf-8", + shell=False, + errors="replace", + timeout=timeout, + capture_output=True, + check=False, + ) + + return CommandArtifact( + command=display, + stdout=res.stdout.strip() if strip else res.stdout, + stderr=res.stderr.strip() if strip else res.stderr, + exit_code=res.returncode, + ) + def read_file( self, filename: str, encoding: str = "utf-8", strip: bool = True ) -> BaseFileArtifact: diff --git a/nodescraper/connection/inband/inbandremote.py b/nodescraper/connection/inband/inbandremote.py index 9e2415ed..cca2acea 100644 --- a/nodescraper/connection/inband/inbandremote.py +++ b/nodescraper/connection/inband/inbandremote.py @@ -39,6 +39,12 @@ CommandArtifact, InBandConnection, ) +from .shellcommand import ( + ShellCommand, + build_exec_argv, + build_sudo_password_argv, + format_argv_display, +) from .sshparams import SSHConnectionParams @@ -123,15 +129,15 @@ def read_file( def run_command( self, - command: str, + command: ShellCommand, sudo=False, timeout: int = 30, strip: bool = True, ) -> CommandArtifact: - """Run a shell command over ssh + """Run a shell command over ssh. Args: - command (str): command to run + command (ShellCommand): shell string (legacy) or argv list (preferred). sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. @@ -140,13 +146,24 @@ def run_command( CommandArtifact: Command artifact with stdout, stderr, which have been decoded and stripped as well as exit code """ write_password = sudo and self.ssh_params.username != "root" and self.ssh_params.password - if write_password: - command = f"sudo -S -p '' {command}" - elif sudo: - command = f"sudo {command}" + + if isinstance(command, str): + cmd_str = command + if write_password: + cmd_str = f"sudo -S -p '' {cmd_str}" + elif sudo: + cmd_str = f"sudo {cmd_str}" + else: + argv = list(command) + if write_password: + cmd_str = format_argv_display(build_sudo_password_argv(argv)) + elif sudo: + cmd_str = format_argv_display(build_exec_argv(argv, sudo=True)) + else: + cmd_str = format_argv_display(argv) try: - stdin, stdout, stderr = self.client.exec_command(command, timeout=timeout) + stdin, stdout, stderr = self.client.exec_command(cmd_str, timeout=timeout) if write_password: stdin.write( @@ -166,7 +183,7 @@ def run_command( exit_code = 124 return CommandArtifact( - command=command, + command=cmd_str, stdout=stdout_str.strip() if strip else stdout_str, stderr=stderr_str.strip() if strip else stderr_str, exit_code=exit_code, diff --git a/nodescraper/connection/inband/shellcommand.py b/nodescraper/connection/inband/shellcommand.py new file mode 100644 index 00000000..cc9ae7ac --- /dev/null +++ b/nodescraper/connection/inband/shellcommand.py @@ -0,0 +1,70 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Sequence, Union + +from nodescraper.utils import shell_quote + +ShellCommand = Union[str, Sequence[str]] + + +def format_argv_display(argv: Sequence[str]) -> str: + """Return a shell-safe, human-readable command string for logging/artifacts. + + Args: + argv (Sequence[str]): argv tokens passed to subprocess or remote shell. + + Returns: + str: quoted command string suitable for CommandArtifact.command. + """ + return " ".join(shell_quote(arg) for arg in argv) + + +def build_exec_argv(argv: Sequence[str], *, sudo: bool = False) -> list[str]: + """Prepend sudo when requested. + + Args: + argv (Sequence[str]): base command argv. + sudo (bool, optional): whether to run with sudo. Defaults to False. + + Returns: + list[str]: argv for local subprocess.run(shell=False) or remote quoting. + """ + base = list(argv) + if sudo: + return ["sudo", *base] + return base + + +def build_sudo_password_argv(argv: Sequence[str]) -> list[str]: + """Build argv for sudo -S (password on stdin) over SSH. + + Args: + argv (Sequence[str]): base command argv. + + Returns: + list[str]: argv prefixed with sudo -S -p ''. + """ + return ["sudo", "-S", "-p", "", *list(argv)] diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 9163e6c5..c7cfe7e2 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -32,7 +32,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import get_exception_traceback, shell_quote +from nodescraper.utils import get_exception_traceback from .collector_args import NetworkCollectorArgs from .ethtool_vendor import ( @@ -730,16 +730,16 @@ def _check_network_connectivity(self, cmd: str, url: str) -> bool: ) # Determine ping options based on OS - ping_option = "-c 1" if self.system_info.os_family == OSFamily.LINUX else "-n 1" - - # Build command based on cmd parameter using class constants - qu = shell_quote(url) if cmd == "ping": - result = self._run_sut_cmd(f"{self.CMD_PING} {qu} {ping_option}") + if self.system_info.os_family == OSFamily.LINUX: + argv = [self.CMD_PING, url, "-c", "1"] + else: + argv = [self.CMD_PING, url, "-n", "1"] + result = self._run_sut_argv(argv) elif cmd == "wget": - result = self._run_sut_cmd(f"{self.CMD_WGET} {qu}") + result = self._run_sut_argv([self.CMD_WGET, url]) else: # curl - result = self._run_sut_cmd(f"{self.CMD_CURL} {qu}") + result = self._run_sut_argv([self.CMD_CURL, url]) if result.exit_code == 0: self._log_event( diff --git a/test/unit/connection/inband/test_shellcommand.py b/test/unit/connection/inband/test_shellcommand.py new file mode 100644 index 00000000..ae625610 --- /dev/null +++ b/test/unit/connection/inband/test_shellcommand.py @@ -0,0 +1,89 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock, patch + +from nodescraper.connection.inband.inbandlocal import LocalShell +from nodescraper.connection.inband.shellcommand import ( + build_exec_argv, + build_sudo_password_argv, + format_argv_display, +) + + +def test_format_argv_display_quotes_metacharacters(): + assert format_argv_display(["ping", "host; rm -rf /", "-c", "1"]) == ( + "'ping' 'host; rm -rf /' '-c' '1'" + ) + + +def test_build_exec_argv_prepends_sudo(): + assert build_exec_argv(["ip", "addr"], sudo=True) == ["sudo", "ip", "addr"] + + +def test_build_sudo_password_argv(): + assert build_sudo_password_argv(["journalctl", "-n", "1"]) == [ + "sudo", + "-S", + "-p", + "", + "journalctl", + "-n", + "1", + ] + + +@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") +def test_localshell_argv_uses_shell_false(mock_run): + mock_run.return_value = MagicMock(stdout="ok\n", stderr="", returncode=0) + + shell = LocalShell() + result = shell.run_command(["ping", "example.com", "-c", "1"]) + + mock_run.assert_called_once() + assert mock_run.call_args.kwargs["shell"] is False + assert mock_run.call_args.args[0] == ["ping", "example.com", "-c", "1"] + assert result.command == "'ping' 'example.com' '-c' '1'" + assert result.stdout == "ok" + + +@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") +def test_localshell_string_uses_shell_true(mock_run): + mock_run.return_value = MagicMock(stdout="ok\n", stderr="", returncode=0) + + shell = LocalShell() + shell.run_command("ip addr show") + + assert mock_run.call_args.kwargs["shell"] is True + + +@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") +def test_localshell_argv_with_sudo(mock_run): + mock_run.return_value = MagicMock(stdout="", stderr="", returncode=0) + + shell = LocalShell() + shell.run_command(["cat", "/etc/shadow"], sudo=True) + + assert mock_run.call_args.args[0] == ["sudo", "cat", "/etc/shadow"] diff --git a/test/unit/test_plugin_convention_warnings.py b/test/unit/test_plugin_convention_warnings.py new file mode 100644 index 00000000..e76ec733 --- /dev/null +++ b/test/unit/test_plugin_convention_warnings.py @@ -0,0 +1,169 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import ast +import importlib.util +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT_PATH = _REPO_ROOT / ".github" / "scripts" / "plugin_convention_warnings.py" + + +def _load_checker(): + spec = importlib.util.spec_from_file_location("plugin_convention_warnings", _SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +checker = _load_checker() + + +def _check_source(src: str, rel_path: str = "example_collector.py") -> list[str]: + tree = ast.parse(src) + return checker._check_shell_quoting(Path(rel_path), tree) + + +def test_shell_quote_rule_flags_args_in_f_string(): + src = """ +class ExampleCollector: + def collect_data(self, args): + self._run_sut_cmd(f"ping {args.url}") +""" + msgs = _check_source(src) + assert len(msgs) == 1 + assert "shell_quote" in msgs[0] + assert "args.url" in msgs[0] + + +def test_shell_quote_rule_flags_unquoted_format_kwarg(): + src = """ +class ExampleCollector: + CMD_TMPL = "grep . -H -r -i {rocm_path}/.info/*" + + def collect_data(self, args): + self._run_sut_cmd(self.CMD_TMPL.format(rocm_path=args.rocm_path)) +""" + msgs = _check_source(src) + assert len(msgs) == 1 + assert "args.rocm_path" in msgs[0] + + +def test_shell_quote_rule_allows_quoted_format_kwarg(): + src = """ +class ExampleCollector: + CMD_TMPL = "grep . -H -r -i {rocm_path}/.info/*" + + def collect_data(self, args): + rocm_path_q = shell_quote(args.rocm_path) + self._run_sut_cmd(self.CMD_TMPL.format(rocm_path=rocm_path_q)) +""" + assert _check_source(src) == [] + + +def test_shell_quote_rule_allows_inline_shell_quote(): + src = """ +class ExampleCollector: + def _get_afid(self, cper_file_path): + cmd = self.CMD_RAS_AFID.format(cper_file=shell_quote(cper_file_path)) + self._run_amd_smi(cmd) +""" + assert _check_source(src) == [] + + +def test_shell_quote_rule_flags_sensitive_function_parameter(): + src = """ +class ExampleCollector: + def _probe(self, url): + self._run_sut_cmd(f"curl {url}") +""" + msgs = _check_source(src) + assert len(msgs) == 1 + assert "url" in msgs[0] + + +def test_shell_quote_rule_ignores_internal_cmd_parameter(): + src = """ +class ExampleCollector: + def _run_amd_smi(self, cmd): + self._run_sut_cmd(f"{self.AMD_SMI_EXE} {cmd}") +""" + assert _check_source(src) == [] + + +def test_shell_quote_rule_flags_reassigned_cmd_variable(): + src = """ +class ExampleCollector: + CMD = "journalctl --no-pager" + + def read(self, args): + cmd = self.CMD + if args is not None and args.boot: + cmd = f"{self.CMD} -b {args.boot}" + self._run_sut_cmd(cmd, sudo=True) +""" + msgs = _check_source(src) + assert len(msgs) == 1 + assert "args.boot" in msgs[0] + + +def test_shell_quote_rule_skips_generic_collection_collector(): + src = """ +class GenericCollectionCollector: + def collect_data(self, args): + for cmd_spec in args.commands: + command = cmd_spec.command.strip() + self._run_sut_cmd(command) +""" + assert _check_source(src) == [] + + +def test_shell_quote_rule_allows_system_derived_values(): + src = """ +class ExampleCollector: + def collect(self, iface): + cmd = self.CMD_ETHTOOL_TEMPLATE.format(interface=iface.name) + self._run_sut_cmd(cmd, sudo=True) +""" + assert _check_source(src) == [] + + +@pytest.mark.parametrize( + "collector_file", + [ + "nodescraper/plugins/inband/network/network_collector.py", + "nodescraper/plugins/inband/rocm/rocm_collector.py", + "nodescraper/plugins/inband/amdsmi/amdsmi_collector.py", + ], +) +def test_fixed_collectors_have_no_shell_quote_warnings(collector_file): + path = _REPO_ROOT / collector_file + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + rel = path.relative_to(_REPO_ROOT) + msgs = checker._check_shell_quoting(rel, tree) + assert msgs == [] From 518e93214038d674fb597048c784bc6167094bfc Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Mon, 20 Jul 2026 14:42:09 -0700 Subject: [PATCH 11/21] removed ethtool blanket analyzer arg --- .../plugins/inband/network/analyzer_args.py | 40 --- .../plugins/inband/network/ethtool_vendor.py | 12 +- .../inband/network/network_analyzer.py | 71 +--- .../inband/network/network_collector.py | 9 +- .../plugins/inband/network/network_plugin.py | 5 +- .../plugins/inband/network/networkdata.py | 6 +- test/unit/plugin/test_network_analyzer.py | 302 +----------------- 7 files changed, 19 insertions(+), 426 deletions(-) delete mode 100644 nodescraper/plugins/inband/network/analyzer_args.py diff --git a/nodescraper/plugins/inband/network/analyzer_args.py b/nodescraper/plugins/inband/network/analyzer_args.py deleted file mode 100644 index f2e63047..00000000 --- a/nodescraper/plugins/inband/network/analyzer_args.py +++ /dev/null @@ -1,40 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Optional, Union - -from pydantic import Field - -from nodescraper.base.regexanalyzer import ErrorRegex -from nodescraper.models import AnalyzerArgs - - -class NetworkAnalyzerArgs(AnalyzerArgs): - """Arguments for the network analyzer plugin.""" - - error_regex: Optional[Union[list[ErrorRegex], list[dict]]] = Field( - default=None, - description="Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.", - ) diff --git a/nodescraper/plugins/inband/network/ethtool_vendor.py b/nodescraper/plugins/inband/network/ethtool_vendor.py index 60aaa4ca..a631ba03 100644 --- a/nodescraper/plugins/inband/network/ethtool_vendor.py +++ b/nodescraper/plugins/inband/network/ethtool_vendor.py @@ -675,16 +675,12 @@ class Cx7EthtoolStatistics(BaseModel): Cx7EthtoolStatistics, ] -VendorEthtoolStatisticsCls = Union[ - type[PollaraEthtoolStatistics], - type[Thor2EthtoolStatistics], - type[Cx7EthtoolStatistics], -] + +VendorEthtoolStatisticsCls = type[VendorEthtoolStatisticsModel] -# Map ifname prefixes to vendor-specific statistic models -# If netdev is ens, use Cx7 -# If netdev is benic, check if it starts with ionic or bnxt to determine if it's Pollara or Thor2 +# Map kernel driver name prefixes (from `ethtool -i`) to vendor-specific models +# ionic -> Pollara, bnxt -> Thor2, mlx -> ConnectX-7 VENDOR_PREFIX_MAP: dict[str, VendorEthtoolStatisticsCls] = { "ionic": PollaraEthtoolStatistics, "bnxt": Thor2EthtoolStatistics, diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index eb590651..90819e09 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -24,34 +24,25 @@ # ############################################################################### import re -from typing import Optional -from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer +from nodescraper.base.regexanalyzer import RegexAnalyzer from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.models import TaskResult -from .analyzer_args import NetworkAnalyzerArgs from .networkdata import NetworkDataModel -class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]): +class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, None]): """Check network statistics for errors.""" DATA_MODEL = NetworkDataModel - # No built-in regex patterns: RDMA-scoped counter classification lives in the vendor - # ethtool models (ethtool_vendor.py). This list is only extended by user-supplied - # NetworkAnalyzerArgs.error_regex patterns. - ERROR_REGEX: list[ErrorRegex] = [] - - def analyze_data( - self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None - ) -> TaskResult: - """Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex. + def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: + """Analyze ethtool -S statistics via RDMA-scoped vendor models. Args: - data: Network data model with ethtool_info and/or ethtool_statistics. - args: Optional analyzer arguments with custom error regex support. + data: Network data model with ethtool_statistics. + args: Unused; retained for analyzer interface compatibility. Returns: TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. @@ -61,52 +52,6 @@ def analyze_data( self.result.status = ExecutionStatus.WARNING return self.result - if not args: - args = NetworkAnalyzerArgs() - - final_error_regex = self._convert_and_extend_error_regex(args.error_regex, self.ERROR_REGEX) - - regex_error = False - regex_warning = False - for interface_name, ethtool_info in data.ethtool_info.items(): - matches_on_interface: list[tuple[str, int, EventPriority]] = [] - for stat_name, stat_value in ethtool_info.statistics.items(): - for error_regex_obj in final_error_regex: - if error_regex_obj.regex.match(stat_name): - try: - value = int(stat_value) - except (ValueError, TypeError): - break - - if value > 0: - matches_on_interface.append( - (stat_name, value, error_regex_obj.event_priority) - ) - break - - if matches_on_interface: - has_error = any( - priority == EventPriority.ERROR for _, _, priority in matches_on_interface - ) - if has_error: - regex_error = True - else: - regex_warning = True - priority = EventPriority.ERROR if has_error else EventPriority.WARNING - severity = "error" if has_error else "warning" - match_names = [match[0] for match in matches_on_interface] - matches_data = {name: value for name, value, _ in matches_on_interface} - self._log_event( - category=EventCategory.NETWORK, - description=f"Network {severity} detected on {interface_name}: [{', '.join(match_names)}]", - data={ - "interface": interface_name, - "errors": matches_data, - }, - priority=priority, - console_log=True, - ) - vendor_error = False vendor_warning = False vendor_error_fields: set[str] = set() @@ -214,10 +159,10 @@ def analyze_data( ", ".join(sorted(vendor_queue_warning_fields)), ) - if regex_error or vendor_error: + if vendor_error: self.result.message = "Network errors detected in statistics" self.result.status = ExecutionStatus.ERROR - elif regex_warning or vendor_warning: + elif vendor_warning: self.result.message = "Network warning counters non-zero in statistics" self.result.status = ExecutionStatus.WARNING else: diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 942ae93e..49159e91 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -529,6 +529,9 @@ def _collect_ethtool_statistic(self, netdev: str, driver: str) -> Optional[Ethto The vendor model is selected by matching ``driver`` (from ``ethtool -i``) against the known vendor prefixes in ``VENDOR_PREFIX_MAP``. + + Returns: + EthtoolStatistics, or None on command failure. """ cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=netdev) res = self._run_sut_cmd(cmd_s, sudo=True) @@ -613,11 +616,7 @@ def _collect_ethtool_statistics( """Collect ethtool -S for vendor netdevs discovered from `ip addr`. Each interface's driver is resolved via `ethtool -i`; only interfaces whose - driver matches a known vendor prefix (see VENDOR_PREFIX_MAP) are collected. The - matched driver selects the respective vendor data model used to parse the - `ethtool -S` counters. Interfaces without a known vendor driver (lo, docker, - veth, ...) are skipped, which avoids noisy `ethtool -S` failures. This does not - depend on RDMA. + driver matches a known vendor prefix (see VENDOR_PREFIX_MAP) are collected. Args: interfaces: Interfaces discovered from `ip addr` to consider. diff --git a/nodescraper/plugins/inband/network/network_plugin.py b/nodescraper/plugins/inband/network/network_plugin.py index 7e1fe518..69e7b671 100644 --- a/nodescraper/plugins/inband/network/network_plugin.py +++ b/nodescraper/plugins/inband/network/network_plugin.py @@ -25,14 +25,13 @@ ############################################################################### from nodescraper.base import InBandDataPlugin -from .analyzer_args import NetworkAnalyzerArgs from .collector_args import NetworkCollectorArgs from .network_analyzer import NetworkAnalyzer from .network_collector import NetworkCollector from .networkdata import NetworkDataModel -class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, NetworkAnalyzerArgs]): +class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, None]): """Plugin for collection of network configuration data""" DATA_MODEL = NetworkDataModel @@ -42,5 +41,3 @@ class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, Net COLLECTOR_ARGS = NetworkCollectorArgs ANALYZER = NetworkAnalyzer - - ANALYZER_ARGS = NetworkAnalyzerArgs diff --git a/nodescraper/plugins/inband/network/networkdata.py b/nodescraper/plugins/inband/network/networkdata.py index 25b5f226..86e8cb6f 100644 --- a/nodescraper/plugins/inband/network/networkdata.py +++ b/nodescraper/plugins/inband/network/networkdata.py @@ -105,8 +105,6 @@ class EthtoolInfo(BaseModel): port: Optional[str] = None # Port type (e.g., "Twisted Pair") auto_negotiation: Optional[str] = None # Auto-negotiation status (e.g., "on", "off") link_detected: Optional[str] = None # Link detection status (e.g., "yes", "no") - # ethtool -S (statistics) output: parsed key-value for error/health analysis - statistics: Dict[str, str] = Field(default_factory=dict) class NetworkDataModel(DataModel): @@ -116,9 +114,7 @@ class NetworkDataModel(DataModel): routes: List[Route] = Field(default_factory=list) rules: List[RoutingRule] = Field(default_factory=list) neighbors: List[Neighbor] = Field(default_factory=list) - ethtool_info: Dict[str, EthtoolInfo] = Field( - default_factory=dict - ) # Interface name -> EthtoolInfo mapping + ethtool_info: Dict[str, EthtoolInfo] = Field(default_factory=dict) # ethtool -S vendor-parsed counters for netdevs whose driver matches a known vendor ethtool_netdevs: List[str] = Field(default_factory=list) ethtool_statistics: List[EthtoolStatistics] = Field(default_factory=list) diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 2eb74490..9796a173 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -26,7 +26,6 @@ import pytest from nodescraper.enums import EventPriority, ExecutionStatus -from nodescraper.plugins.inband.network.analyzer_args import NetworkAnalyzerArgs from nodescraper.plugins.inband.network.ethtool_vendor import ( EthtoolStatistics, Thor2EthtoolStatistics, @@ -37,16 +36,6 @@ NetworkDataModel, ) -# Built-in regex defaults were removed; the vendor ethtool models own RDMA counter -# classification. These custom patterns exercise the user-supplied regex path. -CUSTOM_REGEX = [ - { - "regex": r"^custom_err_\d+$", - "message": "custom err counter non-zero", - "event_category": "NETWORK", - } -] - @pytest.fixture def network_analyzer(system_info): @@ -55,31 +44,10 @@ def network_analyzer(system_info): @pytest.fixture def clean_ethtool_info(): - """EthtoolInfo with no errors (all counters zero).""" + """EthtoolInfo with no vendor statistics.""" return EthtoolInfo( interface="eth0", raw_output="dummy output", - statistics={ - "tx_pfc_frames": "0", - "tx_pfc_ena_frames_pri0": "0", - "tx_pfc_ena_frames_pri1": "0", - "tx_pfc_ena_frames_pri2": "0", - "tx_pfc_ena_frames_pri3": "0", - "tx_pfc_ena_frames_pri4": "0", - "tx_pfc_ena_frames_pri5": "0", - "tx_pfc_ena_frames_pri6": "0", - "tx_pfc_ena_frames_pri7": "0", - "pfc_pri0_tx_transitions": "0", - "pfc_pri1_tx_transitions": "0", - "pfc_pri2_tx_transitions": "0", - "pfc_pri3_tx_transitions": "0", - "pfc_pri4_tx_transitions": "0", - "pfc_pri5_tx_transitions": "0", - "pfc_pri6_tx_transitions": "0", - "pfc_pri7_tx_transitions": "0", - "some_other_stat": "100", # Should be ignored - "rx_bytes": "1234567", # Should be ignored - }, ) @@ -101,88 +69,6 @@ def test_no_errors_detected(network_analyzer, clean_network_model): assert len(result.events) == 0 -def test_single_custom_match_detected(network_analyzer): - """A single non-zero counter matched by a user-supplied custom regex reports as ERROR.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_err_0": "5"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert "errors detected" in result.message - assert len(result.events) == 1 - assert result.events[0].description == "Network error detected on eth0: [custom_err_0]" - assert result.events[0].priority == EventPriority.ERROR - assert result.events[0].data["errors"] == {"custom_err_0": 5} - assert result.events[0].data["interface"] == "eth0" - - -def test_multiple_matches_same_interface(network_analyzer): - """Multiple non-zero counters on one interface produce a single grouped event.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_err_0": "10", "custom_err_1": "3", "custom_err_2": "7"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert "errors detected" in result.message - assert len(result.events) == 1 # one event per interface - assert result.events[0].priority == EventPriority.ERROR - # Check all 3 counters are present - assert len(result.events[0].data["errors"]) == 3 - assert result.events[0].data["errors"]["custom_err_0"] == 10 - assert result.events[0].data["errors"]["custom_err_1"] == 3 - assert result.events[0].data["errors"]["custom_err_2"] == 7 - - -def test_multiple_interfaces_with_matches(network_analyzer): - """Test with custom-regex matches across multiple interfaces.""" - eth0 = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "15", - "custom_err_1": "0", - }, - ) - eth1 = EthtoolInfo( - interface="eth1", - raw_output="dummy", - statistics={ - "custom_err_3": "8", - }, - ) - eth2 = EthtoolInfo( - interface="eth2", - raw_output="dummy", - statistics={ - "custom_err_7": "100", - }, - ) - model = NetworkDataModel( - ethtool_info={ - "eth0": eth0, - "eth1": eth1, - "eth2": eth2, - } - ) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 3 - interfaces = {event.data["interface"] for event in result.events} - assert interfaces == {"eth0", "eth1", "eth2"} - - def test_empty_ethtool_info(network_analyzer): """Test with empty ethtool_info and no RDMA ethtool: WARNING and message logged.""" model = NetworkDataModel(ethtool_info={}) @@ -242,163 +128,6 @@ def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): assert len(result.events) == 0 -def test_custom_regex_matches_multi_digit_suffixes(network_analyzer): - """Test that a user-supplied \\d+ pattern matches single- and double-digit suffixes.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "1", - "custom_err_3": "2", - "custom_err_7": "3", - "custom_err_10": "4", # Test double-digit - "custom_err_15": "5", # Test double-digit - }, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - # All 5 counters should be detected - assert len(result.events[0].data["errors"]) == 5 - - -def test_non_numeric_values_ignored(network_analyzer): - """Test that non-numeric values in statistics are gracefully ignored.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "N/A", # Non-numeric - "custom_err_1": "invalid", # Non-numeric - "custom_err_2": "5", # Valid - }, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - # Only the valid numeric counter should be reported - assert len(result.events[0].data["errors"]) == 1 - assert result.events[0].data["errors"]["custom_err_2"] == 5 - - -def test_zero_values_not_reported(network_analyzer): - """Test that zero values are not reported as errors.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "0", - "custom_err_1": "0", - "custom_err_2": "0", - }, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.OK - assert len(result.events) == 0 - - -def test_non_matching_fields_ignored(network_analyzer): - """Test that statistics not matching error patterns are ignored.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "rx_bytes": "999999999", # High value but not a matched field - "tx_bytes": "888888888", # High value but not a matched field - "some_random_counter": "12345", # Not a matched field - "custom_err_0": "5", # This SHOULD be detected - }, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - # Only custom_err_0 should be reported - assert len(result.events[0].data["errors"]) == 1 - assert "custom_err_0" in result.events[0].data["errors"] - - -def test_mixed_interfaces_with_and_without_errors(network_analyzer): - """Test with some interfaces having matches and others clean.""" - eth0_error = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_err_0": "10", - }, - ) - eth1_clean = EthtoolInfo( - interface="eth1", - raw_output="dummy", - statistics={ - "custom_err_0": "0", - "custom_err_1": "0", - }, - ) - eth2_error = EthtoolInfo( - interface="eth2", - raw_output="dummy", - statistics={ - "custom_err_5": "20", - }, - ) - model = NetworkDataModel( - ethtool_info={ - "eth0": eth0_error, - "eth1": eth1_clean, - "eth2": eth2_error, - } - ) - result = network_analyzer.analyze_data( - model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) - ) - assert result.status == ExecutionStatus.ERROR - # Only 2 events (eth0 and eth2), eth1 should not generate an event - assert len(result.events) == 2 - interfaces_with_errors = {event.data["interface"] for event in result.events} - assert interfaces_with_errors == {"eth0", "eth2"} - - -def test_custom_error_regex_detected(network_analyzer): - """Test that user-supplied custom regex patterns are applied (no built-in defaults).""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={ - "custom_tx_drops": "9", # Matched via custom regex only - "tx_pfc_frames": "0", # No built-in pattern; ignored unless RDMA vendor-scoped - }, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - args = NetworkAnalyzerArgs( - error_regex=[ - { - "regex": r"^custom_tx_drops$", - "message": "Custom tx drops", - "event_category": "NETWORK", - } - ] - ) - - result = network_analyzer.analyze_data(model, args=args) - - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - assert result.events[0].data["interface"] == "eth0" - assert result.events[0].data["errors"] == {"custom_tx_drops": 9} - - def test_queue_counter_error_detected(network_analyzer): """A non-zero per-queue error counter (e.g. rx_discards) is flagged as ERROR.""" stat = EthtoolStatistics( @@ -598,32 +327,3 @@ def test_queue_counter_both_lists_empty_skips_all_checks(network_analyzer, monke result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 - - -def test_custom_warning_priority_regex(network_analyzer): - """A user-supplied WARNING-priority custom pattern yields WARNING status via the regex path.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_warn": "3"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - args = NetworkAnalyzerArgs( - error_regex=[ - { - "regex": r"^custom_warn$", - "message": "custom warn counter", - "event_category": "NETWORK", - "event_priority": "WARNING", - } - ] - ) - - result = network_analyzer.analyze_data(model, args=args) - - assert result.status == ExecutionStatus.WARNING - assert "warning counters" in result.message - assert len(result.events) == 1 - assert result.events[0].priority == EventPriority.WARNING - assert result.events[0].description == "Network warning detected on eth0: [custom_warn]" - assert result.events[0].data["errors"] == {"custom_warn": 3} From eed21fe56b4f4296ca38c53a36ada4e55dc0e35e Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 11:33:24 -0500 Subject: [PATCH 12/21] initial commit --- config/redfish_events_daemon.example.json | 43 +++ docs/REDFISH_EVENTS_DAEMON.md | 80 +++++ nodescraper/cli/cli.py | 16 + nodescraper/cli/daemon_cmd.py | 52 +++ .../plugins/serviceability/__init__.py | 3 + .../plugins/serviceability/analysis_window.py | 193 +++++++++++ .../serviceability/mi3xx/mi3xx_analyzer.py | 146 +------- nodescraper/redfish_events/__init__.py | 101 ++++++ nodescraper/redfish_events/client.py | 107 ++++++ nodescraper/redfish_events/config.py | 87 +++++ nodescraper/redfish_events/daemon_config.py | 88 +++++ nodescraper/redfish_events/daemon_http.py | 202 +++++++++++ nodescraper/redfish_events/daemon_runner.py | 143 ++++++++ nodescraper/redfish_events/log_baseline.py | 252 ++++++++++++++ nodescraper/redfish_events/models.py | 94 +++++ nodescraper/redfish_events/parsing.py | 89 +++++ nodescraper/redfish_events/se_bridge.py | 58 ++++ nodescraper/redfish_events/sse_capability.py | 206 +++++++++++ nodescraper/redfish_events/sse_subscriber.py | 326 ++++++++++++++++++ .../redfish_events/subscriber_manager.py | 284 +++++++++++++++ nodescraper/redfish_events/trigger_engine.py | 112 ++++++ .../redfish_events/webhook_subscriber.py | 221 ++++++++++++ pyproject.toml | 12 + test/unit/plugins/test_analysis_window.py | 54 +++ test/unit/redfish_events/test_daemon_flow.py | 138 ++++++++ test/unit/redfish_events/test_daemon_http.py | 88 +++++ 26 files changed, 3066 insertions(+), 129 deletions(-) create mode 100644 config/redfish_events_daemon.example.json create mode 100644 docs/REDFISH_EVENTS_DAEMON.md create mode 100644 nodescraper/cli/daemon_cmd.py create mode 100644 nodescraper/plugins/serviceability/analysis_window.py create mode 100644 nodescraper/redfish_events/__init__.py create mode 100644 nodescraper/redfish_events/client.py create mode 100644 nodescraper/redfish_events/config.py create mode 100644 nodescraper/redfish_events/daemon_config.py create mode 100644 nodescraper/redfish_events/daemon_http.py create mode 100644 nodescraper/redfish_events/daemon_runner.py create mode 100644 nodescraper/redfish_events/log_baseline.py create mode 100644 nodescraper/redfish_events/models.py create mode 100644 nodescraper/redfish_events/parsing.py create mode 100644 nodescraper/redfish_events/se_bridge.py create mode 100644 nodescraper/redfish_events/sse_capability.py create mode 100644 nodescraper/redfish_events/sse_subscriber.py create mode 100644 nodescraper/redfish_events/subscriber_manager.py create mode 100644 nodescraper/redfish_events/trigger_engine.py create mode 100644 nodescraper/redfish_events/webhook_subscriber.py create mode 100644 test/unit/plugins/test_analysis_window.py create mode 100644 test/unit/redfish_events/test_daemon_flow.py create mode 100644 test/unit/redfish_events/test_daemon_http.py diff --git a/config/redfish_events_daemon.example.json b/config/redfish_events_daemon.example.json new file mode 100644 index 00000000..e50b2d3f --- /dev/null +++ b/config/redfish_events_daemon.example.json @@ -0,0 +1,43 @@ +{ + "stream": { + "event_types": ["Alert", "StatusChange"], + "severities": ["Warning", "Critical"], + "reconnect_delay_seconds": 30, + "baseline_pull_enabled": true, + "enable_webhook_fallback": true, + "allow_loopback_webhook": true, + "dedupe_events": true + }, + "targets": [ + { + "target_key": "node-a", + "name": "MI300X Node A", + "host": "10.0.0.10", + "username": "root", + "password": "changeme", + "verify_ssl": false, + "transport": "auto", + "webhook_url": "http://collector-host:8081/hook/node-a" + } + ], + "trigger": { + "min_events": 3, + "window_seconds": 10, + "cooldown_seconds": 60 + }, + "analysis": { + "hub_python_module": "your_hub_package.hub_module", + "hub_display_name": "MI3XX Service Hub", + "afid_sag_path": "/path/to/AFID_SAG.json", + "hub_analyze_method": "get_service_info", + "skip_hub": false + }, + "http": { + "enabled": true, + "host": "0.0.0.0", + "port": 8081, + "webhook_path_prefix": "/hook", + "recommendations_path": "/recommendations", + "status_path": "/status" + } +} diff --git a/docs/REDFISH_EVENTS_DAEMON.md b/docs/REDFISH_EVENTS_DAEMON.md new file mode 100644 index 00000000..f21dd1bf --- /dev/null +++ b/docs/REDFISH_EVENTS_DAEMON.md @@ -0,0 +1,80 @@ +# Redfish event daemon + +The Redfish event daemon is a long-running process that subscribes to BMC event streams (SSE or webhook fallback), batches bursts of events, and runs the same serviceability analysis path as on-demand `node-scraper run-plugins`. + +## Install + +```bash +cd ~/node-scraper_public +python3 -m venv .venv && source .venv/bin/activate +pip install -e ".[dev,events]" +``` + +The `[events]` extra pulls in `httpx`, required for async Redfish ingest. + +## Run + +```bash +node-scraper daemon --daemon-config config/redfish_events_daemon.example.json +``` + +Copy the example config, set BMC credentials, hub module paths, and (for webhook transport) point each target's `webhook_url` at this daemon's HTTP listener. + +## On-demand CLI vs daemon + +| | On-demand (`run-plugins`) | Daemon (`daemon`) | +|---|---|---| +| Lifecycle | One-shot collection + analysis | Long-lived background process | +| Event source | Redfish log pull during plugin run | Continuous SSE or webhook ingest | +| Analysis trigger | After collection completes | Sliding window (default: 3 events in 10s) | +| Output | Log directory + artifacts | Logs + HTTP `/recommendations` JSON | +| Use when | Ad-hoc debug, CI, field capture | Live monitoring, NOC integration | + +The on-demand MI3XX plugin path is unchanged. Both flows call `analyze_serviceability_window()` so hub configuration stays aligned. + +## Configuration + +Top-level sections in the daemon JSON: + +- **stream** — global ingest settings (`EventStreamConfig`): severities, dedupe, baseline re-pull, webhook fallback. +- **targets** — one entry per BMC (`EventTargetConfig`): host, credentials, transport (`auto`, `sse`, or `webhook`), optional `webhook_url`. +- **trigger** — sliding window thresholds (`TriggerConfig`): `min_events`, `window_seconds`, `cooldown_seconds`. +- **analysis** — same fields as `ServiceabilityAnalyzerArgs` in plugin configs (`hub_python_module`, `afid_sag_path`, etc.). +- **http** — optional listener for webhook ingest and live recommendations. + +See `config/redfish_events_daemon.example.json`. + +### Webhook transport + +When SSE is unavailable, set `transport` to `webhook` (or rely on `auto` + `enable_webhook_fallback`) and configure: + +1. `http.enabled: true` on the daemon with a reachable host/port. +2. Each target's `webhook_url` → `http://:/hook/`. +3. `allow_loopback_webhook: true` only for local testing (BMCs cannot reach loopback). + +### Trigger engine + +Events accumulate per `target_key`. When at least `min_events` arrive within `window_seconds`, the daemon runs serviceability analysis on that batch and ignores further triggers until `cooldown_seconds` elapse. + +## HTTP endpoints + +When `http.enabled` is true (default): + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/status` | Daemon health and per-target transport state | +| GET | `/recommendations` | Latest analysis snapshot for all targets | +| GET | `/recommendations/{target_key}` | Latest snapshot for one target | +| POST | `/hook/{target_key}` | Webhook payload → `handle_webhook_payload()` | + +Recommendations responses include `serviceability` (hub output), `afid_events`, and trigger metadata. + +## Tests + +```bash +pytest test/unit/redfish_events/ test/unit/plugins/test_analysis_window.py -v +``` + +## Deployment notes + +Systemd unit files and Docker images are not bundled yet. Run the daemon under your process supervisor of choice; ensure the HTTP port is reachable from BMCs when using webhook transport. diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 30dc8792..452ab123 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -335,6 +335,16 @@ def build_parser( help="Redfish path to LogService (e.g. redfish/v1/Systems/UBB/LogServices/DiagLogs)", ) + daemon_parser = subparsers.add_parser( + "daemon", + help="Run the long-lived Redfish event daemon (requires amd-node-scraper[events])", + ) + daemon_parser.add_argument( + "--daemon-config", + required=True, + help="Path to daemon JSON config (see config/redfish_events_daemon.example.json)", + ) + config_builder_parser.add_argument( "--plugins", nargs="*", @@ -547,6 +557,12 @@ def main( ) sys.exit(0) + if parsed_args.subcmd == "daemon": + from nodescraper.cli.daemon_cmd import run_daemon_cli + + run_daemon_cli(parsed_args.daemon_config, logger) + sys.exit(0) + if parsed_args.subcmd == "show-redfish-oem-allowable": if not parsed_args.connection_config: parser.error("show-redfish-oem-allowable requires --connection-config") diff --git a/nodescraper/cli/daemon_cmd.py b/nodescraper/cli/daemon_cmd.py new file mode 100644 index 00000000..ae51f99b --- /dev/null +++ b/nodescraper/cli/daemon_cmd.py @@ -0,0 +1,52 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import logging +import sys + +from pydantic import ValidationError + +from nodescraper.redfish_events.daemon_config import load_daemon_config + + +def run_daemon_cli(config_path: str, logger: logging.Logger) -> None: + """Load daemon config and run until interrupted.""" + from nodescraper.redfish_events.daemon_runner import run_event_daemon + + try: + config = load_daemon_config(config_path) + except FileNotFoundError as exc: + logger.error("%s", exc) + sys.exit(1) + except ValueError as exc: + logger.error("Invalid daemon config: %s", exc) + sys.exit(1) + except ValidationError as exc: + logger.error("Invalid daemon config: %s", exc) + sys.exit(1) + logger.info("Starting Redfish event daemon from %s", config_path) + run_event_daemon(config) diff --git a/nodescraper/plugins/serviceability/__init__.py b/nodescraper/plugins/serviceability/__init__.py index c5e9f857..e8bf8916 100644 --- a/nodescraper/plugins/serviceability/__init__.py +++ b/nodescraper/plugins/serviceability/__init__.py @@ -24,6 +24,7 @@ # ############################################################################### from .afid_events import build_afid_events_from_data +from .analysis_window import ServiceabilityWindowResult, analyze_serviceability_window from .analyzer_args import ServiceabilityAnalyzerArgs from .mi3xx import ( MI3XXAnalyzer, @@ -75,7 +76,9 @@ "ServiceabilityPluginMI3XX", "ServiceabilityResult", "ServiceabilitySolution", + "ServiceabilityWindowResult", "TimeOperator", + "analyze_serviceability_window", "build_afid_events_from_data", "build_mi3xx_reporting_version_fields", "compare_iso_datetime", diff --git a/nodescraper/plugins/serviceability/analysis_window.py b/nodescraper/plugins/serviceability/analysis_window.py new file mode 100644 index 00000000..4d605e5e --- /dev/null +++ b/nodescraper/plugins/serviceability/analysis_window.py @@ -0,0 +1,193 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Shared serviceability analysis path for on-demand plugins and the event daemon.""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, Optional + +from .afid_events import build_afid_events_from_data +from .analyzer_args import ServiceabilityAnalyzerArgs +from .cper_decode import CperDecodeError, decode_cper_raw_attachments +from .se_models import AfidEvent, ServiceabilityBlock +from .se_runner import SeRunError, run_service_hub +from .serviceability_data import ServiceabilityDataModel + + +@dataclass +class ServiceabilityWindowResult: + """Outcome of analyze_serviceability_window for CLI plugins or the event daemon.""" + + ok: bool + message: str + afid_events: list[AfidEvent] + serviceability: Optional[ServiceabilityBlock] = None + error: Optional[str] = None + + +def _cper_raw_needing_decode(data: ServiceabilityDataModel) -> dict[str, str]: + """Return CPER attachments that still need configured decode.""" + from .mi3xx.mi3xx_cper_utils import should_skip_cper_fetch_or_decode + + raw = data.cper_raw or {} + if not raw: + return {} + by_id: dict[str, dict[str, Any]] = {} + for member in data.rf_events: + if not isinstance(member, dict): + continue + eid = member.get("Id") + if eid is not None: + by_id[str(eid)] = member + out: dict[str, str] = {} + for event_id, blob in raw.items(): + ev = by_id.get(str(event_id)) + if ev is not None and should_skip_cper_fetch_or_decode(ev): + continue + out[str(event_id)] = blob + return out + + +def analyze_serviceability_window( + data: ServiceabilityDataModel, + args: ServiceabilityAnalyzerArgs, + *, + logger: Optional[logging.Logger] = None, + parent: str = "analyze_serviceability_window", +) -> ServiceabilityWindowResult: + """Build AFID events and optionally run the configured service hub on rf_events.""" + log = logger or logging.getLogger(__name__) + events = data.afid_events or build_afid_events_from_data(data) + data.afid_events = events + + if args.skip_hub: + block = ServiceabilityBlock(afid_events=events) + data.serviceability = block + return ServiceabilityWindowResult( + ok=True, + message=f"Built {len(events)} AFID event(s); hub skipped", + afid_events=events, + serviceability=block, + ) + + cper_data = data.cper_data or {} + cper_raw_to_decode = _cper_raw_needing_decode(data) + skipped_cper = len(data.cper_raw or {}) - len(cper_raw_to_decode) + if skipped_cper: + from .mi3xx.mi3xx_cper_utils import CPER_METHOD_AFID_MAX + + log.info( + "(%s) Skipping CPER decode for %d CPER attachment(s); Redfish log " + "already has usable ACA fields (CPER-method AFID<=%s or no serial on decode)", + parent, + skipped_cper, + CPER_METHOD_AFID_MAX, + ) + if cper_raw_to_decode and not cper_data: + if not args.cper_decode_module: + log.warning( + "(%s) %d CPER attachment(s) collected but cper_decode_module is " + "not set in analysis_args; skipping CPER decode", + parent, + len(cper_raw_to_decode), + ) + else: + log.info( + "(%s) Decoding %d CPER attachment(s) via %s.%s", + parent, + len(cper_raw_to_decode), + args.cper_decode_module, + args.cper_decode_method, + ) + try: + cper_data = decode_cper_raw_attachments( + cper_raw_to_decode, + cper_decode_module=args.cper_decode_module, + cper_decode_method=args.cper_decode_method, + logger=log, + ) + data.cper_data = cper_data + log.info( + "(%s) CPER decode finished: %d of %d attachment(s) decoded", + parent, + len(cper_data), + len(cper_raw_to_decode), + ) + except CperDecodeError as exc: + log.warning("(%s) %s; continuing without decoded CPER", parent, exc) + elif cper_data: + log.info( + "(%s) Using %d pre-decoded CPER record(s) from collection", + parent, + len(cper_data), + ) + + try: + block = run_service_hub( + hub_python_module=args.hub_python_module, # type: ignore[arg-type] + hub_display_name=args.hub_display_name, + afid_events=events, + afid_sag_path=args.afid_sag_path, # type: ignore[arg-type] + rf_events=data.rf_events, + cper_data=cper_data or None, + hub_options=args.resolved_hub_options(), + hub_analyze_method=args.hub_analyze_method, + hub_init_path_kwarg=args.hub_init_path_kwarg, + ) + except (SeRunError, ValueError) as exc: + return ServiceabilityWindowResult( + ok=False, + message=str(exc), + afid_events=events, + error=str(exc), + ) + + data.serviceability = block + hub_label = args.hub_display_name or args.hub_python_module + cper_summary = "" + if cper_data: + cper_summary = f", {len(cper_data)} decoded CPER(s)" + elif cper_raw_to_decode: + cper_summary = f", {len(cper_raw_to_decode)} CPER attachment(s) not decoded" + elif data.cper_raw: + cper_summary = f", {len(data.cper_raw)} CPER attachment(s) omitted (ACA on log entry)" + ver_bits: list[str] = [] + if block.hub_version: + ver_bits.append(f"hub {block.hub_version}") + if block.afid_sag_file_version: + ver_bits.append(f"AFID_SAG {block.afid_sag_file_version}") + ver_suffix = f" [{'; '.join(ver_bits)}]" if ver_bits else "" + message = ( + f"{hub_label}: {len(block.solution)} solution(s) " + f"from {len(data.rf_events)} Redfish event(s){cper_summary}{ver_suffix}" + ) + return ServiceabilityWindowResult( + ok=True, + message=message, + afid_events=events, + serviceability=block, + ) diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py index 87bbaa3f..9ed45a86 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py @@ -7,7 +7,7 @@ # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights -# to use, copy, modify, distribute, sublicense, and/or sell +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # @@ -32,23 +32,18 @@ from nodescraper.enums import ExecutionStatus from nodescraper.interfaces import DataAnalyzer from nodescraper.models import TaskResult -from nodescraper.plugins.serviceability.afid_events import build_afid_events_from_data -from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs -from nodescraper.plugins.serviceability.cper_decode import ( - CperDecodeError, - decode_cper_raw_attachments, +from nodescraper.plugins.serviceability.analysis_window import ( + analyze_serviceability_window, ) +from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs from nodescraper.plugins.serviceability.se_adapter import ( format_serviceability_solution_lines, ) from nodescraper.plugins.serviceability.se_models import ServiceabilityBlock -from nodescraper.plugins.serviceability.se_runner import SeRunError, run_service_hub from nodescraper.plugins.serviceability.serviceability_data import ( ServiceabilityDataModel, ) -from .mi3xx_cper_utils import CPER_METHOD_AFID_MAX, should_skip_cper_fetch_or_decode - class AfidSagMetadataArtifact(BaseModel): """Hub AFID_SAG metadata snapshot; written to ``afid_sag_metadata.json``.""" @@ -80,133 +75,26 @@ def analyze_data( self.result.message = "ServiceabilityAnalyzerArgs are required" return self.result - events = data.afid_events or build_afid_events_from_data(data) - data.afid_events = events - - if args.skip_hub: - data.serviceability = ServiceabilityBlock(afid_events=events) - self.result.status = ExecutionStatus.OK - self.result.message = f"Built {len(events)} AFID event(s); hub skipped" - self._log_serviceability_solutions(data.serviceability) - return self.result - parent = self.parent or self.__class__.__name__ - cper_data = data.cper_data or {} - cper_raw_to_decode = self._cper_raw_needing_decode(data) - skipped_cper = len(data.cper_raw or {}) - len(cper_raw_to_decode) - if skipped_cper: - self.logger.info( - "(%s) Skipping CPER decode for %d CPER attachment(s); Redfish log " - "already has usable ACA fields (CPER-method AFID<=%s or no serial on decode)", - parent, - skipped_cper, - CPER_METHOD_AFID_MAX, - ) - if cper_raw_to_decode and not cper_data: - if not args.cper_decode_module: - self.logger.warning( - "(%s) %d CPER attachment(s) collected but cper_decode_module is " - "not set in analysis_args; skipping CPER decode", - parent, - len(cper_raw_to_decode), - ) - else: - self.logger.info( - "(%s) Decoding %d CPER attachment(s) via %s.%s", - parent, - len(cper_raw_to_decode), - args.cper_decode_module, - args.cper_decode_method, - ) - try: - cper_data = decode_cper_raw_attachments( - cper_raw_to_decode, - cper_decode_module=args.cper_decode_module, - cper_decode_method=args.cper_decode_method, - logger=self.logger, - ) - data.cper_data = cper_data - self.logger.info( - "(%s) CPER decode finished: %d of %d attachment(s) decoded", - parent, - len(cper_data), - len(cper_raw_to_decode), - ) - except CperDecodeError as exc: - self.logger.warning( - "(%s) %s; continuing without decoded CPER", - parent, - exc, - ) - elif cper_data: - self.logger.info( - "(%s) Using %d pre-decoded CPER record(s) from collection", - parent, - len(cper_data), - ) - - try: - block = run_service_hub( - hub_python_module=args.hub_python_module, # type: ignore[arg-type] - hub_display_name=args.hub_display_name, - afid_events=events, - afid_sag_path=args.afid_sag_path, # type: ignore[arg-type] - rf_events=data.rf_events, - cper_data=cper_data or None, - hub_options=args.resolved_hub_options(), - hub_analyze_method=args.hub_analyze_method, - hub_init_path_kwarg=args.hub_init_path_kwarg, - ) - except (SeRunError, ValueError) as exc: + result = analyze_serviceability_window( + data, + args, + logger=self.logger, + parent=parent, + ) + if not result.ok: self.result.status = ExecutionStatus.ERROR - self.result.message = str(exc) + self.result.message = result.message return self.result - data.serviceability = block - self._append_afid_sag_metadata_artifact(block) - self._log_serviceability_solutions(block) - hub_label = args.hub_display_name or args.hub_python_module + if result.serviceability is not None: + self._append_afid_sag_metadata_artifact(result.serviceability) + self._log_serviceability_solutions(result.serviceability) + self.result.status = ExecutionStatus.OK - cper_summary = "" - if cper_data: - cper_summary = f", {len(cper_data)} decoded CPER(s)" - elif cper_raw_to_decode: - cper_summary = f", {len(cper_raw_to_decode)} CPER attachment(s) not decoded" - elif data.cper_raw: - cper_summary = f", {len(data.cper_raw)} CPER attachment(s) omitted (ACA on log entry)" - ver_bits: list[str] = [] - if block.hub_version: - ver_bits.append(f"hub {block.hub_version}") - if block.afid_sag_file_version: - ver_bits.append(f"AFID_SAG {block.afid_sag_file_version}") - ver_suffix = f" [{'; '.join(ver_bits)}]" if ver_bits else "" - self.result.message = ( - f"{hub_label}: {len(block.solution)} solution(s) " - f"from {len(data.rf_events)} Redfish event(s){cper_summary}{ver_suffix}" - ) + self.result.message = result.message return self.result - @staticmethod - def _cper_raw_needing_decode(data: ServiceabilityDataModel) -> dict[str, str]: - """Subset of ``cper_raw`` that still needs configured CPER decode (not already on the log).""" - raw = data.cper_raw or {} - if not raw: - return {} - by_id: dict[str, dict[str, Any]] = {} - for member in data.rf_events: - if not isinstance(member, dict): - continue - eid = member.get("Id") - if eid is not None: - by_id[str(eid)] = member - out: dict[str, str] = {} - for event_id, blob in raw.items(): - ev = by_id.get(str(event_id)) - if ev is not None and should_skip_cper_fetch_or_decode(ev): - continue - out[str(event_id)] = blob - return out - def _append_afid_sag_metadata_artifact(self, block: ServiceabilityBlock) -> None: if block.afid_sag_metadata is None: return diff --git a/nodescraper/redfish_events/__init__.py b/nodescraper/redfish_events/__init__.py new file mode 100644 index 00000000..133bfed9 --- /dev/null +++ b/nodescraper/redfish_events/__init__.py @@ -0,0 +1,101 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Continuous Redfish event ingest for node-scraper background services. + +This package is separate from on-demand CLI plugin runs. It provides SSE and +webhook-based event subscriptions, baseline log pulls, a trigger engine, and +``node-scraper daemon`` for long-running serviceability monitoring. + +Install the optional dependency: pip install amd-node-scraper[events] +""" +from __future__ import annotations + +import importlib +from typing import Any + +from .config import EventStreamConfig, EventTargetConfig +from .daemon_config import DaemonConfig, HttpServerConfig, load_daemon_config +from .models import ( + EventCallback, + EventSource, + RedfishEvent, + SubscriptionState, + TransportMode, +) +from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed +from .se_bridge import redfish_event_to_log_member, redfish_events_to_log_members +from .trigger_engine import TriggerConfig, TriggerEngine + +_LAZY_EXPORTS: dict[str, tuple[str, str]] = { + "AsyncRedfishClient": (".client", "AsyncRedfishClient"), + "SSECapabilityResult": (".sse_capability", "SSECapabilityResult"), + "SSESupport": (".sse_capability", "SSESupport"), + "SseEventSubscriber": (".sse_subscriber", "SseEventSubscriber"), + "SubscriberManager": (".subscriber_manager", "SubscriberManager"), + "WebhookEventSubscriber": (".webhook_subscriber", "WebhookEventSubscriber"), + "WebhookSubscriptionResult": (".webhook_subscriber", "WebhookSubscriptionResult"), + "check_sse_capability": (".sse_capability", "check_sse_capability"), + "pull_baseline_events": (".log_baseline", "pull_baseline_events"), +} + +__all__ = [ + "AsyncRedfishClient", + "DaemonConfig", + "EventCallback", + "EventSource", + "EventStreamConfig", + "EventTargetConfig", + "HttpServerConfig", + "RedfishEvent", + "SSECapabilityResult", + "SSESupport", + "SseEventSubscriber", + "SubscriberManager", + "SubscriptionState", + "TransportMode", + "TriggerConfig", + "TriggerEngine", + "WebhookEventSubscriber", + "WebhookSubscriptionResult", + "check_sse_capability", + "load_daemon_config", + "normalize_severity", + "parse_redfish_timestamp", + "pull_baseline_events", + "redfish_event_to_log_member", + "redfish_events_to_log_members", + "severity_allowed", +] + + +def __getattr__(name: str) -> Any: + if name not in _LAZY_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, attr_name = _LAZY_EXPORTS[name] + module = importlib.import_module(module_name, __name__) + value = getattr(module, attr_name) + globals()[name] = value + return value diff --git a/nodescraper/redfish_events/client.py b/nodescraper/redfish_events/client.py new file mode 100644 index 00000000..a85e122d --- /dev/null +++ b/nodescraper/redfish_events/client.py @@ -0,0 +1,107 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Minimal async Redfish GET client for event ingest follow-up requests.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import httpx + + +@dataclass +class RedfishGetResponse: + """Result of an async Redfish GET.""" + + ok: bool + status_code: int + data: Optional[dict[str, Any]] = None + content: bytes = b"" + error: Optional[str] = None + + +class AsyncRedfishClient: + """Small httpx wrapper for Redfish JSON and binary GETs.""" + + def __init__( + self, + base_url: str, + username: str, + password: str, + *, + verify_ssl: bool = False, + timeout_seconds: float = 30.0, + ) -> None: + self.base_url = base_url.rstrip("/") + self._auth = httpx.BasicAuth(username, password) + self._verify_ssl = verify_ssl + self._timeout = timeout_seconds + + def _abs_url(self, path: str) -> str: + if path.startswith("http://") or path.startswith("https://"): + return path + return f"{self.base_url}/{path.lstrip('/')}" + + async def get_json(self, path: str) -> RedfishGetResponse: + try: + async with httpx.AsyncClient( + auth=self._auth, + verify=self._verify_ssl, + timeout=self._timeout, + follow_redirects=True, + ) as client: + response = await client.get(self._abs_url(path)) + except Exception as exc: # noqa: BLE001 + return RedfishGetResponse(ok=False, status_code=0, error=str(exc)) + if response.status_code != 200: + return RedfishGetResponse( + ok=False, + status_code=response.status_code, + error=response.text[:500], + ) + try: + return RedfishGetResponse(ok=True, status_code=200, data=response.json()) + except Exception as exc: # noqa: BLE001 + return RedfishGetResponse(ok=False, status_code=200, error=str(exc)) + + async def get_bytes(self, path: str) -> RedfishGetResponse: + try: + async with httpx.AsyncClient( + auth=self._auth, + verify=self._verify_ssl, + timeout=self._timeout, + follow_redirects=True, + ) as client: + response = await client.get(self._abs_url(path)) + except Exception as exc: # noqa: BLE001 + return RedfishGetResponse(ok=False, status_code=0, error=str(exc)) + if response.status_code != 200: + return RedfishGetResponse( + ok=False, + status_code=response.status_code, + error=response.text[:500], + ) + return RedfishGetResponse(ok=True, status_code=200, content=response.content) diff --git a/nodescraper/redfish_events/config.py b/nodescraper/redfish_events/config.py new file mode 100644 index 00000000..2d80e0e5 --- /dev/null +++ b/nodescraper/redfish_events/config.py @@ -0,0 +1,87 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Configuration models for Redfish event subscriptions.""" +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, Field, model_validator + +from .models import TransportMode + +DEFAULT_SSE_ENDPOINT = "/redfish/v1/EventService/SSE" + + +class EventTargetConfig(BaseModel): + """One BMC target for continuous event ingest.""" + + target_key: str = Field(..., min_length=1, description="Stable unique id for this target") + name: str = Field(..., min_length=1, description="Display name") + host: str = Field(..., min_length=1, description="BMC hostname or IP") + username: str + password: str + base_url: Optional[str] = Field( + default=None, + description="Redfish base URL; defaults to https://", + ) + verify_ssl: bool = False + sse_endpoint: str = DEFAULT_SSE_ENDPOINT + transport: TransportMode = TransportMode.AUTO + webhook_url: Optional[str] = Field( + default=None, + description="Collector webhook URL when transport is webhook or auto fallback", + ) + + def resolved_base_url(self) -> str: + if self.base_url: + return self.base_url.rstrip("/") + return f"https://{self.host}" + + +class EventStreamConfig(BaseModel): + """Global settings for SubscriberManager.""" + + event_types: list[str] = Field(default_factory=lambda: ["Alert", "StatusChange"]) + severities: list[str] = Field(default_factory=lambda: ["Warning", "Critical"]) + reconnect_delay_seconds: int = 30 + max_retry_duration_hours: float = 24.0 + cooldown_duration_hours: float = 6.0 + degraded_threshold_hours: float = 1.0 + baseline_pull_enabled: bool = True + baseline_max_entries_per_log: int = 200 + baseline_repull_interval_minutes: int = 60 + enable_webhook_fallback: bool = True + allow_loopback_webhook: bool = False + dedupe_events: bool = True + max_dedupe_entries: int = 10000 + + @model_validator(mode="after") + def _validate_lists(self) -> EventStreamConfig: + if not self.event_types: + raise ValueError("event_types must not be empty") + if not self.severities: + raise ValueError("severities must not be empty") + return self diff --git a/nodescraper/redfish_events/daemon_config.py b/nodescraper/redfish_events/daemon_config.py new file mode 100644 index 00000000..dbe53095 --- /dev/null +++ b/nodescraper/redfish_events/daemon_config.py @@ -0,0 +1,88 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Configuration for the long-running Redfish event daemon.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field, model_validator + +from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs + +from .config import EventStreamConfig, EventTargetConfig +from .trigger_engine import TriggerConfig + + +class HttpServerConfig(BaseModel): + """Optional HTTP endpoints for webhook ingest and live recommendations.""" + + enabled: bool = True + host: str = "0.0.0.0" + port: int = Field(default=8081, ge=1, le=65535) + webhook_path_prefix: str = "/hook" + recommendations_path: str = "/recommendations" + status_path: str = "/status" + + @model_validator(mode="after") + def _normalize_paths(self) -> HttpServerConfig: + self.webhook_path_prefix = _normalize_path(self.webhook_path_prefix) + self.recommendations_path = _normalize_path(self.recommendations_path) + self.status_path = _normalize_path(self.status_path) + return self + + +class DaemonConfig(BaseModel): + """Full daemon configuration loaded from JSON.""" + + stream: EventStreamConfig = Field(default_factory=EventStreamConfig) + targets: list[EventTargetConfig] + trigger: TriggerConfig = Field(default_factory=TriggerConfig) + analysis: ServiceabilityAnalyzerArgs + http: HttpServerConfig = Field(default_factory=HttpServerConfig) + + @model_validator(mode="after") + def _require_targets(self) -> DaemonConfig: + if not self.targets: + raise ValueError("At least one target is required") + return self + + +def _normalize_path(value: str) -> str: + text = str(value).strip() or "/" + if not text.startswith("/"): + text = f"/{text}" + return text.rstrip("/") or "/" + + +def load_daemon_config(path: str | Path) -> DaemonConfig: + """Load and validate a daemon JSON config file.""" + config_path = Path(path) + if not config_path.is_file(): + raise FileNotFoundError(f"Daemon config not found: {config_path}") + raw: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8")) + return DaemonConfig.model_validate(raw) diff --git a/nodescraper/redfish_events/daemon_http.py b/nodescraper/redfish_events/daemon_http.py new file mode 100644 index 00000000..f1e5b693 --- /dev/null +++ b/nodescraper/redfish_events/daemon_http.py @@ -0,0 +1,202 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Minimal asyncio HTTP server for webhook ingest and live recommendations.""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +from typing import Any, Optional + +from .daemon_config import HttpServerConfig +from .subscriber_manager import SubscriberManager + +logger = logging.getLogger(__name__) + + +class DaemonHttpServer: + """Serve webhook POST endpoints and JSON recommendation snapshots.""" + + def __init__( + self, + manager: SubscriberManager, + recommendations: dict[str, dict[str, Any]], + config: HttpServerConfig, + ) -> None: + self._manager = manager + self._recommendations = recommendations + self._config = config + self._server: Optional[asyncio.AbstractServer] = None + + async def start(self) -> None: + """Bind and start accepting HTTP connections.""" + self._server = await asyncio.start_server( + self._handle_client, + host=self._config.host, + port=self._config.port, + ) + logger.info( + "Daemon HTTP listening on http://%s:%d", + self._config.host, + self._config.port, + ) + + async def stop(self) -> None: + """Stop the HTTP server.""" + if self._server is None: + return + self._server.close() + await self._server.wait_closed() + self._server = None + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + try: + request_line = (await reader.readline()).decode("utf-8", errors="replace").strip() + if not request_line: + return + parts = request_line.split() + if len(parts) < 2: + await self._write_response(writer, 400, {"error": "bad request line"}) + return + method = parts[0].upper() + path = parts[1].split("?", 1)[0] + + headers: dict[str, str] = {} + while True: + line = (await reader.readline()).decode("utf-8", errors="replace").strip() + if not line: + break + if ":" in line: + key, value = line.split(":", 1) + headers[key.strip().lower()] = value.strip() + + body = b"" + content_length = int(headers.get("content-length", "0") or "0") + if content_length > 0: + body = await reader.readexactly(content_length) + + if method == "GET": + await self._handle_get(writer, path) + return + if method == "POST": + await self._handle_post(writer, path, body) + return + await self._write_response(writer, 405, {"error": "method not allowed"}) + except asyncio.IncompleteReadError: + return + except Exception as exc: # noqa: BLE001 + logger.warning("HTTP handler error: %s", exc) + await self._write_response(writer, 500, {"error": "internal error"}) + finally: + writer.close() + with contextlib.suppress(Exception): + await writer.wait_closed() + + async def _handle_get(self, writer: asyncio.StreamWriter, path: str) -> None: + if path == self._config.status_path: + targets: dict[str, dict[str, Any]] = {} + for key in self._manager.target_keys(): + state = self._manager.subscription_state(key) + targets[key] = { + "transport": self._manager.transport_for(key), + "subscription_state": state.value if state is not None else None, + } + payload = { + "status": "ok", + "targets": targets, + } + await self._write_response(writer, 200, payload) + return + + if path == self._config.recommendations_path: + await self._write_response(writer, 200, self._recommendations) + return + + prefix = self._config.recommendations_path.rstrip("/") + "/" + if path.startswith(prefix): + target_key = path[len(prefix) :] + snapshot = self._recommendations.get(target_key) + if snapshot is None: + await self._write_response(writer, 404, {"error": f"unknown target {target_key}"}) + return + await self._write_response(writer, 200, snapshot) + return + + await self._write_response(writer, 404, {"error": "not found"}) + + async def _handle_post( + self, + writer: asyncio.StreamWriter, + path: str, + body: bytes, + ) -> None: + prefix = self._config.webhook_path_prefix.rstrip("/") + "/" + if not path.startswith(prefix): + await self._write_response(writer, 404, {"error": "not found"}) + return + target_key = path[len(prefix) :] + if not target_key: + await self._write_response(writer, 400, {"error": "missing target_key in path"}) + return + try: + payload = json.loads(body.decode("utf-8") if body else "{}") + except json.JSONDecodeError: + await self._write_response(writer, 400, {"error": "invalid JSON body"}) + return + if not isinstance(payload, dict): + await self._write_response(writer, 400, {"error": "JSON body must be an object"}) + return + emitted = await self._manager.handle_webhook_payload(target_key, payload) + await self._write_response(writer, 200, {"accepted": True, "events_emitted": emitted}) + + async def _write_response( + self, + writer: asyncio.StreamWriter, + status: int, + payload: dict[str, Any], + ) -> None: + body = json.dumps(payload).encode("utf-8") + reason = { + 200: "OK", + 400: "Bad Request", + 404: "Not Found", + 405: "Method Not Allowed", + 500: "Internal Server Error", + }.get(status, "OK") + header = ( + f"HTTP/1.1 {status} {reason}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n" + "\r\n" + ).encode("utf-8") + writer.write(header + body) + await writer.drain() diff --git a/nodescraper/redfish_events/daemon_runner.py b/nodescraper/redfish_events/daemon_runner.py new file mode 100644 index 00000000..a70e4968 --- /dev/null +++ b/nodescraper/redfish_events/daemon_runner.py @@ -0,0 +1,143 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Long-running Redfish event daemon orchestration.""" +from __future__ import annotations + +import asyncio +import logging +import signal +from datetime import UTC, datetime +from typing import Any, Optional + +from nodescraper.plugins.serviceability.analysis_window import ( + analyze_serviceability_window, +) +from nodescraper.plugins.serviceability.se_adapter import ( + format_serviceability_solution_lines, +) +from nodescraper.plugins.serviceability.serviceability_data import ( + ServiceabilityDataModel, +) + +from .daemon_config import DaemonConfig +from .daemon_http import DaemonHttpServer +from .models import RedfishEvent +from .se_bridge import redfish_events_to_log_members +from .subscriber_manager import SubscriberManager +from .trigger_engine import TriggerEngine + +logger = logging.getLogger(__name__) + + +class EventDaemon: + """Wire SubscriberManager, trigger engine, analysis, and optional HTTP endpoints.""" + + def __init__(self, config: DaemonConfig) -> None: + self.config = config + self._recommendations: dict[str, dict[str, Any]] = {} + self._manager = SubscriberManager(config.stream, self._on_event) + self._manager.set_targets(config.targets) + self._trigger = TriggerEngine(config.trigger, self._on_trigger) + self._http: Optional[DaemonHttpServer] = None + self._stop_event = asyncio.Event() + + async def run(self) -> None: + """Start subscriptions and block until SIGINT or SIGTERM.""" + loop = asyncio.get_running_loop() + try: + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, self._stop_event.set) + except NotImplementedError: + signal.signal(signal.SIGINT, lambda *_args: self._stop_event.set()) + + if self.config.http.enabled: + self._http = DaemonHttpServer(self._manager, self._recommendations, self.config.http) + await self._http.start() + + await self._manager.start() + logger.info( + "Redfish event daemon running for %d target(s); Ctrl+C to stop", + len(self.config.targets), + ) + await self._stop_event.wait() + await self.stop() + + async def stop(self) -> None: + """Stop subscriptions and the HTTP server.""" + await self._manager.stop() + if self._http is not None: + await self._http.stop() + self._http = None + logger.info("Redfish event daemon stopped") + + async def _on_event(self, event: RedfishEvent) -> None: + await self._trigger.handle_event(event) + + async def _on_trigger(self, target_key: str, events: list[RedfishEvent]) -> None: + target_name = events[0].target_name if events else target_key + rf_members = redfish_events_to_log_members(events) + bmc_host = events[0].target_host if events else None + data = ServiceabilityDataModel(rf_events=rf_members, bmc_host=bmc_host) + parent = f"EventDaemon:{target_key}" + result = await asyncio.to_thread( + analyze_serviceability_window, + data, + self.config.analysis, + logger=logger, + parent=parent, + ) + snapshot = { + "target_key": target_key, + "target_name": target_name, + "triggered_at": datetime.now(UTC).isoformat(), + "event_count": len(events), + "ok": result.ok, + "message": result.message, + "afid_events": [event.model_dump(mode="json") for event in result.afid_events], + } + if result.serviceability is not None: + snapshot["serviceability"] = result.serviceability.model_dump(mode="json") + for line in format_serviceability_solution_lines(result.serviceability): + logger.info("(%s) %s", parent, line) + elif result.error: + logger.error("(%s) analysis failed: %s", parent, result.error) + else: + logger.info("(%s) %s", parent, result.message) + self._recommendations[target_key] = snapshot + + +def run_event_daemon(config: DaemonConfig) -> None: + """Run the daemon until interrupted.""" + try: + import httpx # noqa: F401 + except ImportError as exc: + raise RuntimeError( + "Redfish event daemon requires the optional [events] extra: " + "pip install amd-node-scraper[events]" + ) from exc + + daemon = EventDaemon(config) + asyncio.run(daemon.run()) diff --git a/nodescraper/redfish_events/log_baseline.py b/nodescraper/redfish_events/log_baseline.py new file mode 100644 index 00000000..37238234 --- /dev/null +++ b/nodescraper/redfish_events/log_baseline.py @@ -0,0 +1,252 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Baseline pull of existing Redfish LogService entries (adapted from Gyanam).""" +from __future__ import annotations + +import logging +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Optional + +import httpx + +from .models import EventSource, RedfishEvent +from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed + +logger = logging.getLogger(__name__) + +BaselineCallback = Callable[[RedfishEvent], None] +_LOG_ROOTS = ("/redfish/v1/Systems", "/redfish/v1/Managers") +_MAX_PAGES = 20 + + +def _abs(base_url: str, uri: str) -> str: + if not uri: + return "" + if uri.startswith("http://") or uri.startswith("https://"): + return uri + return f"{base_url.rstrip('/')}/{uri.lstrip('/')}" + + +def _extract_origin(entry: dict) -> Optional[str]: + origin = entry.get("OriginOfCondition") + if origin is None: + links = entry.get("Links") + if isinstance(links, dict): + origin = links.get("OriginOfCondition") + if isinstance(origin, dict): + return origin.get("@odata.id") + if isinstance(origin, str): + return origin + return None + + +async def _get_json(client: httpx.AsyncClient, url: str) -> Optional[dict]: + try: + resp = await client.get(url) + except Exception as exc: # noqa: BLE001 + logger.debug("Baseline GET failed for %s: %s", url, type(exc).__name__) + return None + if resp.status_code != 200: + return None + try: + return resp.json() + except Exception: # noqa: BLE001 + return None + + +async def _discover_entry_collections(client: httpx.AsyncClient, base_url: str) -> list[str]: + collections: list[str] = [] + seen: set[str] = set() + for root in _LOG_ROOTS: + root_doc = await _get_json(client, _abs(base_url, root)) + if not root_doc: + continue + for member in root_doc.get("Members", []): + member_uri = member.get("@odata.id") if isinstance(member, dict) else None + if not member_uri: + continue + member_doc = await _get_json(client, _abs(base_url, member_uri)) + if not member_doc: + continue + ls_ref = member_doc.get("LogServices", {}) + ls_uri = ls_ref.get("@odata.id") if isinstance(ls_ref, dict) else None + if not ls_uri: + continue + ls_doc = await _get_json(client, _abs(base_url, ls_uri)) + if not ls_doc: + continue + for ls_member in ls_doc.get("Members", []): + ls_member_uri = ls_member.get("@odata.id") if isinstance(ls_member, dict) else None + if not ls_member_uri: + continue + ls_detail = await _get_json(client, _abs(base_url, ls_member_uri)) + if not ls_detail: + continue + entries_ref = ls_detail.get("Entries", {}) + entries_uri = ( + entries_ref.get("@odata.id") if isinstance(entries_ref, dict) else None + ) + if entries_uri and entries_uri not in seen: + seen.add(entries_uri) + collections.append(entries_uri) + return collections + + +def order_members_newest_first(members: list, max_entries: int) -> list[dict]: + dicts = [item for item in members if isinstance(item, dict)] + + def _sort_key(member: dict): + ts = parse_redfish_timestamp(member.get("Created")) + if ts is not None: + return (2, ts.timestamp()) + oid = member.get("@odata.id", "") or "" + try: + return (1, float(oid.rstrip("/").split("/")[-1])) + except (ValueError, IndexError): + return (0, 0.0) + + dicts.sort(key=_sort_key, reverse=True) + return dicts[:max_entries] + + +def _naive_utc(dt: Optional[datetime]) -> Optional[datetime]: + if dt is not None and dt.tzinfo is not None: + return dt.astimezone(UTC).replace(tzinfo=None) + return dt + + +async def _collect_members( + client: httpx.AsyncClient, + base_url: str, + entries_uri: str, + max_entries: int, +) -> list[dict]: + members: list[dict] = [] + seen_ids: set[str] = set() + visited: set[str] = set() + uri: Optional[str] = entries_uri + pages = 0 + while uri: + if uri in visited: + break + visited.add(uri) + doc = await _get_json(client, _abs(base_url, uri)) + if not doc: + break + for member in doc.get("Members", []): + if not isinstance(member, dict): + continue + mid = member.get("@odata.id") + if mid is not None: + if mid in seen_ids: + continue + seen_ids.add(mid) + members.append(member) + pages += 1 + uri = doc.get("Members@odata.nextLink") + if len(members) >= max_entries * 3 or pages >= _MAX_PAGES: + break + return members + + +async def pull_baseline_events( + *, + target_key: str, + target_name: str, + target_host: str, + base_url: str, + username: str, + password: str, + verify_ssl: bool, + callback: BaselineCallback, + severities: Optional[list[str]] = None, + max_entries_per_log: int = 200, + timeout: float = 30.0, +) -> int: + """Pull existing log entries and emit RedfishEvent objects via callback.""" + auth = httpx.BasicAuth(username, password) + emitted = 0 + try: + async with httpx.AsyncClient( + auth=auth, + verify=verify_ssl, + timeout=timeout, + follow_redirects=True, + ) as client: + collections = await _discover_entry_collections(client, base_url) + for entries_uri in collections: + members = await _collect_members( + client, + base_url, + entries_uri, + max_entries_per_log, + ) + for member in order_members_newest_first(members, max_entries_per_log): + entry = member + if any(k not in entry for k in ("Message", "Severity", "Created")): + ref = member.get("@odata.id") + if ref: + fetched = await _get_json(client, _abs(base_url, ref)) + if fetched: + entry = fetched + if ( + "Message" not in entry + and "Severity" not in entry + and "MessageSeverity" not in entry + ): + continue + severity, sev_present = normalize_severity(entry) + if not severity_allowed(severity, sev_present, severities): + continue + source_id = entry.get("@odata.id") or member.get("@odata.id") + event = RedfishEvent( + target_key=target_key, + target_name=target_name, + target_host=target_host, + severity=severity, + message=entry.get("Message", "") or "", + message_id=entry.get("MessageId"), + event_type=entry.get("EntryType") or "Alert", + origin_of_condition=_extract_origin(entry), + event_timestamp=_naive_utc(parse_redfish_timestamp(entry.get("Created"))), + received_at=datetime.now(UTC), + source=EventSource.BASELINE, + source_id=source_id, + raw=dict(entry), + ) + callback(event) + emitted += 1 + except Exception as exc: # noqa: BLE001 + logger.warning( + "Baseline pull failed for %s: %s: %s", + target_name, + type(exc).__name__, + exc, + ) + if emitted: + logger.info("Baseline pull for %s emitted %d log entries", target_name, emitted) + return emitted diff --git a/nodescraper/redfish_events/models.py b/nodescraper/redfish_events/models.py new file mode 100644 index 00000000..237278b7 --- /dev/null +++ b/nodescraper/redfish_events/models.py @@ -0,0 +1,94 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Data models for continuous Redfish event ingest.""" +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any, Optional, Union + +EventCallback = Callable[["RedfishEvent"], Union[None, Awaitable[None]]] + + +class EventSource(str, Enum): + """How an event entered the ingest pipeline.""" + + SSE = "sse" + WEBHOOK = "webhook" + BASELINE = "baseline" + + +class TransportMode(str, Enum): + """Preferred Redfish event transport for a target.""" + + AUTO = "auto" + SSE = "sse" + WEBHOOK = "webhook" + + +class SubscriptionState(str, Enum): + """Connection state for a target event subscription.""" + + CONNECTED = "connected" + RECONNECTING = "reconnecting" + DEGRADED = "degraded" + ON_COOLDOWN = "on_cooldown" + FAILED_PERMANENT = "failed_permanent" + STOPPED = "stopped" + + +@dataclass +class RedfishEvent: + """Normalized Redfish alert or log entry for downstream consumers.""" + + target_key: str + target_name: str + target_host: str + severity: str + message: str + event_type: str + received_at: datetime + source: EventSource + message_id: Optional[str] = None + origin_of_condition: Optional[str] = None + event_timestamp: Optional[datetime] = None + source_id: Optional[str] = None + raw: Optional[dict[str, Any]] = field(default_factory=dict) + + def dedupe_key(self) -> str: + """Return a stable key for in-memory deduplication.""" + if self.source_id: + return self.source_id + parts = ( + self.target_key, + self.message_id or "", + self.event_type, + self.message, + self.event_timestamp.isoformat() if self.event_timestamp else "", + ) + return "|".join(parts) diff --git a/nodescraper/redfish_events/parsing.py b/nodescraper/redfish_events/parsing.py new file mode 100644 index 00000000..c98c9a55 --- /dev/null +++ b/nodescraper/redfish_events/parsing.py @@ -0,0 +1,89 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Redfish event parsing helpers (adapted from Gyanam alert ingest).""" +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone +from typing import Optional + +UTC = timezone.utc + +_TS_RE = re.compile( + r"(\d{4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})" + r"(?:\.(\d+))?\s*(Z|[+-]\d{2}:?\d{2})?" +) + + +def parse_redfish_timestamp(value: Optional[str]) -> Optional[datetime]: + """Parse Redfish EventTimestamp / LogEntry Created values.""" + if not value or not isinstance(value, str): + return None + s = value.strip() + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + except (ValueError, AttributeError): + pass + + match = _TS_RE.search(s) + if not match: + return None + year, mon, day, hour, minute, sec, frac, tz = match.groups() + try: + micro = int((frac or "0").ljust(6, "0")[:6]) + dt = datetime(int(year), int(mon), int(day), int(hour), int(minute), int(sec), micro) + except ValueError: + return None + + if tz and tz != "Z": + sign = 1 if tz[0] == "+" else -1 + digits = tz[1:].replace(":", "") + offset = timedelta(hours=int(digits[:2]), minutes=int(digits[2:4])) + return dt.replace(tzinfo=timezone(sign * offset)) + return dt.replace(tzinfo=UTC) + + +def normalize_severity(event: dict) -> tuple[str, bool]: + """Return severity and whether the field was present on the event.""" + raw = event.get("MessageSeverity") or event.get("Severity") + if raw: + return str(raw), True + return "OK", False + + +def severity_allowed( + severity: str, + present: bool, + allow_list: Optional[list[str]], +) -> bool: + """Apply case-insensitive severity filtering.""" + if not present: + return True + if not allow_list: + return True + allowed = {item.casefold() for item in allow_list} + return severity.casefold() in allowed diff --git a/nodescraper/redfish_events/se_bridge.py b/nodescraper/redfish_events/se_bridge.py new file mode 100644 index 00000000..201dba62 --- /dev/null +++ b/nodescraper/redfish_events/se_bridge.py @@ -0,0 +1,58 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Bridge RedfishEvent objects into serviceability plugin rf_events shape.""" +from __future__ import annotations + +from typing import Any + +from .models import RedfishEvent + + +def redfish_event_to_log_member(event: RedfishEvent) -> dict[str, Any]: + """Return a dict suitable for ServiceabilityDataModel.rf_events entries.""" + if event.raw: + return dict(event.raw) + member: dict[str, Any] = { + "Message": event.message, + "Severity": event.severity, + "EventType": event.event_type, + } + if event.message_id: + member["MessageId"] = event.message_id + if event.origin_of_condition: + member["OriginOfCondition"] = {"@odata.id": event.origin_of_condition} + if event.event_timestamp: + member["EventTimestamp"] = event.event_timestamp.isoformat() + member["Created"] = member["EventTimestamp"] + if event.source_id: + member["@odata.id"] = event.source_id + member["Id"] = event.source_id.rstrip("/").split("/")[-1] + return member + + +def redfish_events_to_log_members(events: list[RedfishEvent]) -> list[dict[str, Any]]: + """Convert a batch of ingest events for on-demand serviceability analysis.""" + return [redfish_event_to_log_member(event) for event in events] diff --git a/nodescraper/redfish_events/sse_capability.py b/nodescraper/redfish_events/sse_capability.py new file mode 100644 index 00000000..d454f8ea --- /dev/null +++ b/nodescraper/redfish_events/sse_capability.py @@ -0,0 +1,206 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Detect whether a BMC supports Redfish EventService SSE (adapted from Gyanam).""" +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass +from enum import Enum +from typing import Optional + +import httpx + +logger = logging.getLogger(__name__) + + +class SSESupport(str, Enum): + SUPPORTED = "supported" + NOT_SUPPORTED = "not_supported" + BROKEN = "broken" + UNKNOWN = "unknown" + + +@dataclass +class SSECapabilityResult: + support: SSESupport + reason: str + event_service_enabled: bool = False + sse_endpoint: Optional[str] = None + test_duration_ms: float = 0.0 + + +async def check_sse_capability( + base_url: str, + username: str, + password: str, + verify_ssl: bool = False, + test_duration_seconds: float = 5.0, +) -> SSECapabilityResult: + start = time.time() + try: + async with httpx.AsyncClient( + auth=httpx.BasicAuth(username, password), + verify=verify_ssl, + timeout=10.0, + ) as client: + response = await client.get(f"{base_url.rstrip('/')}/redfish/v1/EventService") + if response.status_code == 404: + return SSECapabilityResult( + support=SSESupport.NOT_SUPPORTED, + reason="EventService not found (404)", + test_duration_ms=(time.time() - start) * 1000, + ) + if response.status_code != 200: + return SSECapabilityResult( + support=SSESupport.UNKNOWN, + reason=f"EventService returned HTTP {response.status_code}", + test_duration_ms=(time.time() - start) * 1000, + ) + event_service = response.json() + enabled = bool(event_service.get("ServiceEnabled", False)) + sse_uri = event_service.get("ServerSentEventUri") + if not sse_uri: + return SSECapabilityResult( + support=SSESupport.NOT_SUPPORTED, + reason="ServerSentEventUri not advertised", + event_service_enabled=enabled, + test_duration_ms=(time.time() - start) * 1000, + ) + sse_url = sse_uri if sse_uri.startswith("http") else f"{base_url.rstrip('/')}{sse_uri}" + result = await _test_sse_endpoint( + sse_url, + username, + password, + verify_ssl, + test_duration_seconds, + ) + result.event_service_enabled = enabled + result.sse_endpoint = sse_uri + result.test_duration_ms = (time.time() - start) * 1000 + return result + except httpx.TimeoutException: + return SSECapabilityResult( + support=SSESupport.UNKNOWN, + reason="Timeout checking EventService", + test_duration_ms=(time.time() - start) * 1000, + ) + except Exception as exc: # noqa: BLE001 + return SSECapabilityResult( + support=SSESupport.UNKNOWN, + reason=f"{type(exc).__name__}: {exc}", + test_duration_ms=(time.time() - start) * 1000, + ) + + +async def _test_sse_endpoint( + sse_url: str, + username: str, + password: str, + verify_ssl: bool, + test_duration: float, +) -> SSECapabilityResult: + try: + timeout = httpx.Timeout(10.0, read=test_duration + 2.0) + async with ( + httpx.AsyncClient( + auth=httpx.BasicAuth(username, password), + verify=verify_ssl, + timeout=timeout, + ) as client, + client.stream("GET", sse_url) as response, + ): + if response.status_code in (404, 501): + return SSECapabilityResult( + support=SSESupport.NOT_SUPPORTED, + reason=f"SSE endpoint HTTP {response.status_code}", + ) + if response.status_code != 200: + return SSECapabilityResult( + support=SSESupport.BROKEN, + reason=f"SSE endpoint HTTP {response.status_code}", + ) + content_type = response.headers.get("content-type", "") + if "text/event-stream" not in content_type.lower(): + return SSECapabilityResult( + support=SSESupport.BROKEN, + reason=f"Unexpected content-type: {content_type}", + ) + + lines_received = 0 + keepalives = 0 + events = 0 + + async def _read_lines() -> None: + nonlocal lines_received, keepalives, events + async for line in response.aiter_lines(): + lines_received += 1 + stripped = line.strip() + if not stripped: + continue + if stripped.startswith(":"): + keepalives += 1 + elif stripped.startswith("data:"): + events += 1 + + try: + await asyncio.wait_for(_read_lines(), timeout=test_duration) + except asyncio.TimeoutError: + pass + + if lines_received == 0: + return SSECapabilityResult( + support=SSESupport.BROKEN, + reason="SSE stream opened but no data received", + ) + if keepalives or events: + return SSECapabilityResult( + support=SSESupport.SUPPORTED, + reason=( + f"SSE working ({events} events, {keepalives} keep-alives " + f"in {test_duration}s test)" + ), + ) + return SSECapabilityResult( + support=SSESupport.BROKEN, + reason=f"SSE active but invalid format ({lines_received} lines)", + ) + except httpx.ConnectError as exc: + return SSECapabilityResult( + support=SSESupport.UNKNOWN, + reason=f"Cannot connect: {exc}", + ) + except httpx.TimeoutException: + return SSECapabilityResult( + support=SSESupport.UNKNOWN, + reason="Timeout connecting to SSE endpoint", + ) + except Exception as exc: # noqa: BLE001 + return SSECapabilityResult( + support=SSESupport.BROKEN, + reason=f"{type(exc).__name__}: {exc}", + ) diff --git a/nodescraper/redfish_events/sse_subscriber.py b/nodescraper/redfish_events/sse_subscriber.py new file mode 100644 index 00000000..7750dd81 --- /dev/null +++ b/nodescraper/redfish_events/sse_subscriber.py @@ -0,0 +1,326 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""SSE alert subscriber for a single Redfish target (adapted from Gyanam).""" +from __future__ import annotations + +import asyncio +import json +import logging +import random +from collections.abc import Callable +from contextlib import suppress +from datetime import UTC, datetime, timedelta +from enum import Enum +from typing import Optional + +import httpx + +from .models import EventSource, RedfishEvent, SubscriptionState +from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed + +logger = logging.getLogger(__name__) + +SSE_READ_TIMEOUT_SECONDS = 300.0 +EventHandler = Callable[[RedfishEvent], None] + + +class ErrorCategory(str, Enum): + PERMANENT = "permanent" + TRANSIENT = "transient" + + +class SseEventSubscriber: + """Maintain one long-lived Redfish EventService SSE stream.""" + + def __init__( + self, + *, + target_key: str, + target_name: str, + target_host: str, + base_url: str, + username: str, + password: str, + sse_endpoint: str = "/redfish/v1/EventService/SSE", + verify_ssl: bool = False, + callback: Optional[EventHandler] = None, + reconnect_delay: int = 30, + max_retry_duration_hours: float = 24, + cooldown_duration_hours: float = 6, + degraded_threshold_hours: float = 1, + event_types: Optional[list[str]] = None, + severities: Optional[list[str]] = None, + ) -> None: + self.target_key = target_key + self.target_name = target_name + self.target_host = target_host + self.base_url = base_url.rstrip("/") + self.username = username + self.password = password + self.sse_endpoint = sse_endpoint + self.verify_ssl = verify_ssl + self.callback = callback + self.reconnect_delay = reconnect_delay + self.max_retry_duration_hours = max_retry_duration_hours + self.cooldown_duration_hours = cooldown_duration_hours + self.degraded_threshold_hours = degraded_threshold_hours + self.event_types = event_types or ["Alert", "StatusChange"] + self.severities = severities or ["Warning", "Critical"] + + self._running = False + self._task: Optional[asyncio.Task] = None + self._consecutive_failures = 0 + self._last_event_time: Optional[datetime] = None + self._first_failure_time: Optional[datetime] = None + self._cooldown_start_time: Optional[datetime] = None + self._state = SubscriptionState.STOPPED + self._failure_reason: Optional[str] = None + self._next_retry_time: Optional[datetime] = None + + async def start(self) -> None: + if self._running: + return + self._running = True + self._state = SubscriptionState.RECONNECTING + self._task = asyncio.create_task(self._subscribe_loop()) + logger.info("Started SSE event subscription for %s", self.target_name) + + async def stop(self) -> None: + self._running = False + self._state = SubscriptionState.STOPPED + if self._task: + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + logger.info("Stopped SSE event subscription for %s", self.target_name) + + async def _subscribe_loop(self) -> None: + while self._running: + try: + if self._cooldown_start_time: + elapsed_hours = ( + datetime.now(UTC) - self._cooldown_start_time + ).total_seconds() / 3600 + if elapsed_hours < self.cooldown_duration_hours: + self._state = SubscriptionState.ON_COOLDOWN + await asyncio.sleep(60) + continue + self._cooldown_start_time = None + self._first_failure_time = None + self._consecutive_failures = 0 + self._state = SubscriptionState.RECONNECTING + + await self._connect_and_listen() + except asyncio.CancelledError: + break + except Exception as exc: # noqa: BLE001 + category, reason = self._classify_error(exc) + if self._consecutive_failures == 0: + self._first_failure_time = datetime.now(UTC) + self._consecutive_failures += 1 + self._failure_reason = reason + + if category == ErrorCategory.PERMANENT: + logger.warning( + "Permanent SSE error for %s: %s", + self.target_name, + reason, + ) + self._state = SubscriptionState.FAILED_PERMANENT + self._running = False + break + + if self._first_failure_time: + elapsed_hours = ( + datetime.now(UTC) - self._first_failure_time + ).total_seconds() / 3600 + if elapsed_hours >= self.max_retry_duration_hours: + self._cooldown_start_time = datetime.now(UTC) + self._state = SubscriptionState.ON_COOLDOWN + continue + if elapsed_hours >= self.degraded_threshold_hours: + self._state = SubscriptionState.DEGRADED + else: + self._state = SubscriptionState.RECONNECTING + + delay = self._calculate_backoff_delay() + self._next_retry_time = datetime.now(UTC) + timedelta(seconds=delay) + await asyncio.sleep(delay) + + async def _connect_and_listen(self) -> None: + url = f"{self.base_url}{self.sse_endpoint}" + auth = httpx.BasicAuth(self.username, self.password) + timeout = httpx.Timeout(30.0, read=SSE_READ_TIMEOUT_SECONDS) + + async with ( + httpx.AsyncClient( + auth=auth, + verify=self.verify_ssl, + timeout=timeout, + follow_redirects=True, + ) as client, + client.stream("GET", url) as response, + ): + if response.status_code != 200: + preview = "" + try: + preview = (await response.aread())[:200].decode("utf-8", errors="ignore") + except Exception: # noqa: BLE001 + pass + raise RuntimeError(f"SSE connection failed: HTTP {response.status_code} {preview}") + + self._consecutive_failures = 0 + self._first_failure_time = None + self._failure_reason = None + self._next_retry_time = None + self._state = SubscriptionState.CONNECTED + + connect_time = datetime.now(UTC) + event_count = 0 + data_buffer: list[str] = [] + + def _dispatch() -> None: + nonlocal event_count + if not data_buffer: + return + payload = "\n".join(data_buffer) + data_buffer.clear() + if not payload.strip(): + return + event_count += 1 + try: + self._process_event_data(payload) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to process SSE event from %s: %s", + self.target_name, + exc, + ) + + async for line in response.aiter_lines(): + if not self._running: + break + line = line.rstrip("\r\n") + if line == "": + _dispatch() + continue + if line.startswith(":"): + connect_time = datetime.now(UTC) + continue + if ":" in line: + field, _, value = line.partition(":") + if value.startswith(" "): + value = value[1:] + else: + field, value = line, "" + if field == "data": + data_buffer.append(value) + + _dispatch() + + elapsed = (datetime.now(UTC) - connect_time).total_seconds() + if elapsed < 30.0 and event_count == 0: + raise RuntimeError(f"SSE stream closed after {elapsed:.1f}s without sending events") + + def _process_event_data(self, data_str: str) -> None: + data = json.loads(data_str) + for event in data.get("Events", []): + if not isinstance(event, dict): + continue + event_type = event.get("EventType") or "" + severity, sev_present = normalize_severity(event) + if event_type and self.event_types and event_type not in self.event_types: + continue + if not severity_allowed(severity, sev_present, self.severities): + continue + if not event_type: + event_type = "Alert" + + origin = event.get("OriginOfCondition", {}) + origin_uri = None + if isinstance(origin, dict): + origin_uri = origin.get("@odata.id") + elif isinstance(origin, str): + origin_uri = origin + + redfish_event = RedfishEvent( + target_key=self.target_key, + target_name=self.target_name, + target_host=self.target_host, + severity=severity, + message=event.get("Message", ""), + message_id=event.get("MessageId"), + event_type=event_type, + origin_of_condition=origin_uri, + event_timestamp=parse_redfish_timestamp(event.get("EventTimestamp")), + received_at=datetime.now(UTC), + source=EventSource.SSE, + raw=dict(event), + ) + self._last_event_time = datetime.now(UTC) + if self.callback: + self.callback(redfish_event) + + def _classify_error(self, error: Exception) -> tuple[ErrorCategory, str]: + error_str = str(error).lower() + if isinstance(error, httpx.HTTPStatusError): + status = error.response.status_code + if status in (401, 403, 404, 405, 501): + return ErrorCategory.PERMANENT, f"HTTP {status}" + if isinstance(error, RuntimeError) and "http" in error_str: + for code in (401, 403, 404, 405, 501): + if f"http {code}" in error_str: + return ErrorCategory.PERMANENT, f"HTTP {code}" + if "closed after" in error_str and "without sending events" in error_str: + return ErrorCategory.PERMANENT, "Invalid SSE endpoint" + if isinstance(error, httpx.TimeoutException | asyncio.TimeoutError): + return ErrorCategory.TRANSIENT, "Connection timeout" + if isinstance(error, httpx.ConnectError | OSError): + return ErrorCategory.TRANSIENT, "Network connection error" + return ErrorCategory.TRANSIENT, f"{type(error).__name__}: {error}" + + def _calculate_backoff_delay(self) -> int: + delays = [30, 60, 120, 300, 600, 1800, 3600, 7200] + idx = min(max(self._consecutive_failures - 1, 0), len(delays) - 1) + base = delays[idx] + return int(base * random.uniform(0.8, 1.2)) + + @property + def state(self) -> SubscriptionState: + return self._state + + @property + def is_running(self) -> bool: + return self._running + + @property + def last_event_time(self) -> Optional[datetime]: + return self._last_event_time + + @property + def failure_reason(self) -> Optional[str]: + return self._failure_reason diff --git a/nodescraper/redfish_events/subscriber_manager.py b/nodescraper/redfish_events/subscriber_manager.py new file mode 100644 index 00000000..a7f2b972 --- /dev/null +++ b/nodescraper/redfish_events/subscriber_manager.py @@ -0,0 +1,284 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Manage continuous Redfish event subscriptions for multiple BMC targets.""" +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import logging +from collections import OrderedDict +from ipaddress import ip_address +from typing import Optional +from urllib.parse import urlparse + +from .config import EventStreamConfig, EventTargetConfig +from .log_baseline import pull_baseline_events +from .models import EventCallback, RedfishEvent, SubscriptionState, TransportMode +from .sse_capability import SSESupport, check_sse_capability +from .sse_subscriber import SseEventSubscriber +from .webhook_subscriber import WebhookEventSubscriber, WebhookFailureType + +logger = logging.getLogger(__name__) + + +def _webhook_unreachable(url: str) -> tuple[bool, str]: + try: + parsed = urlparse(url) + except Exception as exc: # noqa: BLE001 + return True, f"unparseable URL: {exc}" + if not parsed.scheme or not parsed.hostname: + return True, "missing scheme or host" + host = parsed.hostname + if host in {"localhost", "0.0.0.0"}: + return True, f"host '{host}' is unreachable from a BMC" + try: + ip = ip_address(host) + if ip.is_loopback or ip.is_unspecified: + return True, f"host '{host}' is loopback/unspecified" + except ValueError: + pass + return False, "" + + +class SubscriberManager: + """Start SSE or webhook ingest for configured targets and dispatch events.""" + + def __init__( + self, + config: EventStreamConfig, + on_event: EventCallback, + ) -> None: + self.config = config + self.on_event = on_event + self._targets: dict[str, EventTargetConfig] = {} + self._sse: dict[str, SseEventSubscriber] = {} + self._webhooks: dict[str, WebhookEventSubscriber] = {} + self._transport: dict[str, str] = {} + self._running = False + self._baseline_task: Optional[asyncio.Task] = None + self._dedupe: OrderedDict[str, None] = OrderedDict() + + def set_targets(self, targets: list[EventTargetConfig]) -> None: + self._targets = {target.target_key: target for target in targets} + + async def start(self) -> None: + if self._running: + return + self._running = True + for target in self._targets.values(): + await self._start_target(target) + if self.config.baseline_repull_interval_minutes > 0: + self._baseline_task = asyncio.create_task(self._baseline_repull_loop()) + logger.info("SubscriberManager started for %d target(s)", len(self._targets)) + + async def stop(self) -> None: + self._running = False + if self._baseline_task: + self._baseline_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._baseline_task + self._baseline_task = None + for key, subscriber in list(self._sse.items()): + await subscriber.stop() + del self._sse[key] + for key, webhook in list(self._webhooks.items()): + await webhook.delete_subscription() + del self._webhooks[key] + self._transport.clear() + logger.info("SubscriberManager stopped") + + async def handle_webhook_payload( + self, + target_key: str, + payload: dict, + ) -> int: + webhook = self._webhooks.get(target_key) + if webhook is None: + logger.warning("Webhook payload for unknown target_key=%s", target_key) + return 0 + emitted = 0 + for event in webhook.parse_webhook_payload(payload): + if await self._dispatch(event): + emitted += 1 + return emitted + + def subscription_state(self, target_key: str) -> Optional[SubscriptionState]: + subscriber = self._sse.get(target_key) + if subscriber is not None: + return subscriber.state + if target_key in self._webhooks: + return SubscriptionState.CONNECTED + return None + + def transport_for(self, target_key: str) -> Optional[str]: + return self._transport.get(target_key) + + def target_keys(self) -> list[str]: + """Return configured target keys in registration order.""" + return list(self._targets.keys()) + + async def _start_target(self, target: EventTargetConfig) -> None: + if self.config.baseline_pull_enabled: + await pull_baseline_events( + target_key=target.target_key, + target_name=target.name, + target_host=target.host, + base_url=target.resolved_base_url(), + username=target.username, + password=target.password, + verify_ssl=target.verify_ssl, + callback=self._baseline_callback, + severities=self.config.severities, + max_entries_per_log=self.config.baseline_max_entries_per_log, + ) + + mode = target.transport + if mode == TransportMode.AUTO: + cap = await check_sse_capability( + target.resolved_base_url(), + target.username, + target.password, + target.verify_ssl, + ) + if cap.support == SSESupport.SUPPORTED: + mode = TransportMode.SSE + elif target.webhook_url and self.config.enable_webhook_fallback: + mode = TransportMode.WEBHOOK + else: + logger.warning( + "No supported event transport for %s: %s", + target.name, + cap.reason, + ) + return + + if mode == TransportMode.SSE: + subscriber = SseEventSubscriber( + target_key=target.target_key, + target_name=target.name, + target_host=target.host, + base_url=target.resolved_base_url(), + username=target.username, + password=target.password, + sse_endpoint=target.sse_endpoint, + verify_ssl=target.verify_ssl, + callback=self._sse_callback, + reconnect_delay=self.config.reconnect_delay_seconds, + max_retry_duration_hours=self.config.max_retry_duration_hours, + cooldown_duration_hours=self.config.cooldown_duration_hours, + degraded_threshold_hours=self.config.degraded_threshold_hours, + event_types=self.config.event_types, + severities=self.config.severities, + ) + self._sse[target.target_key] = subscriber + self._transport[target.target_key] = "sse" + await subscriber.start() + return + + if mode == TransportMode.WEBHOOK: + if not target.webhook_url: + logger.error("Target %s requires webhook_url for webhook transport", target.name) + return + unreachable, reason = _webhook_unreachable(target.webhook_url) + if unreachable and not self.config.allow_loopback_webhook: + logger.error( + "Webhook URL for %s is unreachable from BMCs: %s", + target.name, + reason, + ) + return + webhook = WebhookEventSubscriber( + target_key=target.target_key, + target_name=target.name, + target_host=target.host, + base_url=target.resolved_base_url(), + username=target.username, + password=target.password, + webhook_url=target.webhook_url, + verify_ssl=target.verify_ssl, + event_types=self.config.event_types, + severities=self.config.severities, + ) + result = await webhook.create_subscription() + if not result.success: + level = ( + logging.ERROR + if result.failure_type == WebhookFailureType.PERMANENT + else logging.WARNING + ) + logger.log( + level, + "Webhook subscription failed for %s: %s", + target.name, + result.error_message, + ) + return + self._webhooks[target.target_key] = webhook + self._transport[target.target_key] = "webhook" + + def _sse_callback(self, event: RedfishEvent) -> None: + asyncio.create_task(self._dispatch(event)) + + def _baseline_callback(self, event: RedfishEvent) -> None: + asyncio.create_task(self._dispatch(event)) + + async def _dispatch(self, event: RedfishEvent) -> bool: + if self.config.dedupe_events: + key = f"{event.target_key}:{event.dedupe_key()}" + if key in self._dedupe: + return False + self._dedupe[key] = None + while len(self._dedupe) > self.config.max_dedupe_entries: + self._dedupe.popitem(last=False) + + result = self.on_event(event) + if inspect.isawaitable(result): + await result + return True + + async def _baseline_repull_loop(self) -> None: + interval = self.config.baseline_repull_interval_minutes * 60 + while self._running: + try: + await asyncio.sleep(interval) + for target in self._targets.values(): + await pull_baseline_events( + target_key=target.target_key, + target_name=target.name, + target_host=target.host, + base_url=target.resolved_base_url(), + username=target.username, + password=target.password, + verify_ssl=target.verify_ssl, + callback=self._baseline_callback, + severities=self.config.severities, + max_entries_per_log=self.config.baseline_max_entries_per_log, + ) + except asyncio.CancelledError: + break + except Exception as exc: # noqa: BLE001 + logger.warning("Baseline re-pull loop error: %s", exc) diff --git a/nodescraper/redfish_events/trigger_engine.py b/nodescraper/redfish_events/trigger_engine.py new file mode 100644 index 00000000..d4d2e46b --- /dev/null +++ b/nodescraper/redfish_events/trigger_engine.py @@ -0,0 +1,112 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Sliding-window trigger logic for batched serviceability analysis.""" +from __future__ import annotations + +import asyncio +import inspect +import logging +from collections import deque +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from typing import Optional, Union + +from pydantic import BaseModel, Field + +from .models import RedfishEvent + +logger = logging.getLogger(__name__) + +TriggerCallback = Callable[[str, list[RedfishEvent]], Union[None, Awaitable[None]]] +NowFn = Callable[[], datetime] + + +class TriggerConfig(BaseModel): + """When to run serviceability analysis for a burst of ingest events.""" + + min_events: int = Field(default=3, ge=1, description="Events required within the window") + window_seconds: float = Field(default=10.0, gt=0, description="Sliding window size in seconds") + cooldown_seconds: float = Field( + default=60.0, + ge=0, + description="Minimum seconds between analysis runs for the same target", + ) + + +@dataclass +class _TargetWindow: + events: deque[tuple[datetime, RedfishEvent]] = field(default_factory=deque) + cooldown_until: Optional[datetime] = None + + +class TriggerEngine: + """Fire analysis when enough events arrive within a sliding time window.""" + + def __init__( + self, + config: TriggerConfig, + on_trigger: TriggerCallback, + *, + clock_fn: NowFn = lambda: datetime.now(UTC), + ) -> None: + self.config = config + self.on_trigger = on_trigger + self._now = clock_fn + self._targets: dict[str, _TargetWindow] = {} + self._lock = asyncio.Lock() + + async def handle_event(self, event: RedfishEvent) -> bool: + """Record one event and invoke on_trigger when the window threshold is met.""" + async with self._lock: + window = self._targets.setdefault(event.target_key, _TargetWindow()) + now = self._now() + if window.cooldown_until and now < window.cooldown_until: + return False + + window.events.append((now, event)) + cutoff = now - timedelta(seconds=self.config.window_seconds) + while window.events and window.events[0][0] < cutoff: + window.events.popleft() + + if len(window.events) < self.config.min_events: + return False + + batch = [item[1] for item in window.events] + window.events.clear() + if self.config.cooldown_seconds > 0: + window.cooldown_until = now + timedelta(seconds=self.config.cooldown_seconds) + + logger.info( + "Trigger fired for %s with %d event(s) in %.1fs window", + event.target_key, + len(batch), + self.config.window_seconds, + ) + result = self.on_trigger(event.target_key, batch) + if inspect.isawaitable(result): + await result + return True diff --git a/nodescraper/redfish_events/webhook_subscriber.py b/nodescraper/redfish_events/webhook_subscriber.py new file mode 100644 index 00000000..dc128879 --- /dev/null +++ b/nodescraper/redfish_events/webhook_subscriber.py @@ -0,0 +1,221 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Webhook-based Redfish event subscriptions (adapted from Gyanam).""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import Enum +from typing import Optional + +import httpx + +from .models import EventSource, RedfishEvent +from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed + +logger = logging.getLogger(__name__) + + +class WebhookFailureType(str, Enum): + TEMPORARY = "temporary" + PERMANENT = "permanent" + + +@dataclass +class WebhookSubscriptionResult: + success: bool + failure_type: Optional[WebhookFailureType] = None + error_message: Optional[str] = None + + +class WebhookEventSubscriber: + """Create and parse Redfish webhook subscriptions for one target.""" + + def __init__( + self, + *, + target_key: str, + target_name: str, + target_host: str, + base_url: str, + username: str, + password: str, + webhook_url: str, + verify_ssl: bool = False, + event_types: Optional[list[str]] = None, + severities: Optional[list[str]] = None, + ) -> None: + self.target_key = target_key + self.target_name = target_name + self.target_host = target_host + self.base_url = base_url.rstrip("/") + self.username = username + self.password = password + self.webhook_url = webhook_url + self.verify_ssl = verify_ssl + self.event_types = event_types or ["Alert", "StatusChange"] + self.severities = severities or ["Warning", "Critical"] + self._subscription_id: Optional[str] = None + self._subscription_url: Optional[str] = None + + async def create_subscription(self) -> WebhookSubscriptionResult: + payload = { + "Destination": self.webhook_url, + "Protocol": "Redfish", + "EventTypes": self.event_types, + "Context": self.target_key, + } + try: + async with httpx.AsyncClient( + auth=httpx.BasicAuth(self.username, self.password), + verify=self.verify_ssl, + timeout=30.0, + ) as client: + response = await client.post( + f"{self.base_url}/redfish/v1/EventService/Subscriptions", + json=payload, + ) + if response.status_code in (200, 201): + data = response.json() + self._subscription_id = data.get("Id") + location = response.headers.get("Location") + if location: + self._subscription_url = location + elif self._subscription_id: + self._subscription_url = ( + f"{self.base_url}/redfish/v1/EventService/Subscriptions/" + f"{self._subscription_id}" + ) + return WebhookSubscriptionResult(success=True) + if response.status_code == 409: + await self._find_existing_subscription() + if self._subscription_id: + return WebhookSubscriptionResult(success=True) + if response.status_code in (400, 405, 501): + return WebhookSubscriptionResult( + success=False, + failure_type=WebhookFailureType.PERMANENT, + error_message=response.text[:200], + ) + return WebhookSubscriptionResult( + success=False, + failure_type=WebhookFailureType.TEMPORARY, + error_message=f"HTTP {response.status_code}", + ) + except (httpx.ConnectError, httpx.TimeoutException) as exc: + return WebhookSubscriptionResult( + success=False, + failure_type=WebhookFailureType.TEMPORARY, + error_message=str(exc), + ) + + async def delete_subscription(self) -> bool: + if not self._subscription_url: + return False + try: + async with httpx.AsyncClient( + auth=httpx.BasicAuth(self.username, self.password), + verify=self.verify_ssl, + timeout=10.0, + ) as client: + response = await client.delete(self._subscription_url) + if response.status_code in (200, 204, 404): + self._subscription_id = None + self._subscription_url = None + return True + except Exception as exc: # noqa: BLE001 + logger.debug("Webhook delete failed for %s: %s", self.target_name, exc) + return False + + async def _find_existing_subscription(self) -> None: + try: + async with httpx.AsyncClient( + auth=httpx.BasicAuth(self.username, self.password), + verify=self.verify_ssl, + timeout=10.0, + ) as client: + response = await client.get( + f"{self.base_url}/redfish/v1/EventService/Subscriptions" + ) + if response.status_code != 200: + return + for member in response.json().get("Members", []): + ref = member.get("@odata.id") + if not ref: + continue + detail = await client.get(f"{self.base_url}{ref}") + if detail.status_code != 200: + continue + data = detail.json() + if data.get("Destination") == self.webhook_url: + self._subscription_id = ref.rstrip("/").split("/")[-1] + self._subscription_url = f"{self.base_url}{ref}" + return + except Exception as exc: # noqa: BLE001 + logger.debug("Webhook lookup failed for %s: %s", self.target_name, exc) + + def parse_webhook_payload(self, event_data: dict) -> list[RedfishEvent]: + events = event_data.get("Events") + if not isinstance(events, list): + events = ( + [event_data] if (event_data.get("MessageId") or event_data.get("Message")) else [] + ) + parsed: list[RedfishEvent] = [] + for event in events: + if not isinstance(event, dict): + continue + event_type = event.get("EventType") or "" + severity, sev_present = normalize_severity(event) + if event_type and self.event_types and event_type not in self.event_types: + continue + if not severity_allowed(severity, sev_present, self.severities): + continue + if not event_type: + event_type = "Alert" + origin = event.get("OriginOfCondition", {}) + origin_uri = origin.get("@odata.id") if isinstance(origin, dict) else origin + parsed.append( + RedfishEvent( + target_key=self.target_key, + target_name=self.target_name, + target_host=self.target_host, + severity=severity, + message=event.get("Message", ""), + message_id=event.get("MessageId"), + event_type=event_type, + origin_of_condition=origin_uri if isinstance(origin_uri, str) else None, + event_timestamp=parse_redfish_timestamp(event.get("EventTimestamp")), + received_at=datetime.now(UTC), + source=EventSource.WEBHOOK, + raw=dict(event), + ) + ) + return parsed + + @property + def is_subscribed(self) -> bool: + return self._subscription_id is not None diff --git a/pyproject.toml b/pyproject.toml index d2f1bdef..245548de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ dependencies = [ ] [project.optional-dependencies] +events = [ + "httpx>=0.27.0", +] dev = [ "build", "black", @@ -44,6 +47,7 @@ dev = [ "types-paramiko", "types-requests", "types-setuptools", + "httpx>=0.27.0", ] [project.urls] @@ -89,5 +93,13 @@ python_version = "3.9" mypy_path = ["test/unit"] explicit_package_bases = true +[[tool.mypy.overrides]] +module = "httpx" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "httpx.*" +ignore_missing_imports = true + [tool.setuptools_scm] version_scheme = "post-release" diff --git a/test/unit/plugins/test_analysis_window.py b/test/unit/plugins/test_analysis_window.py new file mode 100644 index 00000000..63465f9b --- /dev/null +++ b/test/unit/plugins/test_analysis_window.py @@ -0,0 +1,54 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.plugins.serviceability.analysis_window import ( + analyze_serviceability_window, +) +from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs +from nodescraper.plugins.serviceability.serviceability_data import ( + ServiceabilityDataModel, +) + + +def test_analyze_serviceability_window_skip_hub(): + data = ServiceabilityDataModel( + rf_events=[ + { + "MessageId": "X", + "Message": "fail", + "Severity": "Critical", + "Created": "2026-01-01T00:00:00Z", + "Afid": 1, + "serviceable_unit": "GPU0", + } + ] + ) + result = analyze_serviceability_window( + data, + ServiceabilityAnalyzerArgs(skip_hub=True), + ) + assert result.ok is True + assert result.serviceability is not None + assert len(result.afid_events) == 1 diff --git a/test/unit/redfish_events/test_daemon_flow.py b/test/unit/redfish_events/test_daemon_flow.py new file mode 100644 index 00000000..3b44049e --- /dev/null +++ b/test/unit/redfish_events/test_daemon_flow.py @@ -0,0 +1,138 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import asyncio +import json +from datetime import UTC, datetime, timedelta + +import pytest + +from nodescraper.redfish_events.daemon_config import DaemonConfig, load_daemon_config +from nodescraper.redfish_events.models import EventSource, RedfishEvent +from nodescraper.redfish_events.trigger_engine import TriggerConfig, TriggerEngine + + +def _event(target_key: str = "bmc-1", message: str = "evt") -> RedfishEvent: + return RedfishEvent( + target_key=target_key, + target_name="node-a", + target_host="10.0.0.1", + severity="Warning", + message=message, + event_type="Alert", + received_at=datetime.now(UTC), + source=EventSource.SSE, + ) + + +def test_trigger_engine_fires_after_min_events_in_window(): + fired: list[tuple[str, list[RedfishEvent]]] = [] + times = [datetime(2026, 1, 1, tzinfo=UTC)] + + def clock_fn() -> datetime: + return times[0] + + async def on_trigger(target_key: str, events: list[RedfishEvent]) -> None: + fired.append((target_key, list(events))) + + engine = TriggerEngine( + TriggerConfig(min_events=2, window_seconds=10, cooldown_seconds=0), + on_trigger, + clock_fn=clock_fn, + ) + + async def _run() -> None: + assert await engine.handle_event(_event(message="a")) is False + times[0] += timedelta(seconds=1) + assert await engine.handle_event(_event(message="b")) is True + + asyncio.run(_run()) + assert len(fired) == 1 + assert fired[0][0] == "bmc-1" + assert [event.message for event in fired[0][1]] == ["a", "b"] + + +def test_trigger_engine_respects_cooldown(): + fired: list[str] = [] + times = [datetime(2026, 1, 1, tzinfo=UTC)] + + def clock_fn() -> datetime: + return times[0] + + async def on_trigger(target_key: str, events: list[RedfishEvent]) -> None: + fired.append(target_key) + + engine = TriggerEngine( + TriggerConfig(min_events=1, window_seconds=10, cooldown_seconds=30), + on_trigger, + clock_fn=clock_fn, + ) + + async def _run() -> None: + assert await engine.handle_event(_event()) is True + times[0] += timedelta(seconds=5) + assert await engine.handle_event(_event()) is False + times[0] += timedelta(seconds=30) + assert await engine.handle_event(_event()) is True + + asyncio.run(_run()) + assert fired == ["bmc-1", "bmc-1"] + + +def test_load_daemon_config_example(tmp_path): + payload = { + "targets": [ + { + "target_key": "node-a", + "name": "Node A", + "host": "10.0.0.1", + "username": "root", + "password": "secret", + } + ], + "analysis": { + "hub_python_module": "hub.mod", + "afid_sag_path": "/tmp/AFID_SAG.json", + }, + } + path = tmp_path / "daemon.json" + path.write_text(json.dumps(payload), encoding="utf-8") + config = load_daemon_config(path) + assert isinstance(config, DaemonConfig) + assert config.targets[0].target_key == "node-a" + assert config.trigger.min_events == 3 + + +def test_daemon_config_requires_targets(): + with pytest.raises(ValueError, match="At least one target"): + DaemonConfig.model_validate( + { + "targets": [], + "analysis": { + "hub_python_module": "hub.mod", + "afid_sag_path": "/tmp/AFID_SAG.json", + }, + } + ) diff --git a/test/unit/redfish_events/test_daemon_http.py b/test/unit/redfish_events/test_daemon_http.py new file mode 100644 index 00000000..0e18823e --- /dev/null +++ b/test/unit/redfish_events/test_daemon_http.py @@ -0,0 +1,88 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import asyncio +import json +from typing import cast +from unittest.mock import AsyncMock, MagicMock + +from nodescraper.redfish_events.daemon_config import HttpServerConfig +from nodescraper.redfish_events.daemon_http import DaemonHttpServer +from nodescraper.redfish_events.models import SubscriptionState + + +def test_daemon_http_status_and_webhook_post(): + asyncio.run(_run()) + + +async def _run() -> None: + manager = MagicMock() + manager.target_keys.return_value = ["node-a"] + manager.transport_for.return_value = "webhook" + manager.subscription_state.return_value = SubscriptionState.CONNECTED + manager.handle_webhook_payload = AsyncMock(return_value=1) + + recommendations: dict = {} + server = DaemonHttpServer( + manager, + recommendations, + HttpServerConfig.model_construct(host="127.0.0.1", port=0), + ) + await server.start() + assert server._server is not None + bound_server = cast(asyncio.Server, server._server) + port = bound_server.sockets[0].getsockname()[1] + + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"GET /status HTTP/1.1\r\nHost: localhost\r\n\r\n") + await writer.drain() + response = await reader.readuntil(b"\r\n\r\n") + body = await reader.read() + writer.close() + await writer.wait_closed() + assert b"200 OK" in response + payload = json.loads(body.decode("utf-8")) + assert payload["status"] == "ok" + assert "node-a" in payload["targets"] + + reader, writer = await asyncio.open_connection("127.0.0.1", port) + body_bytes = json.dumps({"Message": "hot", "MessageSeverity": "Critical"}).encode("utf-8") + request = ( + b"POST /hook/node-a HTTP/1.1\r\n" + b"Host: localhost\r\n" + + f"Content-Length: {len(body_bytes)}\r\n\r\n".encode("utf-8") + + body_bytes + ) + writer.write(request) + await writer.drain() + response = await reader.readuntil(b"\r\n\r\n") + body = await reader.read() + writer.close() + await writer.wait_closed() + assert b"200 OK" in response + assert json.loads(body.decode("utf-8"))["events_emitted"] == 1 + manager.handle_webhook_payload.assert_awaited_once() + + await server.stop() From 62efcb18ee8cb3d5d949c3693b04f8c99ac4a1b9 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 11:37:34 -0500 Subject: [PATCH 13/21] Revert "initial commit" This reverts commit eed21fe56b4f4296ca38c53a36ada4e55dc0e35e. --- config/redfish_events_daemon.example.json | 43 --- docs/REDFISH_EVENTS_DAEMON.md | 80 ----- nodescraper/cli/cli.py | 16 - nodescraper/cli/daemon_cmd.py | 52 --- .../plugins/serviceability/__init__.py | 3 - .../plugins/serviceability/analysis_window.py | 193 ----------- .../serviceability/mi3xx/mi3xx_analyzer.py | 146 +++++++- nodescraper/redfish_events/__init__.py | 101 ------ nodescraper/redfish_events/client.py | 107 ------ nodescraper/redfish_events/config.py | 87 ----- nodescraper/redfish_events/daemon_config.py | 88 ----- nodescraper/redfish_events/daemon_http.py | 202 ----------- nodescraper/redfish_events/daemon_runner.py | 143 -------- nodescraper/redfish_events/log_baseline.py | 252 -------------- nodescraper/redfish_events/models.py | 94 ----- nodescraper/redfish_events/parsing.py | 89 ----- nodescraper/redfish_events/se_bridge.py | 58 ---- nodescraper/redfish_events/sse_capability.py | 206 ----------- nodescraper/redfish_events/sse_subscriber.py | 326 ------------------ .../redfish_events/subscriber_manager.py | 284 --------------- nodescraper/redfish_events/trigger_engine.py | 112 ------ .../redfish_events/webhook_subscriber.py | 221 ------------ pyproject.toml | 12 - test/unit/plugins/test_analysis_window.py | 54 --- test/unit/redfish_events/test_daemon_flow.py | 138 -------- test/unit/redfish_events/test_daemon_http.py | 88 ----- 26 files changed, 129 insertions(+), 3066 deletions(-) delete mode 100644 config/redfish_events_daemon.example.json delete mode 100644 docs/REDFISH_EVENTS_DAEMON.md delete mode 100644 nodescraper/cli/daemon_cmd.py delete mode 100644 nodescraper/plugins/serviceability/analysis_window.py delete mode 100644 nodescraper/redfish_events/__init__.py delete mode 100644 nodescraper/redfish_events/client.py delete mode 100644 nodescraper/redfish_events/config.py delete mode 100644 nodescraper/redfish_events/daemon_config.py delete mode 100644 nodescraper/redfish_events/daemon_http.py delete mode 100644 nodescraper/redfish_events/daemon_runner.py delete mode 100644 nodescraper/redfish_events/log_baseline.py delete mode 100644 nodescraper/redfish_events/models.py delete mode 100644 nodescraper/redfish_events/parsing.py delete mode 100644 nodescraper/redfish_events/se_bridge.py delete mode 100644 nodescraper/redfish_events/sse_capability.py delete mode 100644 nodescraper/redfish_events/sse_subscriber.py delete mode 100644 nodescraper/redfish_events/subscriber_manager.py delete mode 100644 nodescraper/redfish_events/trigger_engine.py delete mode 100644 nodescraper/redfish_events/webhook_subscriber.py delete mode 100644 test/unit/plugins/test_analysis_window.py delete mode 100644 test/unit/redfish_events/test_daemon_flow.py delete mode 100644 test/unit/redfish_events/test_daemon_http.py diff --git a/config/redfish_events_daemon.example.json b/config/redfish_events_daemon.example.json deleted file mode 100644 index e50b2d3f..00000000 --- a/config/redfish_events_daemon.example.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "stream": { - "event_types": ["Alert", "StatusChange"], - "severities": ["Warning", "Critical"], - "reconnect_delay_seconds": 30, - "baseline_pull_enabled": true, - "enable_webhook_fallback": true, - "allow_loopback_webhook": true, - "dedupe_events": true - }, - "targets": [ - { - "target_key": "node-a", - "name": "MI300X Node A", - "host": "10.0.0.10", - "username": "root", - "password": "changeme", - "verify_ssl": false, - "transport": "auto", - "webhook_url": "http://collector-host:8081/hook/node-a" - } - ], - "trigger": { - "min_events": 3, - "window_seconds": 10, - "cooldown_seconds": 60 - }, - "analysis": { - "hub_python_module": "your_hub_package.hub_module", - "hub_display_name": "MI3XX Service Hub", - "afid_sag_path": "/path/to/AFID_SAG.json", - "hub_analyze_method": "get_service_info", - "skip_hub": false - }, - "http": { - "enabled": true, - "host": "0.0.0.0", - "port": 8081, - "webhook_path_prefix": "/hook", - "recommendations_path": "/recommendations", - "status_path": "/status" - } -} diff --git a/docs/REDFISH_EVENTS_DAEMON.md b/docs/REDFISH_EVENTS_DAEMON.md deleted file mode 100644 index f21dd1bf..00000000 --- a/docs/REDFISH_EVENTS_DAEMON.md +++ /dev/null @@ -1,80 +0,0 @@ -# Redfish event daemon - -The Redfish event daemon is a long-running process that subscribes to BMC event streams (SSE or webhook fallback), batches bursts of events, and runs the same serviceability analysis path as on-demand `node-scraper run-plugins`. - -## Install - -```bash -cd ~/node-scraper_public -python3 -m venv .venv && source .venv/bin/activate -pip install -e ".[dev,events]" -``` - -The `[events]` extra pulls in `httpx`, required for async Redfish ingest. - -## Run - -```bash -node-scraper daemon --daemon-config config/redfish_events_daemon.example.json -``` - -Copy the example config, set BMC credentials, hub module paths, and (for webhook transport) point each target's `webhook_url` at this daemon's HTTP listener. - -## On-demand CLI vs daemon - -| | On-demand (`run-plugins`) | Daemon (`daemon`) | -|---|---|---| -| Lifecycle | One-shot collection + analysis | Long-lived background process | -| Event source | Redfish log pull during plugin run | Continuous SSE or webhook ingest | -| Analysis trigger | After collection completes | Sliding window (default: 3 events in 10s) | -| Output | Log directory + artifacts | Logs + HTTP `/recommendations` JSON | -| Use when | Ad-hoc debug, CI, field capture | Live monitoring, NOC integration | - -The on-demand MI3XX plugin path is unchanged. Both flows call `analyze_serviceability_window()` so hub configuration stays aligned. - -## Configuration - -Top-level sections in the daemon JSON: - -- **stream** — global ingest settings (`EventStreamConfig`): severities, dedupe, baseline re-pull, webhook fallback. -- **targets** — one entry per BMC (`EventTargetConfig`): host, credentials, transport (`auto`, `sse`, or `webhook`), optional `webhook_url`. -- **trigger** — sliding window thresholds (`TriggerConfig`): `min_events`, `window_seconds`, `cooldown_seconds`. -- **analysis** — same fields as `ServiceabilityAnalyzerArgs` in plugin configs (`hub_python_module`, `afid_sag_path`, etc.). -- **http** — optional listener for webhook ingest and live recommendations. - -See `config/redfish_events_daemon.example.json`. - -### Webhook transport - -When SSE is unavailable, set `transport` to `webhook` (or rely on `auto` + `enable_webhook_fallback`) and configure: - -1. `http.enabled: true` on the daemon with a reachable host/port. -2. Each target's `webhook_url` → `http://:/hook/`. -3. `allow_loopback_webhook: true` only for local testing (BMCs cannot reach loopback). - -### Trigger engine - -Events accumulate per `target_key`. When at least `min_events` arrive within `window_seconds`, the daemon runs serviceability analysis on that batch and ignores further triggers until `cooldown_seconds` elapse. - -## HTTP endpoints - -When `http.enabled` is true (default): - -| Method | Path | Purpose | -|--------|------|---------| -| GET | `/status` | Daemon health and per-target transport state | -| GET | `/recommendations` | Latest analysis snapshot for all targets | -| GET | `/recommendations/{target_key}` | Latest snapshot for one target | -| POST | `/hook/{target_key}` | Webhook payload → `handle_webhook_payload()` | - -Recommendations responses include `serviceability` (hub output), `afid_events`, and trigger metadata. - -## Tests - -```bash -pytest test/unit/redfish_events/ test/unit/plugins/test_analysis_window.py -v -``` - -## Deployment notes - -Systemd unit files and Docker images are not bundled yet. Run the daemon under your process supervisor of choice; ensure the HTTP port is reachable from BMCs when using webhook transport. diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 452ab123..30dc8792 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -335,16 +335,6 @@ def build_parser( help="Redfish path to LogService (e.g. redfish/v1/Systems/UBB/LogServices/DiagLogs)", ) - daemon_parser = subparsers.add_parser( - "daemon", - help="Run the long-lived Redfish event daemon (requires amd-node-scraper[events])", - ) - daemon_parser.add_argument( - "--daemon-config", - required=True, - help="Path to daemon JSON config (see config/redfish_events_daemon.example.json)", - ) - config_builder_parser.add_argument( "--plugins", nargs="*", @@ -557,12 +547,6 @@ def main( ) sys.exit(0) - if parsed_args.subcmd == "daemon": - from nodescraper.cli.daemon_cmd import run_daemon_cli - - run_daemon_cli(parsed_args.daemon_config, logger) - sys.exit(0) - if parsed_args.subcmd == "show-redfish-oem-allowable": if not parsed_args.connection_config: parser.error("show-redfish-oem-allowable requires --connection-config") diff --git a/nodescraper/cli/daemon_cmd.py b/nodescraper/cli/daemon_cmd.py deleted file mode 100644 index ae51f99b..00000000 --- a/nodescraper/cli/daemon_cmd.py +++ /dev/null @@ -1,52 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from __future__ import annotations - -import logging -import sys - -from pydantic import ValidationError - -from nodescraper.redfish_events.daemon_config import load_daemon_config - - -def run_daemon_cli(config_path: str, logger: logging.Logger) -> None: - """Load daemon config and run until interrupted.""" - from nodescraper.redfish_events.daemon_runner import run_event_daemon - - try: - config = load_daemon_config(config_path) - except FileNotFoundError as exc: - logger.error("%s", exc) - sys.exit(1) - except ValueError as exc: - logger.error("Invalid daemon config: %s", exc) - sys.exit(1) - except ValidationError as exc: - logger.error("Invalid daemon config: %s", exc) - sys.exit(1) - logger.info("Starting Redfish event daemon from %s", config_path) - run_event_daemon(config) diff --git a/nodescraper/plugins/serviceability/__init__.py b/nodescraper/plugins/serviceability/__init__.py index e8bf8916..c5e9f857 100644 --- a/nodescraper/plugins/serviceability/__init__.py +++ b/nodescraper/plugins/serviceability/__init__.py @@ -24,7 +24,6 @@ # ############################################################################### from .afid_events import build_afid_events_from_data -from .analysis_window import ServiceabilityWindowResult, analyze_serviceability_window from .analyzer_args import ServiceabilityAnalyzerArgs from .mi3xx import ( MI3XXAnalyzer, @@ -76,9 +75,7 @@ "ServiceabilityPluginMI3XX", "ServiceabilityResult", "ServiceabilitySolution", - "ServiceabilityWindowResult", "TimeOperator", - "analyze_serviceability_window", "build_afid_events_from_data", "build_mi3xx_reporting_version_fields", "compare_iso_datetime", diff --git a/nodescraper/plugins/serviceability/analysis_window.py b/nodescraper/plugins/serviceability/analysis_window.py deleted file mode 100644 index 4d605e5e..00000000 --- a/nodescraper/plugins/serviceability/analysis_window.py +++ /dev/null @@ -1,193 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Shared serviceability analysis path for on-demand plugins and the event daemon.""" -from __future__ import annotations - -import logging -from dataclasses import dataclass -from typing import Any, Optional - -from .afid_events import build_afid_events_from_data -from .analyzer_args import ServiceabilityAnalyzerArgs -from .cper_decode import CperDecodeError, decode_cper_raw_attachments -from .se_models import AfidEvent, ServiceabilityBlock -from .se_runner import SeRunError, run_service_hub -from .serviceability_data import ServiceabilityDataModel - - -@dataclass -class ServiceabilityWindowResult: - """Outcome of analyze_serviceability_window for CLI plugins or the event daemon.""" - - ok: bool - message: str - afid_events: list[AfidEvent] - serviceability: Optional[ServiceabilityBlock] = None - error: Optional[str] = None - - -def _cper_raw_needing_decode(data: ServiceabilityDataModel) -> dict[str, str]: - """Return CPER attachments that still need configured decode.""" - from .mi3xx.mi3xx_cper_utils import should_skip_cper_fetch_or_decode - - raw = data.cper_raw or {} - if not raw: - return {} - by_id: dict[str, dict[str, Any]] = {} - for member in data.rf_events: - if not isinstance(member, dict): - continue - eid = member.get("Id") - if eid is not None: - by_id[str(eid)] = member - out: dict[str, str] = {} - for event_id, blob in raw.items(): - ev = by_id.get(str(event_id)) - if ev is not None and should_skip_cper_fetch_or_decode(ev): - continue - out[str(event_id)] = blob - return out - - -def analyze_serviceability_window( - data: ServiceabilityDataModel, - args: ServiceabilityAnalyzerArgs, - *, - logger: Optional[logging.Logger] = None, - parent: str = "analyze_serviceability_window", -) -> ServiceabilityWindowResult: - """Build AFID events and optionally run the configured service hub on rf_events.""" - log = logger or logging.getLogger(__name__) - events = data.afid_events or build_afid_events_from_data(data) - data.afid_events = events - - if args.skip_hub: - block = ServiceabilityBlock(afid_events=events) - data.serviceability = block - return ServiceabilityWindowResult( - ok=True, - message=f"Built {len(events)} AFID event(s); hub skipped", - afid_events=events, - serviceability=block, - ) - - cper_data = data.cper_data or {} - cper_raw_to_decode = _cper_raw_needing_decode(data) - skipped_cper = len(data.cper_raw or {}) - len(cper_raw_to_decode) - if skipped_cper: - from .mi3xx.mi3xx_cper_utils import CPER_METHOD_AFID_MAX - - log.info( - "(%s) Skipping CPER decode for %d CPER attachment(s); Redfish log " - "already has usable ACA fields (CPER-method AFID<=%s or no serial on decode)", - parent, - skipped_cper, - CPER_METHOD_AFID_MAX, - ) - if cper_raw_to_decode and not cper_data: - if not args.cper_decode_module: - log.warning( - "(%s) %d CPER attachment(s) collected but cper_decode_module is " - "not set in analysis_args; skipping CPER decode", - parent, - len(cper_raw_to_decode), - ) - else: - log.info( - "(%s) Decoding %d CPER attachment(s) via %s.%s", - parent, - len(cper_raw_to_decode), - args.cper_decode_module, - args.cper_decode_method, - ) - try: - cper_data = decode_cper_raw_attachments( - cper_raw_to_decode, - cper_decode_module=args.cper_decode_module, - cper_decode_method=args.cper_decode_method, - logger=log, - ) - data.cper_data = cper_data - log.info( - "(%s) CPER decode finished: %d of %d attachment(s) decoded", - parent, - len(cper_data), - len(cper_raw_to_decode), - ) - except CperDecodeError as exc: - log.warning("(%s) %s; continuing without decoded CPER", parent, exc) - elif cper_data: - log.info( - "(%s) Using %d pre-decoded CPER record(s) from collection", - parent, - len(cper_data), - ) - - try: - block = run_service_hub( - hub_python_module=args.hub_python_module, # type: ignore[arg-type] - hub_display_name=args.hub_display_name, - afid_events=events, - afid_sag_path=args.afid_sag_path, # type: ignore[arg-type] - rf_events=data.rf_events, - cper_data=cper_data or None, - hub_options=args.resolved_hub_options(), - hub_analyze_method=args.hub_analyze_method, - hub_init_path_kwarg=args.hub_init_path_kwarg, - ) - except (SeRunError, ValueError) as exc: - return ServiceabilityWindowResult( - ok=False, - message=str(exc), - afid_events=events, - error=str(exc), - ) - - data.serviceability = block - hub_label = args.hub_display_name or args.hub_python_module - cper_summary = "" - if cper_data: - cper_summary = f", {len(cper_data)} decoded CPER(s)" - elif cper_raw_to_decode: - cper_summary = f", {len(cper_raw_to_decode)} CPER attachment(s) not decoded" - elif data.cper_raw: - cper_summary = f", {len(data.cper_raw)} CPER attachment(s) omitted (ACA on log entry)" - ver_bits: list[str] = [] - if block.hub_version: - ver_bits.append(f"hub {block.hub_version}") - if block.afid_sag_file_version: - ver_bits.append(f"AFID_SAG {block.afid_sag_file_version}") - ver_suffix = f" [{'; '.join(ver_bits)}]" if ver_bits else "" - message = ( - f"{hub_label}: {len(block.solution)} solution(s) " - f"from {len(data.rf_events)} Redfish event(s){cper_summary}{ver_suffix}" - ) - return ServiceabilityWindowResult( - ok=True, - message=message, - afid_events=events, - serviceability=block, - ) diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py index 9ed45a86..87bbaa3f 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py @@ -7,7 +7,7 @@ # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# to use, copy, modify, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # @@ -32,18 +32,23 @@ from nodescraper.enums import ExecutionStatus from nodescraper.interfaces import DataAnalyzer from nodescraper.models import TaskResult -from nodescraper.plugins.serviceability.analysis_window import ( - analyze_serviceability_window, -) +from nodescraper.plugins.serviceability.afid_events import build_afid_events_from_data from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs +from nodescraper.plugins.serviceability.cper_decode import ( + CperDecodeError, + decode_cper_raw_attachments, +) from nodescraper.plugins.serviceability.se_adapter import ( format_serviceability_solution_lines, ) from nodescraper.plugins.serviceability.se_models import ServiceabilityBlock +from nodescraper.plugins.serviceability.se_runner import SeRunError, run_service_hub from nodescraper.plugins.serviceability.serviceability_data import ( ServiceabilityDataModel, ) +from .mi3xx_cper_utils import CPER_METHOD_AFID_MAX, should_skip_cper_fetch_or_decode + class AfidSagMetadataArtifact(BaseModel): """Hub AFID_SAG metadata snapshot; written to ``afid_sag_metadata.json``.""" @@ -75,26 +80,133 @@ def analyze_data( self.result.message = "ServiceabilityAnalyzerArgs are required" return self.result + events = data.afid_events or build_afid_events_from_data(data) + data.afid_events = events + + if args.skip_hub: + data.serviceability = ServiceabilityBlock(afid_events=events) + self.result.status = ExecutionStatus.OK + self.result.message = f"Built {len(events)} AFID event(s); hub skipped" + self._log_serviceability_solutions(data.serviceability) + return self.result + parent = self.parent or self.__class__.__name__ - result = analyze_serviceability_window( - data, - args, - logger=self.logger, - parent=parent, - ) - if not result.ok: + cper_data = data.cper_data or {} + cper_raw_to_decode = self._cper_raw_needing_decode(data) + skipped_cper = len(data.cper_raw or {}) - len(cper_raw_to_decode) + if skipped_cper: + self.logger.info( + "(%s) Skipping CPER decode for %d CPER attachment(s); Redfish log " + "already has usable ACA fields (CPER-method AFID<=%s or no serial on decode)", + parent, + skipped_cper, + CPER_METHOD_AFID_MAX, + ) + if cper_raw_to_decode and not cper_data: + if not args.cper_decode_module: + self.logger.warning( + "(%s) %d CPER attachment(s) collected but cper_decode_module is " + "not set in analysis_args; skipping CPER decode", + parent, + len(cper_raw_to_decode), + ) + else: + self.logger.info( + "(%s) Decoding %d CPER attachment(s) via %s.%s", + parent, + len(cper_raw_to_decode), + args.cper_decode_module, + args.cper_decode_method, + ) + try: + cper_data = decode_cper_raw_attachments( + cper_raw_to_decode, + cper_decode_module=args.cper_decode_module, + cper_decode_method=args.cper_decode_method, + logger=self.logger, + ) + data.cper_data = cper_data + self.logger.info( + "(%s) CPER decode finished: %d of %d attachment(s) decoded", + parent, + len(cper_data), + len(cper_raw_to_decode), + ) + except CperDecodeError as exc: + self.logger.warning( + "(%s) %s; continuing without decoded CPER", + parent, + exc, + ) + elif cper_data: + self.logger.info( + "(%s) Using %d pre-decoded CPER record(s) from collection", + parent, + len(cper_data), + ) + + try: + block = run_service_hub( + hub_python_module=args.hub_python_module, # type: ignore[arg-type] + hub_display_name=args.hub_display_name, + afid_events=events, + afid_sag_path=args.afid_sag_path, # type: ignore[arg-type] + rf_events=data.rf_events, + cper_data=cper_data or None, + hub_options=args.resolved_hub_options(), + hub_analyze_method=args.hub_analyze_method, + hub_init_path_kwarg=args.hub_init_path_kwarg, + ) + except (SeRunError, ValueError) as exc: self.result.status = ExecutionStatus.ERROR - self.result.message = result.message + self.result.message = str(exc) return self.result - if result.serviceability is not None: - self._append_afid_sag_metadata_artifact(result.serviceability) - self._log_serviceability_solutions(result.serviceability) - + data.serviceability = block + self._append_afid_sag_metadata_artifact(block) + self._log_serviceability_solutions(block) + hub_label = args.hub_display_name or args.hub_python_module self.result.status = ExecutionStatus.OK - self.result.message = result.message + cper_summary = "" + if cper_data: + cper_summary = f", {len(cper_data)} decoded CPER(s)" + elif cper_raw_to_decode: + cper_summary = f", {len(cper_raw_to_decode)} CPER attachment(s) not decoded" + elif data.cper_raw: + cper_summary = f", {len(data.cper_raw)} CPER attachment(s) omitted (ACA on log entry)" + ver_bits: list[str] = [] + if block.hub_version: + ver_bits.append(f"hub {block.hub_version}") + if block.afid_sag_file_version: + ver_bits.append(f"AFID_SAG {block.afid_sag_file_version}") + ver_suffix = f" [{'; '.join(ver_bits)}]" if ver_bits else "" + self.result.message = ( + f"{hub_label}: {len(block.solution)} solution(s) " + f"from {len(data.rf_events)} Redfish event(s){cper_summary}{ver_suffix}" + ) return self.result + @staticmethod + def _cper_raw_needing_decode(data: ServiceabilityDataModel) -> dict[str, str]: + """Subset of ``cper_raw`` that still needs configured CPER decode (not already on the log).""" + raw = data.cper_raw or {} + if not raw: + return {} + by_id: dict[str, dict[str, Any]] = {} + for member in data.rf_events: + if not isinstance(member, dict): + continue + eid = member.get("Id") + if eid is not None: + by_id[str(eid)] = member + out: dict[str, str] = {} + for event_id, blob in raw.items(): + ev = by_id.get(str(event_id)) + if ev is not None and should_skip_cper_fetch_or_decode(ev): + continue + out[str(event_id)] = blob + return out + def _append_afid_sag_metadata_artifact(self, block: ServiceabilityBlock) -> None: if block.afid_sag_metadata is None: return diff --git a/nodescraper/redfish_events/__init__.py b/nodescraper/redfish_events/__init__.py deleted file mode 100644 index 133bfed9..00000000 --- a/nodescraper/redfish_events/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Continuous Redfish event ingest for node-scraper background services. - -This package is separate from on-demand CLI plugin runs. It provides SSE and -webhook-based event subscriptions, baseline log pulls, a trigger engine, and -``node-scraper daemon`` for long-running serviceability monitoring. - -Install the optional dependency: pip install amd-node-scraper[events] -""" -from __future__ import annotations - -import importlib -from typing import Any - -from .config import EventStreamConfig, EventTargetConfig -from .daemon_config import DaemonConfig, HttpServerConfig, load_daemon_config -from .models import ( - EventCallback, - EventSource, - RedfishEvent, - SubscriptionState, - TransportMode, -) -from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed -from .se_bridge import redfish_event_to_log_member, redfish_events_to_log_members -from .trigger_engine import TriggerConfig, TriggerEngine - -_LAZY_EXPORTS: dict[str, tuple[str, str]] = { - "AsyncRedfishClient": (".client", "AsyncRedfishClient"), - "SSECapabilityResult": (".sse_capability", "SSECapabilityResult"), - "SSESupport": (".sse_capability", "SSESupport"), - "SseEventSubscriber": (".sse_subscriber", "SseEventSubscriber"), - "SubscriberManager": (".subscriber_manager", "SubscriberManager"), - "WebhookEventSubscriber": (".webhook_subscriber", "WebhookEventSubscriber"), - "WebhookSubscriptionResult": (".webhook_subscriber", "WebhookSubscriptionResult"), - "check_sse_capability": (".sse_capability", "check_sse_capability"), - "pull_baseline_events": (".log_baseline", "pull_baseline_events"), -} - -__all__ = [ - "AsyncRedfishClient", - "DaemonConfig", - "EventCallback", - "EventSource", - "EventStreamConfig", - "EventTargetConfig", - "HttpServerConfig", - "RedfishEvent", - "SSECapabilityResult", - "SSESupport", - "SseEventSubscriber", - "SubscriberManager", - "SubscriptionState", - "TransportMode", - "TriggerConfig", - "TriggerEngine", - "WebhookEventSubscriber", - "WebhookSubscriptionResult", - "check_sse_capability", - "load_daemon_config", - "normalize_severity", - "parse_redfish_timestamp", - "pull_baseline_events", - "redfish_event_to_log_member", - "redfish_events_to_log_members", - "severity_allowed", -] - - -def __getattr__(name: str) -> Any: - if name not in _LAZY_EXPORTS: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - module_name, attr_name = _LAZY_EXPORTS[name] - module = importlib.import_module(module_name, __name__) - value = getattr(module, attr_name) - globals()[name] = value - return value diff --git a/nodescraper/redfish_events/client.py b/nodescraper/redfish_events/client.py deleted file mode 100644 index a85e122d..00000000 --- a/nodescraper/redfish_events/client.py +++ /dev/null @@ -1,107 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Minimal async Redfish GET client for event ingest follow-up requests.""" -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Optional - -import httpx - - -@dataclass -class RedfishGetResponse: - """Result of an async Redfish GET.""" - - ok: bool - status_code: int - data: Optional[dict[str, Any]] = None - content: bytes = b"" - error: Optional[str] = None - - -class AsyncRedfishClient: - """Small httpx wrapper for Redfish JSON and binary GETs.""" - - def __init__( - self, - base_url: str, - username: str, - password: str, - *, - verify_ssl: bool = False, - timeout_seconds: float = 30.0, - ) -> None: - self.base_url = base_url.rstrip("/") - self._auth = httpx.BasicAuth(username, password) - self._verify_ssl = verify_ssl - self._timeout = timeout_seconds - - def _abs_url(self, path: str) -> str: - if path.startswith("http://") or path.startswith("https://"): - return path - return f"{self.base_url}/{path.lstrip('/')}" - - async def get_json(self, path: str) -> RedfishGetResponse: - try: - async with httpx.AsyncClient( - auth=self._auth, - verify=self._verify_ssl, - timeout=self._timeout, - follow_redirects=True, - ) as client: - response = await client.get(self._abs_url(path)) - except Exception as exc: # noqa: BLE001 - return RedfishGetResponse(ok=False, status_code=0, error=str(exc)) - if response.status_code != 200: - return RedfishGetResponse( - ok=False, - status_code=response.status_code, - error=response.text[:500], - ) - try: - return RedfishGetResponse(ok=True, status_code=200, data=response.json()) - except Exception as exc: # noqa: BLE001 - return RedfishGetResponse(ok=False, status_code=200, error=str(exc)) - - async def get_bytes(self, path: str) -> RedfishGetResponse: - try: - async with httpx.AsyncClient( - auth=self._auth, - verify=self._verify_ssl, - timeout=self._timeout, - follow_redirects=True, - ) as client: - response = await client.get(self._abs_url(path)) - except Exception as exc: # noqa: BLE001 - return RedfishGetResponse(ok=False, status_code=0, error=str(exc)) - if response.status_code != 200: - return RedfishGetResponse( - ok=False, - status_code=response.status_code, - error=response.text[:500], - ) - return RedfishGetResponse(ok=True, status_code=200, content=response.content) diff --git a/nodescraper/redfish_events/config.py b/nodescraper/redfish_events/config.py deleted file mode 100644 index 2d80e0e5..00000000 --- a/nodescraper/redfish_events/config.py +++ /dev/null @@ -1,87 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Configuration models for Redfish event subscriptions.""" -from __future__ import annotations - -from typing import Optional - -from pydantic import BaseModel, Field, model_validator - -from .models import TransportMode - -DEFAULT_SSE_ENDPOINT = "/redfish/v1/EventService/SSE" - - -class EventTargetConfig(BaseModel): - """One BMC target for continuous event ingest.""" - - target_key: str = Field(..., min_length=1, description="Stable unique id for this target") - name: str = Field(..., min_length=1, description="Display name") - host: str = Field(..., min_length=1, description="BMC hostname or IP") - username: str - password: str - base_url: Optional[str] = Field( - default=None, - description="Redfish base URL; defaults to https://", - ) - verify_ssl: bool = False - sse_endpoint: str = DEFAULT_SSE_ENDPOINT - transport: TransportMode = TransportMode.AUTO - webhook_url: Optional[str] = Field( - default=None, - description="Collector webhook URL when transport is webhook or auto fallback", - ) - - def resolved_base_url(self) -> str: - if self.base_url: - return self.base_url.rstrip("/") - return f"https://{self.host}" - - -class EventStreamConfig(BaseModel): - """Global settings for SubscriberManager.""" - - event_types: list[str] = Field(default_factory=lambda: ["Alert", "StatusChange"]) - severities: list[str] = Field(default_factory=lambda: ["Warning", "Critical"]) - reconnect_delay_seconds: int = 30 - max_retry_duration_hours: float = 24.0 - cooldown_duration_hours: float = 6.0 - degraded_threshold_hours: float = 1.0 - baseline_pull_enabled: bool = True - baseline_max_entries_per_log: int = 200 - baseline_repull_interval_minutes: int = 60 - enable_webhook_fallback: bool = True - allow_loopback_webhook: bool = False - dedupe_events: bool = True - max_dedupe_entries: int = 10000 - - @model_validator(mode="after") - def _validate_lists(self) -> EventStreamConfig: - if not self.event_types: - raise ValueError("event_types must not be empty") - if not self.severities: - raise ValueError("severities must not be empty") - return self diff --git a/nodescraper/redfish_events/daemon_config.py b/nodescraper/redfish_events/daemon_config.py deleted file mode 100644 index dbe53095..00000000 --- a/nodescraper/redfish_events/daemon_config.py +++ /dev/null @@ -1,88 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Configuration for the long-running Redfish event daemon.""" -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -from pydantic import BaseModel, Field, model_validator - -from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs - -from .config import EventStreamConfig, EventTargetConfig -from .trigger_engine import TriggerConfig - - -class HttpServerConfig(BaseModel): - """Optional HTTP endpoints for webhook ingest and live recommendations.""" - - enabled: bool = True - host: str = "0.0.0.0" - port: int = Field(default=8081, ge=1, le=65535) - webhook_path_prefix: str = "/hook" - recommendations_path: str = "/recommendations" - status_path: str = "/status" - - @model_validator(mode="after") - def _normalize_paths(self) -> HttpServerConfig: - self.webhook_path_prefix = _normalize_path(self.webhook_path_prefix) - self.recommendations_path = _normalize_path(self.recommendations_path) - self.status_path = _normalize_path(self.status_path) - return self - - -class DaemonConfig(BaseModel): - """Full daemon configuration loaded from JSON.""" - - stream: EventStreamConfig = Field(default_factory=EventStreamConfig) - targets: list[EventTargetConfig] - trigger: TriggerConfig = Field(default_factory=TriggerConfig) - analysis: ServiceabilityAnalyzerArgs - http: HttpServerConfig = Field(default_factory=HttpServerConfig) - - @model_validator(mode="after") - def _require_targets(self) -> DaemonConfig: - if not self.targets: - raise ValueError("At least one target is required") - return self - - -def _normalize_path(value: str) -> str: - text = str(value).strip() or "/" - if not text.startswith("/"): - text = f"/{text}" - return text.rstrip("/") or "/" - - -def load_daemon_config(path: str | Path) -> DaemonConfig: - """Load and validate a daemon JSON config file.""" - config_path = Path(path) - if not config_path.is_file(): - raise FileNotFoundError(f"Daemon config not found: {config_path}") - raw: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8")) - return DaemonConfig.model_validate(raw) diff --git a/nodescraper/redfish_events/daemon_http.py b/nodescraper/redfish_events/daemon_http.py deleted file mode 100644 index f1e5b693..00000000 --- a/nodescraper/redfish_events/daemon_http.py +++ /dev/null @@ -1,202 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Minimal asyncio HTTP server for webhook ingest and live recommendations.""" -from __future__ import annotations - -import asyncio -import contextlib -import json -import logging -from typing import Any, Optional - -from .daemon_config import HttpServerConfig -from .subscriber_manager import SubscriberManager - -logger = logging.getLogger(__name__) - - -class DaemonHttpServer: - """Serve webhook POST endpoints and JSON recommendation snapshots.""" - - def __init__( - self, - manager: SubscriberManager, - recommendations: dict[str, dict[str, Any]], - config: HttpServerConfig, - ) -> None: - self._manager = manager - self._recommendations = recommendations - self._config = config - self._server: Optional[asyncio.AbstractServer] = None - - async def start(self) -> None: - """Bind and start accepting HTTP connections.""" - self._server = await asyncio.start_server( - self._handle_client, - host=self._config.host, - port=self._config.port, - ) - logger.info( - "Daemon HTTP listening on http://%s:%d", - self._config.host, - self._config.port, - ) - - async def stop(self) -> None: - """Stop the HTTP server.""" - if self._server is None: - return - self._server.close() - await self._server.wait_closed() - self._server = None - - async def _handle_client( - self, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - ) -> None: - try: - request_line = (await reader.readline()).decode("utf-8", errors="replace").strip() - if not request_line: - return - parts = request_line.split() - if len(parts) < 2: - await self._write_response(writer, 400, {"error": "bad request line"}) - return - method = parts[0].upper() - path = parts[1].split("?", 1)[0] - - headers: dict[str, str] = {} - while True: - line = (await reader.readline()).decode("utf-8", errors="replace").strip() - if not line: - break - if ":" in line: - key, value = line.split(":", 1) - headers[key.strip().lower()] = value.strip() - - body = b"" - content_length = int(headers.get("content-length", "0") or "0") - if content_length > 0: - body = await reader.readexactly(content_length) - - if method == "GET": - await self._handle_get(writer, path) - return - if method == "POST": - await self._handle_post(writer, path, body) - return - await self._write_response(writer, 405, {"error": "method not allowed"}) - except asyncio.IncompleteReadError: - return - except Exception as exc: # noqa: BLE001 - logger.warning("HTTP handler error: %s", exc) - await self._write_response(writer, 500, {"error": "internal error"}) - finally: - writer.close() - with contextlib.suppress(Exception): - await writer.wait_closed() - - async def _handle_get(self, writer: asyncio.StreamWriter, path: str) -> None: - if path == self._config.status_path: - targets: dict[str, dict[str, Any]] = {} - for key in self._manager.target_keys(): - state = self._manager.subscription_state(key) - targets[key] = { - "transport": self._manager.transport_for(key), - "subscription_state": state.value if state is not None else None, - } - payload = { - "status": "ok", - "targets": targets, - } - await self._write_response(writer, 200, payload) - return - - if path == self._config.recommendations_path: - await self._write_response(writer, 200, self._recommendations) - return - - prefix = self._config.recommendations_path.rstrip("/") + "/" - if path.startswith(prefix): - target_key = path[len(prefix) :] - snapshot = self._recommendations.get(target_key) - if snapshot is None: - await self._write_response(writer, 404, {"error": f"unknown target {target_key}"}) - return - await self._write_response(writer, 200, snapshot) - return - - await self._write_response(writer, 404, {"error": "not found"}) - - async def _handle_post( - self, - writer: asyncio.StreamWriter, - path: str, - body: bytes, - ) -> None: - prefix = self._config.webhook_path_prefix.rstrip("/") + "/" - if not path.startswith(prefix): - await self._write_response(writer, 404, {"error": "not found"}) - return - target_key = path[len(prefix) :] - if not target_key: - await self._write_response(writer, 400, {"error": "missing target_key in path"}) - return - try: - payload = json.loads(body.decode("utf-8") if body else "{}") - except json.JSONDecodeError: - await self._write_response(writer, 400, {"error": "invalid JSON body"}) - return - if not isinstance(payload, dict): - await self._write_response(writer, 400, {"error": "JSON body must be an object"}) - return - emitted = await self._manager.handle_webhook_payload(target_key, payload) - await self._write_response(writer, 200, {"accepted": True, "events_emitted": emitted}) - - async def _write_response( - self, - writer: asyncio.StreamWriter, - status: int, - payload: dict[str, Any], - ) -> None: - body = json.dumps(payload).encode("utf-8") - reason = { - 200: "OK", - 400: "Bad Request", - 404: "Not Found", - 405: "Method Not Allowed", - 500: "Internal Server Error", - }.get(status, "OK") - header = ( - f"HTTP/1.1 {status} {reason}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {len(body)}\r\n" - "Connection: close\r\n" - "\r\n" - ).encode("utf-8") - writer.write(header + body) - await writer.drain() diff --git a/nodescraper/redfish_events/daemon_runner.py b/nodescraper/redfish_events/daemon_runner.py deleted file mode 100644 index a70e4968..00000000 --- a/nodescraper/redfish_events/daemon_runner.py +++ /dev/null @@ -1,143 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Long-running Redfish event daemon orchestration.""" -from __future__ import annotations - -import asyncio -import logging -import signal -from datetime import UTC, datetime -from typing import Any, Optional - -from nodescraper.plugins.serviceability.analysis_window import ( - analyze_serviceability_window, -) -from nodescraper.plugins.serviceability.se_adapter import ( - format_serviceability_solution_lines, -) -from nodescraper.plugins.serviceability.serviceability_data import ( - ServiceabilityDataModel, -) - -from .daemon_config import DaemonConfig -from .daemon_http import DaemonHttpServer -from .models import RedfishEvent -from .se_bridge import redfish_events_to_log_members -from .subscriber_manager import SubscriberManager -from .trigger_engine import TriggerEngine - -logger = logging.getLogger(__name__) - - -class EventDaemon: - """Wire SubscriberManager, trigger engine, analysis, and optional HTTP endpoints.""" - - def __init__(self, config: DaemonConfig) -> None: - self.config = config - self._recommendations: dict[str, dict[str, Any]] = {} - self._manager = SubscriberManager(config.stream, self._on_event) - self._manager.set_targets(config.targets) - self._trigger = TriggerEngine(config.trigger, self._on_trigger) - self._http: Optional[DaemonHttpServer] = None - self._stop_event = asyncio.Event() - - async def run(self) -> None: - """Start subscriptions and block until SIGINT or SIGTERM.""" - loop = asyncio.get_running_loop() - try: - for sig in (signal.SIGINT, signal.SIGTERM): - loop.add_signal_handler(sig, self._stop_event.set) - except NotImplementedError: - signal.signal(signal.SIGINT, lambda *_args: self._stop_event.set()) - - if self.config.http.enabled: - self._http = DaemonHttpServer(self._manager, self._recommendations, self.config.http) - await self._http.start() - - await self._manager.start() - logger.info( - "Redfish event daemon running for %d target(s); Ctrl+C to stop", - len(self.config.targets), - ) - await self._stop_event.wait() - await self.stop() - - async def stop(self) -> None: - """Stop subscriptions and the HTTP server.""" - await self._manager.stop() - if self._http is not None: - await self._http.stop() - self._http = None - logger.info("Redfish event daemon stopped") - - async def _on_event(self, event: RedfishEvent) -> None: - await self._trigger.handle_event(event) - - async def _on_trigger(self, target_key: str, events: list[RedfishEvent]) -> None: - target_name = events[0].target_name if events else target_key - rf_members = redfish_events_to_log_members(events) - bmc_host = events[0].target_host if events else None - data = ServiceabilityDataModel(rf_events=rf_members, bmc_host=bmc_host) - parent = f"EventDaemon:{target_key}" - result = await asyncio.to_thread( - analyze_serviceability_window, - data, - self.config.analysis, - logger=logger, - parent=parent, - ) - snapshot = { - "target_key": target_key, - "target_name": target_name, - "triggered_at": datetime.now(UTC).isoformat(), - "event_count": len(events), - "ok": result.ok, - "message": result.message, - "afid_events": [event.model_dump(mode="json") for event in result.afid_events], - } - if result.serviceability is not None: - snapshot["serviceability"] = result.serviceability.model_dump(mode="json") - for line in format_serviceability_solution_lines(result.serviceability): - logger.info("(%s) %s", parent, line) - elif result.error: - logger.error("(%s) analysis failed: %s", parent, result.error) - else: - logger.info("(%s) %s", parent, result.message) - self._recommendations[target_key] = snapshot - - -def run_event_daemon(config: DaemonConfig) -> None: - """Run the daemon until interrupted.""" - try: - import httpx # noqa: F401 - except ImportError as exc: - raise RuntimeError( - "Redfish event daemon requires the optional [events] extra: " - "pip install amd-node-scraper[events]" - ) from exc - - daemon = EventDaemon(config) - asyncio.run(daemon.run()) diff --git a/nodescraper/redfish_events/log_baseline.py b/nodescraper/redfish_events/log_baseline.py deleted file mode 100644 index 37238234..00000000 --- a/nodescraper/redfish_events/log_baseline.py +++ /dev/null @@ -1,252 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Baseline pull of existing Redfish LogService entries (adapted from Gyanam).""" -from __future__ import annotations - -import logging -from collections.abc import Callable -from datetime import UTC, datetime -from typing import Optional - -import httpx - -from .models import EventSource, RedfishEvent -from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed - -logger = logging.getLogger(__name__) - -BaselineCallback = Callable[[RedfishEvent], None] -_LOG_ROOTS = ("/redfish/v1/Systems", "/redfish/v1/Managers") -_MAX_PAGES = 20 - - -def _abs(base_url: str, uri: str) -> str: - if not uri: - return "" - if uri.startswith("http://") or uri.startswith("https://"): - return uri - return f"{base_url.rstrip('/')}/{uri.lstrip('/')}" - - -def _extract_origin(entry: dict) -> Optional[str]: - origin = entry.get("OriginOfCondition") - if origin is None: - links = entry.get("Links") - if isinstance(links, dict): - origin = links.get("OriginOfCondition") - if isinstance(origin, dict): - return origin.get("@odata.id") - if isinstance(origin, str): - return origin - return None - - -async def _get_json(client: httpx.AsyncClient, url: str) -> Optional[dict]: - try: - resp = await client.get(url) - except Exception as exc: # noqa: BLE001 - logger.debug("Baseline GET failed for %s: %s", url, type(exc).__name__) - return None - if resp.status_code != 200: - return None - try: - return resp.json() - except Exception: # noqa: BLE001 - return None - - -async def _discover_entry_collections(client: httpx.AsyncClient, base_url: str) -> list[str]: - collections: list[str] = [] - seen: set[str] = set() - for root in _LOG_ROOTS: - root_doc = await _get_json(client, _abs(base_url, root)) - if not root_doc: - continue - for member in root_doc.get("Members", []): - member_uri = member.get("@odata.id") if isinstance(member, dict) else None - if not member_uri: - continue - member_doc = await _get_json(client, _abs(base_url, member_uri)) - if not member_doc: - continue - ls_ref = member_doc.get("LogServices", {}) - ls_uri = ls_ref.get("@odata.id") if isinstance(ls_ref, dict) else None - if not ls_uri: - continue - ls_doc = await _get_json(client, _abs(base_url, ls_uri)) - if not ls_doc: - continue - for ls_member in ls_doc.get("Members", []): - ls_member_uri = ls_member.get("@odata.id") if isinstance(ls_member, dict) else None - if not ls_member_uri: - continue - ls_detail = await _get_json(client, _abs(base_url, ls_member_uri)) - if not ls_detail: - continue - entries_ref = ls_detail.get("Entries", {}) - entries_uri = ( - entries_ref.get("@odata.id") if isinstance(entries_ref, dict) else None - ) - if entries_uri and entries_uri not in seen: - seen.add(entries_uri) - collections.append(entries_uri) - return collections - - -def order_members_newest_first(members: list, max_entries: int) -> list[dict]: - dicts = [item for item in members if isinstance(item, dict)] - - def _sort_key(member: dict): - ts = parse_redfish_timestamp(member.get("Created")) - if ts is not None: - return (2, ts.timestamp()) - oid = member.get("@odata.id", "") or "" - try: - return (1, float(oid.rstrip("/").split("/")[-1])) - except (ValueError, IndexError): - return (0, 0.0) - - dicts.sort(key=_sort_key, reverse=True) - return dicts[:max_entries] - - -def _naive_utc(dt: Optional[datetime]) -> Optional[datetime]: - if dt is not None and dt.tzinfo is not None: - return dt.astimezone(UTC).replace(tzinfo=None) - return dt - - -async def _collect_members( - client: httpx.AsyncClient, - base_url: str, - entries_uri: str, - max_entries: int, -) -> list[dict]: - members: list[dict] = [] - seen_ids: set[str] = set() - visited: set[str] = set() - uri: Optional[str] = entries_uri - pages = 0 - while uri: - if uri in visited: - break - visited.add(uri) - doc = await _get_json(client, _abs(base_url, uri)) - if not doc: - break - for member in doc.get("Members", []): - if not isinstance(member, dict): - continue - mid = member.get("@odata.id") - if mid is not None: - if mid in seen_ids: - continue - seen_ids.add(mid) - members.append(member) - pages += 1 - uri = doc.get("Members@odata.nextLink") - if len(members) >= max_entries * 3 or pages >= _MAX_PAGES: - break - return members - - -async def pull_baseline_events( - *, - target_key: str, - target_name: str, - target_host: str, - base_url: str, - username: str, - password: str, - verify_ssl: bool, - callback: BaselineCallback, - severities: Optional[list[str]] = None, - max_entries_per_log: int = 200, - timeout: float = 30.0, -) -> int: - """Pull existing log entries and emit RedfishEvent objects via callback.""" - auth = httpx.BasicAuth(username, password) - emitted = 0 - try: - async with httpx.AsyncClient( - auth=auth, - verify=verify_ssl, - timeout=timeout, - follow_redirects=True, - ) as client: - collections = await _discover_entry_collections(client, base_url) - for entries_uri in collections: - members = await _collect_members( - client, - base_url, - entries_uri, - max_entries_per_log, - ) - for member in order_members_newest_first(members, max_entries_per_log): - entry = member - if any(k not in entry for k in ("Message", "Severity", "Created")): - ref = member.get("@odata.id") - if ref: - fetched = await _get_json(client, _abs(base_url, ref)) - if fetched: - entry = fetched - if ( - "Message" not in entry - and "Severity" not in entry - and "MessageSeverity" not in entry - ): - continue - severity, sev_present = normalize_severity(entry) - if not severity_allowed(severity, sev_present, severities): - continue - source_id = entry.get("@odata.id") or member.get("@odata.id") - event = RedfishEvent( - target_key=target_key, - target_name=target_name, - target_host=target_host, - severity=severity, - message=entry.get("Message", "") or "", - message_id=entry.get("MessageId"), - event_type=entry.get("EntryType") or "Alert", - origin_of_condition=_extract_origin(entry), - event_timestamp=_naive_utc(parse_redfish_timestamp(entry.get("Created"))), - received_at=datetime.now(UTC), - source=EventSource.BASELINE, - source_id=source_id, - raw=dict(entry), - ) - callback(event) - emitted += 1 - except Exception as exc: # noqa: BLE001 - logger.warning( - "Baseline pull failed for %s: %s: %s", - target_name, - type(exc).__name__, - exc, - ) - if emitted: - logger.info("Baseline pull for %s emitted %d log entries", target_name, emitted) - return emitted diff --git a/nodescraper/redfish_events/models.py b/nodescraper/redfish_events/models.py deleted file mode 100644 index 237278b7..00000000 --- a/nodescraper/redfish_events/models.py +++ /dev/null @@ -1,94 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Data models for continuous Redfish event ingest.""" -from __future__ import annotations - -from collections.abc import Awaitable, Callable -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from typing import Any, Optional, Union - -EventCallback = Callable[["RedfishEvent"], Union[None, Awaitable[None]]] - - -class EventSource(str, Enum): - """How an event entered the ingest pipeline.""" - - SSE = "sse" - WEBHOOK = "webhook" - BASELINE = "baseline" - - -class TransportMode(str, Enum): - """Preferred Redfish event transport for a target.""" - - AUTO = "auto" - SSE = "sse" - WEBHOOK = "webhook" - - -class SubscriptionState(str, Enum): - """Connection state for a target event subscription.""" - - CONNECTED = "connected" - RECONNECTING = "reconnecting" - DEGRADED = "degraded" - ON_COOLDOWN = "on_cooldown" - FAILED_PERMANENT = "failed_permanent" - STOPPED = "stopped" - - -@dataclass -class RedfishEvent: - """Normalized Redfish alert or log entry for downstream consumers.""" - - target_key: str - target_name: str - target_host: str - severity: str - message: str - event_type: str - received_at: datetime - source: EventSource - message_id: Optional[str] = None - origin_of_condition: Optional[str] = None - event_timestamp: Optional[datetime] = None - source_id: Optional[str] = None - raw: Optional[dict[str, Any]] = field(default_factory=dict) - - def dedupe_key(self) -> str: - """Return a stable key for in-memory deduplication.""" - if self.source_id: - return self.source_id - parts = ( - self.target_key, - self.message_id or "", - self.event_type, - self.message, - self.event_timestamp.isoformat() if self.event_timestamp else "", - ) - return "|".join(parts) diff --git a/nodescraper/redfish_events/parsing.py b/nodescraper/redfish_events/parsing.py deleted file mode 100644 index c98c9a55..00000000 --- a/nodescraper/redfish_events/parsing.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Redfish event parsing helpers (adapted from Gyanam alert ingest).""" -from __future__ import annotations - -import re -from datetime import datetime, timedelta, timezone -from typing import Optional - -UTC = timezone.utc - -_TS_RE = re.compile( - r"(\d{4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})" - r"(?:\.(\d+))?\s*(Z|[+-]\d{2}:?\d{2})?" -) - - -def parse_redfish_timestamp(value: Optional[str]) -> Optional[datetime]: - """Parse Redfish EventTimestamp / LogEntry Created values.""" - if not value or not isinstance(value, str): - return None - s = value.strip() - try: - dt = datetime.fromisoformat(s.replace("Z", "+00:00")) - return dt if dt.tzinfo else dt.replace(tzinfo=UTC) - except (ValueError, AttributeError): - pass - - match = _TS_RE.search(s) - if not match: - return None - year, mon, day, hour, minute, sec, frac, tz = match.groups() - try: - micro = int((frac or "0").ljust(6, "0")[:6]) - dt = datetime(int(year), int(mon), int(day), int(hour), int(minute), int(sec), micro) - except ValueError: - return None - - if tz and tz != "Z": - sign = 1 if tz[0] == "+" else -1 - digits = tz[1:].replace(":", "") - offset = timedelta(hours=int(digits[:2]), minutes=int(digits[2:4])) - return dt.replace(tzinfo=timezone(sign * offset)) - return dt.replace(tzinfo=UTC) - - -def normalize_severity(event: dict) -> tuple[str, bool]: - """Return severity and whether the field was present on the event.""" - raw = event.get("MessageSeverity") or event.get("Severity") - if raw: - return str(raw), True - return "OK", False - - -def severity_allowed( - severity: str, - present: bool, - allow_list: Optional[list[str]], -) -> bool: - """Apply case-insensitive severity filtering.""" - if not present: - return True - if not allow_list: - return True - allowed = {item.casefold() for item in allow_list} - return severity.casefold() in allowed diff --git a/nodescraper/redfish_events/se_bridge.py b/nodescraper/redfish_events/se_bridge.py deleted file mode 100644 index 201dba62..00000000 --- a/nodescraper/redfish_events/se_bridge.py +++ /dev/null @@ -1,58 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Bridge RedfishEvent objects into serviceability plugin rf_events shape.""" -from __future__ import annotations - -from typing import Any - -from .models import RedfishEvent - - -def redfish_event_to_log_member(event: RedfishEvent) -> dict[str, Any]: - """Return a dict suitable for ServiceabilityDataModel.rf_events entries.""" - if event.raw: - return dict(event.raw) - member: dict[str, Any] = { - "Message": event.message, - "Severity": event.severity, - "EventType": event.event_type, - } - if event.message_id: - member["MessageId"] = event.message_id - if event.origin_of_condition: - member["OriginOfCondition"] = {"@odata.id": event.origin_of_condition} - if event.event_timestamp: - member["EventTimestamp"] = event.event_timestamp.isoformat() - member["Created"] = member["EventTimestamp"] - if event.source_id: - member["@odata.id"] = event.source_id - member["Id"] = event.source_id.rstrip("/").split("/")[-1] - return member - - -def redfish_events_to_log_members(events: list[RedfishEvent]) -> list[dict[str, Any]]: - """Convert a batch of ingest events for on-demand serviceability analysis.""" - return [redfish_event_to_log_member(event) for event in events] diff --git a/nodescraper/redfish_events/sse_capability.py b/nodescraper/redfish_events/sse_capability.py deleted file mode 100644 index d454f8ea..00000000 --- a/nodescraper/redfish_events/sse_capability.py +++ /dev/null @@ -1,206 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Detect whether a BMC supports Redfish EventService SSE (adapted from Gyanam).""" -from __future__ import annotations - -import asyncio -import logging -import time -from dataclasses import dataclass -from enum import Enum -from typing import Optional - -import httpx - -logger = logging.getLogger(__name__) - - -class SSESupport(str, Enum): - SUPPORTED = "supported" - NOT_SUPPORTED = "not_supported" - BROKEN = "broken" - UNKNOWN = "unknown" - - -@dataclass -class SSECapabilityResult: - support: SSESupport - reason: str - event_service_enabled: bool = False - sse_endpoint: Optional[str] = None - test_duration_ms: float = 0.0 - - -async def check_sse_capability( - base_url: str, - username: str, - password: str, - verify_ssl: bool = False, - test_duration_seconds: float = 5.0, -) -> SSECapabilityResult: - start = time.time() - try: - async with httpx.AsyncClient( - auth=httpx.BasicAuth(username, password), - verify=verify_ssl, - timeout=10.0, - ) as client: - response = await client.get(f"{base_url.rstrip('/')}/redfish/v1/EventService") - if response.status_code == 404: - return SSECapabilityResult( - support=SSESupport.NOT_SUPPORTED, - reason="EventService not found (404)", - test_duration_ms=(time.time() - start) * 1000, - ) - if response.status_code != 200: - return SSECapabilityResult( - support=SSESupport.UNKNOWN, - reason=f"EventService returned HTTP {response.status_code}", - test_duration_ms=(time.time() - start) * 1000, - ) - event_service = response.json() - enabled = bool(event_service.get("ServiceEnabled", False)) - sse_uri = event_service.get("ServerSentEventUri") - if not sse_uri: - return SSECapabilityResult( - support=SSESupport.NOT_SUPPORTED, - reason="ServerSentEventUri not advertised", - event_service_enabled=enabled, - test_duration_ms=(time.time() - start) * 1000, - ) - sse_url = sse_uri if sse_uri.startswith("http") else f"{base_url.rstrip('/')}{sse_uri}" - result = await _test_sse_endpoint( - sse_url, - username, - password, - verify_ssl, - test_duration_seconds, - ) - result.event_service_enabled = enabled - result.sse_endpoint = sse_uri - result.test_duration_ms = (time.time() - start) * 1000 - return result - except httpx.TimeoutException: - return SSECapabilityResult( - support=SSESupport.UNKNOWN, - reason="Timeout checking EventService", - test_duration_ms=(time.time() - start) * 1000, - ) - except Exception as exc: # noqa: BLE001 - return SSECapabilityResult( - support=SSESupport.UNKNOWN, - reason=f"{type(exc).__name__}: {exc}", - test_duration_ms=(time.time() - start) * 1000, - ) - - -async def _test_sse_endpoint( - sse_url: str, - username: str, - password: str, - verify_ssl: bool, - test_duration: float, -) -> SSECapabilityResult: - try: - timeout = httpx.Timeout(10.0, read=test_duration + 2.0) - async with ( - httpx.AsyncClient( - auth=httpx.BasicAuth(username, password), - verify=verify_ssl, - timeout=timeout, - ) as client, - client.stream("GET", sse_url) as response, - ): - if response.status_code in (404, 501): - return SSECapabilityResult( - support=SSESupport.NOT_SUPPORTED, - reason=f"SSE endpoint HTTP {response.status_code}", - ) - if response.status_code != 200: - return SSECapabilityResult( - support=SSESupport.BROKEN, - reason=f"SSE endpoint HTTP {response.status_code}", - ) - content_type = response.headers.get("content-type", "") - if "text/event-stream" not in content_type.lower(): - return SSECapabilityResult( - support=SSESupport.BROKEN, - reason=f"Unexpected content-type: {content_type}", - ) - - lines_received = 0 - keepalives = 0 - events = 0 - - async def _read_lines() -> None: - nonlocal lines_received, keepalives, events - async for line in response.aiter_lines(): - lines_received += 1 - stripped = line.strip() - if not stripped: - continue - if stripped.startswith(":"): - keepalives += 1 - elif stripped.startswith("data:"): - events += 1 - - try: - await asyncio.wait_for(_read_lines(), timeout=test_duration) - except asyncio.TimeoutError: - pass - - if lines_received == 0: - return SSECapabilityResult( - support=SSESupport.BROKEN, - reason="SSE stream opened but no data received", - ) - if keepalives or events: - return SSECapabilityResult( - support=SSESupport.SUPPORTED, - reason=( - f"SSE working ({events} events, {keepalives} keep-alives " - f"in {test_duration}s test)" - ), - ) - return SSECapabilityResult( - support=SSESupport.BROKEN, - reason=f"SSE active but invalid format ({lines_received} lines)", - ) - except httpx.ConnectError as exc: - return SSECapabilityResult( - support=SSESupport.UNKNOWN, - reason=f"Cannot connect: {exc}", - ) - except httpx.TimeoutException: - return SSECapabilityResult( - support=SSESupport.UNKNOWN, - reason="Timeout connecting to SSE endpoint", - ) - except Exception as exc: # noqa: BLE001 - return SSECapabilityResult( - support=SSESupport.BROKEN, - reason=f"{type(exc).__name__}: {exc}", - ) diff --git a/nodescraper/redfish_events/sse_subscriber.py b/nodescraper/redfish_events/sse_subscriber.py deleted file mode 100644 index 7750dd81..00000000 --- a/nodescraper/redfish_events/sse_subscriber.py +++ /dev/null @@ -1,326 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""SSE alert subscriber for a single Redfish target (adapted from Gyanam).""" -from __future__ import annotations - -import asyncio -import json -import logging -import random -from collections.abc import Callable -from contextlib import suppress -from datetime import UTC, datetime, timedelta -from enum import Enum -from typing import Optional - -import httpx - -from .models import EventSource, RedfishEvent, SubscriptionState -from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed - -logger = logging.getLogger(__name__) - -SSE_READ_TIMEOUT_SECONDS = 300.0 -EventHandler = Callable[[RedfishEvent], None] - - -class ErrorCategory(str, Enum): - PERMANENT = "permanent" - TRANSIENT = "transient" - - -class SseEventSubscriber: - """Maintain one long-lived Redfish EventService SSE stream.""" - - def __init__( - self, - *, - target_key: str, - target_name: str, - target_host: str, - base_url: str, - username: str, - password: str, - sse_endpoint: str = "/redfish/v1/EventService/SSE", - verify_ssl: bool = False, - callback: Optional[EventHandler] = None, - reconnect_delay: int = 30, - max_retry_duration_hours: float = 24, - cooldown_duration_hours: float = 6, - degraded_threshold_hours: float = 1, - event_types: Optional[list[str]] = None, - severities: Optional[list[str]] = None, - ) -> None: - self.target_key = target_key - self.target_name = target_name - self.target_host = target_host - self.base_url = base_url.rstrip("/") - self.username = username - self.password = password - self.sse_endpoint = sse_endpoint - self.verify_ssl = verify_ssl - self.callback = callback - self.reconnect_delay = reconnect_delay - self.max_retry_duration_hours = max_retry_duration_hours - self.cooldown_duration_hours = cooldown_duration_hours - self.degraded_threshold_hours = degraded_threshold_hours - self.event_types = event_types or ["Alert", "StatusChange"] - self.severities = severities or ["Warning", "Critical"] - - self._running = False - self._task: Optional[asyncio.Task] = None - self._consecutive_failures = 0 - self._last_event_time: Optional[datetime] = None - self._first_failure_time: Optional[datetime] = None - self._cooldown_start_time: Optional[datetime] = None - self._state = SubscriptionState.STOPPED - self._failure_reason: Optional[str] = None - self._next_retry_time: Optional[datetime] = None - - async def start(self) -> None: - if self._running: - return - self._running = True - self._state = SubscriptionState.RECONNECTING - self._task = asyncio.create_task(self._subscribe_loop()) - logger.info("Started SSE event subscription for %s", self.target_name) - - async def stop(self) -> None: - self._running = False - self._state = SubscriptionState.STOPPED - if self._task: - self._task.cancel() - with suppress(asyncio.CancelledError): - await self._task - self._task = None - logger.info("Stopped SSE event subscription for %s", self.target_name) - - async def _subscribe_loop(self) -> None: - while self._running: - try: - if self._cooldown_start_time: - elapsed_hours = ( - datetime.now(UTC) - self._cooldown_start_time - ).total_seconds() / 3600 - if elapsed_hours < self.cooldown_duration_hours: - self._state = SubscriptionState.ON_COOLDOWN - await asyncio.sleep(60) - continue - self._cooldown_start_time = None - self._first_failure_time = None - self._consecutive_failures = 0 - self._state = SubscriptionState.RECONNECTING - - await self._connect_and_listen() - except asyncio.CancelledError: - break - except Exception as exc: # noqa: BLE001 - category, reason = self._classify_error(exc) - if self._consecutive_failures == 0: - self._first_failure_time = datetime.now(UTC) - self._consecutive_failures += 1 - self._failure_reason = reason - - if category == ErrorCategory.PERMANENT: - logger.warning( - "Permanent SSE error for %s: %s", - self.target_name, - reason, - ) - self._state = SubscriptionState.FAILED_PERMANENT - self._running = False - break - - if self._first_failure_time: - elapsed_hours = ( - datetime.now(UTC) - self._first_failure_time - ).total_seconds() / 3600 - if elapsed_hours >= self.max_retry_duration_hours: - self._cooldown_start_time = datetime.now(UTC) - self._state = SubscriptionState.ON_COOLDOWN - continue - if elapsed_hours >= self.degraded_threshold_hours: - self._state = SubscriptionState.DEGRADED - else: - self._state = SubscriptionState.RECONNECTING - - delay = self._calculate_backoff_delay() - self._next_retry_time = datetime.now(UTC) + timedelta(seconds=delay) - await asyncio.sleep(delay) - - async def _connect_and_listen(self) -> None: - url = f"{self.base_url}{self.sse_endpoint}" - auth = httpx.BasicAuth(self.username, self.password) - timeout = httpx.Timeout(30.0, read=SSE_READ_TIMEOUT_SECONDS) - - async with ( - httpx.AsyncClient( - auth=auth, - verify=self.verify_ssl, - timeout=timeout, - follow_redirects=True, - ) as client, - client.stream("GET", url) as response, - ): - if response.status_code != 200: - preview = "" - try: - preview = (await response.aread())[:200].decode("utf-8", errors="ignore") - except Exception: # noqa: BLE001 - pass - raise RuntimeError(f"SSE connection failed: HTTP {response.status_code} {preview}") - - self._consecutive_failures = 0 - self._first_failure_time = None - self._failure_reason = None - self._next_retry_time = None - self._state = SubscriptionState.CONNECTED - - connect_time = datetime.now(UTC) - event_count = 0 - data_buffer: list[str] = [] - - def _dispatch() -> None: - nonlocal event_count - if not data_buffer: - return - payload = "\n".join(data_buffer) - data_buffer.clear() - if not payload.strip(): - return - event_count += 1 - try: - self._process_event_data(payload) - except Exception as exc: # noqa: BLE001 - logger.warning( - "Failed to process SSE event from %s: %s", - self.target_name, - exc, - ) - - async for line in response.aiter_lines(): - if not self._running: - break - line = line.rstrip("\r\n") - if line == "": - _dispatch() - continue - if line.startswith(":"): - connect_time = datetime.now(UTC) - continue - if ":" in line: - field, _, value = line.partition(":") - if value.startswith(" "): - value = value[1:] - else: - field, value = line, "" - if field == "data": - data_buffer.append(value) - - _dispatch() - - elapsed = (datetime.now(UTC) - connect_time).total_seconds() - if elapsed < 30.0 and event_count == 0: - raise RuntimeError(f"SSE stream closed after {elapsed:.1f}s without sending events") - - def _process_event_data(self, data_str: str) -> None: - data = json.loads(data_str) - for event in data.get("Events", []): - if not isinstance(event, dict): - continue - event_type = event.get("EventType") or "" - severity, sev_present = normalize_severity(event) - if event_type and self.event_types and event_type not in self.event_types: - continue - if not severity_allowed(severity, sev_present, self.severities): - continue - if not event_type: - event_type = "Alert" - - origin = event.get("OriginOfCondition", {}) - origin_uri = None - if isinstance(origin, dict): - origin_uri = origin.get("@odata.id") - elif isinstance(origin, str): - origin_uri = origin - - redfish_event = RedfishEvent( - target_key=self.target_key, - target_name=self.target_name, - target_host=self.target_host, - severity=severity, - message=event.get("Message", ""), - message_id=event.get("MessageId"), - event_type=event_type, - origin_of_condition=origin_uri, - event_timestamp=parse_redfish_timestamp(event.get("EventTimestamp")), - received_at=datetime.now(UTC), - source=EventSource.SSE, - raw=dict(event), - ) - self._last_event_time = datetime.now(UTC) - if self.callback: - self.callback(redfish_event) - - def _classify_error(self, error: Exception) -> tuple[ErrorCategory, str]: - error_str = str(error).lower() - if isinstance(error, httpx.HTTPStatusError): - status = error.response.status_code - if status in (401, 403, 404, 405, 501): - return ErrorCategory.PERMANENT, f"HTTP {status}" - if isinstance(error, RuntimeError) and "http" in error_str: - for code in (401, 403, 404, 405, 501): - if f"http {code}" in error_str: - return ErrorCategory.PERMANENT, f"HTTP {code}" - if "closed after" in error_str and "without sending events" in error_str: - return ErrorCategory.PERMANENT, "Invalid SSE endpoint" - if isinstance(error, httpx.TimeoutException | asyncio.TimeoutError): - return ErrorCategory.TRANSIENT, "Connection timeout" - if isinstance(error, httpx.ConnectError | OSError): - return ErrorCategory.TRANSIENT, "Network connection error" - return ErrorCategory.TRANSIENT, f"{type(error).__name__}: {error}" - - def _calculate_backoff_delay(self) -> int: - delays = [30, 60, 120, 300, 600, 1800, 3600, 7200] - idx = min(max(self._consecutive_failures - 1, 0), len(delays) - 1) - base = delays[idx] - return int(base * random.uniform(0.8, 1.2)) - - @property - def state(self) -> SubscriptionState: - return self._state - - @property - def is_running(self) -> bool: - return self._running - - @property - def last_event_time(self) -> Optional[datetime]: - return self._last_event_time - - @property - def failure_reason(self) -> Optional[str]: - return self._failure_reason diff --git a/nodescraper/redfish_events/subscriber_manager.py b/nodescraper/redfish_events/subscriber_manager.py deleted file mode 100644 index a7f2b972..00000000 --- a/nodescraper/redfish_events/subscriber_manager.py +++ /dev/null @@ -1,284 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Manage continuous Redfish event subscriptions for multiple BMC targets.""" -from __future__ import annotations - -import asyncio -import contextlib -import inspect -import logging -from collections import OrderedDict -from ipaddress import ip_address -from typing import Optional -from urllib.parse import urlparse - -from .config import EventStreamConfig, EventTargetConfig -from .log_baseline import pull_baseline_events -from .models import EventCallback, RedfishEvent, SubscriptionState, TransportMode -from .sse_capability import SSESupport, check_sse_capability -from .sse_subscriber import SseEventSubscriber -from .webhook_subscriber import WebhookEventSubscriber, WebhookFailureType - -logger = logging.getLogger(__name__) - - -def _webhook_unreachable(url: str) -> tuple[bool, str]: - try: - parsed = urlparse(url) - except Exception as exc: # noqa: BLE001 - return True, f"unparseable URL: {exc}" - if not parsed.scheme or not parsed.hostname: - return True, "missing scheme or host" - host = parsed.hostname - if host in {"localhost", "0.0.0.0"}: - return True, f"host '{host}' is unreachable from a BMC" - try: - ip = ip_address(host) - if ip.is_loopback or ip.is_unspecified: - return True, f"host '{host}' is loopback/unspecified" - except ValueError: - pass - return False, "" - - -class SubscriberManager: - """Start SSE or webhook ingest for configured targets and dispatch events.""" - - def __init__( - self, - config: EventStreamConfig, - on_event: EventCallback, - ) -> None: - self.config = config - self.on_event = on_event - self._targets: dict[str, EventTargetConfig] = {} - self._sse: dict[str, SseEventSubscriber] = {} - self._webhooks: dict[str, WebhookEventSubscriber] = {} - self._transport: dict[str, str] = {} - self._running = False - self._baseline_task: Optional[asyncio.Task] = None - self._dedupe: OrderedDict[str, None] = OrderedDict() - - def set_targets(self, targets: list[EventTargetConfig]) -> None: - self._targets = {target.target_key: target for target in targets} - - async def start(self) -> None: - if self._running: - return - self._running = True - for target in self._targets.values(): - await self._start_target(target) - if self.config.baseline_repull_interval_minutes > 0: - self._baseline_task = asyncio.create_task(self._baseline_repull_loop()) - logger.info("SubscriberManager started for %d target(s)", len(self._targets)) - - async def stop(self) -> None: - self._running = False - if self._baseline_task: - self._baseline_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._baseline_task - self._baseline_task = None - for key, subscriber in list(self._sse.items()): - await subscriber.stop() - del self._sse[key] - for key, webhook in list(self._webhooks.items()): - await webhook.delete_subscription() - del self._webhooks[key] - self._transport.clear() - logger.info("SubscriberManager stopped") - - async def handle_webhook_payload( - self, - target_key: str, - payload: dict, - ) -> int: - webhook = self._webhooks.get(target_key) - if webhook is None: - logger.warning("Webhook payload for unknown target_key=%s", target_key) - return 0 - emitted = 0 - for event in webhook.parse_webhook_payload(payload): - if await self._dispatch(event): - emitted += 1 - return emitted - - def subscription_state(self, target_key: str) -> Optional[SubscriptionState]: - subscriber = self._sse.get(target_key) - if subscriber is not None: - return subscriber.state - if target_key in self._webhooks: - return SubscriptionState.CONNECTED - return None - - def transport_for(self, target_key: str) -> Optional[str]: - return self._transport.get(target_key) - - def target_keys(self) -> list[str]: - """Return configured target keys in registration order.""" - return list(self._targets.keys()) - - async def _start_target(self, target: EventTargetConfig) -> None: - if self.config.baseline_pull_enabled: - await pull_baseline_events( - target_key=target.target_key, - target_name=target.name, - target_host=target.host, - base_url=target.resolved_base_url(), - username=target.username, - password=target.password, - verify_ssl=target.verify_ssl, - callback=self._baseline_callback, - severities=self.config.severities, - max_entries_per_log=self.config.baseline_max_entries_per_log, - ) - - mode = target.transport - if mode == TransportMode.AUTO: - cap = await check_sse_capability( - target.resolved_base_url(), - target.username, - target.password, - target.verify_ssl, - ) - if cap.support == SSESupport.SUPPORTED: - mode = TransportMode.SSE - elif target.webhook_url and self.config.enable_webhook_fallback: - mode = TransportMode.WEBHOOK - else: - logger.warning( - "No supported event transport for %s: %s", - target.name, - cap.reason, - ) - return - - if mode == TransportMode.SSE: - subscriber = SseEventSubscriber( - target_key=target.target_key, - target_name=target.name, - target_host=target.host, - base_url=target.resolved_base_url(), - username=target.username, - password=target.password, - sse_endpoint=target.sse_endpoint, - verify_ssl=target.verify_ssl, - callback=self._sse_callback, - reconnect_delay=self.config.reconnect_delay_seconds, - max_retry_duration_hours=self.config.max_retry_duration_hours, - cooldown_duration_hours=self.config.cooldown_duration_hours, - degraded_threshold_hours=self.config.degraded_threshold_hours, - event_types=self.config.event_types, - severities=self.config.severities, - ) - self._sse[target.target_key] = subscriber - self._transport[target.target_key] = "sse" - await subscriber.start() - return - - if mode == TransportMode.WEBHOOK: - if not target.webhook_url: - logger.error("Target %s requires webhook_url for webhook transport", target.name) - return - unreachable, reason = _webhook_unreachable(target.webhook_url) - if unreachable and not self.config.allow_loopback_webhook: - logger.error( - "Webhook URL for %s is unreachable from BMCs: %s", - target.name, - reason, - ) - return - webhook = WebhookEventSubscriber( - target_key=target.target_key, - target_name=target.name, - target_host=target.host, - base_url=target.resolved_base_url(), - username=target.username, - password=target.password, - webhook_url=target.webhook_url, - verify_ssl=target.verify_ssl, - event_types=self.config.event_types, - severities=self.config.severities, - ) - result = await webhook.create_subscription() - if not result.success: - level = ( - logging.ERROR - if result.failure_type == WebhookFailureType.PERMANENT - else logging.WARNING - ) - logger.log( - level, - "Webhook subscription failed for %s: %s", - target.name, - result.error_message, - ) - return - self._webhooks[target.target_key] = webhook - self._transport[target.target_key] = "webhook" - - def _sse_callback(self, event: RedfishEvent) -> None: - asyncio.create_task(self._dispatch(event)) - - def _baseline_callback(self, event: RedfishEvent) -> None: - asyncio.create_task(self._dispatch(event)) - - async def _dispatch(self, event: RedfishEvent) -> bool: - if self.config.dedupe_events: - key = f"{event.target_key}:{event.dedupe_key()}" - if key in self._dedupe: - return False - self._dedupe[key] = None - while len(self._dedupe) > self.config.max_dedupe_entries: - self._dedupe.popitem(last=False) - - result = self.on_event(event) - if inspect.isawaitable(result): - await result - return True - - async def _baseline_repull_loop(self) -> None: - interval = self.config.baseline_repull_interval_minutes * 60 - while self._running: - try: - await asyncio.sleep(interval) - for target in self._targets.values(): - await pull_baseline_events( - target_key=target.target_key, - target_name=target.name, - target_host=target.host, - base_url=target.resolved_base_url(), - username=target.username, - password=target.password, - verify_ssl=target.verify_ssl, - callback=self._baseline_callback, - severities=self.config.severities, - max_entries_per_log=self.config.baseline_max_entries_per_log, - ) - except asyncio.CancelledError: - break - except Exception as exc: # noqa: BLE001 - logger.warning("Baseline re-pull loop error: %s", exc) diff --git a/nodescraper/redfish_events/trigger_engine.py b/nodescraper/redfish_events/trigger_engine.py deleted file mode 100644 index d4d2e46b..00000000 --- a/nodescraper/redfish_events/trigger_engine.py +++ /dev/null @@ -1,112 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Sliding-window trigger logic for batched serviceability analysis.""" -from __future__ import annotations - -import asyncio -import inspect -import logging -from collections import deque -from collections.abc import Awaitable, Callable -from dataclasses import dataclass, field -from datetime import UTC, datetime, timedelta -from typing import Optional, Union - -from pydantic import BaseModel, Field - -from .models import RedfishEvent - -logger = logging.getLogger(__name__) - -TriggerCallback = Callable[[str, list[RedfishEvent]], Union[None, Awaitable[None]]] -NowFn = Callable[[], datetime] - - -class TriggerConfig(BaseModel): - """When to run serviceability analysis for a burst of ingest events.""" - - min_events: int = Field(default=3, ge=1, description="Events required within the window") - window_seconds: float = Field(default=10.0, gt=0, description="Sliding window size in seconds") - cooldown_seconds: float = Field( - default=60.0, - ge=0, - description="Minimum seconds between analysis runs for the same target", - ) - - -@dataclass -class _TargetWindow: - events: deque[tuple[datetime, RedfishEvent]] = field(default_factory=deque) - cooldown_until: Optional[datetime] = None - - -class TriggerEngine: - """Fire analysis when enough events arrive within a sliding time window.""" - - def __init__( - self, - config: TriggerConfig, - on_trigger: TriggerCallback, - *, - clock_fn: NowFn = lambda: datetime.now(UTC), - ) -> None: - self.config = config - self.on_trigger = on_trigger - self._now = clock_fn - self._targets: dict[str, _TargetWindow] = {} - self._lock = asyncio.Lock() - - async def handle_event(self, event: RedfishEvent) -> bool: - """Record one event and invoke on_trigger when the window threshold is met.""" - async with self._lock: - window = self._targets.setdefault(event.target_key, _TargetWindow()) - now = self._now() - if window.cooldown_until and now < window.cooldown_until: - return False - - window.events.append((now, event)) - cutoff = now - timedelta(seconds=self.config.window_seconds) - while window.events and window.events[0][0] < cutoff: - window.events.popleft() - - if len(window.events) < self.config.min_events: - return False - - batch = [item[1] for item in window.events] - window.events.clear() - if self.config.cooldown_seconds > 0: - window.cooldown_until = now + timedelta(seconds=self.config.cooldown_seconds) - - logger.info( - "Trigger fired for %s with %d event(s) in %.1fs window", - event.target_key, - len(batch), - self.config.window_seconds, - ) - result = self.on_trigger(event.target_key, batch) - if inspect.isawaitable(result): - await result - return True diff --git a/nodescraper/redfish_events/webhook_subscriber.py b/nodescraper/redfish_events/webhook_subscriber.py deleted file mode 100644 index dc128879..00000000 --- a/nodescraper/redfish_events/webhook_subscriber.py +++ /dev/null @@ -1,221 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Webhook-based Redfish event subscriptions (adapted from Gyanam).""" -from __future__ import annotations - -import logging -from dataclasses import dataclass -from datetime import UTC, datetime -from enum import Enum -from typing import Optional - -import httpx - -from .models import EventSource, RedfishEvent -from .parsing import normalize_severity, parse_redfish_timestamp, severity_allowed - -logger = logging.getLogger(__name__) - - -class WebhookFailureType(str, Enum): - TEMPORARY = "temporary" - PERMANENT = "permanent" - - -@dataclass -class WebhookSubscriptionResult: - success: bool - failure_type: Optional[WebhookFailureType] = None - error_message: Optional[str] = None - - -class WebhookEventSubscriber: - """Create and parse Redfish webhook subscriptions for one target.""" - - def __init__( - self, - *, - target_key: str, - target_name: str, - target_host: str, - base_url: str, - username: str, - password: str, - webhook_url: str, - verify_ssl: bool = False, - event_types: Optional[list[str]] = None, - severities: Optional[list[str]] = None, - ) -> None: - self.target_key = target_key - self.target_name = target_name - self.target_host = target_host - self.base_url = base_url.rstrip("/") - self.username = username - self.password = password - self.webhook_url = webhook_url - self.verify_ssl = verify_ssl - self.event_types = event_types or ["Alert", "StatusChange"] - self.severities = severities or ["Warning", "Critical"] - self._subscription_id: Optional[str] = None - self._subscription_url: Optional[str] = None - - async def create_subscription(self) -> WebhookSubscriptionResult: - payload = { - "Destination": self.webhook_url, - "Protocol": "Redfish", - "EventTypes": self.event_types, - "Context": self.target_key, - } - try: - async with httpx.AsyncClient( - auth=httpx.BasicAuth(self.username, self.password), - verify=self.verify_ssl, - timeout=30.0, - ) as client: - response = await client.post( - f"{self.base_url}/redfish/v1/EventService/Subscriptions", - json=payload, - ) - if response.status_code in (200, 201): - data = response.json() - self._subscription_id = data.get("Id") - location = response.headers.get("Location") - if location: - self._subscription_url = location - elif self._subscription_id: - self._subscription_url = ( - f"{self.base_url}/redfish/v1/EventService/Subscriptions/" - f"{self._subscription_id}" - ) - return WebhookSubscriptionResult(success=True) - if response.status_code == 409: - await self._find_existing_subscription() - if self._subscription_id: - return WebhookSubscriptionResult(success=True) - if response.status_code in (400, 405, 501): - return WebhookSubscriptionResult( - success=False, - failure_type=WebhookFailureType.PERMANENT, - error_message=response.text[:200], - ) - return WebhookSubscriptionResult( - success=False, - failure_type=WebhookFailureType.TEMPORARY, - error_message=f"HTTP {response.status_code}", - ) - except (httpx.ConnectError, httpx.TimeoutException) as exc: - return WebhookSubscriptionResult( - success=False, - failure_type=WebhookFailureType.TEMPORARY, - error_message=str(exc), - ) - - async def delete_subscription(self) -> bool: - if not self._subscription_url: - return False - try: - async with httpx.AsyncClient( - auth=httpx.BasicAuth(self.username, self.password), - verify=self.verify_ssl, - timeout=10.0, - ) as client: - response = await client.delete(self._subscription_url) - if response.status_code in (200, 204, 404): - self._subscription_id = None - self._subscription_url = None - return True - except Exception as exc: # noqa: BLE001 - logger.debug("Webhook delete failed for %s: %s", self.target_name, exc) - return False - - async def _find_existing_subscription(self) -> None: - try: - async with httpx.AsyncClient( - auth=httpx.BasicAuth(self.username, self.password), - verify=self.verify_ssl, - timeout=10.0, - ) as client: - response = await client.get( - f"{self.base_url}/redfish/v1/EventService/Subscriptions" - ) - if response.status_code != 200: - return - for member in response.json().get("Members", []): - ref = member.get("@odata.id") - if not ref: - continue - detail = await client.get(f"{self.base_url}{ref}") - if detail.status_code != 200: - continue - data = detail.json() - if data.get("Destination") == self.webhook_url: - self._subscription_id = ref.rstrip("/").split("/")[-1] - self._subscription_url = f"{self.base_url}{ref}" - return - except Exception as exc: # noqa: BLE001 - logger.debug("Webhook lookup failed for %s: %s", self.target_name, exc) - - def parse_webhook_payload(self, event_data: dict) -> list[RedfishEvent]: - events = event_data.get("Events") - if not isinstance(events, list): - events = ( - [event_data] if (event_data.get("MessageId") or event_data.get("Message")) else [] - ) - parsed: list[RedfishEvent] = [] - for event in events: - if not isinstance(event, dict): - continue - event_type = event.get("EventType") or "" - severity, sev_present = normalize_severity(event) - if event_type and self.event_types and event_type not in self.event_types: - continue - if not severity_allowed(severity, sev_present, self.severities): - continue - if not event_type: - event_type = "Alert" - origin = event.get("OriginOfCondition", {}) - origin_uri = origin.get("@odata.id") if isinstance(origin, dict) else origin - parsed.append( - RedfishEvent( - target_key=self.target_key, - target_name=self.target_name, - target_host=self.target_host, - severity=severity, - message=event.get("Message", ""), - message_id=event.get("MessageId"), - event_type=event_type, - origin_of_condition=origin_uri if isinstance(origin_uri, str) else None, - event_timestamp=parse_redfish_timestamp(event.get("EventTimestamp")), - received_at=datetime.now(UTC), - source=EventSource.WEBHOOK, - raw=dict(event), - ) - ) - return parsed - - @property - def is_subscribed(self) -> bool: - return self._subscription_id is not None diff --git a/pyproject.toml b/pyproject.toml index 245548de..d2f1bdef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,6 @@ dependencies = [ ] [project.optional-dependencies] -events = [ - "httpx>=0.27.0", -] dev = [ "build", "black", @@ -47,7 +44,6 @@ dev = [ "types-paramiko", "types-requests", "types-setuptools", - "httpx>=0.27.0", ] [project.urls] @@ -93,13 +89,5 @@ python_version = "3.9" mypy_path = ["test/unit"] explicit_package_bases = true -[[tool.mypy.overrides]] -module = "httpx" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "httpx.*" -ignore_missing_imports = true - [tool.setuptools_scm] version_scheme = "post-release" diff --git a/test/unit/plugins/test_analysis_window.py b/test/unit/plugins/test_analysis_window.py deleted file mode 100644 index 63465f9b..00000000 --- a/test/unit/plugins/test_analysis_window.py +++ /dev/null @@ -1,54 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from nodescraper.plugins.serviceability.analysis_window import ( - analyze_serviceability_window, -) -from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs -from nodescraper.plugins.serviceability.serviceability_data import ( - ServiceabilityDataModel, -) - - -def test_analyze_serviceability_window_skip_hub(): - data = ServiceabilityDataModel( - rf_events=[ - { - "MessageId": "X", - "Message": "fail", - "Severity": "Critical", - "Created": "2026-01-01T00:00:00Z", - "Afid": 1, - "serviceable_unit": "GPU0", - } - ] - ) - result = analyze_serviceability_window( - data, - ServiceabilityAnalyzerArgs(skip_hub=True), - ) - assert result.ok is True - assert result.serviceability is not None - assert len(result.afid_events) == 1 diff --git a/test/unit/redfish_events/test_daemon_flow.py b/test/unit/redfish_events/test_daemon_flow.py deleted file mode 100644 index 3b44049e..00000000 --- a/test/unit/redfish_events/test_daemon_flow.py +++ /dev/null @@ -1,138 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import asyncio -import json -from datetime import UTC, datetime, timedelta - -import pytest - -from nodescraper.redfish_events.daemon_config import DaemonConfig, load_daemon_config -from nodescraper.redfish_events.models import EventSource, RedfishEvent -from nodescraper.redfish_events.trigger_engine import TriggerConfig, TriggerEngine - - -def _event(target_key: str = "bmc-1", message: str = "evt") -> RedfishEvent: - return RedfishEvent( - target_key=target_key, - target_name="node-a", - target_host="10.0.0.1", - severity="Warning", - message=message, - event_type="Alert", - received_at=datetime.now(UTC), - source=EventSource.SSE, - ) - - -def test_trigger_engine_fires_after_min_events_in_window(): - fired: list[tuple[str, list[RedfishEvent]]] = [] - times = [datetime(2026, 1, 1, tzinfo=UTC)] - - def clock_fn() -> datetime: - return times[0] - - async def on_trigger(target_key: str, events: list[RedfishEvent]) -> None: - fired.append((target_key, list(events))) - - engine = TriggerEngine( - TriggerConfig(min_events=2, window_seconds=10, cooldown_seconds=0), - on_trigger, - clock_fn=clock_fn, - ) - - async def _run() -> None: - assert await engine.handle_event(_event(message="a")) is False - times[0] += timedelta(seconds=1) - assert await engine.handle_event(_event(message="b")) is True - - asyncio.run(_run()) - assert len(fired) == 1 - assert fired[0][0] == "bmc-1" - assert [event.message for event in fired[0][1]] == ["a", "b"] - - -def test_trigger_engine_respects_cooldown(): - fired: list[str] = [] - times = [datetime(2026, 1, 1, tzinfo=UTC)] - - def clock_fn() -> datetime: - return times[0] - - async def on_trigger(target_key: str, events: list[RedfishEvent]) -> None: - fired.append(target_key) - - engine = TriggerEngine( - TriggerConfig(min_events=1, window_seconds=10, cooldown_seconds=30), - on_trigger, - clock_fn=clock_fn, - ) - - async def _run() -> None: - assert await engine.handle_event(_event()) is True - times[0] += timedelta(seconds=5) - assert await engine.handle_event(_event()) is False - times[0] += timedelta(seconds=30) - assert await engine.handle_event(_event()) is True - - asyncio.run(_run()) - assert fired == ["bmc-1", "bmc-1"] - - -def test_load_daemon_config_example(tmp_path): - payload = { - "targets": [ - { - "target_key": "node-a", - "name": "Node A", - "host": "10.0.0.1", - "username": "root", - "password": "secret", - } - ], - "analysis": { - "hub_python_module": "hub.mod", - "afid_sag_path": "/tmp/AFID_SAG.json", - }, - } - path = tmp_path / "daemon.json" - path.write_text(json.dumps(payload), encoding="utf-8") - config = load_daemon_config(path) - assert isinstance(config, DaemonConfig) - assert config.targets[0].target_key == "node-a" - assert config.trigger.min_events == 3 - - -def test_daemon_config_requires_targets(): - with pytest.raises(ValueError, match="At least one target"): - DaemonConfig.model_validate( - { - "targets": [], - "analysis": { - "hub_python_module": "hub.mod", - "afid_sag_path": "/tmp/AFID_SAG.json", - }, - } - ) diff --git a/test/unit/redfish_events/test_daemon_http.py b/test/unit/redfish_events/test_daemon_http.py deleted file mode 100644 index 0e18823e..00000000 --- a/test/unit/redfish_events/test_daemon_http.py +++ /dev/null @@ -1,88 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import asyncio -import json -from typing import cast -from unittest.mock import AsyncMock, MagicMock - -from nodescraper.redfish_events.daemon_config import HttpServerConfig -from nodescraper.redfish_events.daemon_http import DaemonHttpServer -from nodescraper.redfish_events.models import SubscriptionState - - -def test_daemon_http_status_and_webhook_post(): - asyncio.run(_run()) - - -async def _run() -> None: - manager = MagicMock() - manager.target_keys.return_value = ["node-a"] - manager.transport_for.return_value = "webhook" - manager.subscription_state.return_value = SubscriptionState.CONNECTED - manager.handle_webhook_payload = AsyncMock(return_value=1) - - recommendations: dict = {} - server = DaemonHttpServer( - manager, - recommendations, - HttpServerConfig.model_construct(host="127.0.0.1", port=0), - ) - await server.start() - assert server._server is not None - bound_server = cast(asyncio.Server, server._server) - port = bound_server.sockets[0].getsockname()[1] - - reader, writer = await asyncio.open_connection("127.0.0.1", port) - writer.write(b"GET /status HTTP/1.1\r\nHost: localhost\r\n\r\n") - await writer.drain() - response = await reader.readuntil(b"\r\n\r\n") - body = await reader.read() - writer.close() - await writer.wait_closed() - assert b"200 OK" in response - payload = json.loads(body.decode("utf-8")) - assert payload["status"] == "ok" - assert "node-a" in payload["targets"] - - reader, writer = await asyncio.open_connection("127.0.0.1", port) - body_bytes = json.dumps({"Message": "hot", "MessageSeverity": "Critical"}).encode("utf-8") - request = ( - b"POST /hook/node-a HTTP/1.1\r\n" - b"Host: localhost\r\n" - + f"Content-Length: {len(body_bytes)}\r\n\r\n".encode("utf-8") - + body_bytes - ) - writer.write(request) - await writer.drain() - response = await reader.readuntil(b"\r\n\r\n") - body = await reader.read() - writer.close() - await writer.wait_closed() - assert b"200 OK" in response - assert json.loads(body.decode("utf-8"))["events_emitted"] == 1 - manager.handle_webhook_payload.assert_awaited_once() - - await server.stop() From f3397106d3645069605a44a4bbe9df2c59b4b202 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 13:37:41 -0500 Subject: [PATCH 14/21] updated to ignore the bank and only pick up the actual matched line --- .../plugins/inband/dmesg/dmesg_analyzer.py | 12 +- nodescraper/plugins/inband/dmesg/mce_utils.py | 138 ++++++------------ test/unit/plugin/test_dmesg_analyzer.py | 87 +++++++++-- test/unit/plugin/test_mce_utils.py | 61 +++++++- 4 files changed, 185 insertions(+), 113 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 04c8737b..5ae53f77 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -38,8 +38,8 @@ from .mce_utils import ( compile_mce_ce_status_regex, compile_mce_uc_status_regex, - ignored_mce_block_line_indices, - mce_block_all_line_indices, + mce_known_regex_skip_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, trim_mce_status_match_content, @@ -718,8 +718,8 @@ def analyze_data( dmesg_content = data.dmesg_content ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) - ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) - mce_block_lines = mce_block_all_line_indices(dmesg_content) + known_skip_lines = mce_known_regex_skip_line_indices(dmesg_content, ignore_mce_banks) + unknown_skip_lines = mce_unknown_suppress_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -728,7 +728,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=ignored_mce_block_lines, + skip_line_indices=known_skip_lines, ) for event in known_err_events: if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"): @@ -764,7 +764,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=mce_block_lines, + skip_line_indices=unknown_skip_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index a8eeccf3..55222aaa 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -35,29 +35,6 @@ _MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE) _MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:") -_CORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+correctable hardware errors detected in total in (?P\w+) block" - r"(?:\s+on\s+(?PCPU:?\d+))?", - re.IGNORECASE, -) - -_UNCORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+uncorrectable hardware errors detected in (?P\w+) block", - re.IGNORECASE, -) - -_GPU_CORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+correctable hardware errors detected in total in " - r"(?P\w+) block", - re.IGNORECASE, -) - -_GPU_UNCORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+uncorrectable hardware errors detected in " - r"(?P\w+) block", - re.IGNORECASE, -) - _MCE_CE_STATUS_RE = re.compile( r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\]", re.IGNORECASE, @@ -116,33 +93,6 @@ def _add_count(counts: dict[str, int], part: str, amount: int) -> None: counts[part] = counts.get(part, 0) + amount -def _part_label( - *, - cpu: Optional[str] = None, - block: Optional[str] = None, - bdf: Optional[str] = None, - gpu_index: Optional[int] = None, -) -> str: - if bdf is not None: - block_suffix = f"/{block}" if block else "" - if gpu_index is not None: - return f"GPU{gpu_index}{block_suffix}" - return f"GPU {bdf}{block_suffix}" - if cpu and block: - return f"{cpu}/{block}" - if cpu: - return cpu - if block: - return block - return "unknown" - - -def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int: - if bdf not in bdf_order: - bdf_order.append(bdf) - return bdf_order.index(bdf) - - def _is_mce_primary_starter(line: str) -> bool: return _MCE_PRIMARY_START_RE.search(line) is not None @@ -192,6 +142,48 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: return False +def mce_defining_status_line_indices(content: str) -> frozenset[int]: + """Return line indices for MCn_STATUS rows that define a corrected or uncorrected MCE.""" + lines = content.splitlines() + indices: set[int] = set() + for index, line in enumerate(lines): + if _MCE_CE_STATUS_LINE_RE.search(line) or _MCE_UC_STATUS_LINE_RE.search(line): + indices.add(index) + return frozenset(indices) + + +def mce_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return every line index containing [Hardware Error]:.""" + lines = content.splitlines() + return frozenset(index for index, line in enumerate(lines) if _is_mce_detail_line(line)) + + +def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return [Hardware Error]: detail lines that are not defining MCn_STATUS CE/UC rows.""" + defining = mce_defining_status_line_indices(content) + return frozenset( + index for index in mce_hardware_error_line_indices(content) if index not in defining + ) + + +def mce_known_regex_skip_line_indices( + content: str, + ignore_banks: Optional[FrozenSet[int]] = None, +) -> frozenset[int]: + """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan.""" + ignored = ignore_banks or frozenset() + skipped = set(mce_non_status_hardware_error_line_indices(content)) + skipped.update(ignored_mce_block_line_indices(content, ignored)) + return frozenset(skipped) + + +def mce_unknown_suppress_line_indices(content: str) -> frozenset[int]: + """Suppress MCE block context and every [Hardware Error]: line from unknown scan.""" + suppressed = set(mce_block_all_line_indices(content)) + suppressed.update(mce_hardware_error_line_indices(content)) + return frozenset(suppressed) + + def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: """Return (start, end) line index ranges for MCE incident blocks. @@ -264,39 +256,14 @@ def parse_correctable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count correctable MCE / RAS hardware errors per component from dmesg text. - - Handles summary lines (for example ``mce: 3 correctable ... on CPU1``), - amdgpu block summaries, and per-event ``MCn_STATUS[|CE|]`` hardware error lines. - """ + """Count correctable MCE hardware errors per CPU from MCn_STATUS[|CE|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_CORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _CORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - cpu = summary_match.group("cpu") - part = _part_label( - cpu=_normalize_cpu_label(cpu) if cpu else None, - block=summary_match.group("block"), - ) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_CE_STATUS_RE.search(line) if status_match: @@ -314,31 +281,14 @@ def parse_uncorrectable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count uncorrectable MCE / RAS hardware errors per component from dmesg text.""" + """Count uncorrectable MCE hardware errors per CPU from MCn_STATUS[|UC|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_UNCORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _UNCORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - part = _part_label(block=summary_match.group("block")) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_UC_STATUS_RE.search(line) if status_match: diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 87477acf..9b009785 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1022,11 +1022,11 @@ def test_priority_override_updates_unkown_dmesg_error(system_info): assert res.events[0].priority == EventPriority.ERROR -def test_mce_threshold_raises_error_for_gpu(system_info): +def test_mce_threshold_raises_error_for_status_lines(system_info): dmesg_content = ( - "kern :err : 2024-10-07T10:17:15,145363-04:00 " - "amdgpu 0000:c1:00.0: amdgpu: socket: 4, die: 0 " - "3 correctable hardware errors detected in total in gfx block\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x2\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x3\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1038,7 +1038,7 @@ def test_mce_threshold_raises_error_for_gpu(system_info): threshold_events = [e for e in res.events if e.data.get("mce_threshold") == 3] assert len(threshold_events) == 1 assert threshold_events[0].priority == EventPriority.ERROR - assert threshold_events[0].data["part"] == "GPU0/gfx" + assert threshold_events[0].data["part"] == "CPU0" assert threshold_events[0].data["correctable_mce_count"] == 3 assert res.status == ExecutionStatus.ERROR @@ -1065,8 +1065,8 @@ def test_mce_threshold_raises_error_for_cpu_colon_status(system_info): def test_mce_threshold_not_triggered_below_limit(system_info): dmesg_content = ( - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " - "mce: 2 correctable hardware errors detected in total in mc0 block on CPU1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x2\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1260,7 +1260,11 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1270,8 +1274,14 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): ) descriptions = {event.description for event in res.events} + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + mce_events = [event for event in res.events if event.description == "MCE Corrected Error"] + assert unknown_events == [] + assert len(mce_events) == 1 + assert "MC60_STATUS" in str(mce_events[0].data["match_content"]) assert "Unknown dmesg error" not in descriptions - assert "MCE Corrected Error" in descriptions or "RAS Corrected Error" in descriptions + assert "MCE Corrected Error" in descriptions + assert "RAS Corrected Error" not in descriptions def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): @@ -1328,6 +1338,65 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" +def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): + """When analysis window omits MCn_STATUS, trailing detail lines must not become unknowns.""" + dmesg_content = ( + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs(check_unknown_dmesg_errors=True), + ) + + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "unrelated plugin failure" + + +def test_filtered_window_orphan_mce_tail_not_unknown(system_info): + """i198-maxcorestim: analysis window can start after MCn_STATUS but before IPID/cache tail.""" + dmesg_content = ( + "kern :info : 2026-07-14T22:21:10,254124-07:00 mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2026-07-14T22:21:10,266984-07:00 [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " + "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xd8202000000c080b\n" + "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + ) + analysis_range_start = datetime.datetime.fromisoformat( + "2026-07-14 22:21:10.314-07:00" + ).astimezone(datetime.timezone.utc) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + analysis_range_start=analysis_range_start, + ), + ) + + filtered_artifact = next( + artifact for artifact in res.artifacts if artifact.filename == "filtered_dmesg.log" + ) + assert "MC60_STATUS" not in filtered_artifact.contents + assert "IPID:" in filtered_artifact.contents + + descriptions = {event.description for event in res.events} + assert "Unknown dmesg error" not in descriptions + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + + def test_mce_match_content_is_single_status_line(system_info): """Adjacent incidents: events.json match_content must be one MCn_STATUS row.""" dmesg_content = ( diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index 2e60c73a..e62d6115 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -27,6 +27,10 @@ hardware_error_block_line_indices, ignored_mce_block_line_indices, iter_hardware_error_block_ranges, + mce_defining_status_line_indices, + mce_hardware_error_line_indices, + mce_non_status_hardware_error_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, trim_mce_status_match_content, @@ -43,7 +47,7 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): counts = parse_correctable_mce_counts(content) - assert counts == {"CPU1/mc0": 3, "CPU0": 1} + assert counts == {"CPU0": 1} def test_parse_correctable_mce_counts_gpu_summary(): @@ -55,7 +59,7 @@ def test_parse_correctable_mce_counts_gpu_summary(): counts = parse_correctable_mce_counts(content) - assert counts == {"GPU0/gfx": 3} + assert counts == {} def test_parse_uncorrectable_mce_counts(): @@ -67,7 +71,7 @@ def test_parse_uncorrectable_mce_counts(): counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU1": 1, "GPU0/gfx": 2} + assert counts == {"CPU1": 1} def test_parse_correctable_mce_counts_skips_ignored_banks(): @@ -192,6 +196,55 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) +def test_mce_non_status_hardware_error_line_indices(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + ) + + assert mce_defining_status_line_indices(content) == frozenset({1}) + assert mce_non_status_hardware_error_line_indices(content) == frozenset({0, 2}) + + +def test_mce_unknown_suppress_orphan_detail_lines(): + """Tail of an MCE block without the defining MCn_STATUS row still suppresses unknowns.""" + content = ( + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert 0 in suppressed + assert 1 in suppressed + assert 2 not in suppressed + + +def test_mce_unknown_suppress_status_and_tail_lines(): + """Status row plus PPIN/IPID/cache tail must never reach unknown dmesg matching.""" + content = ( + "kern :emerg : 2026-07-14T22:21:10,266984-07:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " + "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xabc\n" + "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert suppressed == frozenset({0, 1, 2, 3, 4}) + assert mce_hardware_error_line_indices(content) == frozenset({0, 1, 2, 3, 4}) + + def test_trim_mce_status_match_content_keeps_status_row_only(): multiline = ( "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" @@ -228,7 +281,7 @@ def test_parse_correctable_mce_counts_both_cpu_formats(): counts = parse_correctable_mce_counts(content) - assert counts == {"CPU0": 1, "CPU72": 1, "CPU1/mc0": 2} + assert counts == {"CPU0": 1, "CPU72": 1} def test_parse_uncorrectable_mce_counts_cpu_colon_status(): From ac1137f96b558802525b3d611b8b77ef7f4fe3cc Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 14:57:17 -0500 Subject: [PATCH 15/21] cleanup --- test/unit/test_plugin_convention_warnings.py | 169 ------------------- 1 file changed, 169 deletions(-) delete mode 100644 test/unit/test_plugin_convention_warnings.py diff --git a/test/unit/test_plugin_convention_warnings.py b/test/unit/test_plugin_convention_warnings.py deleted file mode 100644 index e76ec733..00000000 --- a/test/unit/test_plugin_convention_warnings.py +++ /dev/null @@ -1,169 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import ast -import importlib.util -from pathlib import Path - -import pytest - -_REPO_ROOT = Path(__file__).resolve().parents[2] -_SCRIPT_PATH = _REPO_ROOT / ".github" / "scripts" / "plugin_convention_warnings.py" - - -def _load_checker(): - spec = importlib.util.spec_from_file_location("plugin_convention_warnings", _SCRIPT_PATH) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -checker = _load_checker() - - -def _check_source(src: str, rel_path: str = "example_collector.py") -> list[str]: - tree = ast.parse(src) - return checker._check_shell_quoting(Path(rel_path), tree) - - -def test_shell_quote_rule_flags_args_in_f_string(): - src = """ -class ExampleCollector: - def collect_data(self, args): - self._run_sut_cmd(f"ping {args.url}") -""" - msgs = _check_source(src) - assert len(msgs) == 1 - assert "shell_quote" in msgs[0] - assert "args.url" in msgs[0] - - -def test_shell_quote_rule_flags_unquoted_format_kwarg(): - src = """ -class ExampleCollector: - CMD_TMPL = "grep . -H -r -i {rocm_path}/.info/*" - - def collect_data(self, args): - self._run_sut_cmd(self.CMD_TMPL.format(rocm_path=args.rocm_path)) -""" - msgs = _check_source(src) - assert len(msgs) == 1 - assert "args.rocm_path" in msgs[0] - - -def test_shell_quote_rule_allows_quoted_format_kwarg(): - src = """ -class ExampleCollector: - CMD_TMPL = "grep . -H -r -i {rocm_path}/.info/*" - - def collect_data(self, args): - rocm_path_q = shell_quote(args.rocm_path) - self._run_sut_cmd(self.CMD_TMPL.format(rocm_path=rocm_path_q)) -""" - assert _check_source(src) == [] - - -def test_shell_quote_rule_allows_inline_shell_quote(): - src = """ -class ExampleCollector: - def _get_afid(self, cper_file_path): - cmd = self.CMD_RAS_AFID.format(cper_file=shell_quote(cper_file_path)) - self._run_amd_smi(cmd) -""" - assert _check_source(src) == [] - - -def test_shell_quote_rule_flags_sensitive_function_parameter(): - src = """ -class ExampleCollector: - def _probe(self, url): - self._run_sut_cmd(f"curl {url}") -""" - msgs = _check_source(src) - assert len(msgs) == 1 - assert "url" in msgs[0] - - -def test_shell_quote_rule_ignores_internal_cmd_parameter(): - src = """ -class ExampleCollector: - def _run_amd_smi(self, cmd): - self._run_sut_cmd(f"{self.AMD_SMI_EXE} {cmd}") -""" - assert _check_source(src) == [] - - -def test_shell_quote_rule_flags_reassigned_cmd_variable(): - src = """ -class ExampleCollector: - CMD = "journalctl --no-pager" - - def read(self, args): - cmd = self.CMD - if args is not None and args.boot: - cmd = f"{self.CMD} -b {args.boot}" - self._run_sut_cmd(cmd, sudo=True) -""" - msgs = _check_source(src) - assert len(msgs) == 1 - assert "args.boot" in msgs[0] - - -def test_shell_quote_rule_skips_generic_collection_collector(): - src = """ -class GenericCollectionCollector: - def collect_data(self, args): - for cmd_spec in args.commands: - command = cmd_spec.command.strip() - self._run_sut_cmd(command) -""" - assert _check_source(src) == [] - - -def test_shell_quote_rule_allows_system_derived_values(): - src = """ -class ExampleCollector: - def collect(self, iface): - cmd = self.CMD_ETHTOOL_TEMPLATE.format(interface=iface.name) - self._run_sut_cmd(cmd, sudo=True) -""" - assert _check_source(src) == [] - - -@pytest.mark.parametrize( - "collector_file", - [ - "nodescraper/plugins/inband/network/network_collector.py", - "nodescraper/plugins/inband/rocm/rocm_collector.py", - "nodescraper/plugins/inband/amdsmi/amdsmi_collector.py", - ], -) -def test_fixed_collectors_have_no_shell_quote_warnings(collector_file): - path = _REPO_ROOT / collector_file - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - rel = path.relative_to(_REPO_ROOT) - msgs = checker._check_shell_quoting(rel, tree) - assert msgs == [] From b718ea95dca6151f85bdb71302e6114fbc8268db Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 15:13:03 -0500 Subject: [PATCH 16/21] utest dummies --- test/unit/plugin/test_dmesg_analyzer.py | 32 ++++---- test/unit/plugin/test_mce_utils.py | 102 +++++++++++++----------- 2 files changed, 74 insertions(+), 60 deletions(-) diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 9b009785..784b5453 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1285,7 +1285,7 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): - """Dummy excerpt modeled on dmesg_mce_interleave.log: warn/blank lines inside MCE blocks.""" + """Dummy excerpt: warn/blank lines interleaved inside MCE hardware-error blocks.""" dmesg_content = ( "kern :info : 2038-01-19T00:00:00,000000+00:00 " "mce: [Hardware Error]: Machine check events logged\n" @@ -1341,11 +1341,11 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): """When analysis window omits MCn_STATUS, trailing detail lines must not become unknowns.""" dmesg_content = ( - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" - "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 unrelated plugin failure\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1360,20 +1360,22 @@ def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): def test_filtered_window_orphan_mce_tail_not_unknown(system_info): - """i198-maxcorestim: analysis window can start after MCn_STATUS but before IPID/cache tail.""" + """Analysis window can start after MCn_STATUS but before IPID/cache tail lines.""" dmesg_content = ( - "kern :info : 2026-07-14T22:21:10,254124-07:00 mce: [Hardware Error]: Machine check events logged\n" - "kern :emerg : 2026-07-14T22:21:10,266984-07:00 [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " - "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xd8202000000c080b\n" - "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :info : 2038-01-19T00:00:10,254124+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:10,266984+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:10,280615+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:10,303837+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:10,315164+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:10,335516+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) analysis_range_start = datetime.datetime.fromisoformat( - "2026-07-14 22:21:10.314-07:00" + "2038-01-19T00:00:10.314000+00:00" ).astimezone(datetime.timezone.utc) analyzer = DmesgAnalyzer(system_info=system_info) diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index e62d6115..4a3034e6 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -42,7 +42,7 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): "kern :warn : 2038-01-19T00:00:00,000000+00:00 " "mce: 3 correctable hardware errors detected in total in mc0 block on CPU1\n" "kern :warn : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xabc\n" + "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) @@ -65,7 +65,7 @@ def test_parse_correctable_mce_counts_gpu_summary(): def test_parse_uncorrectable_mce_counts(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xfeed|UC|AddrV]: 0x0\n" + "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xbbb|UC|AddrV]: 0x0\n" "amdgpu 0000:de:ad.0: amdgpu: socket: 0 2 uncorrectable hardware errors detected in gfx block\n" ) @@ -101,10 +101,12 @@ def test_parse_correctable_mce_counts_skips_ignored_banks_in_separate_block(): def test_hardware_error_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :info : ts unrelated line\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :info : 2038-01-19T00:00:03,000000+00:00 dummy: unrelated line\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2}) @@ -118,10 +120,10 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" - "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) lines = content.splitlines() @@ -133,11 +135,15 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): def test_hardware_error_block_includes_interleaved_non_mce_lines(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts amdgpu 0000:de:ad.0: amdgpu: unrelated reset notice\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :emerg : ts [Hardware Error]: cache level: L3/GEN\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: dummy unrelated reset notice\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 3, 4}) @@ -146,12 +152,16 @@ def test_hardware_error_block_includes_interleaved_non_mce_lines(): def test_ignored_mce_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "\n" - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :err : ts [Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :err : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0xccc\n" ) ignored = ignored_mce_block_line_indices(content, frozenset({60})) @@ -173,7 +183,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "\n" "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" @@ -182,7 +192,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " - "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" ) lines = content.splitlines() @@ -198,9 +208,11 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): def test_mce_non_status_hardware_error_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" ) assert mce_defining_status_line_indices(content) == frozenset({1}) @@ -210,11 +222,11 @@ def test_mce_non_status_hardware_error_line_indices(): def test_mce_unknown_suppress_orphan_detail_lines(): """Tail of an MCE block without the defining MCn_STATUS row still suppresses unknowns.""" content = ( - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" - "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" ) suppressed = mce_unknown_suppress_line_indices(content) @@ -227,16 +239,16 @@ def test_mce_unknown_suppress_orphan_detail_lines(): def test_mce_unknown_suppress_status_and_tail_lines(): """Status row plus PPIN/IPID/cache tail must never reach unknown dmesg matching.""" content = ( - "kern :emerg : 2026-07-14T22:21:10,266984-07:00 " + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " - "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xabc\n" - "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" - "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" ) suppressed = mce_unknown_suppress_line_indices(content) @@ -247,14 +259,14 @@ def test_mce_unknown_suppress_status_and_tail_lines(): def test_trim_mce_status_match_content_keeps_status_row_only(): multiline = ( - "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" + "[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" "[Hardware Error]: Corrected error, no action required.\n" - "[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" ) trimmed = trim_mce_status_match_content(multiline) - assert trimmed == ("[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") + assert trimmed == ("[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") assert "MC60_STATUS" not in trimmed assert "\n" not in trimmed @@ -262,34 +274,34 @@ def test_trim_mce_status_match_content_keeps_status_row_only(): def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} def test_parse_correctable_mce_counts_both_cpu_formats(): content = ( "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" "kern :warn : 2038-01-19T00:00:02,000000+00:00 " "mce: 2 correctable hardware errors detected in total in mc0 block on CPU:1\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU0": 1, "CPU72": 1} + assert counts == {"CPU0": 1, "CPU12": 1} def test_parse_uncorrectable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xdef\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xbbb\n" ) counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} From d30ecf4fc50ef46e0f7a85a3a17e40469d0723a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Jul 2026 00:20:23 +0000 Subject: [PATCH 17/21] docs: Update plugin documentation [automated] --- docs/PLUGIN_DOC.md | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index 8e0992c8..4351a912 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -17,14 +17,14 @@ | KernelPlugin | sh -c 'uname -a'
sh -c 'cat /proc/sys/kernel/numa_balancing'
wmic os get Version /Value | **Analyzer Args:**
- `exp_kernel`: Union[str, list] — Expected kernel version string(s) to match (e.g. from uname -a).
- `exp_numa`: Optional[int] — Expected value for kernel.numa_balancing (e.g. 0 or 1).
- `regex_match`: bool — If True, match exp_kernel as regex; otherwise exact match. | - | [KernelDataModel](#KernelDataModel-Model) | [KernelCollector](#Collector-Class-KernelCollector) | [KernelAnalyzer](#Data-Analyzer-Class-KernelAnalyzer) | | KernelModulePlugin | cat /proc/modules
modinfo amdgpu
wmic os get Version /Value | **Analyzer Args:**
- `kernel_modules`: dict[str, dict] — Expected kernel module name -> {version, etc.}. Analyzer checks collected modules match.
- `regex_filter`: list[str] — List of regex patterns to filter which collected modules are checked (default: amd). | - | [KernelModuleDataModel](#KernelModuleDataModel-Model) | [KernelModuleCollector](#Collector-Class-KernelModuleCollector) | [KernelModuleAnalyzer](#Data-Analyzer-Class-KernelModuleAnalyzer) | | MemoryPlugin | free -b
lsmem
numactl -H
wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value | **Analyzer Args:**
- `ratio`: float — Required free-memory ratio (0-1). Analysis fails if free/total < ratio.
- `memory_threshold`: str — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. | - | [MemoryDataModel](#MemoryDataModel-Model) | [MemoryCollector](#Collector-Class-MemoryCollector) | [MemoryAnalyzer](#Data-Analyzer-Class-MemoryAnalyzer) | -| NetworkPlugin | ip addr show
curl
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
rdma link -j
ip route show
ip rule show
wget | **Analyzer Args:**
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | +| NetworkPlugin | ip addr show
curl
ethtool -i {interface}
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
ip route show
ip rule show
wget | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | | NicPlugin | niccli --listdev
niccli --list
niccli --list_devices
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | | NvmePlugin | nvme smart-log {dev}
nvme error-log {dev} --log-entries=256
nvme id-ctrl {dev}
nvme id-ns {dev}{ns}
nvme fw-log {dev}
nvme self-test-log {dev}
nvme get-log {dev} --log-id=6 --log-len=512
nvme telemetry-log {dev} --output-file={dev}_{f_name}
nvme list -o json | - | - | [NvmeDataModel](#NvmeDataModel-Model) | [NvmeCollector](#Collector-Class-NvmeCollector) | - | | OsPlugin | sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/\*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | | PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/\*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | | PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | - | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | | ProcessPlugin | top -b -n 1
rocm-smi --showpids
top -b -n 1 -o %CPU | **Analyzer Args:**
- `max_kfd_processes`: int — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.
- `max_cpu_usage`: float — Maximum allowed CPU usage (percent) for process checks. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `top_n_process`: int — Number of top processes by CPU usage to collect (e.g. for top -b -n 1 -o %%CPU). | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | -| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | - | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | +| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors... | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | | RegexSearchPlugin | No collector step: reads local text from the CLI --data path (file or directory).
Directory scans load each file's contents into RegexSearchData for analysis. | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | | RocmPlugin | {rocm_path}/opencl/bin/\*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/\*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}\*
ls -v -d {rocm_path}-[3-7]\* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/\* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | | StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | **Analyzer Args:**
- `min_required_free_space_abs`: Optional[str] — Minimum required free space per mount (e.g. '10G', '1T').
- `min_required_free_space_prct`: Optional[int] — Minimum required free space as percentage of total (0–100).
- `ignore_devices`: Optional[list[str]] — Mount points or devices to exclude from free-space checks.
- `check_devices`: Optional[list[str]] — If non-empty, only these mount points or devices are checked.
- `regex_match`: bool — If True, match device/mount names with regex; otherwise exact match. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | @@ -442,7 +442,7 @@ Collect network configuration details using ip command - **CMD_NEIGHBOR**: `ip neighbor show` - **CMD_ETHTOOL_TEMPLATE**: `ethtool {interface}` - **CMD_ETHTOOL_S_TEMPLATE**: `ethtool -S {interface}` -- **CMD_RDMA_LINK_JSON**: `rdma link -j` +- **CMD_ETHTOOL_I_TEMPLATE**: `ethtool -i {interface}` - **CMD_PING**: `ping` - **CMD_WGET**: `wget` - **CMD_CURL**: `curl` @@ -457,13 +457,13 @@ NetworkDataModel - ip addr show - curl +- ethtool -i {interface} - ethtool -S {interface} - ethtool {interface} - lldpcli show neighbor - lldpctl - ip neighbor show - ping -- rdma link -j - ip route show - ip rule show - wget @@ -1533,8 +1533,8 @@ Complete network configuration data - **rules**: `List[nodescraper.plugins.inband.network.networkdata.RoutingRule]` - **neighbors**: `List[nodescraper.plugins.inband.network.networkdata.Neighbor]` - **ethtool_info**: `Dict[str, nodescraper.plugins.inband.network.networkdata.EthtoolInfo]` -- **rdma_ethtool_netdevs**: `List[str]` -- **rdma_ethtool_statistics**: `List[nodescraper.plugins.inband.network.ethtool_vendor.EthtoolStatistics]` +- **ethtool_netdevs**: `List[str]` +- **ethtool_statistics**: `List[nodescraper.plugins.inband.network.ethtool_vendor.EthtoolStatistics]` - **accessible**: `Optional[bool]` ## NicDataModel Model @@ -2133,10 +2133,6 @@ Check network statistics for errors. **Link to code**: [network_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/network/network_analyzer.py) -### Class Variables - -- **ERROR_REGEX**: `[]` - ## Data Analyzer Class NicAnalyzer ### Description @@ -2517,20 +2513,6 @@ Arguments for journal analyzer - **ratio**: `float` — Required free-memory ratio (0-1). Analysis fails if free/total < ratio. - **memory_threshold**: `str` — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. -## Analyzer Args Class NetworkAnalyzerArgs - -### Description - -Arguments for the network analyzer plugin. - -**Bases**: ['AnalyzerArgs'] - -**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/network/analyzer_args.py) - -### Annotations / fields - -- **error_regex**: `Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType]` — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. - ## Analyzer Args Class NicAnalyzerArgs ### Description From 24d3448f0c5041f94caccfeb4b91cf2182bc75e8 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 22 Jul 2026 10:18:54 -0500 Subject: [PATCH 18/21] cleanup --- nodescraper/base/inbandcollectortask.py | 38 +------------------ nodescraper/connection/inband/inband.py | 4 +- nodescraper/connection/inband/inbandlocal.py | 2 +- nodescraper/connection/inband/inbandremote.py | 2 +- .../inband/network/network_collector.py | 13 +++---- 5 files changed, 10 insertions(+), 49 deletions(-) diff --git a/nodescraper/base/inbandcollectortask.py b/nodescraper/base/inbandcollectortask.py index 21f6c239..25a3c55b 100644 --- a/nodescraper/base/inbandcollectortask.py +++ b/nodescraper/base/inbandcollectortask.py @@ -24,7 +24,7 @@ # ############################################################################### import logging -from typing import Generic, Optional, Sequence, Union +from typing import Generic, Optional, Union from nodescraper.connection.inband import InBandConnection from nodescraper.connection.inband.inband import BaseFileArtifact, CommandArtifact @@ -107,42 +107,6 @@ def _run_sut_cmd( return command_res - def _run_sut_argv( - self, - argv: Sequence[str], - sudo: bool = False, - timeout: int = 300, - strip: bool = True, - log_artifact: bool = True, - html_view: Optional[bool] = None, - ) -> CommandArtifact: - """Run a command via argv list (shell=False locally; quoted on remote SSH). - - Prefer this over _run_sut_cmd for commands that include config/user values. - Legacy shell strings (pipes, globs, redirection) still use _run_sut_cmd. - - Args: - argv (Sequence[str]): executable and arguments, e.g. ["ping", host, "-c", "1"]. - sudo (bool, optional): whether to run the command with sudo. Defaults to False. - timeout (int, optional): command timeout in seconds. Defaults to 300. - strip (bool, optional): whether output should be stripped. Defaults to True. - log_artifact (bool, optional): whether we should log the command result. Defaults to True. - html_view (Optional[bool], optional): whether to include this command in HTML - artifacts. When omitted, uses collection_args.html_view. - - Returns: - CommandArtifact: The result of the command execution. - """ - command_res = self.connection.run_command( - command=list(argv), sudo=sudo, timeout=timeout, strip=strip - ) - effective_html_view = self._effective_html_view(html_view) - if log_artifact or effective_html_view: - command_res.log_html = effective_html_view - self.result.artifacts.append(command_res) - - return command_res - def _read_sut_file( self, filename: str, encoding="utf-8", strip: bool = True, log_artifact=True ) -> BaseFileArtifact: diff --git a/nodescraper/connection/inband/inband.py b/nodescraper/connection/inband/inband.py index db082bcd..d9dfa041 100644 --- a/nodescraper/connection/inband/inband.py +++ b/nodescraper/connection/inband/inband.py @@ -162,9 +162,7 @@ def run_command( """Run an in band shell command. Args: - command (ShellCommand): full shell string (legacy) or argv list (preferred). - When a sequence is passed, collectors should use shell=False locally and - safely quoted execution remotely. + command (ShellCommand): shell command string, or argv tokens (quoted over SSH). sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/connection/inband/inbandlocal.py b/nodescraper/connection/inband/inbandlocal.py index f3590d6d..f53c1f14 100644 --- a/nodescraper/connection/inband/inbandlocal.py +++ b/nodescraper/connection/inband/inbandlocal.py @@ -47,7 +47,7 @@ def run_command( """Run a local in band shell command. Args: - command (ShellCommand): shell string (legacy) or argv list (preferred). + command (ShellCommand): shell command string, or argv tokens for shell=False execution. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/connection/inband/inbandremote.py b/nodescraper/connection/inband/inbandremote.py index cca2acea..e1bfdf98 100644 --- a/nodescraper/connection/inband/inbandremote.py +++ b/nodescraper/connection/inband/inbandremote.py @@ -137,7 +137,7 @@ def run_command( """Run a shell command over ssh. Args: - command (ShellCommand): shell string (legacy) or argv list (preferred). + command (ShellCommand): shell command string, or argv tokens (quoted for SSH). sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index c7cfe7e2..d2b913c8 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -32,7 +32,7 @@ from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult -from nodescraper.utils import get_exception_traceback +from nodescraper.utils import get_exception_traceback, shell_quote from .collector_args import NetworkCollectorArgs from .ethtool_vendor import ( @@ -729,17 +729,16 @@ def _check_network_connectivity(self, cmd: str, url: str) -> bool: f"Valid options are: 'ping', 'wget', 'curl'" ) - # Determine ping options based on OS + url_q = shell_quote(url) if cmd == "ping": if self.system_info.os_family == OSFamily.LINUX: - argv = [self.CMD_PING, url, "-c", "1"] + result = self._run_sut_cmd(f"{self.CMD_PING} {url_q} -c 1") else: - argv = [self.CMD_PING, url, "-n", "1"] - result = self._run_sut_argv(argv) + result = self._run_sut_cmd(f"{self.CMD_PING} {url_q} -n 1") elif cmd == "wget": - result = self._run_sut_argv([self.CMD_WGET, url]) + result = self._run_sut_cmd(f"{self.CMD_WGET} {url_q}") else: # curl - result = self._run_sut_argv([self.CMD_CURL, url]) + result = self._run_sut_cmd(f"{self.CMD_CURL} {url_q}") if result.exit_code == 0: self._log_event( From ae7ab84c8b0ca1cf772802be93fa32afbcf58d54 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 22 Jul 2026 10:54:20 -0500 Subject: [PATCH 19/21] cleanup --- nodescraper/connection/inband/inband.py | 6 +- nodescraper/connection/inband/inbandlocal.py | 36 +--------- nodescraper/connection/inband/inbandremote.py | 29 ++------ nodescraper/connection/inband/shellcommand.py | 70 ------------------- .../connection/inband/test_shellcommand.py | 47 +------------ 5 files changed, 14 insertions(+), 174 deletions(-) delete mode 100644 nodescraper/connection/inband/shellcommand.py diff --git a/nodescraper/connection/inband/inband.py b/nodescraper/connection/inband/inband.py index d9dfa041..291491b4 100644 --- a/nodescraper/connection/inband/inband.py +++ b/nodescraper/connection/inband/inband.py @@ -29,8 +29,6 @@ from pydantic import BaseModel -from .shellcommand import ShellCommand - class CommandArtifact(BaseModel): """Artifact for the result of shell command execution""" @@ -154,7 +152,7 @@ class InBandConnection(abc.ABC): @abc.abstractmethod def run_command( self, - command: ShellCommand, + command: str, sudo: bool = False, timeout: int = 300, strip: bool = True, @@ -162,7 +160,7 @@ def run_command( """Run an in band shell command. Args: - command (ShellCommand): shell command string, or argv tokens (quoted over SSH). + command (str): shell command string. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. diff --git a/nodescraper/connection/inband/inbandlocal.py b/nodescraper/connection/inband/inbandlocal.py index f53c1f14..66e655f9 100644 --- a/nodescraper/connection/inband/inbandlocal.py +++ b/nodescraper/connection/inband/inbandlocal.py @@ -25,21 +25,19 @@ ############################################################################### import os import subprocess -from typing import Sequence from .inband import ( BaseFileArtifact, CommandArtifact, InBandConnection, ) -from .shellcommand import ShellCommand, build_exec_argv, format_argv_display class LocalShell(InBandConnection): def run_command( self, - command: ShellCommand, + command: str, sudo: bool = False, timeout: int = 300, strip: bool = True, @@ -47,7 +45,7 @@ def run_command( """Run a local in band shell command. Args: - command (ShellCommand): shell command string, or argv tokens for shell=False execution. + command (str): shell command string. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. @@ -55,13 +53,6 @@ def run_command( Returns: CommandArtifact: command result object """ - if isinstance(command, str): - return self._run_shell_string(command, sudo=sudo, timeout=timeout, strip=strip) - return self._run_argv(command, sudo=sudo, timeout=timeout, strip=strip) - - def _run_shell_string( - self, command: str, *, sudo: bool, timeout: int, strip: bool - ) -> CommandArtifact: if sudo: command = f"sudo {command}" @@ -82,29 +73,6 @@ def _run_shell_string( exit_code=res.returncode, ) - def _run_argv( - self, argv: Sequence[str], *, sudo: bool, timeout: int, strip: bool - ) -> CommandArtifact: - exec_argv = build_exec_argv(argv, sudo=sudo) - display = format_argv_display(exec_argv) - - res = subprocess.run( - exec_argv, - encoding="utf-8", - shell=False, - errors="replace", - timeout=timeout, - capture_output=True, - check=False, - ) - - return CommandArtifact( - command=display, - stdout=res.stdout.strip() if strip else res.stdout, - stderr=res.stderr.strip() if strip else res.stderr, - exit_code=res.returncode, - ) - def read_file( self, filename: str, encoding: str = "utf-8", strip: bool = True ) -> BaseFileArtifact: diff --git a/nodescraper/connection/inband/inbandremote.py b/nodescraper/connection/inband/inbandremote.py index e1bfdf98..6b8dbf56 100644 --- a/nodescraper/connection/inband/inbandremote.py +++ b/nodescraper/connection/inband/inbandremote.py @@ -39,12 +39,6 @@ CommandArtifact, InBandConnection, ) -from .shellcommand import ( - ShellCommand, - build_exec_argv, - build_sudo_password_argv, - format_argv_display, -) from .sshparams import SSHConnectionParams @@ -129,7 +123,7 @@ def read_file( def run_command( self, - command: ShellCommand, + command: str, sudo=False, timeout: int = 30, strip: bool = True, @@ -137,7 +131,7 @@ def run_command( """Run a shell command over ssh. Args: - command (ShellCommand): shell command string, or argv tokens (quoted for SSH). + command (str): shell command string. sudo (bool, optional): run command with sudo (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. @@ -147,20 +141,11 @@ def run_command( """ write_password = sudo and self.ssh_params.username != "root" and self.ssh_params.password - if isinstance(command, str): - cmd_str = command - if write_password: - cmd_str = f"sudo -S -p '' {cmd_str}" - elif sudo: - cmd_str = f"sudo {cmd_str}" - else: - argv = list(command) - if write_password: - cmd_str = format_argv_display(build_sudo_password_argv(argv)) - elif sudo: - cmd_str = format_argv_display(build_exec_argv(argv, sudo=True)) - else: - cmd_str = format_argv_display(argv) + cmd_str = command + if write_password: + cmd_str = f"sudo -S -p '' {cmd_str}" + elif sudo: + cmd_str = f"sudo {cmd_str}" try: stdin, stdout, stderr = self.client.exec_command(cmd_str, timeout=timeout) diff --git a/nodescraper/connection/inband/shellcommand.py b/nodescraper/connection/inband/shellcommand.py deleted file mode 100644 index cc9ae7ac..00000000 --- a/nodescraper/connection/inband/shellcommand.py +++ /dev/null @@ -1,70 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Sequence, Union - -from nodescraper.utils import shell_quote - -ShellCommand = Union[str, Sequence[str]] - - -def format_argv_display(argv: Sequence[str]) -> str: - """Return a shell-safe, human-readable command string for logging/artifacts. - - Args: - argv (Sequence[str]): argv tokens passed to subprocess or remote shell. - - Returns: - str: quoted command string suitable for CommandArtifact.command. - """ - return " ".join(shell_quote(arg) for arg in argv) - - -def build_exec_argv(argv: Sequence[str], *, sudo: bool = False) -> list[str]: - """Prepend sudo when requested. - - Args: - argv (Sequence[str]): base command argv. - sudo (bool, optional): whether to run with sudo. Defaults to False. - - Returns: - list[str]: argv for local subprocess.run(shell=False) or remote quoting. - """ - base = list(argv) - if sudo: - return ["sudo", *base] - return base - - -def build_sudo_password_argv(argv: Sequence[str]) -> list[str]: - """Build argv for sudo -S (password on stdin) over SSH. - - Args: - argv (Sequence[str]): base command argv. - - Returns: - list[str]: argv prefixed with sudo -S -p ''. - """ - return ["sudo", "-S", "-p", "", *list(argv)] diff --git a/test/unit/connection/inband/test_shellcommand.py b/test/unit/connection/inband/test_shellcommand.py index ae625610..0a1b3fc2 100644 --- a/test/unit/connection/inband/test_shellcommand.py +++ b/test/unit/connection/inband/test_shellcommand.py @@ -26,47 +26,6 @@ from unittest.mock import MagicMock, patch from nodescraper.connection.inband.inbandlocal import LocalShell -from nodescraper.connection.inband.shellcommand import ( - build_exec_argv, - build_sudo_password_argv, - format_argv_display, -) - - -def test_format_argv_display_quotes_metacharacters(): - assert format_argv_display(["ping", "host; rm -rf /", "-c", "1"]) == ( - "'ping' 'host; rm -rf /' '-c' '1'" - ) - - -def test_build_exec_argv_prepends_sudo(): - assert build_exec_argv(["ip", "addr"], sudo=True) == ["sudo", "ip", "addr"] - - -def test_build_sudo_password_argv(): - assert build_sudo_password_argv(["journalctl", "-n", "1"]) == [ - "sudo", - "-S", - "-p", - "", - "journalctl", - "-n", - "1", - ] - - -@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") -def test_localshell_argv_uses_shell_false(mock_run): - mock_run.return_value = MagicMock(stdout="ok\n", stderr="", returncode=0) - - shell = LocalShell() - result = shell.run_command(["ping", "example.com", "-c", "1"]) - - mock_run.assert_called_once() - assert mock_run.call_args.kwargs["shell"] is False - assert mock_run.call_args.args[0] == ["ping", "example.com", "-c", "1"] - assert result.command == "'ping' 'example.com' '-c' '1'" - assert result.stdout == "ok" @patch("nodescraper.connection.inband.inbandlocal.subprocess.run") @@ -80,10 +39,10 @@ def test_localshell_string_uses_shell_true(mock_run): @patch("nodescraper.connection.inband.inbandlocal.subprocess.run") -def test_localshell_argv_with_sudo(mock_run): +def test_localshell_string_with_sudo(mock_run): mock_run.return_value = MagicMock(stdout="", stderr="", returncode=0) shell = LocalShell() - shell.run_command(["cat", "/etc/shadow"], sudo=True) + shell.run_command("cat /etc/shadow", sudo=True) - assert mock_run.call_args.args[0] == ["sudo", "cat", "/etc/shadow"] + assert mock_run.call_args.args[0] == "sudo cat /etc/shadow" From 9f08eb6d46b9948145605747b2deaf3d87c15378 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 22 Jul 2026 14:25:33 -0500 Subject: [PATCH 20/21] utest fix --- nodescraper/plugins/inband/dmesg/mce_utils.py | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 55222aaa..2b492bcf 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -55,6 +55,21 @@ re.IGNORECASE, ) +# MCE incident rows only +_MCE_INCIDENT_LINE_RE = re.compile( + r"\[Hardware Error\]:\s*(?:" + r"(?:Corrected error|Uncorrected error|Machine check events logged)|" + r"Machine Check:|" + r"CPU:?\d|" + r"MC\d+_STATUS|" + r"PPIN:|" + r"IPID:|" + r"Syndrome:|" + r"cache level:" + r")", + re.IGNORECASE, +) + def compile_mce_ce_status_regex() -> re.Pattern[str]: """Return a single-line regex for corrected MCn_STATUS hardware error rows.""" @@ -110,6 +125,10 @@ def _is_mce_detail_line(line: str) -> bool: return _MCE_DETAIL_LINE_RE.search(line) is not None +def _is_mce_incident_line(line: str) -> bool: + return _MCE_INCIDENT_LINE_RE.search(line) is not None + + def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]: return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} @@ -127,8 +146,8 @@ def _is_mce_status_only_starter(line: str) -> bool: return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None -def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: - """Return True when another [Hardware Error]: line appears before the next incident.""" +def _has_mce_incident_line_ahead(lines: Sequence[str], start_index: int) -> bool: + """Return True when another MCE incident [Hardware Error]: row appears before the next block.""" for idx in range(start_index + 1, len(lines)): line = lines[idx] if not line.strip(): @@ -137,8 +156,10 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: return False if _is_mce_status_only_starter(line): return False - if _is_mce_detail_line(line): + if _is_mce_incident_line(line): return True + if _is_mce_detail_line(line): + return False return False @@ -172,7 +193,12 @@ def mce_known_regex_skip_line_indices( ) -> frozenset[int]: """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan.""" ignored = ignore_banks or frozenset() - skipped = set(mce_non_status_hardware_error_line_indices(content)) + lines = content.splitlines() + defining = mce_defining_status_line_indices(content) + primary_starters = frozenset( + index for index, line in enumerate(lines) if _is_mce_primary_starter(line) + ) + skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters skipped.update(ignored_mce_block_line_indices(content, ignored)) return frozenset(skipped) @@ -190,9 +216,9 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) or at the first MCn_STATUS line when not already inside a block. The block then includes subsequent lines until the next primary header, a blank line before a bare - MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail, - or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail - entries belong to the same block. + MCn_STATUS starter, a non-MCE [Hardware Error]: row, trailing lines with no further + MCE incident detail, or EOF. Blank lines, warn/err noise, and other non-MCE dmesg + lines between MCE incident entries belong to the same block. """ blocks: list[tuple[int, int]] = [] index = 0 @@ -208,9 +234,11 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in next_line = _next_non_blank_line_index(lines, index) if next_line is not None and _is_mce_status_only_starter(lines[next_line]): break + elif _is_mce_detail_line(lines[index]) and not _is_mce_incident_line(lines[index]): + break elif _is_mce_block_starter(lines[index], in_block=True): break - elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead( + elif not _is_mce_incident_line(lines[index]) and not _has_mce_incident_line_ahead( lines, index ): break From ea69717338c024e2ea1e1854329e314a69682498 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 22 Jul 2026 14:49:38 -0500 Subject: [PATCH 21/21] fix for doubling down on same err --- nodescraper/plugins/inband/dmesg/mce_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 2b492bcf..9ec57246 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -187,6 +187,18 @@ def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]: ) +def _primary_starters_in_mce_status_blocks( + lines: Sequence[str], defining: frozenset[int] +) -> frozenset[int]: + """Return primary MCE header lines that belong to blocks with MCn_STATUS CE/UC rows.""" + primary_starters = {index for index, line in enumerate(lines) if _is_mce_primary_starter(line)} + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + if any(index in defining for index in range(start, end)): + suppressed.update(index for index in range(start, end) if index in primary_starters) + return frozenset(suppressed) + + def mce_known_regex_skip_line_indices( content: str, ignore_banks: Optional[FrozenSet[int]] = None, @@ -198,7 +210,9 @@ def mce_known_regex_skip_line_indices( primary_starters = frozenset( index for index, line in enumerate(lines) if _is_mce_primary_starter(line) ) + primary_in_status_blocks = _primary_starters_in_mce_status_blocks(lines, defining) skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters + skipped.update(primary_in_status_blocks) skipped.update(ignored_mce_block_line_indices(content, ignored)) return frozenset(skipped)